From 0b1b4e2d23b432b59b301ad2a91861a2d744baf9 Mon Sep 17 00:00:00 2001 From: Marti Bolivar Date: Thu, 10 Jan 2019 10:13:16 -0700 Subject: [PATCH] scripts: update west to upstream commit f01059a The main reason to copy into upstream is to bring in a new runner. Signed-off-by: Marti Bolivar --- scripts/meta/west/_bootstrap/main.py | 359 ++++++--- scripts/meta/west/_bootstrap/version.py | 2 +- scripts/meta/west/_bootstrap/west-schema.yml | 17 + scripts/meta/west/build.py | 4 +- scripts/meta/west/cmake.py | 4 +- scripts/meta/west/commands/build.py | 36 +- scripts/meta/west/commands/debug.py | 5 +- scripts/meta/west/commands/flash.py | 7 +- scripts/meta/west/commands/project.py | 761 +++++++++++++------ scripts/meta/west/commands/run_common.py | 43 +- scripts/meta/west/commands/schema.yml | 135 ---- scripts/meta/west/config.py | 95 +++ scripts/meta/west/log.py | 40 +- scripts/meta/west/main.py | 122 ++- scripts/meta/west/manifest-schema.yml | 132 ++++ scripts/meta/west/manifest.py | 400 ++++++++++ scripts/meta/west/runners/__init__.py | 29 +- scripts/meta/west/runners/arc.py | 4 +- scripts/meta/west/runners/blackmagicprobe.py | 96 +++ scripts/meta/west/runners/bossac.py | 4 +- scripts/meta/west/runners/core.py | 22 +- scripts/meta/west/runners/dfu.py | 10 +- scripts/meta/west/runners/esp32.py | 6 +- scripts/meta/west/runners/intel_s1000.py | 78 +- scripts/meta/west/runners/jlink.py | 18 +- scripts/meta/west/runners/nios2.py | 8 +- scripts/meta/west/runners/nrfjprog.py | 20 +- scripts/meta/west/runners/nsim.py | 19 +- scripts/meta/west/runners/openocd.py | 4 +- scripts/meta/west/runners/pyocd.py | 45 +- scripts/meta/west/runners/qemu.py | 2 +- scripts/meta/west/runners/xtensa.py | 4 +- 32 files changed, 1856 insertions(+), 675 deletions(-) create mode 100644 scripts/meta/west/_bootstrap/west-schema.yml delete mode 100644 scripts/meta/west/commands/schema.yml create mode 100644 scripts/meta/west/config.py create mode 100644 scripts/meta/west/manifest-schema.yml create mode 100644 scripts/meta/west/manifest.py create mode 100644 scripts/meta/west/runners/blackmagicprobe.py diff --git a/scripts/meta/west/_bootstrap/main.py b/scripts/meta/west/_bootstrap/main.py index d168177444a..6b835da4dda 100644 --- a/scripts/meta/west/_bootstrap/main.py +++ b/scripts/meta/west/_bootstrap/main.py @@ -6,10 +6,13 @@ ''' import argparse +import configparser import os import platform +import pykwalify.core import subprocess import sys +import yaml import west._bootstrap.version as version @@ -29,7 +32,7 @@ WEST_DIR = 'west' # Subdirectory to check out the west source repository into. WEST = 'west' # Default west repository URL. -WEST_DEFAULT = 'https://github.com/zephyrproject-rtos/west' +WEST_URL_DEFAULT = 'https://github.com/zephyrproject-rtos/west' # Default revision to check out of the west repository. WEST_REV_DEFAULT = 'master' # File inside of WEST_DIR which marks it as the top level of the @@ -43,10 +46,11 @@ WEST_MARKER = '.west_topdir' # Manifest repository directory under WEST_DIR. MANIFEST = 'manifest' # Default manifest repository URL. -MANIFEST_DEFAULT = 'https://github.com/zephyrproject-rtos/manifest' +MANIFEST_URL_DEFAULT = 'https://github.com/zephyrproject-rtos/manifest' # Default revision to check out of the manifest repository. MANIFEST_REV_DEFAULT = 'master' +_SCHEMA_PATH = os.path.join(os.path.dirname(__file__), "west-schema.yml") # # Helpers shared between init and wrapper mode @@ -61,13 +65,34 @@ class WestNotFound(WestError): '''Neither the current directory nor any parent has a West installation.''' -def find_west_topdir(start): - '''Find the top-level installation directory, starting at ``start``. +def west_dir(start=None): + ''' + Returns the path to the west/ directory, searching ``start`` and its + parents. - If none is found, raises WestNotFound.''' + Raises WestNotFound if no west directory is found. + ''' + return os.path.join(west_topdir(start), WEST_DIR) + + +def manifest_dir(start=None): + ''' + Returns the path to the manifest/ directory, searching ``start`` and its + parents. + + Raises WestNotFound if no west directory is found. + ''' + return os.path.join(west_topdir(start), MANIFEST) + + +def west_topdir(start=None): + ''' + Like west_dir(), but returns the path to the parent directory of the west/ + directory instead, where project repositories are stored + ''' # If you change this function, make sure to update west.util.west_topdir(). - cur_dir = start + cur_dir = start or os.getcwd() while True: if os.path.isfile(os.path.join(cur_dir, WEST_DIR, WEST_MARKER)): @@ -81,13 +106,11 @@ def find_west_topdir(start): cur_dir = parent_dir -def clone(url, rev, dest): +def clone(desc, url, rev, dest): if os.path.exists(dest): raise WestError('refusing to clone into existing location ' + dest) - if not url.startswith(('http:', 'https:', 'git:', 'git+shh:', 'file:')): - raise WestError('Unknown URL scheme for repository: {}'.format(url)) - + print('=== Cloning {} from {}, rev. {} ==='.format(desc, url, rev)) subprocess.check_call(('git', 'clone', '-b', rev, '--', url, dest)) @@ -100,49 +123,200 @@ def init(argv): '''Command line handler for ``west init`` invocations. This exits the program with a nonzero exit code if fatal errors occur.''' + + # Remember to update scripts/west-completion.bash if you add or remove + # flags + init_parser = argparse.ArgumentParser( prog='west init', - description='Bootstrap initialize a Zephyr installation') + formatter_class=argparse.RawDescriptionHelpFormatter, + description=''' +Initializes a Zephyr installation. Use "west clone" afterwards to fetch the +sources. + +In more detail, does the following: + + 1. Clones the manifest repository to west/manifest, and the west repository + to west/west + + 2. Creates a marker file west/{} + + 3. Creates an initial configuration file west/config + +As an alternative to manually editing west/config, 'west init' can be rerun on +an already initialized West instance to update configuration settings. Only +explicitly passed configuration values (e.g. --mr MANIFEST_REVISION) are +updated. + +Updating the manifest URL or revision via 'west init' automatically runs 'west +update --reset-manifest --reset-projects' afterwards to reset the manifest to +the new revision, and all projects to their new manifest revisions. + +Updating the west URL or revision also runs 'west update --reset-west'. + +To suppress the reset of the manifest, west, and projects, pass --no-reset. +With --no-reset, only the configuration file will be updated, and you will have +to handle any resetting yourself. +'''.format(WEST_MARKER)) + init_parser.add_argument( - '-b', '--base-url', - help='''Base URL for both 'manifest' and 'zephyr' repositories; cannot - be given if either -u or -w are''') + '-m', '--manifest-url', + help='Manifest repository URL (default: {})' + .format(MANIFEST_URL_DEFAULT)) + init_parser.add_argument( - '-u', '--manifest-url', - help='Zephyr manifest fetch URL, default ' + MANIFEST_DEFAULT) + '--mr', '--manifest-rev', dest='manifest_rev', + help='Manifest revision to fetch (default: {})' + .format(MANIFEST_REV_DEFAULT)) + init_parser.add_argument( - '--mr', '--manifest-rev', default=MANIFEST_REV_DEFAULT, - dest='manifest_rev', - help='Manifest revision to fetch, default ' + MANIFEST_REV_DEFAULT) - init_parser.add_argument( - '-w', '--west-url', - help='West fetch URL, default ' + WEST_DEFAULT) - init_parser.add_argument( - '--wr', '--west-rev', default=WEST_REV_DEFAULT, dest='west_rev', - help='West revision to fetch, default ' + WEST_REV_DEFAULT) + '--nr', '--no-reset', dest='reset', action='store_false', + help='''Suppress the automatic reset of the manifest, west, and project + repositories when re-running 'west init' in an existing + installation to update the manifest or west URL/revision''') + init_parser.add_argument( 'directory', nargs='?', default=None, - help='Initializes in this directory, creating it if necessary') + help='''Directory to initialize West in. Missing directories will be + created automatically. (default: current directory)''') args = init_parser.parse_args(args=argv) - directory = args.directory or os.getcwd() - - if args.base_url: - if args.manifest_url or args.west_url: - sys.exit('fatal error: -b is incompatible with -u and -w') - args.manifest_url = args.base_url.rstrip('/') + '/manifest' - args.west_url = args.base_url.rstrip('/') + '/west' - else: - if not args.manifest_url: - args.manifest_url = MANIFEST_DEFAULT - if not args.west_url: - args.west_url = WEST_DEFAULT try: - topdir = find_west_topdir(directory) - init_reinit(topdir, args) + reinit(os.path.join(west_dir(args.directory), 'config'), args) except WestNotFound: - init_bootstrap(directory, args) + bootstrap(args) + + +def bootstrap(args): + '''Bootstrap a new manifest + West installation.''' + + west_url = WEST_URL_DEFAULT + manifest_url = args.manifest_url or MANIFEST_URL_DEFAULT + + west_rev = WEST_REV_DEFAULT + manifest_rev = args.manifest_rev or MANIFEST_REV_DEFAULT + + directory = args.directory or os.getcwd() + + if not os.path.isdir(directory): + try: + print('Initializing in new directory', directory) + os.makedirs(directory, exist_ok=False) + except PermissionError: + sys.exit('Cannot initialize in {}: permission denied'.format( + directory)) + except FileExistsError: + sys.exit('Something else created {} concurrently; quitting'.format( + directory)) + except Exception as e: + sys.exit("Can't create directory {}: {}".format( + directory, e.args)) + else: + print('Initializing in', directory) + + # Clone the west source code and the manifest into west/. Git will create + # the west/ directory if it does not exist. + + clone('manifest repository', manifest_url, manifest_rev, + os.path.join(directory, WEST_DIR, MANIFEST)) + + # Parse the manifest and look for a section named "west" + manifest_file = os.path.join(directory, WEST_DIR, MANIFEST, 'default.yml') + with open(manifest_file, 'r') as f: + data = yaml.safe_load(f.read()) + + if 'west' in data: + wdata = data['west'] + try: + pykwalify.core.Core( + source_data=wdata, + schema_files=[_SCHEMA_PATH] + ).validate() + except pykwalify.errors.SchemaError as e: + sys.exit("Error: Failed to parse manifest file '{}': {}" + .format(manifest_file, e)) + + if 'url' in wdata: + west_url = wdata['url'] + if 'revision' in wdata: + west_rev = wdata['revision'] + + print("cloning {} at revision {}".format(west_url, west_rev)) + clone('west repository', west_url, west_rev, + os.path.join(directory, WEST_DIR, WEST)) + + # Create an initial configuration file + + config_path = os.path.join(directory, WEST_DIR, 'config') + update_conf(config_path, manifest_url, manifest_rev) + print('=== Initial configuration written to {} ==='.format(config_path)) + + # Create a dotfile to mark the installation. Hide it on Windows. + + with open(os.path.join(directory, WEST_DIR, WEST_MARKER), 'w') as f: + hide_file(f.name) + + print('=== West initialized. Now run "west clone" in {}. ==='. + format(directory)) + + +def reinit(config_path, args): + ''' + Reinitialize an existing installation. + + This updates the west/config configuration file, and optionally resets the + manifest, west, and project repositories to the new revision. + ''' + manifest_url = args.manifest_url + + if not (manifest_url or args.manifest_rev): + sys.exit('West already initialized. Please pass any settings you ' + 'want to change.') + + update_conf(config_path, manifest_url, args.manifest_rev) + + print('=== Updated configuration written to {} ==='.format(config_path)) + + if args.reset: + cmd = ['update', '--reset-manifest', '--reset-projects', + '--reset-west'] + print("=== Running 'west {}' to update repositories ===" + .format(' '.join(cmd))) + wrap(cmd) + + +def update_conf(config_path, manifest_url, manifest_rev): + ''' + Creates or updates the configuration file at 'config_path' with the + specified values. Values that are None/empty are ignored. + ''' + config = configparser.ConfigParser() + + # This is a no-op if the file doesn't exist, so no need to check + config.read(config_path) + + update_key(config, 'manifest', 'remote', manifest_url) + update_key(config, 'manifest', 'revision', manifest_rev) + + with open(config_path, 'w') as f: + config.write(f) + + +def update_key(config, section, key, value): + ''' + Updates 'key' in section 'section' in ConfigParser 'config', creating + 'section' if it does not exist. + + If value is None/empty, 'key' is left as-is. + ''' + if not value: + return + + if section not in config: + config[section] = {} + + config[section][key] = value def hide_file(path): @@ -168,66 +342,52 @@ def hide_file(path): .format(system, path), file=sys.stderr) -def init_bootstrap(directory, args): - '''Bootstrap a new manifest + West installation in the given directory.''' - if not os.path.isdir(directory): - try: - print('Initializing in new directory', directory) - os.makedirs(directory, exist_ok=False) - except PermissionError: - sys.exit('Cannot initialize in {}: permission denied'.format( - directory)) - except FileExistsError: - sys.exit('Something else created {} concurrently; quitting'.format( - directory)) - except Exception as e: - sys.exit("Can't create directory {}: {}".format( - directory, e.args)) - else: - print('Initializing in', directory) - - # Clone the west source code and the manifest into west/. Git will create - # the west/ directory if it does not exist. - - clone(args.west_url, args.west_rev, - os.path.join(directory, WEST_DIR, WEST)) - - clone(args.manifest_url, args.manifest_rev, - os.path.join(directory, WEST_DIR, MANIFEST)) - - # Create a dotfile to mark the installation. Hide it on Windows. - - with open(os.path.join(directory, WEST_DIR, WEST_MARKER), 'w') as f: - hide_file(f.name) - - -def init_reinit(directory, args): - # TODO - sys.exit('Re-initializing an existing installation is not yet supported.') - - # # Wrap a West command # +def append_to_pythonpath(directory): + pp = os.environ.get('PYTHONPATH') + os.environ['PYTHONPATH'] = ':'.join(([pp] if pp else []) + [directory]) + def wrap(argv): printing_version = False + printing_help_only = False - if argv and argv[0] in ('-V', '--version'): - print('West bootstrapper version: v{} ({})'.format(version.__version__, - os.path.dirname(__file__))) - printing_version = True + if argv: + if argv[0] in ('-V', '--version'): + print('West bootstrapper version: v{} ({})'. + format(version.__version__, os.path.dirname(__file__))) + printing_version = True + elif len(argv) == 1 and argv[0] in ('-h', '--help'): + # This only matters if we're called outside of an + # installation directory. We delegate to the main help if + # called from within one, because it includes a list of + # available commands, etc. + printing_help_only = True start = os.getcwd() try: - topdir = find_west_topdir(start) + topdir = west_topdir(start) except WestNotFound: if printing_version: sys.exit(0) # run outside of an installation directory + elif printing_help_only: + # We call print multiple times here and below instead of using + # \n to be newline agnostic. + print('To set up a Zephyr installation here, run "west init".') + print('Run "west init -h" for additional information.') + sys.exit(0) else: - sys.exit('Error: not a Zephyr directory (or any parent): {}\n' - 'Use "west init" to install Zephyr here'.format(start)) + print('Error: "{}" is not a Zephyr installation directory.'. + format(start), file=sys.stderr) + print('Things to try:', file=sys.stderr) + print(' - Run "west init" to set up an installation here.', + file=sys.stderr) + print(' - Run "west init -h" for additional information.', + file=sys.stderr) + sys.exit(1) west_git_repo = os.path.join(topdir, WEST_DIR, WEST) if printing_version: @@ -239,20 +399,23 @@ def wrap(argv): print('West repository version: {} ({})'.format(git_describe, west_git_repo)) except subprocess.CalledProcessError: - print('West repository verison: unknown; no tags were found') + print('West repository version: unknown; no tags were found') sys.exit(0) - # Replace the wrapper process with the "real" west - - # sys.argv[1:] strips the argv[0] of the wrapper script itself - argv = ([sys.executable, - os.path.join(west_git_repo, 'src', 'west', 'main.py')] + - argv) - - try: - subprocess.check_call(argv) - except subprocess.CalledProcessError as e: - sys.exit(1) + # Import the west package from the installation and run its main + # function with the given command-line arguments. + # + # This can't be done as a subprocess: that would break the + # runners' debug handling for GDB, which needs to block the usual + # control-C signal handling. GDB uses Ctrl-C to halt the debug + # target. So we really do need to import west and delegate within + # this bootstrap process. + # + # Put this at position 1 to make sure it comes before random stuff + # that might be on a developer's PYTHONPATH in the import order. + sys.path.insert(1, os.path.join(west_git_repo, 'src')) + import west.main + west.main.main(argv) # diff --git a/scripts/meta/west/_bootstrap/version.py b/scripts/meta/west/_bootstrap/version.py index b864b23e1ed..5c057ff9376 100644 --- a/scripts/meta/west/_bootstrap/version.py +++ b/scripts/meta/west/_bootstrap/version.py @@ -2,4 +2,4 @@ # # This is the Python 3 version of option 3 in: # https://packaging.python.org/guides/single-sourcing-package-version/#single-sourcing-the-version -__version__ = '0.2.0rc1' +__version__ = '0.4.1' diff --git a/scripts/meta/west/_bootstrap/west-schema.yml b/scripts/meta/west/_bootstrap/west-schema.yml new file mode 100644 index 00000000000..78b44dae017 --- /dev/null +++ b/scripts/meta/west/_bootstrap/west-schema.yml @@ -0,0 +1,17 @@ +## A pykwalify schema for basic validation of the structure of a +## west YAML file. (Full validation would require additional work, +## e.g. to validate that remote URLs obey the URL format specified in +## rfc1738.) +## + +# The top-level west yaml is a map. The only top-level element is +# 'west'. All other elements are contained within it. This allows +# us a bit of future-proofing. +type: map +mapping: + url: + required: false + type: str + revision: + required: false + type: str diff --git a/scripts/meta/west/build.py b/scripts/meta/west/build.py index e2a5bf5e6e6..bc41dab3097 100644 --- a/scripts/meta/west/build.py +++ b/scripts/meta/west/build.py @@ -10,8 +10,8 @@ building Zephyr applications needed by multiple commands. See west.cmd.build for the build command itself. ''' -import cmake -import log +from west import cmake +from west import log DEFAULT_BUILD_DIR = 'build' '''Name of the default Zephyr build directory.''' diff --git a/scripts/meta/west/cmake.py b/scripts/meta/west/cmake.py index 53b7b47950c..37452e306ed 100644 --- a/scripts/meta/west/cmake.py +++ b/scripts/meta/west/cmake.py @@ -10,8 +10,8 @@ import re import subprocess import shutil -import log -from util import quote_sh_list +from west import log +from west.util import quote_sh_list __all__ = ['run_cmake', 'run_build', 'make_c_identifier', diff --git a/scripts/meta/west/commands/build.py b/scripts/meta/west/commands/build.py index 4625dafd602..a7405e9a63c 100644 --- a/scripts/meta/west/commands/build.py +++ b/scripts/meta/west/commands/build.py @@ -5,10 +5,11 @@ import argparse import os -import log -import cmake -from build import DEFAULT_BUILD_DIR, DEFAULT_CMAKE_GENERATOR, is_zephyr_build -from commands import WestCommand +from west import log +from west import cmake +from west.build import DEFAULT_BUILD_DIR, DEFAULT_CMAKE_GENERATOR, \ + is_zephyr_build +from west.commands import WestCommand BUILD_HELP = '''\ Convenience wrapper for building Zephyr applications. @@ -72,24 +73,33 @@ class Build(WestCommand): formatter_class=argparse.RawDescriptionHelpFormatter, description=self.description) + # Remember to update scripts/west-completion.bash if you add or remove + # flags + parser.add_argument('-b', '--board', - help='''board to build for (must be given for the + help='''Board to build for (must be given for the first build, can be omitted later)''') parser.add_argument('-s', '--source-dir', - help='''explicitly sets the source directory; - if not given, infer it from directory context''') + help='''Explicitly set the source directory. + If not given and rebuilding an existing Zephyr + build directory, this is taken from the CMake + cache. Otherwise, the current directory is + assumed.''') parser.add_argument('-d', '--build-dir', - help='''explicitly sets the build directory; - if not given, infer it from directory context''') + help='''Explicitly sets the build directory. + If not given and the current directory is a Zephyr + build directory, it will be used; otherwise, "{}" + is assumed. The directory will be created if + it doesn't exist.'''.format(DEFAULT_BUILD_DIR)) parser.add_argument('-t', '--target', - help='''override the build system target (e.g. + help='''Override the build system target (e.g. 'clean', 'pristine', etc.)''') parser.add_argument('-c', '--cmake', action='store_true', - help='force CMake to run') + help='Force CMake to run') parser.add_argument('-f', '--force', action='store_true', - help='ignore any errors and try to build anyway') + help='Ignore any errors and try to build anyway') parser.add_argument('cmake_opts', nargs='*', metavar='cmake_opt', - help='extra option to pass to CMake; implies -c') + help='Extra option to pass to CMake; implies -c') return parser diff --git a/scripts/meta/west/commands/debug.py b/scripts/meta/west/commands/debug.py index de93ff072d1..7ce3c1bb480 100644 --- a/scripts/meta/west/commands/debug.py +++ b/scripts/meta/west/commands/debug.py @@ -6,8 +6,9 @@ from textwrap import dedent -from commands.run_common import desc_common, add_parser_common, do_run_common -from commands import WestCommand +from west.commands.run_common import desc_common, add_parser_common, \ + do_run_common +from west.commands import WestCommand class Debug(WestCommand): diff --git a/scripts/meta/west/commands/flash.py b/scripts/meta/west/commands/flash.py index 9dd7e97e21d..72b716a0ff4 100644 --- a/scripts/meta/west/commands/flash.py +++ b/scripts/meta/west/commands/flash.py @@ -4,8 +4,9 @@ '''west "flash" command''' -from commands.run_common import desc_common, add_parser_common, do_run_common -from commands import WestCommand +from west.commands.run_common import desc_common, add_parser_common, \ + do_run_common +from west.commands import WestCommand class Flash(WestCommand): @@ -13,7 +14,7 @@ class Flash(WestCommand): def __init__(self): super(Flash, self).__init__( 'flash', - 'Flash and run a binary onto a board.\n\n' + + 'Flash and run a binary on a board.\n\n' + desc_common('flash'), accepts_unknown_args=True) diff --git a/scripts/meta/west/commands/project.py b/scripts/meta/west/commands/project.py index ddb0d5de4a5..6dc325861b7 100644 --- a/scripts/meta/west/commands/project.py +++ b/scripts/meta/west/commands/project.py @@ -11,12 +11,12 @@ import shutil import subprocess import textwrap -import pykwalify.core -import yaml - -import log -import util -from commands import WestCommand +from west.config import config +from west import log +from west import util +from west.commands import WestCommand +from west.manifest import default_path, SpecialProject, \ + Manifest, MalformedManifest, META_NAMES # Branch that points to the revision specified in the manifest (which might be @@ -25,30 +25,143 @@ from commands import WestCommand _MANIFEST_REV_BRANCH = 'manifest-rev' -class ListProjects(WestCommand): +class List(WestCommand): def __init__(self): super().__init__( - 'list-projects', + 'list', _wrap(''' List projects. - Prints the path to the manifest file and lists all projects along - with their clone paths and manifest revisions. Also includes - information on which projects are currently cloned. - ''')) + Individual projects can be specified by name. + + By default, lists all project names in the manifest, along with + each project's path, revision, URL, and whether it has been cloned. + + The west and manifest repositories in the top-level west directory + are not included by default. Use --all or the special project + names "west" and "manifest" to include them.''')) def do_add_parser(self, parser_adder): - return _add_parser(parser_adder, self) + default_fmt = '{name:14} {path:18} {revision:13} {url} {cloned}' + return _add_parser( + parser_adder, self, + _arg('-a', '--all', action='store_true', + help='''Do not ignore repositories in west/ (i.e. west and the + manifest) in the output. Since these are not part of + the manifest, some of their format values (like "revision") + come from other sources. The behavior of this option is + modeled after the Unix ls -a option.'''), + _arg('-f', '--format', default=default_fmt, + help='''Format string to use to list each project; see + FORMAT STRINGS below.'''), + _project_list_arg, + epilog=textwrap.dedent('''\ + FORMAT STRINGS + + Projects are listed using a Python 3 format string. Arguments + to the format string are accessed by name. + + The default format string is: + + "{}" + + The following arguments are available: + + - name: project name in the manifest + - url: full remote URL as specified by the manifest + - path: the relative path to the project from the top level, + as specified in the manifest where applicable + - abspath: absolute and normalized path to the project + - revision: project's manifest revision + - cloned: "(cloned)" if the project has been cloned, "(not cloned)" + otherwise + - clone_depth: project clone depth if specified, "None" otherwise + '''.format(default_fmt))) def do_run(self, args, user_args): - log.inf("Manifest path: {}\n".format(_manifest_path(args))) + # We should only list the meta projects if they were explicitly + # given by name, or --all was given. + list_meta = bool(args.projects) or args.all - for project in _all_projects(args): - log.inf('{:15} {:30} {:15} {}'.format( - project.name, - os.path.join(project.path, ''), # Add final '/' if missing - project.revision, - "(cloned)" if _cloned(project) else "(not cloned)")) + for project in _projects(args, include_meta=True): + if project.name in META_NAMES and not list_meta: + continue + + # Spelling out the format keys explicitly here gives us + # future-proofing if the internal Project representation + # ever changes. + try: + result = args.format.format( + name=project.name, + url=project.url, + path=project.path, + abspath=project.abspath, + revision=project.revision, + cloned="(cloned)" if _cloned(project) else "(not cloned)", + clone_depth=project.clone_depth or "None") + except KeyError as e: + # The raised KeyError seems to just put the first + # invalid argument in the args tuple, regardless of + # how many unrecognizable keys there were. + log.die('unknown key "{}" in format string "{}"'. + format(e.args[0], args.format)) + + log.inf(result) + + +class Clone(WestCommand): + def __init__(self): + super().__init__( + 'clone', + _wrap(''' + Clone projects. + + Clones each of the specified projects (default: all projects) and + creates a branch in each. The branch is named after the project's + revision, and tracks the 'manifest-rev' branch (see below). + + If the project's revision is an SHA, the branch will simply be + called 'work'. + + This command is really just a shorthand for 'west fetch' + + 'west checkout -b '. If you clone a project with + 'west fetch' instead, you will start in a detached HEAD state at + 'manifest-rev'. + + {} + + {}'''.format(_NO_UPDATE_HELP, _MANIFEST_REV_HELP))) + + def do_add_parser(self, parser_adder): + return _add_parser( + parser_adder, self, + _arg('-b', + dest='branch', + metavar='BRANCH_NAME', + help='an alternative branch name to use, instead of one ' + 'based on the revision'), + _no_update_arg, + _project_list_arg) + + def do_run(self, args, user_args): + if args.update: + _update_manifest(args) + _update_west(args) + + for project in _projects(args, listed_must_be_cloned=False): + if args.branch: + branch = args.branch + elif _is_sha(project.revision): + # Don't name the branch after an SHA + branch = 'work' + else: + # Use the last component of the revision, in case it is a + # qualified ref (refs/heads/foo and the like) + branch = project.revision.split('/')[-1] + + _fetch(project) + _create_branch(project, branch) + _checkout(project, branch) class Fetch(WestCommand): @@ -56,17 +169,15 @@ class Fetch(WestCommand): super().__init__( 'fetch', _wrap(''' - Clone/fetch projects. + Fetch projects. Fetches upstream changes in each of the specified projects (default: all projects). Repositories that do not already exist are cloned. - Unless --no-update is passed, the manifest and West source code - repositories are updated prior to fetching. See the 'update' - command. + {} - ''' + _MANIFEST_REV_HELP)) + {}'''.format(_NO_UPDATE_HELP, _MANIFEST_REV_HELP))) def do_add_parser(self, parser_adder): return _add_parser(parser_adder, self, _no_update_arg, @@ -74,10 +185,10 @@ class Fetch(WestCommand): def do_run(self, args, user_args): if args.update: - _update(True, True) + _update_manifest(args) + _update_west(args) for project in _projects(args, listed_must_be_cloned=False): - log.dbg('fetching:', project, level=log.VERBOSE_VERY) _fetch(project) @@ -94,11 +205,10 @@ class Pull(WestCommand): branch up to date. Repositories that do not already exist are cloned. - Unless --no-update is passed, the manifest and West source code - repositories are updated prior to pulling. See the 'update' - command. + {} - '''.format(_MANIFEST_REV_BRANCH) + _MANIFEST_REV_HELP)) + {}'''.format(_MANIFEST_REV_BRANCH, _NO_UPDATE_HELP, + _MANIFEST_REV_HELP))) def do_add_parser(self, parser_adder): return _add_parser(parser_adder, self, _no_update_arg, @@ -106,11 +216,12 @@ class Pull(WestCommand): def do_run(self, args, user_args): if args.update: - _update(True, True) + _update_manifest(args) + _update_west(args) for project in _projects(args, listed_must_be_cloned=False): - if _fetch(project): - _rebase(project) + _fetch(project) + _rebase(project) class Rebase(WestCommand): @@ -178,12 +289,20 @@ class Checkout(WestCommand): super().__init__( 'checkout', _wrap(''' - Check out topic branch. + Check out local branch. - Checks out the specified branch in each of the specified projects + Checks out a local branch in each of the specified projects (default: all cloned projects). Projects that do not have the branch are left alone. - ''')) + + Note: To check out remote branches, use ordinary Git commands + inside the repositories. This command is meant for switching + between work branches that span multiple repositories, without any + interference from whatever remote branches might exist. + + If '-b BRANCH_NAME' is passed, the new branch will be set to track + '{}', like for 'west branch BRANCH_NAME'. + '''.format(_MANIFEST_REV_BRANCH))) def do_add_parser(self, parser_adder): return _add_parser( @@ -236,7 +355,7 @@ class Diff(WestCommand): for project in _cloned_projects(args): # Use paths that are relative to the base directory to make it # easier to see where the changes are - _git(project, 'diff --src-prefix=(path)/ --dst-prefix=(path)/', + _git(project, 'diff --src-prefix={path}/ --dst-prefix={path}/', extra_args=user_args) @@ -255,7 +374,7 @@ class Status(WestCommand): def do_run(self, args, user_args): for project in _cloned_projects(args): - _inf(project, 'status of (name-and-path)') + _inf(project, 'status of {name_and_path}') _git(project, 'status', extra_args=user_args) @@ -265,7 +384,10 @@ class Update(WestCommand): 'update', _wrap(''' Updates the manifest repository and/or the West source code - repository. + repository. The remote to update from is taken from the + manifest.remote and manifest.remote configuration settings, and the + revision from manifest.revision and west.revision configuration + settings. There is normally no need to run this command manually, because 'west fetch' and 'west pull' automatically update the West and @@ -274,6 +396,10 @@ class Update(WestCommand): Pass --update-west or --update-manifest to update just that repository. With no arguments, both are updated. + + Updates are skipped (with a warning) if they can't be done via + fast-forward, unless --reset-manifest, --reset-west, or + --reset-projects is given. ''')) def do_add_parser(self, parser_adder): @@ -282,17 +408,52 @@ class Update(WestCommand): _arg('--update-west', dest='update_west', action='store_true', - help='update the West source code repository'), + help='update the west source code repository'), _arg('--update-manifest', dest='update_manifest', action='store_true', - help='update the manifest repository')) + help='update the manifest repository'), + _arg('--reset-west', + action='store_true', + help='''Like --update-west, but run 'git reset --keep' + afterwards to reset the west repository to the commit + pointed at by the west.remote and west.revision + configuration settings. This is used internally when + changing west.remote or west.revision via + 'west init'.'''), + _arg('--reset-manifest', + action='store_true', + help='''like --reset-west, for the manifest repository, using + manifest.remote and manifest.revision.'''), + _arg('--reset-projects', + action='store_true', + help='''Fetches upstream data in all projects, then runs 'git + reset --keep' to reset them to the manifest revision. + This is used internally when changing manifest.remote or + manifest.revision via 'west init'.''')) def do_run(self, args, user_args): - if not args.update_west and not args.update_manifest: - _update(True, True) - else: - _update(args.update_west, args.update_manifest) + if not (args.update_manifest or args.reset_manifest or + args.update_west or args.reset_west or + args.reset_projects): + + # No arguments is an alias for --update-west --update-manifest + _update_manifest(args) + _update_west(args) + return + + if args.reset_manifest: + _update_and_reset_special(args, 'manifest') + elif args.update_manifest: + _update_manifest(args) + + if args.reset_west: + _update_and_reset_special(args, 'west') + elif args.update_west: + _update_west(args) + + if args.reset_projects: + _reset_projects(args) class ForAll(WestCommand): @@ -326,7 +487,7 @@ class ForAll(WestCommand): def do_run(self, args, user_args): for project in _cloned_projects(args): - _inf(project, "Running '{}' in (name-and-path)" + _inf(project, "Running '{}' in {{name_and_path}}" .format(args.command)) subprocess.Popen(args.command, shell=True, cwd=project.abspath) \ @@ -359,16 +520,19 @@ _no_update_arg = _arg( _project_list_arg = _arg('projects', metavar='PROJECT', nargs='*') -def _add_parser(parser_adder, cmd, *extra_args): +def _add_parser(parser_adder, cmd, *extra_args, **kwargs): # Adds and returns a subparser for the project-related WestCommand 'cmd'. # All of these commands (currently) take the manifest path flag, so it's - # hardcoded here. + # provided by default here, but any defaults can be overridden with kwargs. - return parser_adder.add_parser( - cmd.name, - description=cmd.description, - formatter_class=argparse.RawDescriptionHelpFormatter, - parents=(_manifest_arg,) + extra_args) + if 'description' not in kwargs: + kwargs['description'] = cmd.description + if 'formatter_class' not in kwargs: + kwargs['formatter_class'] = argparse.RawDescriptionHelpFormatter + if 'parents' not in kwargs: + kwargs['parents'] = (_manifest_arg,) + extra_args + + return parser_adder.add_parser(cmd.name, **kwargs) def _wrap(s): @@ -382,19 +546,18 @@ def _wrap(s): return "\n\n".join(textwrap.fill(paragraph) for paragraph in paragraphs) +_NO_UPDATE_HELP = """ +Unless --no-update is passed, the manifest and West source code repositories +are updated prior to cloning. See the 'update' command. +"""[1:].replace('\n', ' ') + + _MANIFEST_REV_HELP = """ The '{}' branch points to the revision that the manifest specified for the project as of the most recent 'west fetch'/'west pull'. """.format(_MANIFEST_REV_BRANCH)[1:].replace("\n", " ") -# Holds information about a project, taken from the manifest file (or -# constructed manually for "special" projects) -Project = collections.namedtuple( - 'Project', - 'name url revision path abspath clone_depth') - - def _cloned_projects(args): # Returns _projects(args, listed_must_be_cloned=True) if a list of projects # was given by the user (i.e., listed projects are required to be cloned). @@ -405,7 +568,7 @@ def _cloned_projects(args): [project for project in _all_projects(args) if _cloned(project)] -def _projects(args, listed_must_be_cloned=True): +def _projects(args, listed_must_be_cloned=True, include_meta=False): # Returns a list of project instances for the projects requested in 'args' # (the command-line arguments), in the same order that they were listed by # the user. If args.projects is empty, no projects were listed, and all @@ -418,189 +581,191 @@ def _projects(args, listed_must_be_cloned=True): # listed_must_be_cloned (default: True): # If True, an error is raised if an uncloned project was listed. This # only applies to projects listed explicitly on the command line. + # + # include_meta (default: False): + # If True, "meta" projects (i.e. west and the manifest) may be given + # in args.projects without raising errors, and are also included in the + # return value if args.projects is empty. projects = _all_projects(args) + if include_meta: + projects += [_special_project(args, name) for name in META_NAMES] + if not args.projects: # No projects specified. Return all projects. return projects - # Got a list of projects on the command line. First, check that they exist - # in the manifest. + # Sort the projects by the length of their absolute paths, with the longest + # path first. That way, projects within projects (e.g., for submodules) are + # tried before their parent projects, when projects are specified via their + # path. + projects.sort(key=lambda project: len(project.abspath), reverse=True) - project_names = [project.name for project in projects] - nonexistent = set(args.projects) - set(project_names) - if nonexistent: - log.die('Unknown project{} {} (available projects: {})' - .format('s' if len(nonexistent) > 1 else '', - ', '.join(nonexistent), - ', '.join(project_names))) + # Listed but missing projects. Used for error reporting. + missing_projects = [] + + def normalize(path): + # Returns a case-normalized canonical absolute version of 'path', for + # comparisons. The normcase() is a no-op on platforms on case-sensitive + # filesystems. + return os.path.normcase(os.path.realpath(path)) - # Return the projects in the order they were listed res = [] - for name in args.projects: + for project_arg in args.projects: for project in projects: - if project.name == name: + if project.name == project_arg: + # The argument is a project name res.append(project) break + else: + # The argument is not a project name. See if it is a project + # (sub)path. + for project in projects: + # The startswith() means we also detect subdirectories of + # project repositories. Giving a plain file in the repo will + # work here too, but that probably doesn't hurt. + if normalize(project_arg).startswith( + normalize(project.abspath)): + res.append(project) + break + else: + # Neither a project name nor a project path. We will report an + # error below. + missing_projects.append(project_arg) + + if missing_projects: + log.die('Unknown project name{0}/path{0} {1} (available projects: {2})' + .format('s' if len(missing_projects) > 1 else '', + ', '.join(missing_projects), + ', '.join(project.name for project in projects))) # Check that all listed repositories are cloned, if requested if listed_must_be_cloned: - uncloned = [prj.name for prj in res if not _cloned(prj)] + # We could still get here with a missing manifest repository if the + # user gave a --manifest argument. + uncloned_meta = [prj.name for prj in res if not _cloned(prj) and + prj.name in META_NAMES] + if uncloned_meta: + log.die('Missing meta project{}: {}.'. + format('s' if len(uncloned_meta) > 1 else '', + ', '.join(uncloned_meta)), + 'The Zephyr installation has been corrupted.') + + uncloned = [prj.name for prj in res + if not _cloned(prj) and prj.name not in META_NAMES] if uncloned: log.die('The following projects are not cloned: {}. Please clone ' - "them first (with 'west fetch')." + "them first with 'west clone'." .format(", ".join(uncloned))) return res def _all_projects(args): - # Parses the manifest file, returning a list of Project instances. + # Get a list of project objects from the manifest. # - # Before the manifest is parsed, it is validated against a pykwalify - # schema. An error is raised on validation errors. - - manifest_path = _manifest_path(args) - - _validate_manifest(manifest_path) - - with open(manifest_path) as f: - manifest = yaml.safe_load(f)['manifest'] - - projects = [] - # Manifest "defaults" keys whose values get copied to each project - # that doesn't specify its own value. - project_defaults = ('remote', 'revision') - - # mp = manifest project (dictionary with values parsed from the manifest) - for mp in manifest['projects']: - # Fill in any missing fields in 'mp' with values from the 'defaults' - # dictionary - if 'defaults' in manifest: - for key, val in manifest['defaults'].items(): - if key in project_defaults: - mp.setdefault(key, val) - - # Add the repository URL to 'mp' - for remote in manifest['remotes']: - if remote['name'] == mp['remote']: - mp['url'] = remote['url'] + '/' + mp['name'] - break - else: - log.die('Remote {} not defined in {}' - .format(mp['remote'], manifest_path)) - - # If no clone path is specified, the project's name is used - clone_path = mp.get('path', mp['name']) - - # Use named tuples to store project information. That gives nicer - # syntax compared to a dict (project.name instead of project['name'], - # etc.) - projects.append(Project( - mp['name'], - mp['url'], - # If no revision is specified, 'master' is used - mp.get('revision', 'master'), - clone_path, - # Absolute clone path - os.path.join(util.west_topdir(), clone_path), - # If no clone depth is specified, we fetch the entire history - mp.get('clone-depth', None))) - - return projects - - -def _validate_manifest(manifest_path): - # Validates the manifest with pykwalify. schema.yml holds the schema. - - schema_path = os.path.join(os.path.dirname(__file__), "schema.yml") + # If the manifest is malformed, a fatal error occurs and the + # command aborts. try: - pykwalify.core.Core( - source_file=manifest_path, - schema_files=[schema_path] - ).validate() - except pykwalify.errors.SchemaError as e: - log.die('{} malformed (schema: {}):\n{}' - .format(manifest_path, schema_path, e)) + return list(Manifest.from_file(_manifest_path(args), + 'manifest').projects) + except MalformedManifest as m: + log.die(m.args[0]) def _manifest_path(args): # Returns the path to the manifest file. Defaults to # .west/manifest/default.yml if the user didn't specify a manifest. - return args.manifest or os.path.join(util.west_dir(), 'manifest', - 'default.yml') + return args.manifest or default_path() def _fetch(project): # Fetches upstream changes for 'project' and updates the 'manifest-rev' # branch to point to the revision specified in the manifest. If the # project's repository does not already exist, it is created first. - # - # Returns True if the project's repository already existed. - exists = _cloned(project) + if not _cloned(project): + _inf(project, 'Creating repository for {name_and_path}') + _git_base(project, 'init {abspath}') + # This remote is only added for the user's convenience. We always fetch + # directly from the URL specified in the manifest. + _git(project, 'remote add -- {remote_name} {url}') - if not exists: - _inf(project, 'Creating repository for (name-and-path)') - _git_base(project, 'init (abspath)') - _git(project, 'remote add origin (url)') + # Fetch the revision specified in the manifest into the manifest-rev branch + msg = "Fetching changes for {name_and_path}" if project.clone_depth: - _inf(project, - 'Fetching changes for (name-and-path) with --depth (clone-depth)') - - # If 'clone-depth' is specified, fetch just the specified revision - # (probably a branch). That will download the minimum amount of data, - # which is probably what's wanted whenever 'clone-depth is used. The - # default 'git fetch' behavior is to do a shallow clone of all branches - # on the remote. - # - # Note: Many servers won't allow fetching arbitrary commits by SHA. - # Combining --depth with an SHA will break for those. - - # Qualify branch names with refs/heads/, just to be safe. Just the - # branch name is likely to work as well though. - _git(project, - 'fetch --depth=(clone-depth) origin ' + - (project.revision if _is_sha(project.revision) else \ - 'refs/heads/' + project.revision)) - + fetch_cmd = "fetch --depth={clone_depth}" + msg += " with --depth {clone_depth}" else: - _inf(project, 'Fetching changes for (name-and-path)') + fetch_cmd = "fetch" - # If 'clone-depth' is not specified, fetch all branches on the - # remote. This gives a more usable repository. - _git(project, 'fetch origin') + _inf(project, msg) + # This two-step approach avoids a "trying to write non-commit object" error + # when the revision is an annotated tag. ^{commit} type peeling isn't + # supported for the in a : refspec, so we have to do it + # separately. + # + # --tags is required to get tags when the remote is specified as an URL. + if _is_sha(project.revision): + # Don't fetch a SHA directly, as server may restrict from doing so. + _git(project, fetch_cmd + ' --tags -- {url}') + _git(project, 'update-ref {qual_manifest_rev_branch} {revision}') + else: + _git(project, fetch_cmd + ' --tags -- {url} {revision}') + _git(project, + 'update-ref {qual_manifest_rev_branch} FETCH_HEAD^{{commit}}') - # Create/update the 'manifest-rev' branch - _git(project, - 'update-ref refs/heads/(manifest-rev-branch) ' + - (project.revision if _is_sha(project.revision) else - 'remotes/origin/' + project.revision)) - - if not exists: - # If we just initialized the repository, check out 'manifest-rev' in a - # detached HEAD state. + if not _head_ok(project): + # If nothing it checked out (which would usually only happen just after + # we initialize the repository), check out 'manifest-rev' in a detached + # HEAD state. # # Otherwise, the initial state would have nothing checked out, and HEAD # would point to a non-existent refs/heads/master branch (that would # get created if the user makes an initial commit). That state causes # e.g. 'west rebase' to fail, and might look confusing. # - # (The --detach flag is strictly redundant here, because the + # The --detach flag is strictly redundant here, because the # refs/heads/ form already detaches HEAD, but it avoids a - # spammy detached HEAD warning from Git.) - _git(project, 'checkout --detach refs/heads/(manifest-rev-branch)') - - return exists + # spammy detached HEAD warning from Git. + _git(project, 'checkout --detach {qual_manifest_rev_branch}') def _rebase(project): - _inf(project, 'Rebasing (name-and-path) to (manifest-rev-branch)') - _git(project, 'rebase (manifest-rev-branch)') + # Rebases the project against the manifest-rev branch + + if _up_to_date_with(project, _MANIFEST_REV_BRANCH): + _inf(project, + '{name_and_path} is up-to-date with {manifest_rev_branch}') + else: + _inf(project, 'Rebasing {name_and_path} to {manifest_rev_branch}') + _git(project, 'rebase {qual_manifest_rev_branch}') + + +def _sha(project, rev): + # Returns the SHA of a revision (HEAD, v2.0.0, etc.), passed as a string in + # 'rev' + + return _git(project, 'rev-parse ' + rev, capture_stdout=True).stdout + + +def _merge_base(project, rev1, rev2): + # Returns the latest commit in common between 'rev1' and 'rev2' + + return _git(project, 'merge-base -- {} {}'.format(rev1, rev2), + capture_stdout=True).stdout + + +def _up_to_date_with(project, rev): + # Returns True if all commits in 'rev' are also in HEAD. This is used to + # check if 'project' needs rebasing. 'revision' can be anything that + # resolves to a commit. + + return _sha(project, rev) == _merge_base(project, 'HEAD', rev) def _cloned(project): @@ -641,73 +806,104 @@ def _branches(project): def _create_branch(project, branch): if _has_branch(project, branch): - _inf(project, "Branch '{}' already exists in (name-and-path)" + _inf(project, "Branch '{}' already exists in {{name_and_path}}" .format(branch)) else: - _inf(project, "Creating branch '{}' in (name-and-path)" - .format(branch)) - _git(project, 'branch --quiet --track {} (manifest-rev-branch)' + _inf(project, "Creating branch '{}' in {{name_and_path}}" .format(branch)) + _git(project, + 'branch --quiet --track -- {} {{qual_manifest_rev_branch}}' + .format(branch)) + def _has_branch(project, branch): - return _git(project, 'show-ref --quiet --verify refs/heads/' + branch, - check=False).returncode == 0 + return _ref_ok(project, 'refs/heads/' + branch) + + +def _ref_ok(project, ref): + # Returns True if the reference 'ref' exists and can be resolved to a + # commit + return _git(project, 'show-ref --quiet --verify ' + ref, check=False) \ + .returncode == 0 + + +def _head_ok(project): + # Returns True if the reference 'HEAD' exists and is not a tag or remote + # ref (e.g. refs/remotes/origin/HEAD). + # Some versions of git will report 1, when doing + # 'git show-ref --verify HEAD' even if HEAD is valid, see #119. + # 'git show-ref --head ' will always return 0 if HEAD or + # is valid. + # We are only interested in HEAD, thus we must avoid being + # valid. '/' can never point to valid reference, thus 'show-ref --head /' + # will return: + # - 0 if HEAD is present + # - 1 otherwise + return _git(project, 'show-ref --quiet --head /', check=False) \ + .returncode == 0 def _checkout(project, branch): - _inf(project, "Checking out branch '{}' in (name-and-path)".format(branch)) + _inf(project, + "Checking out branch '{}' in {{name_and_path}}".format(branch)) _git(project, 'checkout ' + branch) -def _special_project(name): +def _special_project(args, name): # Returns a Project instance for one of the special repositories in west/, # so that we can reuse the project-related functions for them - return Project( - name, - 'dummy URL for {} repository'.format(name), - 'master', - os.path.join('west', name.lower()), # Path - os.path.join(util.west_dir(), name.lower()), # Absolute path - None # Clone depth - ) + if name == 'manifest': + url = config.get(name, 'remote', fallback='origin') + revision = config.get(name, 'revision', fallback='master') + return SpecialProject(name, revision=revision, + path=os.path.join('west', name), url=url) + + return Manifest.from_file(_manifest_path(args), name).west_project -def _update(update_west, update_manifest): - # 'try' is a keyword - def attempt(project, cmd): - res = _git(project, cmd, capture_stdout=True, check=False) - if res.returncode: - # The Git command's stderr isn't redirected and will also be - # available - _die(project, _FAILED_UPDATE_MSG.format(cmd)) - return res.stdout +def _update_west(args): + _update_special(args, 'west') - projects = [] - if update_west: - projects.append(_special_project('West')) - if update_manifest: - projects.append(_special_project('manifest')) - for project in projects: - _dbg(project, 'Updating (name-and-path)', level=log.VERBOSE_NORMAL) +def _update_manifest(args): + _update_special(args, 'manifest') - # Fetch changes from upstream - attempt(project, 'fetch') - # Get the SHA of the last commit in common with the upstream branch - merge_base = attempt(project, 'merge-base HEAD remotes/origin/master') +def _update_special(args, name): + with _error_context(_FAILED_UPDATE_MSG): + project = _special_project(args, name) + _dbg(project, 'Updating {name_and_path}', level=log.VERBOSE_NORMAL) - # Get the current SHA of the upstream branch - head_sha = attempt(project, 'show-ref --hash remotes/origin/master') + old_sha = _sha(project, 'HEAD') - # If they differ, we need to rebase - if merge_base != head_sha: - attempt(project, 'rebase remotes/origin/master') + # Only update special repositories if possible via fast-forward, as + # automatic rebasing is probably more annoying than useful when working + # directly on them. + # + # --tags is required to get tags when the remote is specified as a URL. + # --ff-only is required to ensure that the merge only takes place if it + # can be fast-forwarded. + if _git(project, + 'fetch --quiet --tags -- {url} {revision}', + check=False).returncode: - _inf(project, 'Updated (rebased) (name-and-path) to the ' - 'latest version') + _wrn(project, + 'Skipping automatic update of {name_and_path}. ' + "{revision} cannot be fetched (from {url}).") + + elif _git(project, + 'merge --quiet --ff-only FETCH_HEAD', + check=False).returncode: + + _wrn(project, + 'Skipping automatic update of {name_and_path}. ' + "Can't be fast-forwarded to {revision} (from {url}).") + + elif old_sha != _sha(project, 'HEAD'): + _inf(project, + 'Updated {name_and_path} to {revision} (from {url}).') if project.name == 'west': # Signal self-update, which will cause a restart. This is a bit @@ -716,10 +912,40 @@ def _update(update_west, update_manifest): raise WestUpdated() +def _update_and_reset_special(args, name): + # Updates one of the special repositories (the manifest and west) by + # resetting to the new revision after fetching it (with 'git reset --keep') + + project = _special_project(args, name) + with _error_context(', while updating/resetting special project'): + _inf(project, + "Fetching and resetting {name_and_path} to '{revision}'") + _git(project, 'fetch -- {url} {revision}') + if _git(project, 'reset --keep FETCH_HEAD', check=False).returncode: + _wrn(project, + 'Failed to reset special project {name_and_path} to ' + "{revision} (with 'git reset --keep')") + + +def _reset_projects(args): + # Fetches changes in all cloned projects and then resets them the manifest + # revision (with 'git reset --keep') + + for project in _all_projects(args): + if _cloned(project): + _fetch(project) + _inf(project, 'Resetting {name_and_path} to {manifest_rev_branch}') + if _git(project, 'reset --keep {manifest_rev_branch}', + check=False).returncode: + + _wrn(project, + 'Failed to reset {name_and_path} to ' + "{manifest_rev_branch} (with 'git reset --keep')") + + _FAILED_UPDATE_MSG = """ -Failed to update (name-and-path), while running command '{}'. Please fix the -state of the repository, or pass --no-update to 'west fetch/pull' to skip -updating the manifest and West for the duration of the command."""[1:] +, while running automatic self-update. Pass --no-update to 'west fetch/pull' to +skip updating the manifest and West for the duration of the command."""[1:] class WestUpdated(Exception): @@ -806,8 +1032,10 @@ def _git_helper(project, cmd, extra_args, cwd, capture_stdout, check): log.dbg(dbg_msg, level=log.VERBOSE_VERY) if check and popen.returncode: - _die(project, "Command '{}' failed for (name-and-path)" - .format(cmd_str)) + msg = "Command '{}' failed for {{name_and_path}}".format(cmd_str) + if _error_context_msg: + msg += _error_context_msg.replace('\n', ' ') + _die(project, msg) if capture_stdout: # Manual UTF-8 decoding and universal newlines. Before Python 3.6, @@ -822,20 +1050,59 @@ def _git_helper(project, cmd, extra_args, cwd, capture_stdout, check): return CompletedProcess(popen.args, popen.returncode, stdout) +# Some Python shenanigans to be able to set up a context with +# +# with _error_context("Doing stuff"): +# Do the stuff +# +# A context is just some extra text that gets printed on Git errors. +# +# Note: If we ever need to support nested contexts, _error_context_msg could be +# turned into a stack. + +_error_context_msg = None + + +class _error_context: + def __init__(self, msg): + self.msg = msg + + def __enter__(self): + global _error_context_msg + _error_context_msg = self.msg + + def __exit__(self, *args): + global _error_context_msg + _error_context_msg = None + + def _expand_shorthands(project, s): # Expands project-related shorthands in 's' to their values, # returning the expanded string - return s.replace('(name)', project.name) \ - .replace('(name-and-path)', - '{} ({})'.format( - project.name, os.path.join(project.path, ""))) \ - .replace('(url)', project.url) \ - .replace('(path)', project.path) \ - .replace('(abspath)', project.abspath) \ - .replace('(revision)', project.revision) \ - .replace('(manifest-rev-branch)', _MANIFEST_REV_BRANCH) \ - .replace('(clone-depth)', str(project.clone_depth)) + # Some of the trickier ones below. 'qual' stands for 'qualified', meaning + # the full path to the ref (e.g. refs/heads/master). + # + # manifest-rev-branch: + # The name of the magic branch that points to the manifest revision + # + # qual-manifest-rev-branch: + # A qualified reference to the magic manifest revision branch, e.g. + # refs/heads/manifest-rev + + return s.format(name=project.name, + name_and_path='{} ({})'.format( + project.name, os.path.join(project.path, "")), + remote_name=('None' if project.remote is None + else project.remote.name), + url=project.url, + path=project.path, + abspath=project.abspath, + revision=project.revision, + manifest_rev_branch=_MANIFEST_REV_BRANCH, + qual_manifest_rev_branch=('refs/heads/' + + _MANIFEST_REV_BRANCH), + clone_depth=str(project.clone_depth)) def _inf(project, msg): diff --git a/scripts/meta/west/commands/run_common.py b/scripts/meta/west/commands/run_common.py index 1bd10f8a90b..22a1f6e7a5b 100644 --- a/scripts/meta/west/commands/run_common.py +++ b/scripts/meta/west/commands/run_common.py @@ -10,13 +10,13 @@ from os import getcwd, path from subprocess import CalledProcessError import textwrap -import cmake -import log -import util -from build import DEFAULT_BUILD_DIR, is_zephyr_build -from runners import get_runner_cls, ZephyrBinaryRunner -from runners.core import RunnerConfig -from commands import CommandContextError +from west import cmake +from west import log +from west import util +from west.build import DEFAULT_BUILD_DIR, is_zephyr_build +from west.runners import get_runner_cls, ZephyrBinaryRunner +from west.runners.core import RunnerConfig +from west.commands import CommandContextError # Context-sensitive help indentation. # Don't change this, or output from argparse won't match up. @@ -29,6 +29,9 @@ def add_parser_common(parser_adder, command): formatter_class=argparse.RawDescriptionHelpFormatter, description=command.description) + # Remember to update scripts/west-completion.bash if you add or remove + # flags + parser.add_argument('-H', '--context', action='store_true', help='''Rebuild application and print context-sensitive help; this may be combined with --runner to restrict @@ -72,14 +75,17 @@ def add_parser_common(parser_adder, command): # # This is how we detect if the user provided them or not when # overriding values from the cached configuration. + + command_verb = "flash" if command == "flash" else "debug" + group.add_argument('--board-dir', help='Zephyr board directory') - group.add_argument('--kernel-elf', - help='Path to kernel binary in .elf format') - group.add_argument('--kernel-hex', - help='Path to kernel binary in .hex format') - group.add_argument('--kernel-bin', - help='Path to kernel binary in .bin format') + group.add_argument('--elf-file', + help='Path to elf file to {0}'.format(command_verb)) + group.add_argument('--hex-file', + help='Path to hex file to {0}'.format(command_verb)) + group.add_argument('--bin-file', + help='Path to binary file to {0}'.format(command_verb)) group.add_argument('--gdb', help='Path to GDB, if applicable') group.add_argument('--openocd', @@ -108,15 +114,18 @@ def desc_common(command_name): def cached_runner_config(build_dir, cache): '''Parse the RunnerConfig from a build directory and CMake Cache.''' board_dir = cache['ZEPHYR_RUNNER_CONFIG_BOARD_DIR'] - kernel_elf = cache['ZEPHYR_RUNNER_CONFIG_KERNEL_ELF'] - kernel_hex = cache['ZEPHYR_RUNNER_CONFIG_KERNEL_HEX'] - kernel_bin = cache['ZEPHYR_RUNNER_CONFIG_KERNEL_BIN'] + elf_file = cache.get('ZEPHYR_RUNNER_CONFIG_ELF_FILE', + cache['ZEPHYR_RUNNER_CONFIG_KERNEL_ELF']) + hex_file = cache.get('ZEPHYR_RUNNER_CONFIG_HEX_FILE', + cache['ZEPHYR_RUNNER_CONFIG_KERNEL_HEX']) + bin_file = cache.get('ZEPHYR_RUNNER_CONFIG_BIN_FILE', + cache['ZEPHYR_RUNNER_CONFIG_KERNEL_BIN']) gdb = cache.get('ZEPHYR_RUNNER_CONFIG_GDB') openocd = cache.get('ZEPHYR_RUNNER_CONFIG_OPENOCD') openocd_search = cache.get('ZEPHYR_RUNNER_CONFIG_OPENOCD_SEARCH') return RunnerConfig(build_dir, board_dir, - kernel_elf, kernel_hex, kernel_bin, + elf_file, hex_file, bin_file, gdb=gdb, openocd=openocd, openocd_search=openocd_search) diff --git a/scripts/meta/west/commands/schema.yml b/scripts/meta/west/commands/schema.yml deleted file mode 100644 index bed0dd12f83..00000000000 --- a/scripts/meta/west/commands/schema.yml +++ /dev/null @@ -1,135 +0,0 @@ -## A pykwalify schema for basic validation of the structure of a -## manifest YAML file. (Full validation would require additional work, -## e.g. to validate that remote URLs obey the URL format specified in -## rfc1738.) -## -## This schema has similar semantics to the repo XML format: -## -## https://gerrit.googlesource.com/git-repo/+/master/docs/manifest-format.txt -## -## However, the features don't map 1:1. - -# The top-level manifest is a map. The only top-level element is -# 'manifest'. All other elements are contained within it. This allows -# us a bit of future-proofing. -type: map -mapping: - manifest: - required: true - type: map - mapping: - # The "defaults" key specifies some default values used in the - # rest of the manifest. - # - # The value is a map with the following keys: - # - # - remote: if given, this is the default remote in each project - # - revision: if given, this is the default revision to check - # out of each project - # - # See below for more information about remotes and projects. - # - # Examples: - # - # default: - # remote: zephyrproject-rtos - # revision: master - defaults: - required: false - type: map - mapping: - remote: - required: false - type: str - revision: - required: false - type: str - - # The "remotes" key specifies a sequence of remotes, each of - # which has a name and a fetch URL. - # - # These work like repo remotes, in that they specify a URL - # prefix which remote-specific Git repositories hang off of. - # (This saves typing and makes it easier to move things around - # when most repositories are on the same server or GitHub - # organization.) - # - # Example: - # - # remotes: - # - name: zephyrproject-rtos - # url: https://github.com/zephyrproject-rtos - # - name: developer-fork - # url: https://github.com/a-developer - remotes: - required: true - type: seq - sequence: - - type: map - mapping: - name: - required: true - type: str - url: - required: true - type: str - - # The "projects" key specifies a sequence of "projects", - # i.e. Git repositories. These work like repo projects, in that - # each project has a name, a remote, and optional additional - # metadata. - # - # Each project is a map with the following keys: - # - # - name: Mandatory, the name of the git repository. The clone - # URL is formed by remote + '/' + name - # - remote: Optional, the name of the remote to pull it from. - # If the remote is missing, the remote'key in the top-level - # defaults key is used instead. If both are missing, it's an error. - # - revision: Optional, the name of the revision to check out. - # If not given, the value from the default element will be used. - # If both are missing, then the default is 'master'. - # - path: Where to clone the repository locally. If missing, - # it's cloned at top level in a directory given by its name. - # - clone-depth: if given, it is a number which creates a shallow - # history in the cloned repository limited to the given number - # of commits. - # - # Example, using default and non-default remotes: - # - # projects: - # # Uses default remote (zephyrproject-rtos), so clone URL is: - # # https://github.com/zephyrproject-rtos/zephyr - # - name: zephyr - # # Manually specified remote; clone URL is: - # # https://github.com/a-developer/west - # - name: west - # remote: developer-fork - # # Manually specified remote, clone URL is: - # # https://github.com/zephyrproject-rtos/some-vendor-hal - # # Local clone path (relative to installation root) is: - # # ext/hal/some-vendor - # - name: some-vendor-hal - # remote: zephyrproject-rtos - # path: ext/hal/some-vendor - projects: - required: true - type: seq - sequence: - - type: map - mapping: - name: - required: true - type: str - remote: - required: false - type: str - revision: - required: false - type: text # SHAs could be only numbers - path: - required: false - type: str - clone-depth: - required: false - type: int diff --git a/scripts/meta/west/config.py b/scripts/meta/west/config.py new file mode 100644 index 00000000000..34be7e4b57b --- /dev/null +++ b/scripts/meta/west/config.py @@ -0,0 +1,95 @@ +# Copyright (c) 2018, Nordic Semiconductor ASA +# +# SPDX-License-Identifier: Apache-2.0 + +''' +Configuration file handling, using the standard configparser module. +''' + +import configparser +import os +import platform + +from west.util import west_dir + + +# Configuration values. +# +# Initially empty, populated in read_config(). Always having this available is +# nice in case something checks configuration values before the configuration +# file has been read (e.g. the log.py functions, to check color settings, and +# tests). +config = configparser.ConfigParser() + + +def read_config(): + ''' + Reads all configuration files, making the configuration values available as + a configparser.ConfigParser object in config.config. This object works + similarly to a dictionary: config.config['foo']['bar'] gets the value for + key 'bar' in section 'foo'. + + Git conventions for configuration file locations are used. See the FILES + section in the git-config(1) man page. + + The following configuration files are read. + + System-wide: + + Linux: /etc/westconfig + Mac OS: /usr/local/etc/westconfig + Windows: %PROGRAMDATA%\\west\\config + + User-specific: + + $XDG_CONFIG_HOME/west/config (on Linux) + and + ~/.westconfig + + ($XDG_CONFIG_DIR defaults to ~/.config/ if unset.) + + Instance-specific: + + /west/config + + Configuration values from later configuration files override configuration + from earlier ones. Instance-specific configuration values have the highest + precedence, and system-wide the lowest. + ''' + + # Gather (potential) configuration file paths + + # System-wide and user-specific + + if platform.system() == 'Linux': + # Probably wouldn't hurt to check $XDG_CONFIG_HOME (defaults to + # ~/.config) on all systems. It's listed in git-config(1). People were + # iffy about it as of writing though. + files = ['/etc/westconfig', + os.path.join(os.environ.get('XDG_CONFIG_HOME', + os.path.expanduser('~/.config')), + 'west', 'config')] + + elif platform.system() == 'Darwin': # Mac OS + # This was seen on a local machine ($(prefix) = /usr/local) + files = ['/usr/local/etc/westconfig'] + elif platform.system() == 'Windows': + # Seen on a local machine + files = [os.path.expandvars('%PROGRAMDATA%\\west\\config')] + + files.append(os.path.expanduser('~/.westconfig')) + + # Repository-specific + + files.append(os.path.join(west_dir(), 'config')) + + # + # Parse all existing configuration files + # + + config.read(files, encoding='utf-8') + + +def use_colors(): + # Convenience function for reading the color.ui setting + return config.getboolean('color', 'ui', fallback=True) diff --git a/scripts/meta/west/log.py b/scripts/meta/west/log.py index f63d20bf20c..76928f3a3df 100644 --- a/scripts/meta/west/log.py +++ b/scripts/meta/west/log.py @@ -6,6 +6,8 @@ Provides common methods for logging messages to display to the user.''' +from west import config + import colorama import sys @@ -47,6 +49,10 @@ def inf(*args, colorize=False): colorize (default: False): If True, the message is printed in bright green if stdout is a terminal. ''' + + if not config.use_colors(): + colorize = False + # This approach colorizes any sep= and end= text too, as expected. # # colorama automatically strips the ANSI escapes when stdout isn't a @@ -57,29 +63,43 @@ def inf(*args, colorize=False): print(*args) if colorize: - # The final flush=True avoids issues with unrelated output from - # commands (usually Git) becoming green, due to the final attribute - # reset ANSI escape getting line-buffered. - print(colorama.Style.RESET_ALL, end='', flush=True) + _reset_colors(sys.stdout) def wrn(*args): '''Print a warning.''' - print(colorama.Fore.LIGHTRED_EX + 'WARNING: ', end='', file=sys.stderr) + + if config.use_colors(): + print(colorama.Fore.LIGHTRED_EX, end='', file=sys.stderr) + + print('WARNING: ', end='', file=sys.stderr) print(*args, file=sys.stderr) - print(colorama.Style.RESET_ALL, end='', file=sys.stderr, flush=True) + + if config.use_colors(): + _reset_colors(sys.stderr) def err(*args, fatal=False): '''Print an error.''' - print(colorama.Fore.LIGHTRED_EX - + ('FATAL ERROR: ' if fatal else 'ERROR: '), - end='', file=sys.stderr) + + if config.use_colors(): + print(colorama.Fore.LIGHTRED_EX, end='', file=sys.stderr) + + print('FATAL ERROR: ' if fatal else 'ERROR: ', end='', file=sys.stderr) print(*args, file=sys.stderr) - print(colorama.Style.RESET_ALL, end='', file=sys.stderr, flush=True) + + if config.use_colors(): + _reset_colors(sys.stderr) def die(*args, exit_code=1): '''Print a fatal error, and abort with the given exit code.''' err(*args, fatal=True) sys.exit(exit_code) + + +def _reset_colors(file): + # The flush=True avoids issues with unrelated output from commands (usually + # Git) becoming colorized, due to the final attribute reset ANSI escape + # getting line-buffered + print(colorama.Style.RESET_ALL, end='', file=file, flush=True) diff --git a/scripts/meta/west/main.py b/scripts/meta/west/main.py index 4fe120bd7d0..fb581c8e0ba 100755 --- a/scripts/meta/west/main.py +++ b/scripts/meta/west/main.py @@ -15,17 +15,19 @@ import os import sys from subprocess import CalledProcessError, check_output, DEVNULL -import log -from commands import CommandContextError -from commands.build import Build -from commands.flash import Flash -from commands.debug import Debug, DebugServer, Attach -from commands.project import ListProjects, Fetch, Pull, Rebase, Branch, \ - Checkout, Diff, Status, Update, ForAll, \ - WestUpdated -from util import quote_sh_list, in_multirepo_install +from west import log +from west import config +from west.commands import CommandContextError +from west.commands.build import Build +from west.commands.flash import Flash +from west.commands.debug import Debug, DebugServer, Attach +from west.commands.project import List, Clone, Fetch, Pull, Rebase, Branch, \ + Checkout, Diff, Status, Update, ForAll, \ + WestUpdated +from west.manifest import Manifest +from west.util import quote_sh_list, in_multirepo_install, west_dir -IN_MULTIREPO_INSTALL = in_multirepo_install(__file__) +IN_MULTIREPO_INSTALL = in_multirepo_install(os.path.dirname(__file__)) BUILD_FLASH_COMMANDS = [ Build(), @@ -36,7 +38,8 @@ BUILD_FLASH_COMMANDS = [ ] PROJECT_COMMANDS = [ - ListProjects(), + List(), + Clone(), Fetch(), Pull(), Rebase(), @@ -65,16 +68,55 @@ def command_handler(command, known_args, unknown_args): command.run(known_args, unknown_args) -def validate_context(args, unknown): - '''Validate the run-time context expected by west.''' +def set_zephyr_base(args): + '''Ensure ZEPHYR_BASE is set, emitting warnings if that's not + possible, or if the user is pointing it somewhere different than + what the manifest expects.''' + zb_env = os.environ.get('ZEPHYR_BASE') + if args.zephyr_base: - os.environ['ZEPHYR_BASE'] = args.zephyr_base + # The command line --zephyr-base takes precedence over + # everything else. + zb = os.path.abspath(args.zephyr_base) + zb_origin = 'command line' else: - if 'ZEPHYR_BASE' not in os.environ: - log.wrn('--zephyr-base missing and no ZEPHYR_BASE', - 'in the environment') + # If the user doesn't specify it concretely, use the project + # with path 'zephyr' if that exists, or the ZEPHYR_BASE value + # in the calling environment. + # + # At some point, we need a more flexible way to set environment + # variables based on manifest contents, but this is good enough + # to get started with and to ask for wider testing. + manifest = Manifest.from_file() + for project in manifest.projects: + if project.path == 'zephyr': + zb = project.abspath + zb_origin = 'manifest file {}'.format(manifest.path) + break else: - args.zephyr_base = os.environ['ZEPHYR_BASE'] + if zb_env is None: + log.wrn('no --zephyr-base given, ZEPHYR_BASE is unset,', + 'and no manifest project has path "zephyr"') + zb = None + zb_origin = None + else: + zb = zb_env + zb_origin = 'environment' + + if zb_env and os.path.abspath(zb) != os.path.abspath(zb_env): + # The environment ZEPHYR_BASE takes precedence over either the + # command line or the manifest, but in normal multi-repo + # operation we shouldn't expect to need to set ZEPHYR_BASE to + # point to some random place. In practice, this is probably + # happening because zephyr-env.sh/cmd was run in some other + # zephyr installation, and the user forgot about that. + log.wrn('ZEPHYR_BASE={}'.format(zb_env), + 'in the calling environment, but has been set to', + zb, 'instead by the', zb_origin) + + os.environ['ZEPHYR_BASE'] = zb + + log.dbg('ZEPHYR_BASE={} (origin: {})'.format(zb, zb_origin)) def print_version_info(): @@ -96,7 +138,7 @@ def print_version_info(): stderr=DEVNULL, cwd=os.path.dirname(__file__)) west_version = desc.decode(sys.getdefaultencoding()).strip() - except CalledProcessError as e: + except CalledProcessError: west_version = 'unknown' else: west_version = 'N/A, monorepo installation' @@ -112,14 +154,21 @@ def parse_args(argv): west_parser = argparse.ArgumentParser( prog='west', description='The Zephyr RTOS meta-tool.', epilog='Run "west -h" for help on each command.') + + # Remember to update scripts/west-completion.bash if you add or remove + # flags + west_parser.add_argument('-z', '--zephyr-base', default=None, - help='''Path to the Zephyr base directory. If not - given, ZEPHYR_BASE must be defined in the - environment, and will be used instead.''') + help='''Override the Zephyr base directory. The + default is the manifest project with path + "zephyr".''') + west_parser.add_argument('-v', '--verbose', default=0, action='count', help='''Display verbose output. May be given multiple times to increase verbosity.''') + west_parser.add_argument('-V', '--version', action='store_true') + subparser_gen = west_parser.add_subparsers(title='commands', dest='command') @@ -138,16 +187,16 @@ def parse_args(argv): # work properly. log.set_verbosity(args.verbose) - try: - validate_context(args, unknown) - except InvalidWestContext as iwc: - log.err(*iwc.args, fatal=True) - west_parser.print_usage(file=sys.stderr) - sys.exit(1) + if IN_MULTIREPO_INSTALL: + set_zephyr_base(args) if 'handler' not in args: - log.err('you must specify a command', fatal=True) - west_parser.print_usage(file=sys.stderr) + if IN_MULTIREPO_INSTALL: + log.err('west installation found (in {}), but no command given'. + format(west_dir())) + else: + log.err('no west command given') + west_parser.print_help(file=sys.stderr) sys.exit(1) return args, unknown @@ -162,6 +211,10 @@ def main(argv=None): argv = sys.argv[1:] args, unknown = parse_args(argv) + if IN_MULTIREPO_INSTALL: + # Read the configuration files + config.read_config() + for_stack_trace = 'run as "west -v ... {} ..." for a stack trace'.format( args.command) try: @@ -169,7 +222,7 @@ def main(argv=None): except WestUpdated: # West has been automatically updated. Restart ourselves to run the # latest version, with the same arguments that we were given. - os.execv(sys.executable, [sys.executable] + sys.argv) + os.execv(sys.executable, [sys.executable] + argv) except KeyboardInterrupt: sys.exit(0) except CalledProcessError as cpe: @@ -182,12 +235,7 @@ def main(argv=None): except CommandContextError as cce: log.die('command', args.command, 'cannot be run in this context:', *cce.args) - except Exception as exc: - log.err(*exc.args, fatal=True) - if args.verbose: - raise - else: - log.inf(for_stack_trace) + if __name__ == "__main__": main() diff --git a/scripts/meta/west/manifest-schema.yml b/scripts/meta/west/manifest-schema.yml new file mode 100644 index 00000000000..c7d4e22dd2d --- /dev/null +++ b/scripts/meta/west/manifest-schema.yml @@ -0,0 +1,132 @@ +## A pykwalify schema for basic validation of the structure of a +## manifest YAML file. (Full validation would require additional work, +## e.g. to validate that remote URLs obey the URL format specified in +## rfc1738.) +## +## This schema has similar semantics to the repo XML format: +## +## https://gerrit.googlesource.com/git-repo/+/master/docs/manifest-format.txt +## +## However, the features don't map 1:1. + +# The top-level manifest is a map. There may be multiple sections in the +# manifest file. Each section can be validated by their own schema. +# This schema validates the 'manifest' section. +type: map +mapping: + # The "defaults" key specifies some default values used in the + # rest of the manifest. + # + # The value is a map with the following keys: + # + # - remote: if given, this is the default remote in each project + # - revision: if given, this is the default revision to check + # out of each project + # + # See below for more information about remotes and projects. + # + # Examples: + # + # default: + # remote: zephyrproject-rtos + # revision: master + defaults: + required: false + type: map + mapping: + remote: + required: false + type: str + revision: + required: false + type: str + + # The "remotes" key specifies a sequence of remotes, each of + # which has a name and a fetch URL. + # + # These work like repo remotes, in that they specify a URL + # prefix which remote-specific Git repositories hang off of. + # (This saves typing and makes it easier to move things around + # when most repositories are on the same server or GitHub + # organization.) + # + # Example: + # + # remotes: + # - name: zephyrproject-rtos + # url-base: https://github.com/zephyrproject-rtos + # - name: developer-fork + # url-base: https://github.com/a-developer + remotes: + required: true + type: seq + sequence: + - type: map + mapping: + name: + required: true + type: str + url-base: + required: true + type: str + + # The "projects" key specifies a sequence of "projects", + # i.e. Git repositories. These work like repo projects, in that + # each project has a name, a remote, and optional additional + # metadata. + # + # Each project is a map with the following keys: + # + # - name: Mandatory, the name of the git repository. The clone + # URL is formed by remote url-base + '/' + name. The name cannot + # be one of the reserved values "west" and "manifest". + # - remote: Optional, the name of the remote to pull it from. + # If the remote is missing, the remote'key in the top-level + # defaults key is used instead. If both are missing, it's an error. + # - revision: Optional, the name of the revision to check out. + # If not given, the value from the default element will be used. + # If both are missing, then the default is 'master'. + # - path: Where to clone the repository locally. If missing, + # it's cloned at top level in a directory given by its name. + # - clone-depth: if given, it is a number which creates a shallow + # history in the cloned repository limited to the given number + # of commits. + # + # Example, using default and non-default remotes: + # + # projects: + # # Uses default remote (zephyrproject-rtos), so clone URL is: + # # https://github.com/zephyrproject-rtos/zephyr + # - name: zephyr + # # Manually specified remote; clone URL is: + # # https://github.com/a-developer/west + # - name: west + # remote: developer-fork + # # Manually specified remote, clone URL is: + # # https://github.com/zephyrproject-rtos/some-vendor-hal + # # Local clone path (relative to installation root) is: + # # ext/hal/some-vendor + # - name: some-vendor-hal + # remote: zephyrproject-rtos + # path: ext/hal/some-vendor + projects: + required: true + type: seq + sequence: + - type: map + mapping: + name: + required: true + type: str + remote: + required: false + type: str + revision: + required: false + type: text # SHAs could be only numbers + path: + required: false + type: str + clone-depth: + required: false + type: int diff --git a/scripts/meta/west/manifest.py b/scripts/meta/west/manifest.py new file mode 100644 index 00000000000..c3aa76a4a8a --- /dev/null +++ b/scripts/meta/west/manifest.py @@ -0,0 +1,400 @@ +# Copyright (c) 2018, Nordic Semiconductor ASA +# Copyright 2018, Foundries.io Ltd +# +# SPDX-License-Identifier: Apache-2.0 + +'''Parser and abstract data types for west manifests. + +The main class is Manifest. The recommended method for creating a +Manifest instance is via its from_file() or from_data() helper +methods. + +There are additionally Defaults, Remote, and Project types defined, +which represent the values by the same names in a west +manifest. (I.e. "Remote" represents one of the elements in the +"remote" sequence in the manifest, and so on.) Some Default values, +such as the default project revision, may be supplied by this module +if they are not present in the manifest data.''' + +import os + +import pykwalify.core +import yaml + +from west import util, log + +# Todo: take from _bootstrap? +# Default west repository URL. +WEST_URL_DEFAULT = 'https://github.com/zephyrproject-rtos/west' +# Default revision to check out of the west repository. +WEST_REV_DEFAULT = 'master' + +META_NAMES = ['west', 'manifest'] +'''Names of the special "meta-projects", which are reserved and cannot +be used to name a project in the manifest file.''' + +MANIFEST_SECTIONS = ['manifest', 'west'] +'''Sections in the manifest file''' + + +def default_path(): + '''Return the path to the default manifest in the west directory. + + Raises WestNotFound if called from outside of a west working directory.''' + return os.path.join(util.west_dir(), 'manifest', 'default.yml') + + +class Manifest: + '''Represents the contents of a West manifest file. + + The most convenient way to construct an instance is using the + from_file and from_data helper methods.''' + + @staticmethod + def from_file(source_file=None, sections=MANIFEST_SECTIONS): + '''Create and return a new Manifest object given a source YAML file. + + :param source_file: Path to a YAML file containing the manifest. + :param sections: Only parse specified sections from YAML file, + default: all sections are parsed. + + If source_file is None, the value returned by default_path() + is used. + + Raises MalformedManifest in case of validation errors.''' + if source_file is None: + source_file = default_path() + return Manifest(source_file=source_file, sections=sections) + + @staticmethod + def from_data(source_data, sections=MANIFEST_SECTIONS): + '''Create and return a new Manifest object given parsed YAML data. + + :param source_data: Parsed YAML data as a Python object. + :param sections: Only parse specified sections from YAML data, + default: all sections are parsed. + + Raises MalformedManifest in case of validation errors.''' + return Manifest(source_data=source_data, sections=sections) + + def __init__(self, source_file=None, source_data=None, + sections=MANIFEST_SECTIONS): + '''Create a new Manifest object. + + :param source_file: Path to a YAML file containing the manifest. + :param source_data: Parsed YAML data as a Python object. + :param sections: Only parse specified sections from YAML file, + default: all sections are parsed. + + Normally, it is more convenient to use the `from_file` and + `from_data` convenience factories than calling the constructor + directly. + + Exactly one of the source_file and source_data parameters must + be given. + + Raises MalformedManifest in case of validation errors.''' + if source_file and source_data: + raise ValueError('both source_file and source_data were given') + + if source_file: + with open(source_file, 'r') as f: + self._data = yaml.safe_load(f.read()) + path = source_file + else: + self._data = source_data + path = None + + self.path = path + '''Path to the file containing the manifest, or None if created + from data rather than the file system.''' + + if not self._data: + self._malformed('manifest contains no data') + + if 'manifest' not in self._data: + self._malformed('manifest contains no manifest element') + + for key in self._data: + if key in sections: + try: + pykwalify.core.Core( + source_data=self._data[key], + schema_files=[_SCHEMA_PATH[key]] + ).validate() + except pykwalify.errors.SchemaError as e: + self._malformed(e, key) + + self.defaults = None + '''west.manifest.Defaults object representing default values + in the manifest, either as specified by the user or west itself.''' + + self.remotes = None + '''Sequence of west.manifest.Remote objects representing manifest + remotes.''' + + self.projects = None + '''Sequence of west.manifest.Project objects representing manifest + projects. + + Each element's values are fully initialized; there is no need + to consult the defaults field to supply missing values.''' + + self.west_project = None + '''west.manifest.SpecialProject object representing the west meta + project.''' + + # Set up the public attributes documented above, as well as + # any internal attributes needed to implement the public API. + self._load(self._data, sections) + + def get_remote(self, name): + '''Get a manifest Remote, given its name.''' + return self._remotes_dict[name] + + def _malformed(self, complaint, section='manifest'): + context = (' file {} '.format(self.path) if self.path + else ' data:\n{}\n'.format(self._data)) + raise MalformedManifest('Malformed manifest{}(schema: {}):\n{}' + .format(context, _SCHEMA_PATH[section], + complaint)) + + def _load(self, data, sections): + # Initialize this instance's fields from values given in the + # manifest data, which must be validated according to the schema. + if 'west' in sections: + west = data.get('west', {}) + + url = west.get('url') or WEST_URL_DEFAULT + revision = west.get('revision') or WEST_REV_DEFAULT + + self.west_project = SpecialProject('west', + url=url, + revision=revision, + path=os.path.join('west', + 'west')) + + # Next is the manifest section + if 'manifest' not in sections: + return + + projects = [] + project_abspaths = set() + + manifest = data.get('manifest') + + # Map from each remote's name onto that remote's data in the manifest. + remotes = tuple(Remote(r['name'], r['url-base']) for r in + manifest['remotes']) + remotes_dict = {r.name: r for r in remotes} + + # Get any defaults out of the manifest. + # + # md = manifest defaults (dictionary with values parsed from + # the manifest) + md = manifest.get('defaults', dict()) + mdrem = md.get('remote') + if mdrem: + # The default remote name, if provided, must refer to a + # well-defined remote. + if mdrem not in remotes_dict: + self._malformed('default remote {} is not defined'. + format(mdrem)) + default_remote = remotes_dict[mdrem] + default_remote_name = mdrem + else: + default_remote = None + default_remote_name = None + defaults = Defaults(remote=default_remote, revision=md.get('revision')) + + # mp = manifest project (dictionary with values parsed from + # the manifest) + for mp in manifest['projects']: + # Validate the project name. + name = mp['name'] + if name in META_NAMES: + self._malformed('the name "{}" is reserved and cannot '. + format(name) + + 'be used to name a manifest project') + + # Validate the project remote. + remote_name = mp.get('remote', default_remote_name) + if remote_name is None: + self._malformed('project {} does not specify a remote'. + format(name)) + if remote_name not in remotes_dict: + self._malformed('project {} remote {} is not defined'. + format(name, remote_name)) + project = Project(name, + remotes_dict[remote_name], + defaults, + path=mp.get('path'), + clone_depth=mp.get('clone-depth'), + revision=mp.get('revision')) + + # Two projects cannot have the same path. We use absolute + # paths to check for collisions to ensure paths are + # normalized (e.g. for case-insensitive file systems or + # in cases like on Windows where / or \ may serve as a + # path component separator). + if project.abspath in project_abspaths: + self._malformed('project {} path {} is already in use'. + format(project.name, project.path)) + + project_abspaths.add(project.abspath) + projects.append(project) + + self.defaults = defaults + self.remotes = remotes + self._remotes_dict = remotes_dict + self.projects = tuple(projects) + + +class MalformedManifest(Exception): + '''Exception indicating that west manifest parsing failed due to a + malformed value.''' + + +# Definitions for Manifest attribute types. + +class Defaults: + '''Represents default values in a manifest, either specified by the + user or by west itself. + + Defaults are neither comparable nor hashable.''' + + __slots__ = 'remote revision'.split() + + def __init__(self, remote=None, revision=None): + '''Initialize a defaults value from manifest data. + + :param remote: Remote instance corresponding to the default remote, + or None (an actual Remote object, not the name of + a remote as a string). + :param revision: Default Git revision; 'master' if not given.''' + if remote is not None: + _wrn_if_not_remote(remote) + if revision is None: + revision = 'master' + + self.remote = remote + self.revision = revision + + def __eq__(self, other): + return NotImplemented + + def __repr__(self): + return 'Defaults(remote={}, revision={})'.format(repr(self.remote), + repr(self.revision)) + + +class Remote: + '''Represents a remote defined in a west manifest. + + Remotes may be compared for equality, but are not hashable.''' + + __slots__ = 'name url_base'.split() + + def __init__(self, name, url_base): + '''Initialize a remote from manifest data. + + :param name: remote's name + :param url_base: remote's URL base.''' + if url_base.endswith('/'): + log.wrn('Remote', name, 'URL base', url_base, + 'ends with a slash ("/"); these are automatically', + 'appended by West') + + self.name = name + self.url_base = url_base + + def __eq__(self, other): + return self.name == other.name and self.url_base == other.url_base + + def __repr__(self): + return 'Remote(name={}, url_base={})'.format(repr(self.name), + repr(self.url_base)) + + +class Project: + '''Represents a project defined in a west manifest. + + Projects are neither comparable nor hashable.''' + + __slots__ = 'name remote url path abspath clone_depth revision'.split() + + def __init__(self, name, remote, defaults, path=None, clone_depth=None, + revision=None): + '''Specify a Project by name, Remote, and optional information. + + :param name: Project's user-defined name in the manifest. + :param remote: Remote instance corresponding to this Project as + specified in the manifest. This is used to build + the project's URL, and is also stored as an attribute. + :param defaults: If the revision parameter is not given, the project's + revision is set to defaults.revision. + :param path: Relative path to the project in the west + installation, if present in the manifest. If not given, + the project's ``name`` is used. + :param clone_depth: Nonnegative integer clone depth if present in + the manifest. + :param revision: Project revision as given in the manifest, if present. + If not given, defaults.revision is used instead. + ''' + _wrn_if_not_remote(remote) + + self.name = name + self.remote = remote + self.url = remote.url_base + '/' + name + self.path = os.path.normpath(path or name) + self.abspath = os.path.realpath(os.path.join(util.west_topdir(), + self.path)) + self.clone_depth = clone_depth + self.revision = revision or defaults.revision + + def __eq__(self, other): + return NotImplemented + + def __repr__(self): + reprs = [repr(x) for x in + (self.name, self.remote, self.url, self.path, + self.abspath, self.clone_depth, self.revision)] + return ('Project(name={}, remote={}, url={}, path={}, abspath={}, ' + 'clone_depth={}, revision={})').format(*reprs) + + +class SpecialProject(Project): + '''Represents a special project, e.g. the west or manifest project. + + Projects are neither comparable nor hashable.''' + + def __init__(self, name, path=None, revision=None, url=None): + '''Specify a Special Project by name, and url, and optional information. + + :param name: Special Project's user-defined name in the manifest + :param path: Relative path to the project in the west + installation, if present in the manifest. If None, + the project's ``name`` is used. + :param revision: Project revision as given in the manifest, if present. + :param url: Complete URL for special project. + ''' + self.name = name + self.url = url + self.path = path or name + self.abspath = os.path.realpath(os.path.join(util.west_topdir(), + self.path)) + self.revision = revision + self.remote = None + self.clone_depth = None + + +def _wrn_if_not_remote(remote): + if not isinstance(remote, Remote): + log.wrn('Remote', remote, 'is not a Remote instance') + + +_SCHEMA_PATH = {'manifest': os.path.join(os.path.dirname(__file__), + "manifest-schema.yml"), + 'west': os.path.join(os.path.dirname(__file__), + "_bootstrap", + "west-schema.yml")} diff --git a/scripts/meta/west/runners/__init__.py b/scripts/meta/west/runners/__init__.py index c642c64dda3..85b6532de77 100644 --- a/scripts/meta/west/runners/__init__.py +++ b/scripts/meta/west/runners/__init__.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -from runners.core import ZephyrBinaryRunner +from west.runners.core import ZephyrBinaryRunner # We import these here to ensure the ZephyrBinaryRunner subclasses are # defined; otherwise, ZephyrBinaryRunner.create_for_shell_script() @@ -10,19 +10,20 @@ from runners.core import ZephyrBinaryRunner # Explicitly silence the unused import warning. # flake8: noqa: F401 -from runners import arc -from runners import bossac -from runners import dfu -from runners import esp32 -from runners import jlink -from runners import nios2 -from runners import nrfjprog -from runners import nsim -from runners import openocd -from runners import pyocd -from runners import qemu -from runners import xtensa -from runners import intel_s1000 +from west.runners import arc +from west.runners import bossac +from west.runners import dfu +from west.runners import esp32 +from west.runners import jlink +from west.runners import nios2 +from west.runners import nrfjprog +from west.runners import nsim +from west.runners import openocd +from west.runners import pyocd +from west.runners import qemu +from west.runners import xtensa +from west.runners import intel_s1000 +from west.runners import blackmagicprobe def get_runner_cls(runner): '''Get a runner's class object, given its name.''' diff --git a/scripts/meta/west/runners/arc.py b/scripts/meta/west/runners/arc.py index 00411949ff3..bb1253facfb 100644 --- a/scripts/meta/west/runners/arc.py +++ b/scripts/meta/west/runners/arc.py @@ -7,7 +7,7 @@ from os import path -from runners.core import ZephyrBinaryRunner +from west.runners.core import ZephyrBinaryRunner DEFAULT_ARC_TCL_PORT = 6333 DEFAULT_ARC_TELNET_PORT = 4444 @@ -93,7 +93,7 @@ class EmStarterKitBinaryRunner(ZephyrBinaryRunner): ['-ex', 'target remote :{}'.format(self.gdb_port), '-ex', 'load'] + continue_arg + - [self.cfg.kernel_elf]) + [self.cfg.elf_file]) self.run_server_and_client(server_cmd, gdb_cmd) diff --git a/scripts/meta/west/runners/blackmagicprobe.py b/scripts/meta/west/runners/blackmagicprobe.py new file mode 100644 index 00000000000..dd19526ad38 --- /dev/null +++ b/scripts/meta/west/runners/blackmagicprobe.py @@ -0,0 +1,96 @@ +# Copyright (c) 2018 Roman Tataurov +# Modified 2018 Tavish Naruka +# +# SPDX-License-Identifier: Apache-2.0 +'''Runner for flashing with Black Magic Probe.''' +# https://github.com/blacksphere/blackmagic/wiki + +from west.runners.core import ZephyrBinaryRunner, RunnerCaps + + +class BlackMagicProbeRunner(ZephyrBinaryRunner): + '''Runner front-end for Black Magic probe.''' + + def __init__(self, cfg, gdb_serial): + super(BlackMagicProbeRunner, self).__init__(cfg) + self.gdb = [cfg.gdb] if cfg.gdb else None + self.elf_file = cfg.elf_file + self.gdb_serial = gdb_serial + + @classmethod + def name(cls): + return 'blackmagicprobe' + + @classmethod + def capabilities(cls): + return RunnerCaps(commands={'flash', 'debug', 'attach'}) + + @classmethod + def create(cls, cfg, args): + return BlackMagicProbeRunner(cfg, args.gdb_serial) + + @classmethod + def do_add_parser(cls, parser): + parser.add_argument('--gdb-serial', default='/dev/ttyACM0', + help='GDB serial port') + + def bmp_flash(self, command, **kwargs): + if self.gdb is None: + raise ValueError('Cannot flash; gdb is missing') + if self.elf_file is None: + raise ValueError('Cannot debug; elf file is missing') + command = (self.gdb + + ['-ex', "set confirm off", + '-ex', "target extended-remote {}".format(self.gdb_serial), + '-ex', "monitor swdp_scan", + '-ex', "attach 1", + '-ex', "load {}".format(self.elf_file), + '-ex', "kill", + '-ex', "quit", + '-silent']) + self.check_call(command) + + def bmp_attach(self, command, **kwargs): + if self.gdb is None: + raise ValueError('Cannot attach; gdb is missing') + if self.elf_file is None: + command = (self.gdb + + ['-ex', "set confirm off", + '-ex', "target extended-remote {}".format( + self.gdb_serial), + '-ex', "monitor swdp_scan", + '-ex', "attach 1"]) + else: + command = (self.gdb + + ['-ex', "set confirm off", + '-ex', "target extended-remote {}".format( + self.gdb_serial), + '-ex', "monitor swdp_scan", + '-ex', "attach 1", + '-ex', "file {}".format(self.elf_file)]) + self.check_call(command) + + def bmp_debug(self, command, **kwargs): + if self.gdb is None: + raise ValueError('Cannot debug; gdb is missing') + if self.elf_file is None: + raise ValueError('Cannot debug; elf file is missing') + command = (self.gdb + + ['-ex', "set confirm off", + '-ex', "target extended-remote {}".format(self.gdb_serial), + '-ex', "monitor swdp_scan", + '-ex', "attach 1", + '-ex', "file {}".format(self.elf_file), + '-ex', "load {}".format(self.elf_file)]) + self.check_call(command) + + def do_run(self, command, **kwargs): + + if command == 'flash': + self.bmp_flash(command, **kwargs) + elif command == 'debug': + self.bmp_debug(command, **kwargs) + elif command == 'attach': + self.bmp_attach(command, **kwargs) + else: + self.bmp_flash(command, **kwargs) diff --git a/scripts/meta/west/runners/bossac.py b/scripts/meta/west/runners/bossac.py index 0173008581c..5ba6c480b12 100644 --- a/scripts/meta/west/runners/bossac.py +++ b/scripts/meta/west/runners/bossac.py @@ -6,7 +6,7 @@ import platform -from runners.core import ZephyrBinaryRunner, RunnerCaps +from west.runners.core import ZephyrBinaryRunner, RunnerCaps DEFAULT_BOSSAC_PORT = '/dev/ttyACM0' @@ -48,7 +48,7 @@ class BossacBinaryRunner(ZephyrBinaryRunner): 'ospeed', '1200', 'cs8', '-cstopb', 'ignpar', 'eol', '255', 'eof', '255'] cmd_flash = [self.bossac, '-p', self.port, '-R', '-e', '-w', '-v', - '-b', self.cfg.kernel_bin] + '-b', self.cfg.bin_file] self.check_call(cmd_stty) self.check_call(cmd_flash) diff --git a/scripts/meta/west/runners/core.py b/scripts/meta/west/runners/core.py index 42401feec6b..60dc6cc5979 100644 --- a/scripts/meta/west/runners/core.py +++ b/scripts/meta/west/runners/core.py @@ -18,8 +18,8 @@ import platform import signal import subprocess -import log -from util import quote_sh_list +from west import log +from west.util import quote_sh_list # Turn on to enable just printing the commands that would be run, # without actually running them. This can break runners that are expecting @@ -191,8 +191,8 @@ class RunnerConfig: This class's __slots__ contains exactly the configuration variables. ''' - __slots__ = ['build_dir', 'board_dir', 'kernel_elf', 'kernel_hex', - 'kernel_bin', 'gdb', 'openocd', 'openocd_search'] + __slots__ = ['build_dir', 'board_dir', 'elf_file', 'hex_file', + 'bin_file', 'gdb', 'openocd', 'openocd_search'] # TODO: revisit whether we can get rid of some of these. Having # tool-specific configuration options here is a layering @@ -200,7 +200,7 @@ class RunnerConfig: # store the locations of tools (like gdb and openocd) that are # needed by multiple ZephyrBinaryRunner subclasses. def __init__(self, build_dir, board_dir, - kernel_elf, kernel_hex, kernel_bin, + elf_file, hex_file, bin_file, gdb=None, openocd=None, openocd_search=None): self.build_dir = build_dir '''Zephyr application build directory''' @@ -208,14 +208,14 @@ class RunnerConfig: self.board_dir = board_dir '''Zephyr board directory''' - self.kernel_elf = kernel_elf - '''Path to kernel binary in .elf format''' + self.elf_file = elf_file + '''Path to the elf file that the runner should operate on''' - self.kernel_hex = kernel_hex - '''Path to kernel binary in .hex format''' + self.hex_file = hex_file + '''Path to the hex file that the runner should operate on''' - self.kernel_bin = kernel_bin - '''Path to kernel binary in .bin format''' + self.bin_file = bin_file + '''Path to the bin file that the runner should operate on''' self.gdb = gdb ''''Path to GDB compatible with the target, may be None.''' diff --git a/scripts/meta/west/runners/dfu.py b/scripts/meta/west/runners/dfu.py index 45445be6468..595409964c8 100644 --- a/scripts/meta/west/runners/dfu.py +++ b/scripts/meta/west/runners/dfu.py @@ -5,12 +5,12 @@ '''Runner for flashing with dfu-util.''' from collections import namedtuple -import os import sys import time -import log -from runners.core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration +from west import log +from west.runners.core import ZephyrBinaryRunner, RunnerCaps, \ + BuildConfiguration DfuSeConfig = namedtuple('DfuSeConfig', ['address', 'options']) @@ -54,7 +54,7 @@ class DfuUtilBinaryRunner(ZephyrBinaryRunner): # Optional: parser.add_argument("--img", - help="binary to flash, default is --kernel-bin") + help="binary to flash, default is --bin-file") parser.add_argument("--dfuse", default=False, action='store_true', help='''set if target is a DfuSe device; implies --dt-flash.''') @@ -69,7 +69,7 @@ class DfuUtilBinaryRunner(ZephyrBinaryRunner): @classmethod def create(cls, cfg, args): if args.img is None: - args.img = cfg.kernel_bin + args.img = cfg.bin_file if args.dfuse: args.dt_flash = True # --dfuse implies --dt-flash. diff --git a/scripts/meta/west/runners/esp32.py b/scripts/meta/west/runners/esp32.py index c12a4500924..e13fe1b8ef6 100644 --- a/scripts/meta/west/runners/esp32.py +++ b/scripts/meta/west/runners/esp32.py @@ -6,8 +6,8 @@ from os import path -import log -from runners.core import ZephyrBinaryRunner, RunnerCaps +from west import log +from west.runners.core import ZephyrBinaryRunner, RunnerCaps class Esp32BinaryRunner(ZephyrBinaryRunner): @@ -17,7 +17,7 @@ class Esp32BinaryRunner(ZephyrBinaryRunner): flash_freq='40m', flash_mode='dio', espidf='espidf', bootloader_bin=None, partition_table_bin=None): super(Esp32BinaryRunner, self).__init__(cfg) - self.elf = cfg.kernel_elf + self.elf = cfg.elf_file self.device = device self.baud = baud self.flash_size = flash_size diff --git a/scripts/meta/west/runners/intel_s1000.py b/scripts/meta/west/runners/intel_s1000.py index 6caf16ec142..c56c9d5953d 100644 --- a/scripts/meta/west/runners/intel_s1000.py +++ b/scripts/meta/west/runners/intel_s1000.py @@ -6,10 +6,9 @@ '''Runner for debugging and flashing Intel S1000 CRB''' from os import path import time -import subprocess - -import log -from runners.core import ZephyrBinaryRunner +import signal +from west import log +from west.runners.core import ZephyrBinaryRunner DEFAULT_XT_GDB_PORT = 20000 @@ -22,7 +21,7 @@ class IntelS1000BinaryRunner(ZephyrBinaryRunner): gdb_port=DEFAULT_XT_GDB_PORT): super(IntelS1000BinaryRunner, self).__init__(cfg) self.board_dir = cfg.board_dir - self.elf_name = cfg.kernel_elf + self.elf_name = cfg.elf_file self.gdb_cmd = cfg.gdb self.xt_ocd_dir = xt_ocd_dir self.ocd_topology = ocd_topology @@ -73,8 +72,7 @@ class IntelS1000BinaryRunner(ZephyrBinaryRunner): elif command == 'debugserver': self.debugserver(**kwargs) else: - self.debugserver(**kwargs) - self.do_debug() + self.do_debug(**kwargs) def flash(self, **kwargs): topology_file = kwargs['ocd-topology'] @@ -87,36 +85,66 @@ class IntelS1000BinaryRunner(ZephyrBinaryRunner): '-I', jtag_instr_file] # Start the server - # Note that XTOCD always fails the first time. It has to be - # relaunched the second time to work. - self.call(server_cmd) + # Note that XTOCD takes a few seconds to execute and always fails the + # first time. It has to be relaunched the second time to work. server_proc = self.popen_ignore_int(server_cmd) - time.sleep(3) + time.sleep(6) + server_proc.terminate() + server_proc = self.popen_ignore_int(server_cmd) + time.sleep(6) - # Start the client, and wait for it to finish flashing the file. + # Start the client gdb_cmd = [self.gdb_cmd, '-x', gdb_flash_file] client_proc = self.popen_ignore_int(gdb_cmd) - client_proc.wait() + + # Wait for 3 seconds (waiting for XTGDB to finish loading the image) + time.sleep(3) # At this point, the ELF image is loaded and the program is in - # execution. Now we can quit the server (xt-ocd); it is not - # needed anymore. The loaded program (ELF) will continue to - # run. + # execution. Now we can quit the client (xt-gdb) and the server + # (xt-ocd) as they are not needed anymore. The loaded program + # (ELF) will continue to run though. + client_proc.terminate() server_proc.terminate() - def do_debug(self): + def do_debug(self, **kwargs): if self.elf_name is None: raise ValueError('Cannot debug; elf is missing') if self.gdb_cmd is None: raise ValueError('Cannot debug; no gdb specified') + topology_file = kwargs['ocd-topology'] + jtag_instr_file = kwargs['ocd-jtag-instr'] + + self.print_gdbserver_message(self.gdb_port) + server_cmd = [self.xt_ocd_dir, + '-c', topology_file, + '-I', jtag_instr_file] + + # Start the server + # Note that XTOCD takes a few seconds to execute and always fails the + # first time. It has to be relaunched the second time to work. + server_proc = self.popen_ignore_int(server_cmd) + time.sleep(6) + server_proc.terminate() + server_proc = self.popen_ignore_int(server_cmd) + time.sleep(6) + gdb_cmd = [self.gdb_cmd, '-ex', 'target remote :{}'.format(self.gdb_port), self.elf_name] - gdb_proc = self.popen_ignore_int(gdb_cmd) - retcode = gdb_proc.wait() - if retcode: - raise subprocess.CalledProcessError((retcode, gdb_cmd)) + + # Start the client + # The below statement will consume the "^C" keypress ensuring + # the python main application doesn't exit. This is important + # since ^C in gdb means a "halt" operation. + previous = signal.signal(signal.SIGINT, signal.SIG_IGN) + try: + self.check_call(gdb_cmd) + finally: + signal.signal(signal.SIGINT, previous) + server_proc.terminate() + server_proc.wait() def print_gdbserver_message(self, gdb_port): log.inf('Intel S1000 GDB server running on port {}'.format(gdb_port)) @@ -130,7 +158,9 @@ class IntelS1000BinaryRunner(ZephyrBinaryRunner): '-c', topology_file, '-I', jtag_instr_file] - # Note that XTOCD always fails the first time. It has to be - # relaunched the second time to work. - self.call(server_cmd) + # Note that XTOCD takes a few seconds to execute and always fails the + # first time. It has to be relaunched the second time to work. + server_proc = self.popen_ignore_int(server_cmd) + time.sleep(6) + server_proc.terminate() self.check_call(server_cmd) diff --git a/scripts/meta/west/runners/jlink.py b/scripts/meta/west/runners/jlink.py index e46d6ef8796..bc5d59b66bc 100644 --- a/scripts/meta/west/runners/jlink.py +++ b/scripts/meta/west/runners/jlink.py @@ -6,10 +6,13 @@ import os import tempfile +import sys -import log -from runners.core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration +from west import log +from west.runners.core import ZephyrBinaryRunner, RunnerCaps, \ + BuildConfiguration +DEFAULT_JLINK_EXE = 'JLink.exe' if sys.platform == 'win32' else 'JLinkExe' DEFAULT_JLINK_GDB_PORT = 2331 @@ -17,14 +20,14 @@ class JLinkBinaryRunner(ZephyrBinaryRunner): '''Runner front-end for the J-Link GDB server.''' def __init__(self, cfg, device, - commander='JLinkExe', + commander=DEFAULT_JLINK_EXE, flash_addr=0x0, erase=True, iface='swd', speed='auto', gdbserver='JLinkGDBServer', gdb_port=DEFAULT_JLINK_GDB_PORT, tui=False): super(JLinkBinaryRunner, self).__init__(cfg) - self.bin_name = cfg.kernel_bin - self.elf_name = cfg.kernel_elf + self.bin_name = cfg.bin_file + self.elf_name = cfg.elf_file self.gdb_cmd = [cfg.gdb] if cfg.gdb else None self.device = device self.commander = commander @@ -62,7 +65,7 @@ class JLinkBinaryRunner(ZephyrBinaryRunner): parser.add_argument('--gdb-port', default=DEFAULT_JLINK_GDB_PORT, help='pyocd gdb port, defaults to {}'.format( DEFAULT_JLINK_GDB_PORT)) - parser.add_argument('--commander', default='JLinkExe', + parser.add_argument('--commander', default=DEFAULT_JLINK_EXE, help='J-Link Commander, default is JLinkExe') parser.add_argument('--erase', default=False, action='store_true', help='if given, mass erase flash before loading') @@ -127,6 +130,9 @@ class JLinkBinaryRunner(ZephyrBinaryRunner): lines.append('g') # Start the CPU lines.append('q') # Close the connection and quit + log.dbg('JLink commander script:') + log.dbg('\n'.join(lines)) + # Don't use NamedTemporaryFile: the resulting file can't be # opened again on Windows. with tempfile.TemporaryDirectory(suffix='jlink') as d: diff --git a/scripts/meta/west/runners/nios2.py b/scripts/meta/west/runners/nios2.py index 4edef1c94de..1298a023879 100644 --- a/scripts/meta/west/runners/nios2.py +++ b/scripts/meta/west/runners/nios2.py @@ -4,8 +4,8 @@ '''Runner for NIOS II, based on quartus-flash.py and GDB.''' -import log -from runners.core import ZephyrBinaryRunner, NetworkPortHelper +from west import log +from west.runners.core import ZephyrBinaryRunner, NetworkPortHelper class Nios2BinaryRunner(ZephyrBinaryRunner): @@ -19,8 +19,8 @@ class Nios2BinaryRunner(ZephyrBinaryRunner): def __init__(self, cfg, quartus_py=None, cpu_sof=None, tui=False): super(Nios2BinaryRunner, self).__init__(cfg) - self.hex_name = cfg.kernel_hex - self.elf_name = cfg.kernel_elf + self.hex_name = cfg.hex_file + self.elf_name = cfg.elf_file self.cpu_sof = cpu_sof self.quartus_py = quartus_py self.gdb_cmd = [cfg.gdb] if cfg.gdb else None diff --git a/scripts/meta/west/runners/nrfjprog.py b/scripts/meta/west/runners/nrfjprog.py index ce163a1a5a4..0aafdc57c7b 100644 --- a/scripts/meta/west/runners/nrfjprog.py +++ b/scripts/meta/west/runners/nrfjprog.py @@ -6,8 +6,8 @@ import sys -import log -from runners.core import ZephyrBinaryRunner, RunnerCaps +from west import log +from west.runners.core import ZephyrBinaryRunner, RunnerCaps class NrfJprogBinaryRunner(ZephyrBinaryRunner): @@ -15,7 +15,7 @@ class NrfJprogBinaryRunner(ZephyrBinaryRunner): def __init__(self, cfg, family, softreset, snr, erase=False): super(NrfJprogBinaryRunner, self).__init__(cfg) - self.hex_ = cfg.kernel_hex + self.hex_ = cfg.hex_file self.family = family self.softreset = softreset self.snr = snr @@ -44,19 +44,21 @@ class NrfJprogBinaryRunner(ZephyrBinaryRunner): @classmethod def create(cls, cfg, args): - return NrfJprogBinaryRunner(cfg, args.nrf_family, args.softreset, args.snr, - erase=args.erase) + return NrfJprogBinaryRunner(cfg, args.nrf_family, args.softreset, + args.snr, erase=args.erase) def get_board_snr_from_user(self): snrs = self.check_output(['nrfjprog', '--ids']) snrs = snrs.decode(sys.getdefaultencoding()).strip().splitlines() if len(snrs) == 0: - raise RuntimeError('"nrfjprog --ids" did not find a board; Is the board connected?') + raise RuntimeError('"nrfjprog --ids" did not find a board; ' + 'is the board connected?') elif len(snrs) == 1: board_snr = snrs[0] if board_snr == '0': - raise RuntimeError('"nrfjprog --ids" returned 0; is a debugger already connected?') + raise RuntimeError('"nrfjprog --ids" returned 0; ' + 'is a debugger already connected?') return board_snr log.dbg("Refusing the temptation to guess a board", @@ -99,14 +101,14 @@ class NrfJprogBinaryRunner(ZephyrBinaryRunner): '-f', self.family, '--snr', board_snr], program_cmd - ]) + ]) else: if self.family == 'NRF51': commands.append(program_cmd + ['--sectorerase']) else: commands.append(program_cmd + ['--sectoranduicrerase']) - if self.family == 'NRF52' and self.softreset == False: + if self.family == 'NRF52' and not self.softreset: commands.extend([ # Enable pin reset ['nrfjprog', '--pinresetenable', '-f', self.family, diff --git a/scripts/meta/west/runners/nsim.py b/scripts/meta/west/runners/nsim.py index c58ff7cf6f8..5eb6dc6a617 100644 --- a/scripts/meta/west/runners/nsim.py +++ b/scripts/meta/west/runners/nsim.py @@ -7,7 +7,7 @@ from os import path -from runners.core import ZephyrBinaryRunner +from west.runners.core import ZephyrBinaryRunner DEFAULT_ARC_GDB_PORT = 3333 DEFAULT_PROPS_FILE = 'nsim.props' @@ -56,7 +56,7 @@ class NsimBinaryRunner(ZephyrBinaryRunner): def do_run(self, command, **kwargs): kwargs['nsim-cfg'] = path.join(self.cfg.board_dir, 'support', - self.props) + self.props) if command == 'flash': self.do_flash(**kwargs) @@ -68,20 +68,19 @@ class NsimBinaryRunner(ZephyrBinaryRunner): def do_flash(self, **kwargs): config = kwargs['nsim-cfg'] - cmd = (self.nsim_cmd + - ['-propsfile', config, self.cfg.kernel_elf]) + cmd = (self.nsim_cmd + ['-propsfile', config, self.cfg.elf_file]) self.check_call(cmd) def do_debug(self, **kwargs): config = kwargs['nsim-cfg'] - server_cmd = (self.nsim_cmd + - ['-gdb', '-port={}'.format(self.gdb_port), - '-propsfile', config]) + server_cmd = (self.nsim_cmd + ['-gdb', + '-port={}'.format(self.gdb_port), + '-propsfile', config]) gdb_cmd = (self.gdb_cmd + ['-ex', 'target remote :{}'.format(self.gdb_port), - '-ex', 'load', self.cfg.kernel_elf]) + '-ex', 'load', self.cfg.elf_file]) self.run_server_and_client(server_cmd, gdb_cmd) @@ -89,7 +88,7 @@ class NsimBinaryRunner(ZephyrBinaryRunner): config = kwargs['nsim-cfg'] cmd = (self.nsim_cmd + - ['-gdb', '-port={}'.format(self.gdb_port), - '-propsfile', config]) + ['-gdb', '-port={}'.format(self.gdb_port), + '-propsfile', config]) self.check_call(cmd) diff --git a/scripts/meta/west/runners/openocd.py b/scripts/meta/west/runners/openocd.py index 8daad3ef55e..32d14da93c5 100644 --- a/scripts/meta/west/runners/openocd.py +++ b/scripts/meta/west/runners/openocd.py @@ -6,7 +6,7 @@ from os import path -from runners.core import ZephyrBinaryRunner +from west.runners.core import ZephyrBinaryRunner DEFAULT_OPENOCD_TCL_PORT = 6333 DEFAULT_OPENOCD_TELNET_PORT = 4444 @@ -30,7 +30,7 @@ class OpenOcdBinaryRunner(ZephyrBinaryRunner): if cfg.openocd_search is not None: search_args = ['-s', cfg.openocd_search] self.openocd_cmd = [cfg.openocd] + search_args - self.elf_name = cfg.kernel_elf + self.elf_name = cfg.elf_file self.load_cmd = load_cmd self.verify_cmd = verify_cmd self.pre_cmd = pre_cmd diff --git a/scripts/meta/west/runners/pyocd.py b/scripts/meta/west/runners/pyocd.py index bda7d027283..1d9d88c94ee 100644 --- a/scripts/meta/west/runners/pyocd.py +++ b/scripts/meta/west/runners/pyocd.py @@ -5,9 +5,10 @@ '''Runner for pyOCD .''' import os -import sys -from runners.core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration -import log + +from west.runners.core import ZephyrBinaryRunner, RunnerCaps, \ + BuildConfiguration +from west import log DEFAULT_PYOCD_GDB_PORT = 3333 @@ -20,7 +21,7 @@ class PyOcdBinaryRunner(ZephyrBinaryRunner): flashtool_opts=None, gdbserver='pyocd-gdbserver', gdb_port=DEFAULT_PYOCD_GDB_PORT, tui=False, - board_id=None, daparg=None): + board_id=None, daparg=None, frequency=None): super(PyOcdBinaryRunner, self).__init__(cfg) self.target_args = ['-t', target] @@ -30,8 +31,9 @@ class PyOcdBinaryRunner(ZephyrBinaryRunner): self.gdbserver = gdbserver self.gdb_port = gdb_port self.tui_args = ['-tui'] if tui else [] - self.bin_name = cfg.kernel_bin - self.elf_name = cfg.kernel_elf + self.hex_name = cfg.hex_file + self.bin_name = cfg.bin_file + self.elf_name = cfg.elf_file board_args = [] if board_id is not None: @@ -43,6 +45,11 @@ class PyOcdBinaryRunner(ZephyrBinaryRunner): daparg_args = ['-da', daparg] self.daparg_args = daparg_args + frequency_args = [] + if frequency is not None: + frequency_args = ['-f', frequency] + self.frequency_args = frequency_args + self.flashtool_extra = flashtool_opts if flashtool_opts else [] @classmethod @@ -66,6 +73,8 @@ class PyOcdBinaryRunner(ZephyrBinaryRunner): parser.add_argument('--flashtool-opt', default=[], action='append', help='''Additional options for pyocd-flashtool, e.g. -ce to chip erase''') + parser.add_argument('--frequency', + help='SWD clock frequency in Hz') parser.add_argument('--gdbserver', default='pyocd-gdbserver', help='GDB server, default is pyocd-gdbserver') parser.add_argument('--gdb-port', default=DEFAULT_PYOCD_GDB_PORT, @@ -94,7 +103,8 @@ class PyOcdBinaryRunner(ZephyrBinaryRunner): cfg, args.target, flashtool=args.flashtool, flash_addr=flash_addr, flashtool_opts=args.flashtool_opt, gdbserver=args.gdbserver, gdb_port=args.gdb_port, tui=args.tui, - board_id=args.board_id, daparg=args.daparg) + board_id=args.board_id, daparg=args.daparg, + frequency=args.frequency) def port_args(self): return ['-p', str(self.gdb_port)] @@ -106,16 +116,23 @@ class PyOcdBinaryRunner(ZephyrBinaryRunner): self.debug_debugserver(command, **kwargs) def flash(self, **kwargs): - if self.bin_name is None: - raise ValueError('Cannot flash; bin_name is missing') + if os.path.isfile(self.hex_name): + fname = self.hex_name + elif os.path.isfile(self.bin_name): + fname = self.bin_name + else: + raise ValueError( + 'Cannot flash; no hex ({}) or bin ({}) files'.format( + self.hex_name, self.bin_name)) cmd = ([self.flashtool] + self.flash_addr_args + self.daparg_args + self.target_args + self.board_args + + self.frequency_args + self.flashtool_extra + - [self.bin_name]) + [fname]) log.inf('Flashing Target Device') self.check_call(cmd) @@ -128,7 +145,8 @@ class PyOcdBinaryRunner(ZephyrBinaryRunner): self.daparg_args + self.port_args() + self.target_args + - self.board_args) + self.board_args + + self.frequency_args) if command == 'debugserver': self.print_gdbserver_message() @@ -143,8 +161,9 @@ class PyOcdBinaryRunner(ZephyrBinaryRunner): [self.elf_name] + ['-ex', 'target remote :{}'.format(self.gdb_port)]) if command == 'debug': - client_cmd += ['-ex', 'load', - '-ex', 'monitor reset halt'] + client_cmd += ['-ex', 'monitor halt', + '-ex', 'monitor reset', + '-ex', 'load'] self.print_gdbserver_message() self.run_server_and_client(server_cmd, client_cmd) diff --git a/scripts/meta/west/runners/qemu.py b/scripts/meta/west/runners/qemu.py index ea39be8ad41..fc62fa4a142 100644 --- a/scripts/meta/west/runners/qemu.py +++ b/scripts/meta/west/runners/qemu.py @@ -4,7 +4,7 @@ '''Runner stub for QEMU.''' -from runners.core import ZephyrBinaryRunner, RunnerCaps +from west.runners.core import ZephyrBinaryRunner, RunnerCaps class QemuBinaryRunner(ZephyrBinaryRunner): diff --git a/scripts/meta/west/runners/xtensa.py b/scripts/meta/west/runners/xtensa.py index 2aeb42d8f69..8c8e0e9c44b 100644 --- a/scripts/meta/west/runners/xtensa.py +++ b/scripts/meta/west/runners/xtensa.py @@ -6,7 +6,7 @@ from os import path -from runners.core import ZephyrBinaryRunner, RunnerCaps +from west.runners.core import ZephyrBinaryRunner, RunnerCaps class XtensaBinaryRunner(ZephyrBinaryRunner): @@ -35,6 +35,6 @@ class XtensaBinaryRunner(ZephyrBinaryRunner): return XtensaBinaryRunner(cfg) def do_run(self, command, **kwargs): - gdb_cmd = [self.cfg.gdb, self.cfg.kernel_elf] + gdb_cmd = [self.cfg.gdb, self.cfg.elf_file] self.check_call(gdb_cmd)