diff --git a/cmake/flash/CMakeLists.txt b/cmake/flash/CMakeLists.txt index 0ccbf0480f0..5558e3d98d7 100644 --- a/cmake/flash/CMakeLists.txt +++ b/cmake/flash/CMakeLists.txt @@ -89,7 +89,7 @@ foreach(target flash debug debugserver) ${CMAKE_COMMAND} -E env PYTHONPATH=${ZEPHYR_BASE}/scripts/meta ${PYTHON_EXECUTABLE} - -m west + ${ZEPHYR_BASE}/scripts/meta/west/main.py ${RUNNER_VERBOSE} ${target} --skip-rebuild diff --git a/doc/conf.py b/doc/conf.py index a1947a11135..a4d263e98fd 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -28,6 +28,9 @@ ZEPHYR_BUILD = os.path.abspath(os.environ["ZEPHYR_BUILD"]) sys.path.insert(0, os.path.join(ZEPHYR_BASE, 'doc', 'extensions')) # Also add west, to be able to pull in its API docs. sys.path.append(os.path.join(ZEPHYR_BASE, 'scripts', 'meta')) +# HACK: also add the runners module, to work around some import issues +# related to west's current packaging. +sys.path.append(os.path.join(ZEPHYR_BASE, 'scripts', 'meta', 'west')) # -- General configuration ------------------------------------------------ diff --git a/doc/west/flash-debug.rst b/doc/west/flash-debug.rst index ffdf3e7f3a3..e45e7166418 100644 --- a/doc/west/flash-debug.rst +++ b/doc/west/flash-debug.rst @@ -191,26 +191,26 @@ For example, to print usage information about the ``jlink`` runner:: .. _west-runner: -Library Backend: ``west.runner`` -******************************** +Library Backend: ``west.runners`` +********************************* In keeping with West's :ref:`west-design-constraints`, the flash and debug commands are wrappers around a separate library that is part of West, and can be used by other code. -This library is the ``west.runner`` subpackage of West itself. The +This library is the ``west.runners`` subpackage of West itself. The central abstraction within this library is ``ZephyrBinaryRunner``, an abstract class which represents *runner* objects, which can flash and/or debug Zephyr programs. The set of available runners is determined by the imported subclasses of ``ZephyrBinaryRunner``. -``ZephyrBinaryRunner`` is available in the ``west.runner.core`` +``ZephyrBinaryRunner`` is available in the ``west.runners.core`` module; individual runner implementations are in other submodules, -such as ``west.runner.nrfjprog``, ``west.runner.openocd``, etc. +such as ``west.runners.nrfjprog``, ``west.runners.openocd``, etc. Developers can add support for new ways to flash and debug Zephyr programs by implementing additional runners. To get this support into upstream Zephyr, the runner should be added into a new or existing -``west.runner`` module, and imported from +``west.runners`` module, and imported from :file:`west/runner/__init__.py`. .. important:: @@ -223,7 +223,7 @@ upstream Zephyr, the runner should be added into a new or existing API documentation for the core module follows. -.. automodule:: west.runner.core +.. automodule:: west.runners.core :members: Doing it By Hand diff --git a/scripts/meta/west/__main__.py b/scripts/meta/west/__main__.py deleted file mode 100644 index 03469bd3d53..00000000000 --- a/scripts/meta/west/__main__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2018 Open Source Foundries Limited. -# -# SPDX-License-Identifier: Apache-2.0 - -'''Zephyr RTOS meta-tool (west) - -Main entry point for running this package as a module, e.g.: - -py -3 west # Windows -python3 -m west # Unix -''' - -from .main import main - -if __name__ == '__main__': - main() diff --git a/scripts/meta/west/__init__.py b/scripts/meta/west/_bootstrap/__init__.py similarity index 78% rename from scripts/meta/west/__init__.py rename to scripts/meta/west/_bootstrap/__init__.py index 22e6ddd8a44..7ee30ca49b4 100644 --- a/scripts/meta/west/__init__.py +++ b/scripts/meta/west/_bootstrap/__init__.py @@ -2,4 +2,4 @@ # # SPDX-License-Identifier: Apache-2.0 -# Nothing here for now. +# Empty file. diff --git a/scripts/meta/west/_bootstrap/main.py b/scripts/meta/west/_bootstrap/main.py new file mode 100644 index 00000000000..d168177444a --- /dev/null +++ b/scripts/meta/west/_bootstrap/main.py @@ -0,0 +1,276 @@ +# Copyright 2018 Open Source Foundries Limited. +# +# SPDX-License-Identifier: Apache-2.0 + +'''West's bootstrap/wrapper script. +''' + +import argparse +import os +import platform +import subprocess +import sys + +import west._bootstrap.version as version + +if sys.version_info < (3,): + sys.exit('fatal error: you are running Python 2') + + +# +# Special files and directories in the west installation. +# +# These are given variable names for clarity, but they can't be +# changed without propagating the changes into west itself. +# + +# Top-level west directory, containing west itself and the manifest. +WEST_DIR = 'west' +# Subdirectory to check out the west source repository into. +WEST = 'west' +# Default west repository URL. +WEST_DEFAULT = 'https://github.com/zephyrproject-rtos/west' +# Default revision to check out of the west repository. +WEST_REV_DEFAULT = 'master' +# File inside of WEST_DIR which marks it as the top level of the +# Zephyr project installation. +# +# (The WEST_DIR name is not distinct enough to use when searching for +# the top level; other directories named "west" may exist elsewhere, +# e.g. zephyr/doc/west.) +WEST_MARKER = '.west_topdir' + +# Manifest repository directory under WEST_DIR. +MANIFEST = 'manifest' +# Default manifest repository URL. +MANIFEST_DEFAULT = 'https://github.com/zephyrproject-rtos/manifest' +# Default revision to check out of the manifest repository. +MANIFEST_REV_DEFAULT = 'master' + + +# +# Helpers shared between init and wrapper mode +# + + +class WestError(RuntimeError): + pass + + +class WestNotFound(WestError): + '''Neither the current directory nor any parent has a West installation.''' + + +def find_west_topdir(start): + '''Find the top-level installation directory, starting at ``start``. + + If none is found, raises WestNotFound.''' + # If you change this function, make sure to update west.util.west_topdir(). + + cur_dir = start + + while True: + if os.path.isfile(os.path.join(cur_dir, WEST_DIR, WEST_MARKER)): + return cur_dir + + parent_dir = os.path.dirname(cur_dir) + if cur_dir == parent_dir: + # At the root + raise WestNotFound('Could not find a West installation ' + 'in this or any parent directory') + cur_dir = parent_dir + + +def clone(url, rev, dest): + if os.path.exists(dest): + raise WestError('refusing to clone into existing location ' + dest) + + if not url.startswith(('http:', 'https:', 'git:', 'git+shh:', 'file:')): + raise WestError('Unknown URL scheme for repository: {}'.format(url)) + + subprocess.check_call(('git', 'clone', '-b', rev, '--', url, dest)) + + +# +# west init +# + + +def init(argv): + '''Command line handler for ``west init`` invocations. + + This exits the program with a nonzero exit code if fatal errors occur.''' + init_parser = argparse.ArgumentParser( + prog='west init', + description='Bootstrap initialize a Zephyr installation') + init_parser.add_argument( + '-b', '--base-url', + help='''Base URL for both 'manifest' and 'zephyr' repositories; cannot + be given if either -u or -w are''') + init_parser.add_argument( + '-u', '--manifest-url', + help='Zephyr manifest fetch URL, default ' + MANIFEST_DEFAULT) + init_parser.add_argument( + '--mr', '--manifest-rev', default=MANIFEST_REV_DEFAULT, + dest='manifest_rev', + help='Manifest revision to fetch, default ' + MANIFEST_REV_DEFAULT) + init_parser.add_argument( + '-w', '--west-url', + help='West fetch URL, default ' + WEST_DEFAULT) + init_parser.add_argument( + '--wr', '--west-rev', default=WEST_REV_DEFAULT, dest='west_rev', + help='West revision to fetch, default ' + WEST_REV_DEFAULT) + init_parser.add_argument( + 'directory', nargs='?', default=None, + help='Initializes in this directory, creating it if necessary') + + args = init_parser.parse_args(args=argv) + directory = args.directory or os.getcwd() + + if args.base_url: + if args.manifest_url or args.west_url: + sys.exit('fatal error: -b is incompatible with -u and -w') + args.manifest_url = args.base_url.rstrip('/') + '/manifest' + args.west_url = args.base_url.rstrip('/') + '/west' + else: + if not args.manifest_url: + args.manifest_url = MANIFEST_DEFAULT + if not args.west_url: + args.west_url = WEST_DEFAULT + + try: + topdir = find_west_topdir(directory) + init_reinit(topdir, args) + except WestNotFound: + init_bootstrap(directory, args) + + +def hide_file(path): + '''Ensure path is a hidden file. + + On Windows, this uses attrib to hide the file manually. + + On UNIX systems, this just checks that the path's basename begins + with a period ('.'), for it to be hidden already. It's a fatal + error if it does not begin with a period in this case. + + On other systems, this just prints a warning. + ''' + system = platform.system() + + if system == 'Windows': + subprocess.check_call(['attrib', '+H', path]) + elif os.name == 'posix': # Try to check for all Unix, not just macOS/Linux + if not os.path.basename(path).startswith('.'): + sys.exit("internal error: {} can't be hidden on UNIX".format(path)) + else: + print("warning: unknown platform {}; {} may not be hidden" + .format(system, path), file=sys.stderr) + + +def init_bootstrap(directory, args): + '''Bootstrap a new manifest + West installation in the given directory.''' + if not os.path.isdir(directory): + try: + print('Initializing in new directory', directory) + os.makedirs(directory, exist_ok=False) + except PermissionError: + sys.exit('Cannot initialize in {}: permission denied'.format( + directory)) + except FileExistsError: + sys.exit('Something else created {} concurrently; quitting'.format( + directory)) + except Exception as e: + sys.exit("Can't create directory {}: {}".format( + directory, e.args)) + else: + print('Initializing in', directory) + + # Clone the west source code and the manifest into west/. Git will create + # the west/ directory if it does not exist. + + clone(args.west_url, args.west_rev, + os.path.join(directory, WEST_DIR, WEST)) + + clone(args.manifest_url, args.manifest_rev, + os.path.join(directory, WEST_DIR, MANIFEST)) + + # Create a dotfile to mark the installation. Hide it on Windows. + + with open(os.path.join(directory, WEST_DIR, WEST_MARKER), 'w') as f: + hide_file(f.name) + + +def init_reinit(directory, args): + # TODO + sys.exit('Re-initializing an existing installation is not yet supported.') + + +# +# Wrap a West command +# + + +def wrap(argv): + printing_version = False + + if argv and argv[0] in ('-V', '--version'): + print('West bootstrapper version: v{} ({})'.format(version.__version__, + os.path.dirname(__file__))) + printing_version = True + + start = os.getcwd() + try: + topdir = find_west_topdir(start) + except WestNotFound: + if printing_version: + sys.exit(0) # run outside of an installation directory + else: + sys.exit('Error: not a Zephyr directory (or any parent): {}\n' + 'Use "west init" to install Zephyr here'.format(start)) + + west_git_repo = os.path.join(topdir, WEST_DIR, WEST) + if printing_version: + try: + git_describe = subprocess.check_output( + ['git', 'describe', '--tags'], + stderr=subprocess.DEVNULL, + cwd=west_git_repo).decode(sys.getdefaultencoding()).strip() + print('West repository version: {} ({})'.format(git_describe, + west_git_repo)) + except subprocess.CalledProcessError: + print('West repository verison: unknown; no tags were found') + sys.exit(0) + + # Replace the wrapper process with the "real" west + + # sys.argv[1:] strips the argv[0] of the wrapper script itself + argv = ([sys.executable, + os.path.join(west_git_repo, 'src', 'west', 'main.py')] + + argv) + + try: + subprocess.check_call(argv) + except subprocess.CalledProcessError as e: + sys.exit(1) + + +# +# Main entry point +# + + +def main(wrap_argv=None): + '''Entry point to the wrapper script.''' + if wrap_argv is None: + wrap_argv = sys.argv[1:] + + if not wrap_argv or wrap_argv[0] != 'init': + wrap(wrap_argv) + else: + init(wrap_argv[1:]) + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/scripts/meta/west/_bootstrap/version.py b/scripts/meta/west/_bootstrap/version.py new file mode 100644 index 00000000000..b864b23e1ed --- /dev/null +++ b/scripts/meta/west/_bootstrap/version.py @@ -0,0 +1,5 @@ +# Don't put anything else in here! +# +# This is the Python 3 version of option 3 in: +# https://packaging.python.org/guides/single-sourcing-package-version/#single-sourcing-the-version +__version__ = '0.2.0rc1' diff --git a/scripts/meta/west/build.py b/scripts/meta/west/build.py new file mode 100644 index 00000000000..e2a5bf5e6e6 --- /dev/null +++ b/scripts/meta/west/build.py @@ -0,0 +1,42 @@ +# Copyright 2018 (c) Foundries.io. +# +# SPDX-License-Identifier: Apache-2.0 + +'''Common definitions for building Zephyr applications. + +This provides some default settings and convenience wrappers for +building Zephyr applications needed by multiple commands. + +See west.cmd.build for the build command itself. +''' + +import cmake +import log + +DEFAULT_BUILD_DIR = 'build' +'''Name of the default Zephyr build directory.''' + +DEFAULT_CMAKE_GENERATOR = 'Ninja' +'''Name of the default CMake generator.''' + + +def is_zephyr_build(path): + '''Return true if and only if `path` appears to be a valid Zephyr + build directory. + + "Valid" means the given path is a directory which contains a CMake + cache with a 'ZEPHYR_TOOLCHAIN_VARIANT' key. + ''' + try: + cache = cmake.CMakeCache.from_build_dir(path) + except FileNotFoundError: + cache = {} + + if 'ZEPHYR_TOOLCHAIN_VARIANT' in cache: + log.dbg('{} is a zephyr build directory'.format(path), + level=log.VERBOSE_EXTREME) + return True + else: + log.dbg('{} is NOT a valid zephyr build directory'.format(path), + level=log.VERBOSE_EXTREME) + return False diff --git a/scripts/meta/west/cmake.py b/scripts/meta/west/cmake.py index 08829b67891..53b7b47950c 100644 --- a/scripts/meta/west/cmake.py +++ b/scripts/meta/west/cmake.py @@ -5,34 +5,41 @@ '''Helpers for dealing with CMake''' from collections import OrderedDict +import os.path import re import subprocess import shutil -from . import log -from .util import quote_sh_list +import log +from util import quote_sh_list -__all__ = ['run_build', 'make_c_identifier', 'CMakeCacheEntry', 'CMakeCache'] +__all__ = ['run_cmake', 'run_build', + 'make_c_identifier', + 'CMakeCacheEntry', 'CMakeCache'] DEFAULT_CACHE = 'CMakeCache.txt' -def run_build(build_directory, extra_args=[], quiet=False): - '''Run cmake in build tool mode in `build_directory`''' +def run_cmake(args, quiet=False): + '''Run cmake to (re)generate a build system''' cmake = shutil.which('cmake') if cmake is None: log.die('CMake is not installed or cannot be found; cannot build.') - cmd = [cmake, '--build', build_directory] + extra_args - kwargs = {} + cmd = [cmake] + args + kwargs = dict() if quiet: kwargs['stdout'] = subprocess.DEVNULL kwargs['stderr'] = subprocess.STDOUT - log.dbg('Re-building', build_directory) - log.dbg('Build command list:', cmd, level=log.VERBOSE_VERY) + log.dbg('Running CMake:', cmd, level=log.VERBOSE_VERY) log.dbg('As command:', quote_sh_list(cmd), level=log.VERBOSE_VERY) subprocess.check_call(cmd, **kwargs) +def run_build(build_directory, extra_args=(), quiet=False): + '''Run cmake in build tool mode in `build_directory`''' + run_cmake(['--build', build_directory] + list(extra_args), quiet=quiet) + + def make_c_identifier(string): '''Make a C identifier from a string in the same way CMake does. ''' @@ -154,6 +161,10 @@ class CMakeCacheEntry: class CMakeCache: '''Parses and represents a CMake cache file.''' + @staticmethod + def from_build_dir(build_dir): + return CMakeCache(os.path.join(build_dir, DEFAULT_CACHE)) + def __init__(self, cache_file): self.cache_file = cache_file self.load(cache_file) @@ -190,6 +201,9 @@ class CMakeCache: else: return default + def __contains__(self, name): + return name in self._entries + def __getitem__(self, name): return self._entries[name].value diff --git a/scripts/meta/west/cmd/__init__.py b/scripts/meta/west/commands/__init__.py similarity index 100% rename from scripts/meta/west/cmd/__init__.py rename to scripts/meta/west/commands/__init__.py diff --git a/scripts/meta/west/commands/build.py b/scripts/meta/west/commands/build.py new file mode 100644 index 00000000000..4625dafd602 --- /dev/null +++ b/scripts/meta/west/commands/build.py @@ -0,0 +1,278 @@ +# Copyright (c) 2018 Foundries.io +# +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import os + +import log +import cmake +from build import DEFAULT_BUILD_DIR, DEFAULT_CMAKE_GENERATOR, is_zephyr_build +from commands import WestCommand + +BUILD_HELP = '''\ +Convenience wrapper for building Zephyr applications. + +This command attempts to do what you mean when run from a Zephyr +application source or a pre-existing build directory: + +- When "west build" is run from a Zephyr build directory, the source + directory is obtained from the CMake cache, and that build directory + is re-compiled. + +- Otherwise, the source directory defaults to the current working + directory, so running "west build" from a Zephyr application's + source directory compiles it. + +The source and build directories can be explicitly set with the +--source-dir and --build-dir options. The build directory defaults to +'build' if it is not auto-detected. The build directory is always +created if it does not exist. + +This command runs CMake to generate a build system if one is not +present in the build directory, then builds the application. +Subsequent builds try to avoid re-running CMake; you can force it +to run by setting --cmake. + +To pass additional options to CMake, give them as extra arguments +after a '--' For example, "west build -- -DOVERLAY_CONFIG=some.conf" sets +an overlay config file. (Doing this forces a CMake run.)''' + + +class Build(WestCommand): + + def __init__(self): + super(Build, self).__init__( + 'build', + BUILD_HELP, + accepts_unknown_args=False) + + self.source_dir = None + '''Source directory for the build, or None on error.''' + + self.build_dir = None + '''Final build directory used to run the build, or None on error.''' + + self.created_build_dir = False + '''True if the build directory was created; False otherwise.''' + + self.run_cmake = False + '''True if CMake was run; False otherwise. + + Note: this only describes CMake runs done by this command. The + build system generated by CMake may also update itself due to + internal logic.''' + + self.cmake_cache = None + '''Final parsed CMake cache for the build, or None on error.''' + + def do_add_parser(self, parser_adder): + parser = parser_adder.add_parser( + self.name, + formatter_class=argparse.RawDescriptionHelpFormatter, + description=self.description) + + parser.add_argument('-b', '--board', + help='''board to build for (must be given for the + first build, can be omitted later)''') + parser.add_argument('-s', '--source-dir', + help='''explicitly sets the source directory; + if not given, infer it from directory context''') + parser.add_argument('-d', '--build-dir', + help='''explicitly sets the build directory; + if not given, infer it from directory context''') + parser.add_argument('-t', '--target', + help='''override the build system target (e.g. + 'clean', 'pristine', etc.)''') + parser.add_argument('-c', '--cmake', action='store_true', + help='force CMake to run') + parser.add_argument('-f', '--force', action='store_true', + help='ignore any errors and try to build anyway') + parser.add_argument('cmake_opts', nargs='*', metavar='cmake_opt', + help='extra option to pass to CMake; implies -c') + + return parser + + def do_run(self, args, ignored): + self.args = args # Avoid having to pass them around + log.dbg('args:', args, level=log.VERBOSE_EXTREME) + self._sanity_precheck() + self._setup_build_dir() + if is_zephyr_build(self.build_dir): + self._update_cache() + if self.args.cmake or self.args.cmake_opts: + self.run_cmake = True + else: + self.run_cmake = True + self._setup_source_dir() + self._sanity_check() + + log.inf('source directory: {}'.format(self.source_dir), colorize=True) + log.inf('build directory: {}{}'. + format(self.build_dir, + (' (created)' if self.created_build_dir + else '')), + colorize=True) + if self.cmake_cache: + board = self.cmake_cache.get('CACHED_BOARD') + elif self.args.board: + board = self.args.board + else: + board = 'UNKNOWN' # shouldn't happen + log.inf('BOARD:', board, colorize=True) + + self._run_cmake(self.args.cmake_opts) + self._sanity_check() + self._update_cache() + + extra_args = ['--target', args.target] if args.target else [] + cmake.run_build(self.build_dir, extra_args=extra_args) + + def _sanity_precheck(self): + app = self.args.source_dir + if app: + if not os.path.isdir(app): + self._check_force('source directory {} does not exist'. + format(app)) + elif 'CMakeLists.txt' not in os.listdir(app): + self._check_force("{} doesn't contain a CMakeLists.txt". + format(app)) + + def _update_cache(self): + try: + self.cmake_cache = cmake.CMakeCache.from_build_dir(self.build_dir) + except FileNotFoundError: + pass + + def _setup_build_dir(self): + # Initialize build_dir and created_build_dir attributes. + log.dbg('setting up build directory', level=log.VERBOSE_EXTREME) + if self.args.build_dir: + build_dir = self.args.build_dir + else: + cwd = os.getcwd() + if is_zephyr_build(cwd): + build_dir = cwd + else: + build_dir = DEFAULT_BUILD_DIR + build_dir = os.path.abspath(build_dir) + + if os.path.exists(build_dir): + if not os.path.isdir(build_dir): + log.die('build directory {} exists and is not a directory'. + format(build_dir)) + else: + os.makedirs(build_dir, exist_ok=False) + self.created_build_dir = True + self.run_cmake = True + + self.build_dir = build_dir + + def _setup_source_dir(self): + # Initialize source_dir attribute, either from command line argument, + # implicitly from the build directory's CMake cache, or using the + # default (current working directory). + log.dbg('setting up source directory', level=log.VERBOSE_EXTREME) + if self.args.source_dir: + source_dir = self.args.source_dir + elif self.cmake_cache: + source_dir = self.cmake_cache.get('APPLICATION_SOURCE_DIR') + if not source_dir: + # Maybe Zephyr changed the key? Give the user a way + # to retry, at least. + log.die("can't determine application from build directory " + "{}, please specify an application to build". + format(self.build_dir)) + else: + source_dir = os.getcwd() + self.source_dir = os.path.abspath(source_dir) + + def _sanity_check(self): + # Sanity check the build configuration. + # Side effect: may update cmake_cache attribute. + log.dbg('sanity checking the build', level=log.VERBOSE_EXTREME) + if self.source_dir == self.build_dir: + # There's no forcing this. + log.die('source and build directory {} cannot be the same; ' + 'use --build-dir {} to specify a build directory'. + format(self.source_dir, self.build_dir)) + + srcrel = os.path.relpath(self.source_dir) + if is_zephyr_build(self.source_dir): + self._check_force('it looks like {srcrel} is a build directory: ' + 'did you mean -build-dir {srcrel} instead?'. + format(srcrel=srcrel)) + elif 'CMakeLists.txt' not in os.listdir(self.source_dir): + self._check_force('source directory "{srcrel}" does not contain ' + 'a CMakeLists.txt; is that really what you ' + 'want to build? (Use -s SOURCE_DIR to specify ' + 'the application source directory)'. + format(srcrel=srcrel)) + + if not is_zephyr_build(self.build_dir) and not self.args.board: + self._check_force('this looks like a new or clean build, ' + 'please provide --board') + + if not self.cmake_cache: + return # That's all we can check without a cache. + + cached_app = self.cmake_cache.get('APPLICATION_SOURCE_DIR') + log.dbg('APPLICATION_SOURCE_DIR:', cached_app, + level=log.VERBOSE_EXTREME) + source_abs = (os.path.abspath(self.args.source_dir) + if self.args.source_dir else None) + cached_abs = os.path.abspath(cached_app) if cached_app else None + if cached_abs and source_abs and source_abs != cached_abs: + self._check_force('build directory "{}" is for application "{}", ' + 'but source directory "{}" was specified; ' + 'please clean it or use --build-dir to set ' + 'another build directory'. + format(self.build_dir, cached_abs, + source_abs)) + self.run_cmake = True # If they insist, we need to re-run cmake. + + cached_board = self.cmake_cache.get('CACHED_BOARD') + log.dbg('CACHED_BOARD:', cached_board, level=log.VERBOSE_EXTREME) + if not cached_board and not self.args.board: + if self.created_build_dir: + self._check_force( + 'Building for the first time: you must provide --board') + else: + self._check_force( + 'Board is missing or unknown, please provide --board') + if self.args.board and cached_board and \ + self.args.board != cached_board: + self._check_force('Build directory {} targets board {}, ' + 'but board {} was specified. (Clean that ' + 'directory or use --build-dir to specify ' + 'a different one.)'. + format(self.build_dir, cached_board, + self.args.board)) + + def _check_force(self, msg): + if not self.args.force: + log.err(msg) + log.die('refusing to proceed without --force due to above error') + + def _run_cmake(self, cmake_opts): + if not self.run_cmake: + log.dbg('not running cmake; build system is present') + return + + # It's unfortunate to have to use the undocumented -B and -H + # options to set the source and binary directories. + # + # However, it's the only known way to set that directory and + # run CMake from the current working directory. This is + # important because users expect invocations like this to Just + # Work: + # + # west build -- -DOVERLAY_CONFIG=relative-path.conf + final_cmake_args = ['-B{}'.format(self.build_dir), + '-H{}'.format(self.source_dir), + '-G{}'.format(DEFAULT_CMAKE_GENERATOR)] + if self.args.board: + final_cmake_args.append('-DBOARD={}'.format(self.args.board)) + if cmake_opts: + final_cmake_args.extend(cmake_opts) + cmake.run_cmake(final_cmake_args) diff --git a/scripts/meta/west/cmd/debug.py b/scripts/meta/west/commands/debug.py similarity index 62% rename from scripts/meta/west/cmd/debug.py rename to scripts/meta/west/commands/debug.py index 871c7d3e8ec..de93ff072d1 100644 --- a/scripts/meta/west/cmd/debug.py +++ b/scripts/meta/west/commands/debug.py @@ -6,8 +6,8 @@ from textwrap import dedent -from .run_common import desc_common, add_parser_common, do_run_common -from . import WestCommand +from commands.run_common import desc_common, add_parser_common, do_run_common +from commands import WestCommand class Debug(WestCommand): @@ -15,7 +15,9 @@ class Debug(WestCommand): def __init__(self): super(Debug, self).__init__( 'debug', - 'Connect to the board and start a debugging session.\n\n' + + dedent(''' + Connect to the board, program the flash, and start a + debugging session.\n\n''') + desc_common('debug'), accepts_unknown_args=True) @@ -47,3 +49,21 @@ class DebugServer(WestCommand): def do_run(self, my_args, runner_args): do_run_common(self, my_args, runner_args, 'ZEPHYR_BOARD_DEBUG_RUNNER') + +class Attach(WestCommand): + + def __init__(self): + super(Attach, self).__init__( + 'attach', + dedent(''' + Connect to the board without programming the flash, and + start a debugging session.\n\n''') + + desc_common('attach'), + accepts_unknown_args=True) + + def do_add_parser(self, parser_adder): + return add_parser_common(parser_adder, self) + + def do_run(self, my_args, runner_args): + do_run_common(self, my_args, runner_args, + 'ZEPHYR_BOARD_DEBUG_RUNNER') diff --git a/scripts/meta/west/cmd/flash.py b/scripts/meta/west/commands/flash.py similarity index 84% rename from scripts/meta/west/cmd/flash.py rename to scripts/meta/west/commands/flash.py index 66567927330..9dd7e97e21d 100644 --- a/scripts/meta/west/cmd/flash.py +++ b/scripts/meta/west/commands/flash.py @@ -4,8 +4,8 @@ '''west "flash" command''' -from .run_common import desc_common, add_parser_common, do_run_common -from . import WestCommand +from commands.run_common import desc_common, add_parser_common, do_run_common +from commands import WestCommand class Flash(WestCommand): diff --git a/scripts/meta/west/commands/project.py b/scripts/meta/west/commands/project.py new file mode 100644 index 00000000000..ddb0d5de4a5 --- /dev/null +++ b/scripts/meta/west/commands/project.py @@ -0,0 +1,876 @@ +# Copyright (c) 2018, Nordic Semiconductor ASA +# +# SPDX-License-Identifier: Apache-2.0 + +'''West project commands''' + +import argparse +import collections +import os +import shutil +import subprocess +import textwrap + +import pykwalify.core +import yaml + +import log +import util +from commands import WestCommand + + +# Branch that points to the revision specified in the manifest (which might be +# an SHA). Local branches created with 'west branch' are set to track this +# branch. +_MANIFEST_REV_BRANCH = 'manifest-rev' + + +class ListProjects(WestCommand): + def __init__(self): + super().__init__( + 'list-projects', + _wrap(''' + List projects. + + Prints the path to the manifest file and lists all projects along + with their clone paths and manifest revisions. Also includes + information on which projects are currently cloned. + ''')) + + def do_add_parser(self, parser_adder): + return _add_parser(parser_adder, self) + + def do_run(self, args, user_args): + log.inf("Manifest path: {}\n".format(_manifest_path(args))) + + for project in _all_projects(args): + log.inf('{:15} {:30} {:15} {}'.format( + project.name, + os.path.join(project.path, ''), # Add final '/' if missing + project.revision, + "(cloned)" if _cloned(project) else "(not cloned)")) + + +class Fetch(WestCommand): + def __init__(self): + super().__init__( + 'fetch', + _wrap(''' + Clone/fetch projects. + + Fetches upstream changes in each of the specified projects + (default: all projects). Repositories that do not already exist are + cloned. + + Unless --no-update is passed, the manifest and West source code + repositories are updated prior to fetching. See the 'update' + command. + + ''' + _MANIFEST_REV_HELP)) + + def do_add_parser(self, parser_adder): + return _add_parser(parser_adder, self, _no_update_arg, + _project_list_arg) + + def do_run(self, args, user_args): + if args.update: + _update(True, True) + + for project in _projects(args, listed_must_be_cloned=False): + log.dbg('fetching:', project, level=log.VERBOSE_VERY) + _fetch(project) + + +class Pull(WestCommand): + def __init__(self): + super().__init__( + 'pull', + _wrap(''' + Clone/fetch and rebase projects. + + Fetches upstream changes in each of the specified projects + (default: all projects) and rebases the checked-out branch (or + detached HEAD state) on top of '{}', effectively bringing the + branch up to date. Repositories that do not already exist are + cloned. + + Unless --no-update is passed, the manifest and West source code + repositories are updated prior to pulling. See the 'update' + command. + + '''.format(_MANIFEST_REV_BRANCH) + _MANIFEST_REV_HELP)) + + def do_add_parser(self, parser_adder): + return _add_parser(parser_adder, self, _no_update_arg, + _project_list_arg) + + def do_run(self, args, user_args): + if args.update: + _update(True, True) + + for project in _projects(args, listed_must_be_cloned=False): + if _fetch(project): + _rebase(project) + + +class Rebase(WestCommand): + def __init__(self): + super().__init__( + 'rebase', + _wrap(''' + Rebase projects. + + Rebases the checked-out branch (or detached HEAD) on top of '{}' in + each of the specified projects (default: all cloned projects), + effectively bringing the branch up to date. + + '''.format(_MANIFEST_REV_BRANCH) + _MANIFEST_REV_HELP)) + + def do_add_parser(self, parser_adder): + return _add_parser(parser_adder, self, _project_list_arg) + + def do_run(self, args, user_args): + for project in _cloned_projects(args): + _rebase(project) + + +class Branch(WestCommand): + def __init__(self): + super().__init__( + 'branch', + _wrap(''' + Create a branch or list branches, in multiple projects. + + Creates a branch in each of the specified projects (default: all + cloned projects). The new branches are set to track '{}'. + + With no arguments, lists all local branches along with the + repositories they appear in. + + '''.format(_MANIFEST_REV_BRANCH) + _MANIFEST_REV_HELP)) + + def do_add_parser(self, parser_adder): + return _add_parser( + parser_adder, self, + _arg('branch', nargs='?', metavar='BRANCH_NAME'), + _project_list_arg) + + def do_run(self, args, user_args): + if args.branch: + # Create a branch in the specified projects + for project in _cloned_projects(args): + _create_branch(project, args.branch) + else: + # No arguments. List local branches from all cloned projects along + # with the projects they appear in. + + branch2projs = collections.defaultdict(list) + for project in _cloned_projects(args): + for branch in _branches(project): + branch2projs[branch].append(project.name) + + for branch, projs in sorted(branch2projs.items()): + log.inf('{:18} {}'.format(branch, ", ".join(projs))) + + +class Checkout(WestCommand): + def __init__(self): + super().__init__( + 'checkout', + _wrap(''' + Check out topic branch. + + Checks out the specified branch in each of the specified projects + (default: all cloned projects). Projects that do not have the + branch are left alone. + ''')) + + def do_add_parser(self, parser_adder): + return _add_parser( + parser_adder, self, + _arg('-b', + dest='create_branch', + action='store_true', + help='create the branch before checking it out'), + _arg('branch', metavar='BRANCH_NAME'), + _project_list_arg) + + def do_run(self, args, user_args): + branch_exists = False + + for project in _cloned_projects(args): + if args.create_branch: + _create_branch(project, args.branch) + _checkout(project, args.branch) + branch_exists = True + elif _has_branch(project, args.branch): + _checkout(project, args.branch) + branch_exists = True + + if not branch_exists: + msg = 'No branch {} exists in any '.format(args.branch) + if args.projects: + log.die(msg + 'of the listed projects') + else: + log.die(msg + 'cloned project') + + +class Diff(WestCommand): + def __init__(self): + super().__init__( + 'diff', + _wrap(''' + 'git diff' projects. + + Runs 'git diff' for each of the specified projects (default: all + cloned projects). + + Extra arguments are passed as-is to 'git diff'. + '''), + accepts_unknown_args=True) + + def do_add_parser(self, parser_adder): + return _add_parser(parser_adder, self, _project_list_arg) + + def do_run(self, args, user_args): + for project in _cloned_projects(args): + # Use paths that are relative to the base directory to make it + # easier to see where the changes are + _git(project, 'diff --src-prefix=(path)/ --dst-prefix=(path)/', + extra_args=user_args) + + +class Status(WestCommand): + def __init__(self): + super().__init__( + 'status', + _wrap(''' + Runs 'git status' for each of the specified projects (default: all + cloned projects). Extra arguments are passed as-is to 'git status'. + '''), + accepts_unknown_args=True) + + def do_add_parser(self, parser_adder): + return _add_parser(parser_adder, self, _project_list_arg) + + def do_run(self, args, user_args): + for project in _cloned_projects(args): + _inf(project, 'status of (name-and-path)') + _git(project, 'status', extra_args=user_args) + + +class Update(WestCommand): + def __init__(self): + super().__init__( + 'update', + _wrap(''' + Updates the manifest repository and/or the West source code + repository. + + There is normally no need to run this command manually, because + 'west fetch' and 'west pull' automatically update the West and + manifest repositories to the latest version before doing anything + else. + + Pass --update-west or --update-manifest to update just that + repository. With no arguments, both are updated. + ''')) + + def do_add_parser(self, parser_adder): + return _add_parser( + parser_adder, self, + _arg('--update-west', + dest='update_west', + action='store_true', + help='update the West source code repository'), + _arg('--update-manifest', + dest='update_manifest', + action='store_true', + help='update the manifest repository')) + + def do_run(self, args, user_args): + if not args.update_west and not args.update_manifest: + _update(True, True) + else: + _update(args.update_west, args.update_manifest) + + +class ForAll(WestCommand): + def __init__(self): + super().__init__( + 'forall', + _wrap(''' + Runs a shell (Linux) or batch (Windows) command within the + repository of each of the specified projects (default: all cloned + projects). Note that you have to quote the command if it consists + of more than one word, to prevent the shell you use to run 'west' + from splitting it up. + + Since the command is run through the shell, you can use wildcards + and the like. + + For example, the following command will list the contents of + proj-1's and proj-2's repositories on Linux, in long form: + + west forall -c 'ls -l' proj-1 proj-2 + ''')) + + def do_add_parser(self, parser_adder): + return _add_parser( + parser_adder, self, + _arg('-c', + dest='command', + metavar='COMMAND', + required=True), + _project_list_arg) + + def do_run(self, args, user_args): + for project in _cloned_projects(args): + _inf(project, "Running '{}' in (name-and-path)" + .format(args.command)) + + subprocess.Popen(args.command, shell=True, cwd=project.abspath) \ + .wait() + + +def _arg(*args, **kwargs): + # Helper for creating a new argument parser for a single argument, + # later passed in parents= to add_parser() + + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument(*args, **kwargs) + return parser + + +# Arguments shared between more than one command + +_manifest_arg = _arg( + '-m', '--manifest', + help='path to manifest file (default: west/manifest/default.yml)') + +# For 'fetch' and 'pull' +_no_update_arg = _arg( + '--no-update', + dest='update', + action='store_false', + help='do not update the manifest or West before fetching project data') + +# List of projects +_project_list_arg = _arg('projects', metavar='PROJECT', nargs='*') + + +def _add_parser(parser_adder, cmd, *extra_args): + # Adds and returns a subparser for the project-related WestCommand 'cmd'. + # All of these commands (currently) take the manifest path flag, so it's + # hardcoded here. + + return parser_adder.add_parser( + cmd.name, + description=cmd.description, + formatter_class=argparse.RawDescriptionHelpFormatter, + parents=(_manifest_arg,) + extra_args) + + +def _wrap(s): + # Wraps help texts for commands. Some of them have variable length (due to + # _MANIFEST_REV_BRANCH), so just a textwrap.dedent() can look a bit wonky. + + # [1:] gets rid of the initial newline. It's turned into a space by + # textwrap.fill() otherwise. + paragraphs = textwrap.dedent(s[1:]).split("\n\n") + + return "\n\n".join(textwrap.fill(paragraph) for paragraph in paragraphs) + + +_MANIFEST_REV_HELP = """ +The '{}' branch points to the revision that the manifest specified for the +project as of the most recent 'west fetch'/'west pull'. +""".format(_MANIFEST_REV_BRANCH)[1:].replace("\n", " ") + + +# Holds information about a project, taken from the manifest file (or +# constructed manually for "special" projects) +Project = collections.namedtuple( + 'Project', + 'name url revision path abspath clone_depth') + + +def _cloned_projects(args): + # Returns _projects(args, listed_must_be_cloned=True) if a list of projects + # was given by the user (i.e., listed projects are required to be cloned). + # If no projects were listed, returns all cloned projects. + + # This approach avoids redundant _cloned() checks + return _projects(args) if args.projects else \ + [project for project in _all_projects(args) if _cloned(project)] + + +def _projects(args, listed_must_be_cloned=True): + # Returns a list of project instances for the projects requested in 'args' + # (the command-line arguments), in the same order that they were listed by + # the user. If args.projects is empty, no projects were listed, and all + # projects will be returned. If a non-existent project was listed by the + # user, an error is raised. + # + # Before the manifest is parsed, it is validated agains a pykwalify schema. + # An error is raised on validation errors. + # + # listed_must_be_cloned (default: True): + # If True, an error is raised if an uncloned project was listed. This + # only applies to projects listed explicitly on the command line. + + projects = _all_projects(args) + + if not args.projects: + # No projects specified. Return all projects. + return projects + + # Got a list of projects on the command line. First, check that they exist + # in the manifest. + + project_names = [project.name for project in projects] + nonexistent = set(args.projects) - set(project_names) + if nonexistent: + log.die('Unknown project{} {} (available projects: {})' + .format('s' if len(nonexistent) > 1 else '', + ', '.join(nonexistent), + ', '.join(project_names))) + + # Return the projects in the order they were listed + res = [] + for name in args.projects: + for project in projects: + if project.name == name: + res.append(project) + break + + # Check that all listed repositories are cloned, if requested + if listed_must_be_cloned: + uncloned = [prj.name for prj in res if not _cloned(prj)] + if uncloned: + log.die('The following projects are not cloned: {}. Please clone ' + "them first (with 'west fetch')." + .format(", ".join(uncloned))) + + return res + + +def _all_projects(args): + # Parses the manifest file, returning a list of Project instances. + # + # Before the manifest is parsed, it is validated against a pykwalify + # schema. An error is raised on validation errors. + + manifest_path = _manifest_path(args) + + _validate_manifest(manifest_path) + + with open(manifest_path) as f: + manifest = yaml.safe_load(f)['manifest'] + + projects = [] + # Manifest "defaults" keys whose values get copied to each project + # that doesn't specify its own value. + project_defaults = ('remote', 'revision') + + # mp = manifest project (dictionary with values parsed from the manifest) + for mp in manifest['projects']: + # Fill in any missing fields in 'mp' with values from the 'defaults' + # dictionary + if 'defaults' in manifest: + for key, val in manifest['defaults'].items(): + if key in project_defaults: + mp.setdefault(key, val) + + # Add the repository URL to 'mp' + for remote in manifest['remotes']: + if remote['name'] == mp['remote']: + mp['url'] = remote['url'] + '/' + mp['name'] + break + else: + log.die('Remote {} not defined in {}' + .format(mp['remote'], manifest_path)) + + # If no clone path is specified, the project's name is used + clone_path = mp.get('path', mp['name']) + + # Use named tuples to store project information. That gives nicer + # syntax compared to a dict (project.name instead of project['name'], + # etc.) + projects.append(Project( + mp['name'], + mp['url'], + # If no revision is specified, 'master' is used + mp.get('revision', 'master'), + clone_path, + # Absolute clone path + os.path.join(util.west_topdir(), clone_path), + # If no clone depth is specified, we fetch the entire history + mp.get('clone-depth', None))) + + return projects + + +def _validate_manifest(manifest_path): + # Validates the manifest with pykwalify. schema.yml holds the schema. + + schema_path = os.path.join(os.path.dirname(__file__), "schema.yml") + + try: + pykwalify.core.Core( + source_file=manifest_path, + schema_files=[schema_path] + ).validate() + except pykwalify.errors.SchemaError as e: + log.die('{} malformed (schema: {}):\n{}' + .format(manifest_path, schema_path, e)) + + +def _manifest_path(args): + # Returns the path to the manifest file. Defaults to + # .west/manifest/default.yml if the user didn't specify a manifest. + + return args.manifest or os.path.join(util.west_dir(), 'manifest', + 'default.yml') + + +def _fetch(project): + # Fetches upstream changes for 'project' and updates the 'manifest-rev' + # branch to point to the revision specified in the manifest. If the + # project's repository does not already exist, it is created first. + # + # Returns True if the project's repository already existed. + + exists = _cloned(project) + + if not exists: + _inf(project, 'Creating repository for (name-and-path)') + _git_base(project, 'init (abspath)') + _git(project, 'remote add origin (url)') + + if project.clone_depth: + _inf(project, + 'Fetching changes for (name-and-path) with --depth (clone-depth)') + + # If 'clone-depth' is specified, fetch just the specified revision + # (probably a branch). That will download the minimum amount of data, + # which is probably what's wanted whenever 'clone-depth is used. The + # default 'git fetch' behavior is to do a shallow clone of all branches + # on the remote. + # + # Note: Many servers won't allow fetching arbitrary commits by SHA. + # Combining --depth with an SHA will break for those. + + # Qualify branch names with refs/heads/, just to be safe. Just the + # branch name is likely to work as well though. + _git(project, + 'fetch --depth=(clone-depth) origin ' + + (project.revision if _is_sha(project.revision) else \ + 'refs/heads/' + project.revision)) + + else: + _inf(project, 'Fetching changes for (name-and-path)') + + # If 'clone-depth' is not specified, fetch all branches on the + # remote. This gives a more usable repository. + _git(project, 'fetch origin') + + # Create/update the 'manifest-rev' branch + _git(project, + 'update-ref refs/heads/(manifest-rev-branch) ' + + (project.revision if _is_sha(project.revision) else + 'remotes/origin/' + project.revision)) + + if not exists: + # If we just initialized the repository, check out 'manifest-rev' in a + # detached HEAD state. + # + # Otherwise, the initial state would have nothing checked out, and HEAD + # would point to a non-existent refs/heads/master branch (that would + # get created if the user makes an initial commit). That state causes + # e.g. 'west rebase' to fail, and might look confusing. + # + # (The --detach flag is strictly redundant here, because the + # refs/heads/ form already detaches HEAD, but it avoids a + # spammy detached HEAD warning from Git.) + _git(project, 'checkout --detach refs/heads/(manifest-rev-branch)') + + return exists + + +def _rebase(project): + _inf(project, 'Rebasing (name-and-path) to (manifest-rev-branch)') + _git(project, 'rebase (manifest-rev-branch)') + + +def _cloned(project): + # Returns True if the project's path is a directory that looks + # like the top-level directory of a Git repository, and False + # otherwise. + + def handle(result): + log.dbg('project', project.name, + 'is {}cloned'.format('' if result else 'not '), + level=log.VERBOSE_EXTREME) + return result + + if not os.path.isdir(project.abspath): + return handle(False) + + # --is-inside-work-tree doesn't require that the directory is the top-level + # directory of a Git repository. Use --show-cdup instead, which prints an + # empty string (i.e., just a newline, which we strip) for the top-level + # directory. + res = _git(project, 'rev-parse --show-cdup', capture_stdout=True, + check=False) + + return handle(not (res.returncode or res.stdout)) + + +def _branches(project): + # Returns a sorted list of all local branches in 'project' + + # refname:lstrip=-1 isn't available before Git 2.8 (introduced by commit + # 'tag: do not show ambiguous tag names as "tags/foo"'). Strip + # 'refs/heads/' manually instead. + return [ref[len('refs/heads/'):] for ref in + _git(project, + 'for-each-ref --sort=refname --format=%(refname) refs/heads', + capture_stdout=True).stdout.split('\n')] + + +def _create_branch(project, branch): + if _has_branch(project, branch): + _inf(project, "Branch '{}' already exists in (name-and-path)" + .format(branch)) + else: + _inf(project, "Creating branch '{}' in (name-and-path)" + .format(branch)) + _git(project, 'branch --quiet --track {} (manifest-rev-branch)' + .format(branch)) + + +def _has_branch(project, branch): + return _git(project, 'show-ref --quiet --verify refs/heads/' + branch, + check=False).returncode == 0 + + +def _checkout(project, branch): + _inf(project, "Checking out branch '{}' in (name-and-path)".format(branch)) + _git(project, 'checkout ' + branch) + + +def _special_project(name): + # Returns a Project instance for one of the special repositories in west/, + # so that we can reuse the project-related functions for them + + return Project( + name, + 'dummy URL for {} repository'.format(name), + 'master', + os.path.join('west', name.lower()), # Path + os.path.join(util.west_dir(), name.lower()), # Absolute path + None # Clone depth + ) + + +def _update(update_west, update_manifest): + # 'try' is a keyword + def attempt(project, cmd): + res = _git(project, cmd, capture_stdout=True, check=False) + if res.returncode: + # The Git command's stderr isn't redirected and will also be + # available + _die(project, _FAILED_UPDATE_MSG.format(cmd)) + return res.stdout + + projects = [] + if update_west: + projects.append(_special_project('West')) + if update_manifest: + projects.append(_special_project('manifest')) + + for project in projects: + _dbg(project, 'Updating (name-and-path)', level=log.VERBOSE_NORMAL) + + # Fetch changes from upstream + attempt(project, 'fetch') + + # Get the SHA of the last commit in common with the upstream branch + merge_base = attempt(project, 'merge-base HEAD remotes/origin/master') + + # Get the current SHA of the upstream branch + head_sha = attempt(project, 'show-ref --hash remotes/origin/master') + + # If they differ, we need to rebase + if merge_base != head_sha: + attempt(project, 'rebase remotes/origin/master') + + _inf(project, 'Updated (rebased) (name-and-path) to the ' + 'latest version') + + if project.name == 'west': + # Signal self-update, which will cause a restart. This is a bit + # nicer than doing the restart here, as callers will have a + # chance to flush file buffers, etc. + raise WestUpdated() + + +_FAILED_UPDATE_MSG = """ +Failed to update (name-and-path), while running command '{}'. Please fix the +state of the repository, or pass --no-update to 'west fetch/pull' to skip +updating the manifest and West for the duration of the command."""[1:] + + +class WestUpdated(Exception): + '''Raised after West has updated its own source code''' + + +def _is_sha(s): + try: + int(s, 16) + except ValueError: + return False + + return len(s) == 40 + + +def _git_base(project, cmd, *, extra_args=(), capture_stdout=False, + check=True): + # Runs a git command in the West top directory. See _git_helper() for + # parameter documentation. + # + # Returns a CompletedProcess instance (see below). + + return _git_helper(project, cmd, extra_args, util.west_topdir(), + capture_stdout, check) + + +def _git(project, cmd, *, extra_args=(), capture_stdout=False, check=True): + # Runs a git command within a particular project. See _git_helper() for + # parameter documentation. + # + # Returns a CompletedProcess instance (see below). + + return _git_helper(project, cmd, extra_args, project.abspath, + capture_stdout, check) + + +def _git_helper(project, cmd, extra_args, cwd, capture_stdout, check): + # Runs a git command. + # + # project: + # The Project instance for the project, derived from the manifest file. + # + # cmd: + # String with git arguments. Supports some "(foo)" shorthands. See below. + # + # extra_args: + # List of additional arguments to pass to the git command (e.g. from the + # user). + # + # cwd: + # Directory to switch to first (None = current directory) + # + # capture_stdout: + # True if stdout should be captured into the returned + # subprocess.CompletedProcess instance instead of being printed. + # + # We never capture stderr, to prevent error messages from being eaten. + # + # check: + # True if an error should be raised if the git command finishes with a + # non-zero return code. + # + # Returns a subprocess.CompletedProcess instance. + + # TODO: Run once somewhere? + if shutil.which('git') is None: + log.die('Git is not installed or cannot be found') + + args = (('git',) + + tuple(_expand_shorthands(project, arg) for arg in cmd.split()) + + tuple(extra_args)) + cmd_str = util.quote_sh_list(args) + + log.dbg("running '{}'".format(cmd_str), 'in', cwd, level=log.VERBOSE_VERY) + popen = subprocess.Popen( + args, stdout=subprocess.PIPE if capture_stdout else None, cwd=cwd) + + stdout, _ = popen.communicate() + + dbg_msg = "'{}' in {} finished with exit status {}" \ + .format(cmd_str, cwd, popen.returncode) + if capture_stdout: + dbg_msg += " and wrote {} to stdout".format(stdout) + log.dbg(dbg_msg, level=log.VERBOSE_VERY) + + if check and popen.returncode: + _die(project, "Command '{}' failed for (name-and-path)" + .format(cmd_str)) + + if capture_stdout: + # Manual UTF-8 decoding and universal newlines. Before Python 3.6, + # Popen doesn't seem to allow using universal newlines mode (which + # enables decoding) with a specific encoding (because the encoding= + # parameter is missing). + # + # Also strip all trailing newlines as convenience. The splitlines() + # already means we lose a final '\n' anyway. + stdout = "\n".join(stdout.decode('utf-8').splitlines()).rstrip("\n") + + return CompletedProcess(popen.args, popen.returncode, stdout) + + +def _expand_shorthands(project, s): + # Expands project-related shorthands in 's' to their values, + # returning the expanded string + + return s.replace('(name)', project.name) \ + .replace('(name-and-path)', + '{} ({})'.format( + project.name, os.path.join(project.path, ""))) \ + .replace('(url)', project.url) \ + .replace('(path)', project.path) \ + .replace('(abspath)', project.abspath) \ + .replace('(revision)', project.revision) \ + .replace('(manifest-rev-branch)', _MANIFEST_REV_BRANCH) \ + .replace('(clone-depth)', str(project.clone_depth)) + + +def _inf(project, msg): + # Print '=== msg' (to clearly separate it from Git output). Supports the + # same (foo) shorthands as the git commands. + # + # Prints the message in green if stdout is a terminal, to clearly separate + # it from command (usually Git) output. + + log.inf('=== ' + _expand_shorthands(project, msg), colorize=True) + + +def _wrn(project, msg): + # Warn with 'msg'. Supports the same (foo) shorthands as the git commands. + + log.wrn(_expand_shorthands(project, msg)) + + +def _dbg(project, msg, level): + # Like _wrn(), for debug messages + + log.dbg(_expand_shorthands(project, msg), level=level) + + +def _die(project, msg): + # Like _wrn(), for dying + + log.die(_expand_shorthands(project, msg)) + + +# subprocess.CompletedProcess-alike, used instead of the real deal for Python +# 3.4 compatibility, and with two small differences: +# +# - Trailing newlines are stripped from stdout +# +# - The 'stderr' attribute is omitted, because we never capture stderr +CompletedProcess = collections.namedtuple( + 'CompletedProcess', 'args returncode stdout') diff --git a/scripts/meta/west/cmd/run_common.py b/scripts/meta/west/commands/run_common.py similarity index 78% rename from scripts/meta/west/cmd/run_common.py rename to scripts/meta/west/commands/run_common.py index daed3015850..1bd10f8a90b 100644 --- a/scripts/meta/west/cmd/run_common.py +++ b/scripts/meta/west/commands/run_common.py @@ -10,12 +10,13 @@ from os import getcwd, path from subprocess import CalledProcessError import textwrap -from .. import cmake -from .. import log -from .. import util -from ..runner import get_runner_cls, ZephyrBinaryRunner -from ..runner.core import RunnerConfig -from . import CommandContextError +import cmake +import log +import util +from build import DEFAULT_BUILD_DIR, is_zephyr_build +from runners import get_runner_cls, ZephyrBinaryRunner +from runners.core import RunnerConfig +from commands import CommandContextError # Context-sensitive help indentation. # Don't change this, or output from argparse won't match up. @@ -37,8 +38,10 @@ def add_parser_common(parser_adder, command): group.add_argument('-d', '--build-dir', help='''Build directory to obtain runner information - from; default is the current working directory.''') - group.add_argument('-c', '--cmake-cache', default=cmake.DEFAULT_CACHE, + from. If not given, this command tries to use build/ + and then the current working directory, in that + order.''') + group.add_argument('-c', '--cmake-cache', help='''Path to CMake cache file containing runner configuration (this is generated by the Zephyr build system when compiling binaries); @@ -127,13 +130,32 @@ def _override_config_from_namespace(cfg, namespace): setattr(cfg, var, val) +def _build_dir(args, die_if_none=True): + # Get the build directory for the given argument list and environment. + if args.build_dir: + return args.build_dir + + cwd = getcwd() + default = path.join(cwd, DEFAULT_BUILD_DIR) + if is_zephyr_build(default): + return default + elif is_zephyr_build(cwd): + return cwd + elif die_if_none: + log.die('--build-dir was not given, and neither {} ' + 'nor {} are zephyr build directories.'. + format(default, cwd)) + else: + return None + + def do_run_common(command, args, runner_args, cached_runner_var): if args.context: _dump_context(command, args, runner_args, cached_runner_var) return command_name = command.name - build_dir = args.build_dir or getcwd() + build_dir = _build_dir(args) if not args.skip_rebuild: try: @@ -153,7 +175,7 @@ def do_run_common(command, args, runner_args, cached_runner_var): # line override. Get the ZephyrBinaryRunner class by name, and # make sure it supports the command. - cache_file = path.join(build_dir, args.cmake_cache) + cache_file = path.join(build_dir, args.cmake_cache or cmake.DEFAULT_CACHE) cache = cmake.CMakeCache(cache_file) board = cache['CACHED_BOARD'] available = cache.get_list('ZEPHYR_RUNNERS') @@ -218,35 +240,33 @@ def do_run_common(command, args, runner_args, cached_runner_var): # def _dump_context(command, args, runner_args, cached_runner_var): - build_dir = args.build_dir or getcwd() + build_dir = _build_dir(args, die_if_none=False) - # If the cache is a file, try to ensure build artifacts are up to - # date. If that doesn't work, still try to print information on a - # best-effort basis. - cache_file = path.abspath(path.join(build_dir, args.cmake_cache)) - cache = None - - if path.isfile(cache_file): - have_cache_file = True + # Try to figure out the CMake cache file based on the build + # directory or an explicit argument. + if build_dir is not None: + cache_file = path.abspath( + path.join(build_dir, args.cmake_cache or cmake.DEFAULT_CACHE)) + elif args.cmake_cache: + cache_file = path.abspath(args.cmake_cache) else: - have_cache_file = False - if args.build_dir: - msg = textwrap.dedent('''\ - CMake cache {}: no such file or directory, --build-dir {} - is invalid'''.format(cache_file, args.build_dir)) - log.die('\n'.join(textwrap.wrap(msg, initial_indent='', - subsequent_indent=INDENT, - break_on_hyphens=False))) - else: - msg = textwrap.dedent('''\ - No cache file {} found; is this a build directory? - (Use --build-dir to set one if not, otherwise, output will be - limited.)'''.format(cache_file)) - log.wrn('\n'.join(textwrap.wrap(msg, initial_indent='', - subsequent_indent=INDENT, - break_on_hyphens=False))) + cache_file = None - if have_cache_file and not args.skip_rebuild: + # Load the cache itself, if possible. + if cache_file is None: + log.wrn('No build directory (--build-dir) or CMake cache ' + '(--cache-file) given or found; output will be limited') + cache = None + else: + try: + cache = cmake.CMakeCache(cache_file) + except Exception: + log.die('Cannot load cache {}.'.format(cache_file)) + + # If we have a build directory, try to ensure build artifacts are + # up to date. If that doesn't work, still try to print information + # on a best-effort basis. + if build_dir and not args.skip_rebuild: try: cmake.run_build(build_dir) except CalledProcessError: @@ -255,18 +275,12 @@ def _dump_context(command, args, runner_args, cached_runner_var): msg += 'Is {} the right --build-dir?'.format(args.build_dir) else: msg += textwrap.dedent('''\ - Use --build-dir (-d) to specify a build directory; the default - is the current directory, {}.'''.format(build_dir)) + Use --build-dir (-d) to specify a build directory; the one + used was {}.'''.format(build_dir)) log.die('\n'.join(textwrap.wrap(msg, initial_indent='', subsequent_indent=INDENT, break_on_hyphens=False))) - if have_cache_file: - try: - cache = cmake.CMakeCache(cache_file) - except Exception: - log.die('Cannot load cache {}.'.format(cache_file)) - if cache is None: _dump_no_context_info(command, args) if not args.runner: @@ -287,19 +301,24 @@ def _dump_context(command, args, runner_args, cached_runner_var): default_runner = cache.get(cached_runner_var) cfg = cached_runner_config(build_dir, cache) - log.inf('All Zephyr runners which support {}:'.format(command.name)) + log.inf('All Zephyr runners which support {}:'.format(command.name), + colorize=True) for line in util.wrap(', '.join(all_cls.keys()), INDENT): log.inf(line) - log.inf('(Not all may work with this build, see available runners below.)') + log.inf('(Not all may work with this build, see available runners below.)', + colorize=True) if cache is None: log.warn('Missing or invalid CMake cache {}; there is no context.', 'Use --build-dir to specify the build directory.') return - log.inf('Build directory:', build_dir) - log.inf('Board:', board) - log.inf('CMake cache:', cache_file) + log.inf('Build directory:', colorize=True) + log.inf(INDENT + build_dir) + log.inf('Board:', colorize=True) + log.inf(INDENT + board) + log.inf('CMake cache:', colorize=True) + log.inf(INDENT + cache_file) if not available: # Bail with a message if no runners are available. @@ -307,33 +326,39 @@ def _dump_context(command, args, runner_args, cached_runner_var): 'Consult the documentation for instructions on how to run ' 'binaries on this target.').format(board) for line in util.wrap(msg, ''): - log.inf(line) + log.inf(line, colorize=True) return - log.inf('Available {} runners:'.format(command.name), ', '.join(available)) - log.inf('Additional options for available', command.name, 'runners:') + log.inf('Available {} runners:'.format(command.name), colorize=True) + log.inf(INDENT + ', '.join(available)) + log.inf('Additional options for available', command.name, 'runners:', + colorize=True) for runner in available: _dump_runner_opt_help(runner, all_cls[runner]) - log.inf('Default {} runner: {}'.format(command.name, default_runner)) + log.inf('Default {} runner:'.format(command.name), colorize=True) + log.inf(INDENT + default_runner) _dump_runner_config(cfg, '', INDENT) - log.inf('Runner-specific information:') + log.inf('Runner-specific information:', colorize=True) for runner in available: - log.inf('{}{}:'.format(INDENT, runner)) + log.inf('{}{}:'.format(INDENT, runner), colorize=True) _dump_runner_cached_opts(cache, runner, INDENT * 2, INDENT * 3) _dump_runner_caps(available_cls[runner], INDENT * 2) if len(available) > 1: - log.inf('(Add -r RUNNER to just print information about one runner.)') + log.inf('(Add -r RUNNER to just print information about one runner.)', + colorize=True) def _dump_no_context_info(command, args): all_cls = {cls.name(): cls for cls in ZephyrBinaryRunner.get_runners() if command.name in cls.capabilities().commands} - log.inf('All Zephyr runners which support {}:'.format(command.name)) + log.inf('All Zephyr runners which support {}:'.format(command.name), + colorize=True) for line in util.wrap(', '.join(all_cls.keys()), INDENT): log.inf(line) if not args.runner: - log.inf('Add -r RUNNER to print more information about any runner.') + log.inf('Add -r RUNNER to print more information about any runner.', + colorize=True) def _dump_one_runner_info(cache, args, build_dir, indent): @@ -348,10 +373,14 @@ def _dump_one_runner_info(cache, args, build_dir, indent): available = runner in cache.get_list('ZEPHYR_RUNNERS') cfg = cached_runner_config(build_dir, cache) - log.inf('Build directory:', build_dir) - log.inf('Board:', cache['CACHED_BOARD']) - log.inf('CMake cache:', cache.cache_file) - log.inf(runner, 'is available:', 'yes' if available else 'no') + log.inf('Build directory:', colorize=True) + log.inf(INDENT + build_dir) + log.inf('Board:', colorize=True) + log.inf(INDENT + cache['CACHED_BOARD']) + log.inf('CMake cache:', colorize=True) + log.inf(INDENT + cache.cache_file) + log.inf(runner, 'is available:', 'yes' if available else 'no', + colorize=True) _dump_runner_opt_help(runner, cls) _dump_runner_config(cfg, '', indent) if available: @@ -362,7 +391,7 @@ def _dump_one_runner_info(cache, args, build_dir, indent): def _dump_runner_caps(cls, base_indent): - log.inf('{}Capabilities:'.format(base_indent)) + log.inf('{}Capabilities:'.format(base_indent), colorize=True) log.inf('{}{}'.format(base_indent + INDENT, cls.capabilities())) @@ -379,15 +408,20 @@ def _dump_runner_opt_help(runner, cls): if len(actions) == 1 and actions[0].dest == 'command': # This is the lone positional argument. Skip it. continue - formatter.start_section('{} option help'.format(runner)) + formatter.start_section('REMOVE ME') formatter.add_text(group.description) formatter.add_arguments(actions) formatter.end_section() - log.inf(formatter.format_help()) + # Get the runner help, with the "REMOVE ME" string gone + runner_help = '\n'.join(formatter.format_help().splitlines()[1:]) + + log.inf('{} options:'.format(runner), colorize=True) + log.inf(runner_help) def _dump_runner_config(cfg, initial_indent, subsequent_indent): - log.inf('{}Cached common runner configuration:'.format(initial_indent)) + log.inf('{}Cached common runner configuration:'.format(initial_indent), + colorize=True) for var in cfg.__slots__: log.inf('{}--{}={}'.format(subsequent_indent, var, getattr(cfg, var))) @@ -397,8 +431,8 @@ def _dump_runner_cached_opts(cache, runner, initial_indent, subsequent_indent): if not runner_args: return - log.inf('{}Cached runner-specific options:'.format( - initial_indent)) + log.inf('{}Cached runner-specific options:'.format(initial_indent), + colorize=True) for arg in runner_args: log.inf('{}{}'.format(subsequent_indent, arg)) diff --git a/scripts/meta/west/commands/schema.yml b/scripts/meta/west/commands/schema.yml new file mode 100644 index 00000000000..bed0dd12f83 --- /dev/null +++ b/scripts/meta/west/commands/schema.yml @@ -0,0 +1,135 @@ +## A pykwalify schema for basic validation of the structure of a +## manifest YAML file. (Full validation would require additional work, +## e.g. to validate that remote URLs obey the URL format specified in +## rfc1738.) +## +## This schema has similar semantics to the repo XML format: +## +## https://gerrit.googlesource.com/git-repo/+/master/docs/manifest-format.txt +## +## However, the features don't map 1:1. + +# The top-level manifest is a map. The only top-level element is +# 'manifest'. All other elements are contained within it. This allows +# us a bit of future-proofing. +type: map +mapping: + manifest: + required: true + type: map + mapping: + # The "defaults" key specifies some default values used in the + # rest of the manifest. + # + # The value is a map with the following keys: + # + # - remote: if given, this is the default remote in each project + # - revision: if given, this is the default revision to check + # out of each project + # + # See below for more information about remotes and projects. + # + # Examples: + # + # default: + # remote: zephyrproject-rtos + # revision: master + defaults: + required: false + type: map + mapping: + remote: + required: false + type: str + revision: + required: false + type: str + + # The "remotes" key specifies a sequence of remotes, each of + # which has a name and a fetch URL. + # + # These work like repo remotes, in that they specify a URL + # prefix which remote-specific Git repositories hang off of. + # (This saves typing and makes it easier to move things around + # when most repositories are on the same server or GitHub + # organization.) + # + # Example: + # + # remotes: + # - name: zephyrproject-rtos + # url: https://github.com/zephyrproject-rtos + # - name: developer-fork + # url: https://github.com/a-developer + remotes: + required: true + type: seq + sequence: + - type: map + mapping: + name: + required: true + type: str + url: + required: true + type: str + + # The "projects" key specifies a sequence of "projects", + # i.e. Git repositories. These work like repo projects, in that + # each project has a name, a remote, and optional additional + # metadata. + # + # Each project is a map with the following keys: + # + # - name: Mandatory, the name of the git repository. The clone + # URL is formed by remote + '/' + name + # - remote: Optional, the name of the remote to pull it from. + # If the remote is missing, the remote'key in the top-level + # defaults key is used instead. If both are missing, it's an error. + # - revision: Optional, the name of the revision to check out. + # If not given, the value from the default element will be used. + # If both are missing, then the default is 'master'. + # - path: Where to clone the repository locally. If missing, + # it's cloned at top level in a directory given by its name. + # - clone-depth: if given, it is a number which creates a shallow + # history in the cloned repository limited to the given number + # of commits. + # + # Example, using default and non-default remotes: + # + # projects: + # # Uses default remote (zephyrproject-rtos), so clone URL is: + # # https://github.com/zephyrproject-rtos/zephyr + # - name: zephyr + # # Manually specified remote; clone URL is: + # # https://github.com/a-developer/west + # - name: west + # remote: developer-fork + # # Manually specified remote, clone URL is: + # # https://github.com/zephyrproject-rtos/some-vendor-hal + # # Local clone path (relative to installation root) is: + # # ext/hal/some-vendor + # - name: some-vendor-hal + # remote: zephyrproject-rtos + # path: ext/hal/some-vendor + projects: + required: true + type: seq + sequence: + - type: map + mapping: + name: + required: true + type: str + remote: + required: false + type: str + revision: + required: false + type: text # SHAs could be only numbers + path: + required: false + type: str + clone-depth: + required: false + type: int diff --git a/scripts/meta/west/log.py b/scripts/meta/west/log.py index 38bbf2ecbf1..f63d20bf20c 100644 --- a/scripts/meta/west/log.py +++ b/scripts/meta/west/log.py @@ -6,6 +6,7 @@ Provides common methods for logging messages to display to the user.''' +import colorama import sys VERBOSE_NONE = 0 @@ -40,27 +41,45 @@ def dbg(*args, level=VERBOSE_NORMAL): print(*args) -def inf(*args): - '''Print an informational message.''' +def inf(*args, colorize=False): + '''Print an informational message. + + colorize (default: False): + If True, the message is printed in bright green if stdout is a terminal. + ''' + # This approach colorizes any sep= and end= text too, as expected. + # + # colorama automatically strips the ANSI escapes when stdout isn't a + # terminal (by wrapping sys.stdout). + if colorize: + print(colorama.Fore.LIGHTGREEN_EX, end='') + print(*args) + if colorize: + # The final flush=True avoids issues with unrelated output from + # commands (usually Git) becoming green, due to the final attribute + # reset ANSI escape getting line-buffered. + print(colorama.Style.RESET_ALL, end='', flush=True) + def wrn(*args): '''Print a warning.''' - print('warning:', end=' ', file=sys.stderr, flush=False) + print(colorama.Fore.LIGHTRED_EX + 'WARNING: ', end='', file=sys.stderr) print(*args, file=sys.stderr) + print(colorama.Style.RESET_ALL, end='', file=sys.stderr, flush=True) def err(*args, fatal=False): '''Print an error.''' - if fatal: - print('fatal', end=' ', file=sys.stderr, flush=False) - print('error:', end=' ', file=sys.stderr, flush=False) + print(colorama.Fore.LIGHTRED_EX + + ('FATAL ERROR: ' if fatal else 'ERROR: '), + end='', file=sys.stderr) print(*args, file=sys.stderr) + print(colorama.Style.RESET_ALL, end='', file=sys.stderr, flush=True) def die(*args, exit_code=1): '''Print a fatal error, and abort with the given exit code.''' - print('fatal error:', end=' ', file=sys.stderr, flush=False) - print(*args, file=sys.stderr) + err(*args, fatal=True) sys.exit(exit_code) diff --git a/scripts/meta/west/main.py b/scripts/meta/west/main.py old mode 100644 new mode 100755 index a51ece0f9e3..4fe120bd7d0 --- a/scripts/meta/west/main.py +++ b/scripts/meta/west/main.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + # Copyright 2018 Open Source Foundries Limited. # # SPDX-License-Identifier: Apache-2.0 @@ -7,20 +9,52 @@ import argparse +import colorama from functools import partial import os import sys -from subprocess import CalledProcessError +from subprocess import CalledProcessError, check_output, DEVNULL -from . import log -from .cmd import CommandContextError -from .cmd.flash import Flash -from .cmd.debug import Debug, DebugServer -from .util import quote_sh_list +import log +from commands import CommandContextError +from commands.build import Build +from commands.flash import Flash +from commands.debug import Debug, DebugServer, Attach +from commands.project import ListProjects, Fetch, Pull, Rebase, Branch, \ + Checkout, Diff, Status, Update, ForAll, \ + WestUpdated +from util import quote_sh_list, in_multirepo_install +IN_MULTIREPO_INSTALL = in_multirepo_install(__file__) -COMMANDS = (Flash(), Debug(), DebugServer()) -'''Supported top-level commands.''' +BUILD_FLASH_COMMANDS = [ + Build(), + Flash(), + Debug(), + DebugServer(), + Attach(), +] + +PROJECT_COMMANDS = [ + ListProjects(), + Fetch(), + Pull(), + Rebase(), + Branch(), + Checkout(), + Diff(), + Status(), + Update(), + ForAll(), +] + +# Built-in commands in this West. For compatibility with monorepo +# installations of West within the Zephyr tree, we only expose the +# project commands if this is a multirepo installation. +COMMANDS = BUILD_FLASH_COMMANDS + +if IN_MULTIREPO_INSTALL: + COMMANDS += PROJECT_COMMANDS class InvalidWestContext(RuntimeError): @@ -43,7 +77,38 @@ def validate_context(args, unknown): args.zephyr_base = os.environ['ZEPHYR_BASE'] +def print_version_info(): + # The bootstrapper will print its own version, as well as that of + # the west repository itself, then exit. So if this file is being + # asked to print the version, it's because it's being run + # directly, and not via the bootstrapper. + # + # Rather than play tricks like invoking "pip show west" (which + # assumes the bootstrapper was installed via pip, the common but + # not universal case), refuse the temptation to make guesses and + # print an honest answer. + log.inf('West bootstrapper version: N/A, not run via bootstrapper') + + # The running west installation. + if IN_MULTIREPO_INSTALL: + try: + desc = check_output(['git', 'describe', '--tags'], + stderr=DEVNULL, + cwd=os.path.dirname(__file__)) + west_version = desc.decode(sys.getdefaultencoding()).strip() + except CalledProcessError as e: + west_version = 'unknown' + else: + west_version = 'N/A, monorepo installation' + west_src_west = os.path.dirname(__file__) + print('West repository version: {} ({})'. + format(west_version, + os.path.dirname(os.path.dirname(west_src_west)))) + + def parse_args(argv): + # The prog='west' override avoids the absolute path of the main.py script + # showing up when West is run via the wrapper west_parser = argparse.ArgumentParser( prog='west', description='The Zephyr RTOS meta-tool.', epilog='Run "west -h" for help on each command.') @@ -54,6 +119,7 @@ def parse_args(argv): west_parser.add_argument('-v', '--verbose', default=0, action='count', help='''Display verbose output. May be given multiple times to increase verbosity.''') + west_parser.add_argument('-V', '--version', action='store_true') subparser_gen = west_parser.add_subparsers(title='commands', dest='command') @@ -63,6 +129,10 @@ def parse_args(argv): args, unknown = west_parser.parse_known_args(args=argv) + if args.version: + print_version_info() + sys.exit(0) + # Set up logging verbosity before doing anything else, so # e.g. verbose messages related to argument handling errors # work properly. @@ -84,6 +154,10 @@ def parse_args(argv): def main(argv=None): + # Makes ANSI color escapes work on Windows, and strips them when + # stdout/stderr isn't a terminal + colorama.init() + if argv is None: argv = sys.argv[1:] args, unknown = parse_args(argv) @@ -92,6 +166,10 @@ def main(argv=None): args.command) try: args.handler(args, unknown) + except WestUpdated: + # West has been automatically updated. Restart ourselves to run the + # latest version, with the same arguments that we were given. + os.execv(sys.executable, [sys.executable] + sys.argv) except KeyboardInterrupt: sys.exit(0) except CalledProcessError as cpe: @@ -110,3 +188,6 @@ def main(argv=None): raise else: log.inf(for_stack_trace) + +if __name__ == "__main__": + main() diff --git a/scripts/meta/west/runner/__init__.py b/scripts/meta/west/runners/__init__.py similarity index 60% rename from scripts/meta/west/runner/__init__.py rename to scripts/meta/west/runners/__init__.py index ccc6ebedd44..c642c64dda3 100644 --- a/scripts/meta/west/runner/__init__.py +++ b/scripts/meta/west/runners/__init__.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -from .core import ZephyrBinaryRunner +from runners.core import ZephyrBinaryRunner # We import these here to ensure the ZephyrBinaryRunner subclasses are # defined; otherwise, ZephyrBinaryRunner.create_for_shell_script() @@ -10,19 +10,19 @@ from .core import ZephyrBinaryRunner # Explicitly silence the unused import warning. # flake8: noqa: F401 -from . import arc -from . import bossac -from . import dfu -from . import esp32 -from . import jlink -from . import nios2 -from . import nrfjprog -from . import nsim -from . import openocd -from . import pyocd -from . import qemu -from . import xtensa -from . import intel_s1000 +from runners import arc +from runners import bossac +from runners import dfu +from runners import esp32 +from runners import jlink +from runners import nios2 +from runners import nrfjprog +from runners import nsim +from runners import openocd +from runners import pyocd +from runners import qemu +from runners import xtensa +from runners import intel_s1000 def get_runner_cls(runner): '''Get a runner's class object, given its name.''' diff --git a/scripts/meta/west/runner/arc.py b/scripts/meta/west/runners/arc.py similarity index 98% rename from scripts/meta/west/runner/arc.py rename to scripts/meta/west/runners/arc.py index 87b17a2421b..00411949ff3 100644 --- a/scripts/meta/west/runner/arc.py +++ b/scripts/meta/west/runners/arc.py @@ -7,7 +7,7 @@ from os import path -from .core import ZephyrBinaryRunner +from runners.core import ZephyrBinaryRunner DEFAULT_ARC_TCL_PORT = 6333 DEFAULT_ARC_TELNET_PORT = 4444 diff --git a/scripts/meta/west/runner/bossac.py b/scripts/meta/west/runners/bossac.py similarity index 96% rename from scripts/meta/west/runner/bossac.py rename to scripts/meta/west/runners/bossac.py index 3138803846f..0173008581c 100644 --- a/scripts/meta/west/runner/bossac.py +++ b/scripts/meta/west/runners/bossac.py @@ -6,7 +6,7 @@ import platform -from .core import ZephyrBinaryRunner, RunnerCaps +from runners.core import ZephyrBinaryRunner, RunnerCaps DEFAULT_BOSSAC_PORT = '/dev/ttyACM0' diff --git a/scripts/meta/west/runner/core.py b/scripts/meta/west/runners/core.py similarity index 95% rename from scripts/meta/west/runner/core.py rename to scripts/meta/west/runners/core.py index cbd830e4a61..42401feec6b 100644 --- a/scripts/meta/west/runner/core.py +++ b/scripts/meta/west/runners/core.py @@ -18,8 +18,8 @@ import platform import signal import subprocess -from .. import log -from ..util import quote_sh_list +import log +from util import quote_sh_list # Turn on to enable just printing the commands that would be run, # without actually running them. This can break runners that are expecting @@ -163,7 +163,7 @@ class RunnerCaps: Available capabilities: - commands: set of supported commands; default is {'flash', - 'debug', 'debugserver'}. + 'debug', 'debugserver', 'attach'}. - flash_addr: whether the runner supports flashing to an arbitrary address. Default is False. If true, the runner @@ -171,7 +171,7 @@ class RunnerCaps: ''' def __init__(self, - commands={'flash', 'debug', 'debugserver'}, + commands={'flash', 'debug', 'debugserver', 'attach'}, flash_addr=False): self.commands = commands self.flash_addr = bool(flash_addr) @@ -256,15 +256,21 @@ class ZephyrBinaryRunner(abc.ABC): - 'flash': flash a previously configured binary to the board, start execution on the target, then return. - - 'debug': connect to the board via a debugging protocol, then - drop the user into a debugger interface with symbol tables - loaded from the current binary, and block until it exits. + - 'debug': connect to the board via a debugging protocol, program + the flash, then drop the user into a debugger interface with + symbol tables loaded from the current binary, and block until it + exits. - 'debugserver': connect via a board-specific debugging protocol, then reset and halt the target. Ensure the user is now able to connect to a debug server with symbol tables loaded from the binary. + - 'attach': connect to the board via a debugging protocol, then drop + the user into a debugger interface with symbol tables loaded from + the current binary, and block until it exits. Unlike 'debug', this + command does not program the flash. + This class provides an API for these commands. Every runner has a name (like 'pyocd'), and declares commands it can handle (like 'flash'). Zephyr boards (like 'nrf52_pca10040') declare compatible @@ -391,7 +397,7 @@ class ZephyrBinaryRunner(abc.ABC): return default def run(self, command, **kwargs): - '''Runs command ('flash', 'debug', 'debugserver'). + '''Runs command ('flash', 'debug', 'debugserver', 'attach'). This is the main entry point to this runner.''' caps = self.capabilities() diff --git a/scripts/meta/west/runner/dfu.py b/scripts/meta/west/runners/dfu.py similarity index 97% rename from scripts/meta/west/runner/dfu.py rename to scripts/meta/west/runners/dfu.py index 6063ad55622..45445be6468 100644 --- a/scripts/meta/west/runner/dfu.py +++ b/scripts/meta/west/runners/dfu.py @@ -9,8 +9,8 @@ import os import sys import time -from .. import log -from .core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration +import log +from runners.core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration DfuSeConfig = namedtuple('DfuSeConfig', ['address', 'options']) diff --git a/scripts/meta/west/runner/esp32.py b/scripts/meta/west/runners/esp32.py similarity index 98% rename from scripts/meta/west/runner/esp32.py rename to scripts/meta/west/runners/esp32.py index 1dc1a98a92b..c12a4500924 100644 --- a/scripts/meta/west/runner/esp32.py +++ b/scripts/meta/west/runners/esp32.py @@ -6,8 +6,8 @@ from os import path -from .. import log -from .core import ZephyrBinaryRunner, RunnerCaps +import log +from runners.core import ZephyrBinaryRunner, RunnerCaps class Esp32BinaryRunner(ZephyrBinaryRunner): diff --git a/scripts/meta/west/runner/intel_s1000.py b/scripts/meta/west/runners/intel_s1000.py similarity index 98% rename from scripts/meta/west/runner/intel_s1000.py rename to scripts/meta/west/runners/intel_s1000.py index 871ac093b95..47fdb062cc5 100644 --- a/scripts/meta/west/runner/intel_s1000.py +++ b/scripts/meta/west/runners/intel_s1000.py @@ -8,8 +8,8 @@ from os import path import time import subprocess -from .. import log -from .core import ZephyrBinaryRunner +import log +from runners.core import ZephyrBinaryRunner DEFAULT_XT_GDB_PORT = 20000 diff --git a/scripts/meta/west/runner/jlink.py b/scripts/meta/west/runners/jlink.py similarity index 92% rename from scripts/meta/west/runner/jlink.py rename to scripts/meta/west/runners/jlink.py index 7aadf62a6e9..e46d6ef8796 100644 --- a/scripts/meta/west/runner/jlink.py +++ b/scripts/meta/west/runners/jlink.py @@ -7,8 +7,8 @@ import os import tempfile -from .. import log -from .core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration +import log +from runners.core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration DEFAULT_JLINK_GDB_PORT = 2331 @@ -42,7 +42,8 @@ class JLinkBinaryRunner(ZephyrBinaryRunner): @classmethod def capabilities(cls): - return RunnerCaps(flash_addr=True) + return RunnerCaps(commands={'flash', 'debug', 'debugserver', 'attach'}, + flash_addr=True) @classmethod def do_add_parser(cls, parser): @@ -104,10 +105,11 @@ class JLinkBinaryRunner(ZephyrBinaryRunner): client_cmd = (self.gdb_cmd + self.tui_arg + [self.elf_name] + - ['-ex', 'target remote :{}'.format(self.gdb_port), - '-ex', 'monitor halt', - '-ex', 'monitor reset', - '-ex', 'load']) + ['-ex', 'target remote :{}'.format(self.gdb_port)]) + if command == 'debug': + client_cmd += ['-ex', 'monitor halt', + '-ex', 'monitor reset', + '-ex', 'load'] self.print_gdbserver_message() self.run_server_and_client(server_cmd, client_cmd) diff --git a/scripts/meta/west/runner/nios2.py b/scripts/meta/west/runners/nios2.py similarity index 97% rename from scripts/meta/west/runner/nios2.py rename to scripts/meta/west/runners/nios2.py index 3292b66b330..4edef1c94de 100644 --- a/scripts/meta/west/runner/nios2.py +++ b/scripts/meta/west/runners/nios2.py @@ -4,8 +4,8 @@ '''Runner for NIOS II, based on quartus-flash.py and GDB.''' -from .. import log -from .core import ZephyrBinaryRunner, NetworkPortHelper +import log +from runners.core import ZephyrBinaryRunner, NetworkPortHelper class Nios2BinaryRunner(ZephyrBinaryRunner): diff --git a/scripts/meta/west/runner/nrfjprog.py b/scripts/meta/west/runners/nrfjprog.py similarity index 98% rename from scripts/meta/west/runner/nrfjprog.py rename to scripts/meta/west/runners/nrfjprog.py index f236815a87f..ce163a1a5a4 100644 --- a/scripts/meta/west/runner/nrfjprog.py +++ b/scripts/meta/west/runners/nrfjprog.py @@ -6,8 +6,8 @@ import sys -from .. import log -from .core import ZephyrBinaryRunner, RunnerCaps +import log +from runners.core import ZephyrBinaryRunner, RunnerCaps class NrfJprogBinaryRunner(ZephyrBinaryRunner): diff --git a/scripts/meta/west/runner/nsim.py b/scripts/meta/west/runners/nsim.py similarity index 98% rename from scripts/meta/west/runner/nsim.py rename to scripts/meta/west/runners/nsim.py index ee16cdd325a..c58ff7cf6f8 100644 --- a/scripts/meta/west/runner/nsim.py +++ b/scripts/meta/west/runners/nsim.py @@ -7,7 +7,7 @@ from os import path -from .core import ZephyrBinaryRunner +from runners.core import ZephyrBinaryRunner DEFAULT_ARC_GDB_PORT = 3333 DEFAULT_PROPS_FILE = 'nsim.props' diff --git a/scripts/meta/west/runner/openocd.py b/scripts/meta/west/runners/openocd.py similarity index 99% rename from scripts/meta/west/runner/openocd.py rename to scripts/meta/west/runners/openocd.py index d55771dbe3a..8daad3ef55e 100644 --- a/scripts/meta/west/runner/openocd.py +++ b/scripts/meta/west/runners/openocd.py @@ -6,7 +6,7 @@ from os import path -from .core import ZephyrBinaryRunner +from runners.core import ZephyrBinaryRunner DEFAULT_OPENOCD_TCL_PORT = 6333 DEFAULT_OPENOCD_TELNET_PORT = 4444 diff --git a/scripts/meta/west/runner/pyocd.py b/scripts/meta/west/runners/pyocd.py similarity index 93% rename from scripts/meta/west/runner/pyocd.py rename to scripts/meta/west/runners/pyocd.py index b8a6ee4ed4a..bda7d027283 100644 --- a/scripts/meta/west/runner/pyocd.py +++ b/scripts/meta/west/runners/pyocd.py @@ -6,8 +6,8 @@ import os import sys -from .core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration -from .. import log +from runners.core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration +import log DEFAULT_PYOCD_GDB_PORT = 3333 @@ -51,7 +51,8 @@ class PyOcdBinaryRunner(ZephyrBinaryRunner): @classmethod def capabilities(cls): - return RunnerCaps(flash_addr=True) + return RunnerCaps(commands={'flash', 'debug', 'debugserver', 'attach'}, + flash_addr=True) @classmethod def do_add_parser(cls, parser): @@ -140,8 +141,10 @@ class PyOcdBinaryRunner(ZephyrBinaryRunner): client_cmd = (self.gdb_cmd + self.tui_args + [self.elf_name] + - ['-ex', 'target remote :{}'.format(self.gdb_port), - '-ex', 'load', - '-ex', 'monitor reset halt']) + ['-ex', 'target remote :{}'.format(self.gdb_port)]) + if command == 'debug': + client_cmd += ['-ex', 'load', + '-ex', 'monitor reset halt'] + self.print_gdbserver_message() self.run_server_and_client(server_cmd, client_cmd) diff --git a/scripts/meta/west/runner/qemu.py b/scripts/meta/west/runners/qemu.py similarity index 92% rename from scripts/meta/west/runner/qemu.py rename to scripts/meta/west/runners/qemu.py index dd4de5e8e47..ea39be8ad41 100644 --- a/scripts/meta/west/runner/qemu.py +++ b/scripts/meta/west/runners/qemu.py @@ -4,7 +4,7 @@ '''Runner stub for QEMU.''' -from .core import ZephyrBinaryRunner, RunnerCaps +from runners.core import ZephyrBinaryRunner, RunnerCaps class QemuBinaryRunner(ZephyrBinaryRunner): diff --git a/scripts/meta/west/runner/xtensa.py b/scripts/meta/west/runners/xtensa.py similarity index 94% rename from scripts/meta/west/runner/xtensa.py rename to scripts/meta/west/runners/xtensa.py index 931bc1610dd..2aeb42d8f69 100644 --- a/scripts/meta/west/runner/xtensa.py +++ b/scripts/meta/west/runners/xtensa.py @@ -6,7 +6,7 @@ from os import path -from .core import ZephyrBinaryRunner, RunnerCaps +from runners.core import ZephyrBinaryRunner, RunnerCaps class XtensaBinaryRunner(ZephyrBinaryRunner): diff --git a/scripts/meta/west/util.py b/scripts/meta/west/util.py index a9f8180d1e9..458abbabe48 100644 --- a/scripts/meta/west/util.py +++ b/scripts/meta/west/util.py @@ -5,6 +5,7 @@ '''Miscellaneous utilities used by west ''' +import os import shlex import textwrap @@ -20,3 +21,60 @@ def wrap(text, indent): '''Convenience routine for wrapping text to a consistent indent.''' return textwrap.wrap(text, initial_indent=indent, subsequent_indent=indent) + + +class WestNotFound(RuntimeError): + '''Neither the current directory nor any parent has a West installation.''' + + +def west_dir(start=None): + '''Returns the absolute path of the west/ top level directory. + + Starts the search from the start directory, and goes to its + parents. If the start directory is not specified, the current + directory is used. + + Raises WestNotFound if no west top-level directory is found. + ''' + return os.path.join(west_topdir(start), 'west') + + +def west_topdir(start=None): + ''' + Like west_dir(), but returns the path to the parent directory of the west/ + directory instead, where project repositories are stored + ''' + # If you change this function, make sure to update the bootstrap + # script's find_west_topdir(). + + if start is None: + cur_dir = os.getcwd() + else: + cur_dir = start + + while True: + if os.path.isfile(os.path.join(cur_dir, 'west', '.west_topdir')): + return cur_dir + + parent_dir = os.path.dirname(cur_dir) + if cur_dir == parent_dir: + # At the root + raise WestNotFound('Could not find a West installation ' + 'in this or any parent directory') + cur_dir = parent_dir + + +def in_multirepo_install(start=None): + '''Returns True iff the path ``start`` is in a multi-repo installation. + + If start is not given, it defaults to the current working directory. + + This is equivalent to checking if west_dir() raises an exception + when given the same start kwarg. + ''' + try: + west_topdir(start) + result = True + except WestNotFound: + result = False + return result diff --git a/scripts/west b/scripts/west index 964699a0036..ddc0661d604 100755 --- a/scripts/west +++ b/scripts/west @@ -1,5 +1,6 @@ #!/bin/sh -# UNIX operating system entry point to the west tool. -export "PYTHONPATH=${PYTHONPATH:+${PYTHONPATH}:}$ZEPHYR_BASE/scripts/meta" -python3 -m west $@ +# Zephyr meta-tool (west) launcher alias, which keeps +# monorepo Zephyr installations' 'make flash' etc. working. +here=$(readlink -f $(dirname $0)) +python3 "$here/west-launcher.py" $@ diff --git a/scripts/west-launcher.py b/scripts/west-launcher.py new file mode 100644 index 00000000000..c46e1936144 --- /dev/null +++ b/scripts/west-launcher.py @@ -0,0 +1,103 @@ +# Zephyr launcher which is interoperable with: +# +# 1. "mono-repo" Zephyr installations that have 'make flash' +# etc. supplied by a copy of some west code in scripts/meta. +# +# 2. "multi-repo" Zephyr installations where west is provided in a +# separate Git repository elsewhere. +# +# This is basically a copy of the "wrapper" functionality in the west +# bootstrap script for the multi-repo case, plus a fallback onto the +# copy in scripts/meta/west for mono-repo installs. + +import os +import subprocess +import sys + +if sys.version_info < (3,): + sys.exit('fatal error: you are running Python 2') + +# Top-level west directory, containing west itself and the manifest. +WEST_DIR = 'west' +# Subdirectory to check out the west source repository into. +WEST = 'west' +# File inside of WEST_DIR which marks it as the top level of the +# Zephyr project installation. +# +# (The WEST_DIR name is not distinct enough to use when searching for +# the top level; other directories named "west" may exist elsewhere, +# e.g. zephyr/doc/west.) +WEST_MARKER = '.west_topdir' + + +class WestError(RuntimeError): + pass + + +class WestNotFound(WestError): + '''Neither the current directory nor any parent has a West installation.''' + + +def find_west_topdir(start): + '''Find the top-level installation directory, starting at ``start``. + + If none is found, raises WestNotFound.''' + cur_dir = start + + while True: + if os.path.isfile(os.path.join(cur_dir, WEST_DIR, WEST_MARKER)): + return cur_dir + + parent_dir = os.path.dirname(cur_dir) + if cur_dir == parent_dir: + # At the root + raise WestNotFound('Could not find a West installation ' + 'in this or any parent directory') + cur_dir = parent_dir + + +def append_to_pythonpath(directory): + pp = os.environ.get('PYTHONPATH') + os.environ['PYTHONPATH'] = ':'.join(([pp] if pp else []) + [directory]) + + +def wrap(topdir, argv): + # Replace the wrapper process with the "real" west + + # sys.argv[1:] strips the argv[0] of the wrapper script itself + west_git_repo = os.path.join(topdir, WEST_DIR, WEST) + argv = ([sys.executable, + os.path.join(west_git_repo, 'src', 'west', 'main.py')] + + argv) + + try: + append_to_pythonpath(os.path.join(west_git_repo, 'src')) + subprocess.check_call(argv) + except subprocess.CalledProcessError as e: + sys.exit(e.returncode) + + +def run_scripts_meta_west(): + try: + subprocess.check_call([sys.executable, + os.path.join(os.environ['ZEPHYR_BASE'], + 'scripts', 'meta', 'west', + 'main.py')] + sys.argv[1:]) + except subprocess.CalledProcessError as e: + sys.exit(e.returncode) + + +def main(): + try: + topdir = find_west_topdir(__file__) + except WestNotFound: + topdir = None + + if topdir is not None: + wrap(topdir, sys.argv[1:]) + else: + run_scripts_meta_west() + + +if __name__ == '__main__': + main() diff --git a/scripts/west-win.py b/scripts/west-win.py deleted file mode 100644 index 4f0adc397f9..00000000000 --- a/scripts/west-win.py +++ /dev/null @@ -1,11 +0,0 @@ -# Windows-specific launcher alias for west (west wind?). - -import os -import sys - -zephyr_base = os.environ['ZEPHYR_BASE'] -sys.path.append(os.path.join(zephyr_base, 'scripts', 'meta')) - -from west.main import main # noqa E402 (silence flake8 warning) - -main(sys.argv[1:]) diff --git a/zephyr-env.cmd b/zephyr-env.cmd index 89e6e44bf67..1ea7467f272 100644 --- a/zephyr-env.cmd +++ b/zephyr-env.cmd @@ -5,5 +5,13 @@ if exist "%userprofile%\zephyrrc.cmd" ( call "%userprofile%\zephyrrc.cmd" ) -rem Zephyr meta-tool (west) launcher alias -doskey west=py -3 %ZEPHYR_BASE%\scripts\west-win.py $* +rem Zephyr meta-tool (west) launcher alias, which keeps monorepo +rem Zephyr installations' 'make flash' etc. working. See +rem https://www.python.org/dev/peps/pep-0486/ for details on the +rem virtualenv-related pieces. (We need to implement this manually +rem because Zephyr's minimum supported Python version is 3.4.) +if defined VIRTUAL_ENV ( + doskey west=python %ZEPHYR_BASE%\scripts\west-launcher.py $* +) else ( + doskey west=py -3 %ZEPHYR_BASE%\scripts\west-launcher.py $* +)