scripts: update west to latest upstream version.

Update to the latest west. This includes a new 'attach' command. There
are also multi-repo commands, but those won't get exposed to the user
unless they install Zephyr using "west init" + "west fetch" (and not,
say, "git clone").

Replace the launchers; they now detect whether zephyr is part of a
multi-repo installation, and run the west code in its own repository
if that is the case.

This also requires an update to:

- the flash/debug CMakeLists.txt, as the new west package is no longer
  executable as a module and must have its main script run by the
  interpreter instead.

- the documentation, to reflect a rename and with a hack to fix
  the automodule directive in flash-debug.rst for now

Signed-off-by: Marti Bolivar <marti@foundries.io>

cmake/flash/CMakeLists.txt

@@ -89,7 +89,7 @@ foreach(target flash debug debugserver)
     ${CMAKE_COMMAND} -E env
     PYTHONPATH=${ZEPHYR_BASE}/scripts/meta
     ${PYTHON_EXECUTABLE}
-    -m west
+    ${ZEPHYR_BASE}/scripts/meta/west/main.py
     ${RUNNER_VERBOSE}
     ${target}
     --skip-rebuild

doc/conf.py

@@ -28,6 +28,9 @@ ZEPHYR_BUILD = os.path.abspath(os.environ["ZEPHYR_BUILD"])
 sys.path.insert(0, os.path.join(ZEPHYR_BASE, 'doc', 'extensions'))
 # Also add west, to be able to pull in its API docs.
 sys.path.append(os.path.join(ZEPHYR_BASE, 'scripts', 'meta'))
+# HACK: also add the runners module, to work around some import issues
+# related to west's current packaging.
+sys.path.append(os.path.join(ZEPHYR_BASE, 'scripts', 'meta', 'west'))
 
 # -- General configuration ------------------------------------------------
 

doc/west/flash-debug.rst

@@ -191,26 +191,26 @@ For example, to print usage information about the ``jlink`` runner::
 
 .. _west-runner:
 
-Library Backend: ``west.runner``
+Library Backend: ``west.runners``
-********************************
+*********************************
 
 In keeping with West's :ref:`west-design-constraints`, the flash and
 debug commands are wrappers around a separate library that is part of
 West, and can be used by other code.
 
-This library is the ``west.runner`` subpackage of West itself.  The
+This library is the ``west.runners`` subpackage of West itself.  The
 central abstraction within this library is ``ZephyrBinaryRunner``, an
 abstract class which represents *runner* objects, which can flash
 and/or debug Zephyr programs. The set of available runners is
 determined by the imported subclasses of ``ZephyrBinaryRunner``.
-``ZephyrBinaryRunner`` is available in the ``west.runner.core``
+``ZephyrBinaryRunner`` is available in the ``west.runners.core``
 module; individual runner implementations are in other submodules,
-such as ``west.runner.nrfjprog``, ``west.runner.openocd``, etc.
+such as ``west.runners.nrfjprog``, ``west.runners.openocd``, etc.
 
 Developers can add support for new ways to flash and debug Zephyr
 programs by implementing additional runners. To get this support into
 upstream Zephyr, the runner should be added into a new or existing
-``west.runner`` module, and imported from
+``west.runners`` module, and imported from
 :file:`west/runner/__init__.py`.
 
 .. important::
@@ -223,7 +223,7 @@ upstream Zephyr, the runner should be added into a new or existing
 
 API documentation for the core module follows.
 
-.. automodule:: west.runner.core
+.. automodule:: west.runners.core
    :members:
 
 Doing it By Hand

scripts/meta/west/__main__.py

@@ -1,16 +0,0 @@
-# Copyright 2018 Open Source Foundries Limited.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-'''Zephyr RTOS meta-tool (west)
-
-Main entry point for running this package as a module, e.g.:
-
-py -3 west      # Windows
-python3 -m west # Unix
-'''
-
-from .main import main
-
-if __name__ == '__main__':
-    main()

scripts/meta/west/_bootstrap/__init__.py

@@ -2,4 +2,4 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-# Nothing here for now.
+# Empty file.

scripts/meta/west/_bootstrap/main.py

@@ -0,0 +1,276 @@
+# Copyright 2018 Open Source Foundries Limited.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+'''West's bootstrap/wrapper script.
+'''
+
+import argparse
+import os
+import platform
+import subprocess
+import sys
+
+import west._bootstrap.version as version
+
+if sys.version_info < (3,):
+    sys.exit('fatal error: you are running Python 2')
+
+
+#
+# Special files and directories in the west installation.
+#
+# These are given variable names for clarity, but they can't be
+# changed without propagating the changes into west itself.
+#
+
+# Top-level west directory, containing west itself and the manifest.
+WEST_DIR = 'west'
+# Subdirectory to check out the west source repository into.
+WEST = 'west'
+# Default west repository URL.
+WEST_DEFAULT = 'https://github.com/zephyrproject-rtos/west'
+# Default revision to check out of the west repository.
+WEST_REV_DEFAULT = 'master'
+# File inside of WEST_DIR which marks it as the top level of the
+# Zephyr project installation.
+#
+# (The WEST_DIR name is not distinct enough to use when searching for
+# the top level; other directories named "west" may exist elsewhere,
+# e.g. zephyr/doc/west.)
+WEST_MARKER = '.west_topdir'
+
+# Manifest repository directory under WEST_DIR.
+MANIFEST = 'manifest'
+# Default manifest repository URL.
+MANIFEST_DEFAULT = 'https://github.com/zephyrproject-rtos/manifest'
+# Default revision to check out of the manifest repository.
+MANIFEST_REV_DEFAULT = 'master'
+
+
+#
+# Helpers shared between init and wrapper mode
+#
+
+
+class WestError(RuntimeError):
+    pass
+
+
+class WestNotFound(WestError):
+    '''Neither the current directory nor any parent has a West installation.'''
+
+
+def find_west_topdir(start):
+    '''Find the top-level installation directory, starting at ``start``.
+
+    If none is found, raises WestNotFound.'''
+    # If you change this function, make sure to update west.util.west_topdir().
+
+    cur_dir = start
+
+    while True:
+        if os.path.isfile(os.path.join(cur_dir, WEST_DIR, WEST_MARKER)):
+            return cur_dir
+
+        parent_dir = os.path.dirname(cur_dir)
+        if cur_dir == parent_dir:
+            # At the root
+            raise WestNotFound('Could not find a West installation '
+                               'in this or any parent directory')
+        cur_dir = parent_dir
+
+
+def clone(url, rev, dest):
+    if os.path.exists(dest):
+        raise WestError('refusing to clone into existing location ' + dest)
+
+    if not url.startswith(('http:', 'https:', 'git:', 'git+shh:', 'file:')):
+        raise WestError('Unknown URL scheme for repository: {}'.format(url))
+
+    subprocess.check_call(('git', 'clone', '-b', rev, '--', url, dest))
+
+
+#
+# west init
+#
+
+
+def init(argv):
+    '''Command line handler for ``west init`` invocations.
+
+    This exits the program with a nonzero exit code if fatal errors occur.'''
+    init_parser = argparse.ArgumentParser(
+        prog='west init',
+        description='Bootstrap initialize a Zephyr installation')
+    init_parser.add_argument(
+        '-b', '--base-url',
+        help='''Base URL for both 'manifest' and 'zephyr' repositories; cannot
+        be given if either -u or -w are''')
+    init_parser.add_argument(
+        '-u', '--manifest-url',
+        help='Zephyr manifest fetch URL, default ' + MANIFEST_DEFAULT)
+    init_parser.add_argument(
+        '--mr', '--manifest-rev', default=MANIFEST_REV_DEFAULT,
+        dest='manifest_rev',
+        help='Manifest revision to fetch, default ' + MANIFEST_REV_DEFAULT)
+    init_parser.add_argument(
+        '-w', '--west-url',
+        help='West fetch URL, default ' + WEST_DEFAULT)
+    init_parser.add_argument(
+        '--wr', '--west-rev', default=WEST_REV_DEFAULT, dest='west_rev',
+        help='West revision to fetch, default ' + WEST_REV_DEFAULT)
+    init_parser.add_argument(
+        'directory', nargs='?', default=None,
+        help='Initializes in this directory, creating it if necessary')
+
+    args = init_parser.parse_args(args=argv)
+    directory = args.directory or os.getcwd()
+
+    if args.base_url:
+        if args.manifest_url or args.west_url:
+            sys.exit('fatal error: -b is incompatible with -u and -w')
+        args.manifest_url = args.base_url.rstrip('/') + '/manifest'
+        args.west_url = args.base_url.rstrip('/') + '/west'
+    else:
+        if not args.manifest_url:
+            args.manifest_url = MANIFEST_DEFAULT
+        if not args.west_url:
+            args.west_url = WEST_DEFAULT
+
+    try:
+        topdir = find_west_topdir(directory)
+        init_reinit(topdir, args)
+    except WestNotFound:
+        init_bootstrap(directory, args)
+
+
+def hide_file(path):
+    '''Ensure path is a hidden file.
+
+    On Windows, this uses attrib to hide the file manually.
+
+    On UNIX systems, this just checks that the path's basename begins
+    with a period ('.'), for it to be hidden already. It's a fatal
+    error if it does not begin with a period in this case.
+
+    On other systems, this just prints a warning.
+    '''
+    system = platform.system()
+
+    if system == 'Windows':
+        subprocess.check_call(['attrib', '+H', path])
+    elif os.name == 'posix':  # Try to check for all Unix, not just macOS/Linux
+        if not os.path.basename(path).startswith('.'):
+            sys.exit("internal error: {} can't be hidden on UNIX".format(path))
+    else:
+        print("warning: unknown platform {}; {} may not be hidden"
+              .format(system, path), file=sys.stderr)
+
+
+def init_bootstrap(directory, args):
+    '''Bootstrap a new manifest + West installation in the given directory.'''
+    if not os.path.isdir(directory):
+        try:
+            print('Initializing in new directory', directory)
+            os.makedirs(directory, exist_ok=False)
+        except PermissionError:
+            sys.exit('Cannot initialize in {}: permission denied'.format(
+                directory))
+        except FileExistsError:
+            sys.exit('Something else created {} concurrently; quitting'.format(
+                directory))
+        except Exception as e:
+            sys.exit("Can't create directory {}: {}".format(
+                directory, e.args))
+    else:
+        print('Initializing in', directory)
+
+    # Clone the west source code and the manifest into west/. Git will create
+    # the west/ directory if it does not exist.
+
+    clone(args.west_url, args.west_rev,
+          os.path.join(directory, WEST_DIR, WEST))
+
+    clone(args.manifest_url, args.manifest_rev,
+          os.path.join(directory, WEST_DIR, MANIFEST))
+
+    # Create a dotfile to mark the installation. Hide it on Windows.
+
+    with open(os.path.join(directory, WEST_DIR, WEST_MARKER), 'w') as f:
+        hide_file(f.name)
+
+
+def init_reinit(directory, args):
+    # TODO
+    sys.exit('Re-initializing an existing installation is not yet supported.')
+
+
+#
+# Wrap a West command
+#
+
+
+def wrap(argv):
+    printing_version = False
+
+    if argv and argv[0] in ('-V', '--version'):
+        print('West bootstrapper version: v{} ({})'.format(version.__version__,
+                                                    os.path.dirname(__file__)))
+        printing_version = True
+
+    start = os.getcwd()
+    try:
+        topdir = find_west_topdir(start)
+    except WestNotFound:
+        if printing_version:
+            sys.exit(0)         # run outside of an installation directory
+        else:
+            sys.exit('Error: not a Zephyr directory (or any parent): {}\n'
+                     'Use "west init" to install Zephyr here'.format(start))
+
+    west_git_repo = os.path.join(topdir, WEST_DIR, WEST)
+    if printing_version:
+        try:
+            git_describe = subprocess.check_output(
+                ['git', 'describe', '--tags'],
+                stderr=subprocess.DEVNULL,
+                cwd=west_git_repo).decode(sys.getdefaultencoding()).strip()
+            print('West repository version: {} ({})'.format(git_describe,
+                                                            west_git_repo))
+        except subprocess.CalledProcessError:
+            print('West repository verison: unknown; no tags were found')
+        sys.exit(0)
+
+    # Replace the wrapper process with the "real" west
+
+    # sys.argv[1:] strips the argv[0] of the wrapper script itself
+    argv = ([sys.executable,
+             os.path.join(west_git_repo, 'src', 'west', 'main.py')] +
+            argv)
+
+    try:
+        subprocess.check_call(argv)
+    except subprocess.CalledProcessError as e:
+        sys.exit(1)
+
+
+#
+# Main entry point
+#
+
+
+def main(wrap_argv=None):
+    '''Entry point to the wrapper script.'''
+    if wrap_argv is None:
+        wrap_argv = sys.argv[1:]
+
+    if not wrap_argv or wrap_argv[0] != 'init':
+        wrap(wrap_argv)
+    else:
+        init(wrap_argv[1:])
+        sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

scripts/meta/west/_bootstrap/version.py

@@ -0,0 +1,5 @@
+# Don't put anything else in here!
+#
+# This is the Python 3 version of option 3 in:
+# https://packaging.python.org/guides/single-sourcing-package-version/#single-sourcing-the-version
+__version__ = '0.2.0rc1'

scripts/meta/west/build.py

@@ -0,0 +1,42 @@
+# Copyright 2018 (c) Foundries.io.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+'''Common definitions for building Zephyr applications.
+
+This provides some default settings and convenience wrappers for
+building Zephyr applications needed by multiple commands.
+
+See west.cmd.build for the build command itself.
+'''
+
+import cmake
+import log
+
+DEFAULT_BUILD_DIR = 'build'
+'''Name of the default Zephyr build directory.'''
+
+DEFAULT_CMAKE_GENERATOR = 'Ninja'
+'''Name of the default CMake generator.'''
+
+
+def is_zephyr_build(path):
+    '''Return true if and only if `path` appears to be a valid Zephyr
+    build directory.
+
+    "Valid" means the given path is a directory which contains a CMake
+    cache with a 'ZEPHYR_TOOLCHAIN_VARIANT' key.
+    '''
+    try:
+        cache = cmake.CMakeCache.from_build_dir(path)
+    except FileNotFoundError:
+        cache = {}
+
+    if 'ZEPHYR_TOOLCHAIN_VARIANT' in cache:
+        log.dbg('{} is a zephyr build directory'.format(path),
+                level=log.VERBOSE_EXTREME)
+        return True
+    else:
+        log.dbg('{} is NOT a valid zephyr build directory'.format(path),
+                level=log.VERBOSE_EXTREME)
+        return False

scripts/meta/west/cmake.py

@@ -5,34 +5,41 @@
 '''Helpers for dealing with CMake'''
 
 from collections import OrderedDict
+import os.path
 import re
 import subprocess
 import shutil
 
-from . import log
+import log
-from .util import quote_sh_list
+from util import quote_sh_list
 
-__all__ = ['run_build', 'make_c_identifier', 'CMakeCacheEntry', 'CMakeCache']
+__all__ = ['run_cmake', 'run_build',
+           'make_c_identifier',
+           'CMakeCacheEntry', 'CMakeCache']
 
 DEFAULT_CACHE = 'CMakeCache.txt'
 
 
-def run_build(build_directory, extra_args=[], quiet=False):
+def run_cmake(args, quiet=False):
-    '''Run cmake in build tool mode in `build_directory`'''
+    '''Run cmake to (re)generate a build system'''
     cmake = shutil.which('cmake')
     if cmake is None:
         log.die('CMake is not installed or cannot be found; cannot build.')
-    cmd = [cmake, '--build', build_directory] + extra_args
+    cmd = [cmake] + args
-    kwargs = {}
+    kwargs = dict()
     if quiet:
         kwargs['stdout'] = subprocess.DEVNULL
         kwargs['stderr'] = subprocess.STDOUT
-    log.dbg('Re-building', build_directory)
+    log.dbg('Running CMake:', cmd, level=log.VERBOSE_VERY)
-    log.dbg('Build command list:', cmd, level=log.VERBOSE_VERY)
     log.dbg('As command:', quote_sh_list(cmd), level=log.VERBOSE_VERY)
     subprocess.check_call(cmd, **kwargs)
 
 
+def run_build(build_directory, extra_args=(), quiet=False):
+    '''Run cmake in build tool mode in `build_directory`'''
+    run_cmake(['--build', build_directory] + list(extra_args), quiet=quiet)
+
+
 def make_c_identifier(string):
     '''Make a C identifier from a string in the same way CMake does.
     '''
@@ -154,6 +161,10 @@ class CMakeCacheEntry:
 class CMakeCache:
     '''Parses and represents a CMake cache file.'''
 
+    @staticmethod
+    def from_build_dir(build_dir):
+        return CMakeCache(os.path.join(build_dir, DEFAULT_CACHE))
+
     def __init__(self, cache_file):
         self.cache_file = cache_file
         self.load(cache_file)
@@ -190,6 +201,9 @@ class CMakeCache:
         else:
             return default
 
+    def __contains__(self, name):
+        return name in self._entries
+
     def __getitem__(self, name):
         return self._entries[name].value
 

scripts/meta/west/commands/build.py

@@ -0,0 +1,278 @@
+# Copyright (c) 2018 Foundries.io
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import os
+
+import log
+import cmake
+from build import DEFAULT_BUILD_DIR, DEFAULT_CMAKE_GENERATOR, is_zephyr_build
+from commands import WestCommand
+
+BUILD_HELP = '''\
+Convenience wrapper for building Zephyr applications.
+
+This command attempts to do what you mean when run from a Zephyr
+application source or a pre-existing build directory:
+
+- When "west build" is run from a Zephyr build directory, the source
+  directory is obtained from the CMake cache, and that build directory
+  is re-compiled.
+
+- Otherwise, the source directory defaults to the current working
+  directory, so running "west build" from a Zephyr application's
+  source directory compiles it.
+
+The source and build directories can be explicitly set with the
+--source-dir and --build-dir options. The build directory defaults to
+'build' if it is not auto-detected. The build directory is always
+created if it does not exist.
+
+This command runs CMake to generate a build system if one is not
+present in the build directory, then builds the application.
+Subsequent builds try to avoid re-running CMake; you can force it
+to run by setting --cmake.
+
+To pass additional options to CMake, give them as extra arguments
+after a '--' For example, "west build -- -DOVERLAY_CONFIG=some.conf" sets
+an overlay config file. (Doing this forces a CMake run.)'''
+
+
+class Build(WestCommand):
+
+    def __init__(self):
+        super(Build, self).__init__(
+            'build',
+            BUILD_HELP,
+            accepts_unknown_args=False)
+
+        self.source_dir = None
+        '''Source directory for the build, or None on error.'''
+
+        self.build_dir = None
+        '''Final build directory used to run the build, or None on error.'''
+
+        self.created_build_dir = False
+        '''True if the build directory was created; False otherwise.'''
+
+        self.run_cmake = False
+        '''True if CMake was run; False otherwise.
+
+        Note: this only describes CMake runs done by this command. The
+        build system generated by CMake may also update itself due to
+        internal logic.'''
+
+        self.cmake_cache = None
+        '''Final parsed CMake cache for the build, or None on error.'''
+
+    def do_add_parser(self, parser_adder):
+        parser = parser_adder.add_parser(
+            self.name,
+            formatter_class=argparse.RawDescriptionHelpFormatter,
+            description=self.description)
+
+        parser.add_argument('-b', '--board',
+                            help='''board to build for (must be given for the
+                            first build, can be omitted later)''')
+        parser.add_argument('-s', '--source-dir',
+                            help='''explicitly sets the source directory;
+                            if not given, infer it from directory context''')
+        parser.add_argument('-d', '--build-dir',
+                            help='''explicitly sets the build directory;
+                            if not given, infer it from directory context''')
+        parser.add_argument('-t', '--target',
+                            help='''override the build system target (e.g.
+                            'clean', 'pristine', etc.)''')
+        parser.add_argument('-c', '--cmake', action='store_true',
+                            help='force CMake to run')
+        parser.add_argument('-f', '--force', action='store_true',
+                            help='ignore any errors and try to build anyway')
+        parser.add_argument('cmake_opts', nargs='*', metavar='cmake_opt',
+                            help='extra option to pass to CMake; implies -c')
+
+        return parser
+
+    def do_run(self, args, ignored):
+        self.args = args        # Avoid having to pass them around
+        log.dbg('args:', args, level=log.VERBOSE_EXTREME)
+        self._sanity_precheck()
+        self._setup_build_dir()
+        if is_zephyr_build(self.build_dir):
+            self._update_cache()
+            if self.args.cmake or self.args.cmake_opts:
+                self.run_cmake = True
+        else:
+            self.run_cmake = True
+        self._setup_source_dir()
+        self._sanity_check()
+
+        log.inf('source directory: {}'.format(self.source_dir), colorize=True)
+        log.inf('build directory: {}{}'.
+                format(self.build_dir,
+                       (' (created)' if self.created_build_dir
+                        else '')),
+                colorize=True)
+        if self.cmake_cache:
+            board = self.cmake_cache.get('CACHED_BOARD')
+        elif self.args.board:
+            board = self.args.board
+        else:
+            board = 'UNKNOWN'   # shouldn't happen
+        log.inf('BOARD:', board, colorize=True)
+
+        self._run_cmake(self.args.cmake_opts)
+        self._sanity_check()
+        self._update_cache()
+
+        extra_args = ['--target', args.target] if args.target else []
+        cmake.run_build(self.build_dir, extra_args=extra_args)
+
+    def _sanity_precheck(self):
+        app = self.args.source_dir
+        if app:
+            if not os.path.isdir(app):
+                self._check_force('source directory {} does not exist'.
+                                  format(app))
+            elif 'CMakeLists.txt' not in os.listdir(app):
+                self._check_force("{} doesn't contain a CMakeLists.txt".
+                                  format(app))
+
+    def _update_cache(self):
+        try:
+            self.cmake_cache = cmake.CMakeCache.from_build_dir(self.build_dir)
+        except FileNotFoundError:
+            pass
+
+    def _setup_build_dir(self):
+        # Initialize build_dir and created_build_dir attributes.
+        log.dbg('setting up build directory', level=log.VERBOSE_EXTREME)
+        if self.args.build_dir:
+            build_dir = self.args.build_dir
+        else:
+            cwd = os.getcwd()
+            if is_zephyr_build(cwd):
+                build_dir = cwd
+            else:
+                build_dir = DEFAULT_BUILD_DIR
+        build_dir = os.path.abspath(build_dir)
+
+        if os.path.exists(build_dir):
+            if not os.path.isdir(build_dir):
+                log.die('build directory {} exists and is not a directory'.
+                        format(build_dir))
+        else:
+            os.makedirs(build_dir, exist_ok=False)
+            self.created_build_dir = True
+            self.run_cmake = True
+
+        self.build_dir = build_dir
+
+    def _setup_source_dir(self):
+        # Initialize source_dir attribute, either from command line argument,
+        # implicitly from the build directory's CMake cache, or using the
+        # default (current working directory).
+        log.dbg('setting up source directory', level=log.VERBOSE_EXTREME)
+        if self.args.source_dir:
+            source_dir = self.args.source_dir
+        elif self.cmake_cache:
+            source_dir = self.cmake_cache.get('APPLICATION_SOURCE_DIR')
+            if not source_dir:
+                # Maybe Zephyr changed the key? Give the user a way
+                # to retry, at least.
+                log.die("can't determine application from build directory "
+                        "{}, please specify an application to build".
+                        format(self.build_dir))
+        else:
+            source_dir = os.getcwd()
+        self.source_dir = os.path.abspath(source_dir)
+
+    def _sanity_check(self):
+        # Sanity check the build configuration.
+        # Side effect: may update cmake_cache attribute.
+        log.dbg('sanity checking the build', level=log.VERBOSE_EXTREME)
+        if self.source_dir == self.build_dir:
+            # There's no forcing this.
+            log.die('source and build directory {} cannot be the same; '
+                    'use --build-dir {} to specify a build directory'.
+                    format(self.source_dir, self.build_dir))
+
+        srcrel = os.path.relpath(self.source_dir)
+        if is_zephyr_build(self.source_dir):
+            self._check_force('it looks like {srcrel} is a build directory: '
+                              'did you mean -build-dir {srcrel} instead?'.
+                              format(srcrel=srcrel))
+        elif 'CMakeLists.txt' not in os.listdir(self.source_dir):
+            self._check_force('source directory "{srcrel}" does not contain '
+                              'a CMakeLists.txt; is that really what you '
+                              'want to build? (Use -s SOURCE_DIR to specify '
+                              'the application source directory)'.
+                              format(srcrel=srcrel))
+
+        if not is_zephyr_build(self.build_dir) and not self.args.board:
+            self._check_force('this looks like a new or clean build, '
+                              'please provide --board')
+
+        if not self.cmake_cache:
+            return          # That's all we can check without a cache.
+
+        cached_app = self.cmake_cache.get('APPLICATION_SOURCE_DIR')
+        log.dbg('APPLICATION_SOURCE_DIR:', cached_app,
+                level=log.VERBOSE_EXTREME)
+        source_abs = (os.path.abspath(self.args.source_dir)
+                      if self.args.source_dir else None)
+        cached_abs = os.path.abspath(cached_app) if cached_app else None
+        if cached_abs and source_abs and source_abs != cached_abs:
+            self._check_force('build directory "{}" is for application "{}", '
+                              'but source directory "{}" was specified; '
+                              'please clean it or use --build-dir to set '
+                              'another build directory'.
+                              format(self.build_dir, cached_abs,
+                                     source_abs))
+            self.run_cmake = True  # If they insist, we need to re-run cmake.
+
+        cached_board = self.cmake_cache.get('CACHED_BOARD')
+        log.dbg('CACHED_BOARD:', cached_board, level=log.VERBOSE_EXTREME)
+        if not cached_board and not self.args.board:
+            if self.created_build_dir:
+                self._check_force(
+                    'Building for the first time: you must provide --board')
+            else:
+                self._check_force(
+                    'Board is missing or unknown, please provide --board')
+        if self.args.board and cached_board and \
+           self.args.board != cached_board:
+            self._check_force('Build directory {} targets board {}, '
+                              'but board {} was specified. (Clean that '
+                              'directory or use --build-dir to specify '
+                              'a different one.)'.
+                              format(self.build_dir, cached_board,
+                                     self.args.board))
+
+    def _check_force(self, msg):
+        if not self.args.force:
+            log.err(msg)
+            log.die('refusing to proceed without --force due to above error')
+
+    def _run_cmake(self, cmake_opts):
+        if not self.run_cmake:
+            log.dbg('not running cmake; build system is present')
+            return
+
+        # It's unfortunate to have to use the undocumented -B and -H
+        # options to set the source and binary directories.
+        #
+        # However, it's the only known way to set that directory and
+        # run CMake from the current working directory. This is
+        # important because users expect invocations like this to Just
+        # Work:
+        #
+        # west build -- -DOVERLAY_CONFIG=relative-path.conf
+        final_cmake_args = ['-B{}'.format(self.build_dir),
+                            '-H{}'.format(self.source_dir),
+                            '-G{}'.format(DEFAULT_CMAKE_GENERATOR)]
+        if self.args.board:
+            final_cmake_args.append('-DBOARD={}'.format(self.args.board))
+        if cmake_opts:
+            final_cmake_args.extend(cmake_opts)
+        cmake.run_cmake(final_cmake_args)

scripts/meta/west/commands/debug.py

@@ -6,8 +6,8 @@
 
 from textwrap import dedent
 
-from .run_common import desc_common, add_parser_common, do_run_common
+from commands.run_common import desc_common, add_parser_common, do_run_common
-from . import WestCommand
+from commands import WestCommand
 
 
 class Debug(WestCommand):
@@ -15,7 +15,9 @@ class Debug(WestCommand):
     def __init__(self):
         super(Debug, self).__init__(
             'debug',
-            'Connect to the board and start a debugging session.\n\n' +
+            dedent('''
+            Connect to the board, program the flash, and start a
+            debugging session.\n\n''') +
             desc_common('debug'),
             accepts_unknown_args=True)
 
@@ -47,3 +49,21 @@ class DebugServer(WestCommand):
     def do_run(self, my_args, runner_args):
         do_run_common(self, my_args, runner_args,
                       'ZEPHYR_BOARD_DEBUG_RUNNER')
+
+class Attach(WestCommand):
+
+    def __init__(self):
+        super(Attach, self).__init__(
+            'attach',
+            dedent('''
+            Connect to the board without programming the flash, and
+            start a debugging session.\n\n''') +
+            desc_common('attach'),
+            accepts_unknown_args=True)
+
+    def do_add_parser(self, parser_adder):
+        return add_parser_common(parser_adder, self)
+
+    def do_run(self, my_args, runner_args):
+        do_run_common(self, my_args, runner_args,
+                      'ZEPHYR_BOARD_DEBUG_RUNNER')

scripts/meta/west/commands/flash.py

@@ -4,8 +4,8 @@
 
 '''west "flash" command'''
 
-from .run_common import desc_common, add_parser_common, do_run_common
+from commands.run_common import desc_common, add_parser_common, do_run_common
-from . import WestCommand
+from commands import WestCommand
 
 
 class Flash(WestCommand):

scripts/meta/west/commands/project.py

@@ -0,0 +1,876 @@
+# Copyright (c) 2018, Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+
+'''West project commands'''
+
+import argparse
+import collections
+import os
+import shutil
+import subprocess
+import textwrap
+
+import pykwalify.core
+import yaml
+
+import log
+import util
+from commands import WestCommand
+
+
+# Branch that points to the revision specified in the manifest (which might be
+# an SHA). Local branches created with 'west branch' are set to track this
+# branch.
+_MANIFEST_REV_BRANCH = 'manifest-rev'
+
+
+class ListProjects(WestCommand):
+    def __init__(self):
+        super().__init__(
+            'list-projects',
+            _wrap('''
+            List projects.
+
+            Prints the path to the manifest file and lists all projects along
+            with their clone paths and manifest revisions. Also includes
+            information on which projects are currently cloned.
+            '''))
+
+    def do_add_parser(self, parser_adder):
+        return _add_parser(parser_adder, self)
+
+    def do_run(self, args, user_args):
+        log.inf("Manifest path: {}\n".format(_manifest_path(args)))
+
+        for project in _all_projects(args):
+            log.inf('{:15} {:30} {:15} {}'.format(
+                project.name,
+                os.path.join(project.path, ''),  # Add final '/' if missing
+                project.revision,
+                "(cloned)" if _cloned(project) else "(not cloned)"))
+
+
+class Fetch(WestCommand):
+    def __init__(self):
+        super().__init__(
+            'fetch',
+            _wrap('''
+            Clone/fetch projects.
+
+            Fetches upstream changes in each of the specified projects
+            (default: all projects). Repositories that do not already exist are
+            cloned.
+
+            Unless --no-update is passed, the manifest and West source code
+            repositories are updated prior to fetching. See the 'update'
+            command.
+
+            ''' + _MANIFEST_REV_HELP))
+
+    def do_add_parser(self, parser_adder):
+        return _add_parser(parser_adder, self, _no_update_arg,
+                           _project_list_arg)
+
+    def do_run(self, args, user_args):
+        if args.update:
+            _update(True, True)
+
+        for project in _projects(args, listed_must_be_cloned=False):
+            log.dbg('fetching:', project, level=log.VERBOSE_VERY)
+            _fetch(project)
+
+
+class Pull(WestCommand):
+    def __init__(self):
+        super().__init__(
+            'pull',
+            _wrap('''
+            Clone/fetch and rebase projects.
+
+            Fetches upstream changes in each of the specified projects
+            (default: all projects) and rebases the checked-out branch (or
+            detached HEAD state) on top of '{}', effectively bringing the
+            branch up to date. Repositories that do not already exist are
+            cloned.
+
+            Unless --no-update is passed, the manifest and West source code
+            repositories are updated prior to pulling. See the 'update'
+            command.
+
+            '''.format(_MANIFEST_REV_BRANCH) + _MANIFEST_REV_HELP))
+
+    def do_add_parser(self, parser_adder):
+        return _add_parser(parser_adder, self, _no_update_arg,
+                           _project_list_arg)
+
+    def do_run(self, args, user_args):
+        if args.update:
+            _update(True, True)
+
+        for project in _projects(args, listed_must_be_cloned=False):
+            if _fetch(project):
+                _rebase(project)
+
+
+class Rebase(WestCommand):
+    def __init__(self):
+        super().__init__(
+            'rebase',
+            _wrap('''
+            Rebase projects.
+
+            Rebases the checked-out branch (or detached HEAD) on top of '{}' in
+            each of the specified projects (default: all cloned projects),
+            effectively bringing the branch up to date.
+
+            '''.format(_MANIFEST_REV_BRANCH) + _MANIFEST_REV_HELP))
+
+    def do_add_parser(self, parser_adder):
+        return _add_parser(parser_adder, self, _project_list_arg)
+
+    def do_run(self, args, user_args):
+        for project in _cloned_projects(args):
+            _rebase(project)
+
+
+class Branch(WestCommand):
+    def __init__(self):
+        super().__init__(
+            'branch',
+            _wrap('''
+            Create a branch or list branches, in multiple projects.
+
+            Creates a branch in each of the specified projects (default: all
+            cloned projects). The new branches are set to track '{}'.
+
+            With no arguments, lists all local branches along with the
+            repositories they appear in.
+
+            '''.format(_MANIFEST_REV_BRANCH) + _MANIFEST_REV_HELP))
+
+    def do_add_parser(self, parser_adder):
+        return _add_parser(
+            parser_adder, self,
+            _arg('branch', nargs='?', metavar='BRANCH_NAME'),
+            _project_list_arg)
+
+    def do_run(self, args, user_args):
+        if args.branch:
+            # Create a branch in the specified projects
+            for project in _cloned_projects(args):
+                _create_branch(project, args.branch)
+        else:
+            # No arguments. List local branches from all cloned projects along
+            # with the projects they appear in.
+
+            branch2projs = collections.defaultdict(list)
+            for project in _cloned_projects(args):
+                for branch in _branches(project):
+                    branch2projs[branch].append(project.name)
+
+            for branch, projs in sorted(branch2projs.items()):
+                log.inf('{:18} {}'.format(branch, ", ".join(projs)))
+
+
+class Checkout(WestCommand):
+    def __init__(self):
+        super().__init__(
+            'checkout',
+            _wrap('''
+            Check out topic branch.
+
+            Checks out the specified branch in each of the specified projects
+            (default: all cloned projects). Projects that do not have the
+            branch are left alone.
+            '''))
+
+    def do_add_parser(self, parser_adder):
+        return _add_parser(
+            parser_adder, self,
+            _arg('-b',
+                 dest='create_branch',
+                 action='store_true',
+                 help='create the branch before checking it out'),
+            _arg('branch', metavar='BRANCH_NAME'),
+            _project_list_arg)
+
+    def do_run(self, args, user_args):
+        branch_exists = False
+
+        for project in _cloned_projects(args):
+            if args.create_branch:
+                _create_branch(project, args.branch)
+                _checkout(project, args.branch)
+                branch_exists = True
+            elif _has_branch(project, args.branch):
+                _checkout(project, args.branch)
+                branch_exists = True
+
+        if not branch_exists:
+            msg = 'No branch {} exists in any '.format(args.branch)
+            if args.projects:
+                log.die(msg + 'of the listed projects')
+            else:
+                log.die(msg + 'cloned project')
+
+
+class Diff(WestCommand):
+    def __init__(self):
+        super().__init__(
+            'diff',
+            _wrap('''
+            'git diff' projects.
+
+            Runs 'git diff' for each of the specified projects (default: all
+            cloned projects).
+
+            Extra arguments are passed as-is to 'git diff'.
+            '''),
+            accepts_unknown_args=True)
+
+    def do_add_parser(self, parser_adder):
+        return _add_parser(parser_adder, self, _project_list_arg)
+
+    def do_run(self, args, user_args):
+        for project in _cloned_projects(args):
+            # Use paths that are relative to the base directory to make it
+            # easier to see where the changes are
+            _git(project, 'diff --src-prefix=(path)/ --dst-prefix=(path)/',
+                 extra_args=user_args)
+
+
+class Status(WestCommand):
+    def __init__(self):
+        super().__init__(
+            'status',
+            _wrap('''
+            Runs 'git status' for each of the specified projects (default: all
+            cloned projects). Extra arguments are passed as-is to 'git status'.
+            '''),
+            accepts_unknown_args=True)
+
+    def do_add_parser(self, parser_adder):
+        return _add_parser(parser_adder, self, _project_list_arg)
+
+    def do_run(self, args, user_args):
+        for project in _cloned_projects(args):
+            _inf(project, 'status of (name-and-path)')
+            _git(project, 'status', extra_args=user_args)
+
+
+class Update(WestCommand):
+    def __init__(self):
+        super().__init__(
+            'update',
+            _wrap('''
+            Updates the manifest repository and/or the West source code
+            repository.
+
+            There is normally no need to run this command manually, because
+            'west fetch' and 'west pull' automatically update the West and
+            manifest repositories to the latest version before doing anything
+            else.
+
+            Pass --update-west or --update-manifest to update just that
+            repository. With no arguments, both are updated.
+            '''))
+
+    def do_add_parser(self, parser_adder):
+        return _add_parser(
+            parser_adder, self,
+            _arg('--update-west',
+                 dest='update_west',
+                 action='store_true',
+                 help='update the West source code repository'),
+            _arg('--update-manifest',
+                 dest='update_manifest',
+                 action='store_true',
+                 help='update the manifest repository'))
+
+    def do_run(self, args, user_args):
+        if not args.update_west and not args.update_manifest:
+            _update(True, True)
+        else:
+            _update(args.update_west, args.update_manifest)
+
+
+class ForAll(WestCommand):
+    def __init__(self):
+        super().__init__(
+            'forall',
+            _wrap('''
+            Runs a shell (Linux) or batch (Windows) command within the
+            repository of each of the specified projects (default: all cloned
+            projects). Note that you have to quote the command if it consists
+            of more than one word, to prevent the shell you use to run 'west'
+            from splitting it up.
+
+            Since the command is run through the shell, you can use wildcards
+            and the like.
+
+            For example, the following command will list the contents of
+            proj-1's and proj-2's repositories on Linux, in long form:
+
+              west forall -c 'ls -l' proj-1 proj-2
+            '''))
+
+    def do_add_parser(self, parser_adder):
+        return _add_parser(
+            parser_adder, self,
+            _arg('-c',
+                 dest='command',
+                 metavar='COMMAND',
+                 required=True),
+            _project_list_arg)
+
+    def do_run(self, args, user_args):
+        for project in _cloned_projects(args):
+            _inf(project, "Running '{}' in (name-and-path)"
+                          .format(args.command))
+
+            subprocess.Popen(args.command, shell=True, cwd=project.abspath) \
+                .wait()
+
+
+def _arg(*args, **kwargs):
+    # Helper for creating a new argument parser for a single argument,
+    # later passed in parents= to add_parser()
+
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument(*args, **kwargs)
+    return parser
+
+
+# Arguments shared between more than one command
+
+_manifest_arg = _arg(
+    '-m', '--manifest',
+    help='path to manifest file (default: west/manifest/default.yml)')
+
+# For 'fetch' and 'pull'
+_no_update_arg = _arg(
+    '--no-update',
+    dest='update',
+    action='store_false',
+    help='do not update the manifest or West before fetching project data')
+
+# List of projects
+_project_list_arg = _arg('projects', metavar='PROJECT', nargs='*')
+
+
+def _add_parser(parser_adder, cmd, *extra_args):
+    # Adds and returns a subparser for the project-related WestCommand 'cmd'.
+    # All of these commands (currently) take the manifest path flag, so it's
+    # hardcoded here.
+
+    return parser_adder.add_parser(
+        cmd.name,
+        description=cmd.description,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        parents=(_manifest_arg,) + extra_args)
+
+
+def _wrap(s):
+    # Wraps help texts for commands. Some of them have variable length (due to
+    # _MANIFEST_REV_BRANCH), so just a textwrap.dedent() can look a bit wonky.
+
+    # [1:] gets rid of the initial newline. It's turned into a space by
+    # textwrap.fill() otherwise.
+    paragraphs = textwrap.dedent(s[1:]).split("\n\n")
+
+    return "\n\n".join(textwrap.fill(paragraph) for paragraph in paragraphs)
+
+
+_MANIFEST_REV_HELP = """
+The '{}' branch points to the revision that the manifest specified for the
+project as of the most recent 'west fetch'/'west pull'.
+""".format(_MANIFEST_REV_BRANCH)[1:].replace("\n", " ")
+
+
+# Holds information about a project, taken from the manifest file (or
+# constructed manually for "special" projects)
+Project = collections.namedtuple(
+    'Project',
+    'name url revision path abspath clone_depth')
+
+
+def _cloned_projects(args):
+    # Returns _projects(args, listed_must_be_cloned=True) if a list of projects
+    # was given by the user (i.e., listed projects are required to be cloned).
+    # If no projects were listed, returns all cloned projects.
+
+    # This approach avoids redundant _cloned() checks
+    return _projects(args) if args.projects else \
+        [project for project in _all_projects(args) if _cloned(project)]
+
+
+def _projects(args, listed_must_be_cloned=True):
+    # Returns a list of project instances for the projects requested in 'args'
+    # (the command-line arguments), in the same order that they were listed by
+    # the user. If args.projects is empty, no projects were listed, and all
+    # projects will be returned. If a non-existent project was listed by the
+    # user, an error is raised.
+    #
+    # Before the manifest is parsed, it is validated agains a pykwalify schema.
+    # An error is raised on validation errors.
+    #
+    # listed_must_be_cloned (default: True):
+    #   If True, an error is raised if an uncloned project was listed. This
+    #   only applies to projects listed explicitly on the command line.
+
+    projects = _all_projects(args)
+
+    if not args.projects:
+        # No projects specified. Return all projects.
+        return projects
+
+    # Got a list of projects on the command line. First, check that they exist
+    # in the manifest.
+
+    project_names = [project.name for project in projects]
+    nonexistent = set(args.projects) - set(project_names)
+    if nonexistent:
+        log.die('Unknown project{} {} (available projects: {})'
+                .format('s' if len(nonexistent) > 1 else '',
+                        ', '.join(nonexistent),
+                        ', '.join(project_names)))
+
+    # Return the projects in the order they were listed
+    res = []
+    for name in args.projects:
+        for project in projects:
+            if project.name == name:
+                res.append(project)
+                break
+
+    # Check that all listed repositories are cloned, if requested
+    if listed_must_be_cloned:
+        uncloned = [prj.name for prj in res if not _cloned(prj)]
+        if uncloned:
+            log.die('The following projects are not cloned: {}. Please clone '
+                    "them first (with 'west fetch')."
+                    .format(", ".join(uncloned)))
+
+    return res
+
+
+def _all_projects(args):
+    # Parses the manifest file, returning a list of Project instances.
+    #
+    # Before the manifest is parsed, it is validated against a pykwalify
+    # schema. An error is raised on validation errors.
+
+    manifest_path = _manifest_path(args)
+
+    _validate_manifest(manifest_path)
+
+    with open(manifest_path) as f:
+        manifest = yaml.safe_load(f)['manifest']
+
+    projects = []
+    # Manifest "defaults" keys whose values get copied to each project
+    # that doesn't specify its own value.
+    project_defaults = ('remote', 'revision')
+
+    # mp = manifest project (dictionary with values parsed from the manifest)
+    for mp in manifest['projects']:
+        # Fill in any missing fields in 'mp' with values from the 'defaults'
+        # dictionary
+        if 'defaults' in manifest:
+            for key, val in manifest['defaults'].items():
+                if key in project_defaults:
+                    mp.setdefault(key, val)
+
+        # Add the repository URL to 'mp'
+        for remote in manifest['remotes']:
+            if remote['name'] == mp['remote']:
+                mp['url'] = remote['url'] + '/' + mp['name']
+                break
+        else:
+            log.die('Remote {} not defined in {}'
+                    .format(mp['remote'], manifest_path))
+
+        # If no clone path is specified, the project's name is used
+        clone_path = mp.get('path', mp['name'])
+
+        # Use named tuples to store project information. That gives nicer
+        # syntax compared to a dict (project.name instead of project['name'],
+        # etc.)
+        projects.append(Project(
+            mp['name'],
+            mp['url'],
+            # If no revision is specified, 'master' is used
+            mp.get('revision', 'master'),
+            clone_path,
+            # Absolute clone path
+            os.path.join(util.west_topdir(), clone_path),
+            # If no clone depth is specified, we fetch the entire history
+            mp.get('clone-depth', None)))
+
+    return projects
+
+
+def _validate_manifest(manifest_path):
+    # Validates the manifest with pykwalify. schema.yml holds the schema.
+
+    schema_path = os.path.join(os.path.dirname(__file__), "schema.yml")
+
+    try:
+        pykwalify.core.Core(
+            source_file=manifest_path,
+            schema_files=[schema_path]
+        ).validate()
+    except pykwalify.errors.SchemaError as e:
+        log.die('{} malformed (schema: {}):\n{}'
+                .format(manifest_path, schema_path, e))
+
+
+def _manifest_path(args):
+    # Returns the path to the manifest file. Defaults to
+    # .west/manifest/default.yml if the user didn't specify a manifest.
+
+    return args.manifest or os.path.join(util.west_dir(), 'manifest',
+                                         'default.yml')
+
+
+def _fetch(project):
+    # Fetches upstream changes for 'project' and updates the 'manifest-rev'
+    # branch to point to the revision specified in the manifest. If the
+    # project's repository does not already exist, it is created first.
+    #
+    # Returns True if the project's repository already existed.
+
+    exists = _cloned(project)
+
+    if not exists:
+        _inf(project, 'Creating repository for (name-and-path)')
+        _git_base(project, 'init (abspath)')
+        _git(project, 'remote add origin (url)')
+
+    if project.clone_depth:
+        _inf(project,
+             'Fetching changes for (name-and-path) with --depth (clone-depth)')
+
+        # If 'clone-depth' is specified, fetch just the specified revision
+        # (probably a branch). That will download the minimum amount of data,
+        # which is probably what's wanted whenever 'clone-depth is used. The
+        # default 'git fetch' behavior is to do a shallow clone of all branches
+        # on the remote.
+        #
+        # Note: Many servers won't allow fetching arbitrary commits by SHA.
+        # Combining --depth with an SHA will break for those.
+
+        # Qualify branch names with refs/heads/, just to be safe. Just the
+        # branch name is likely to work as well though.
+        _git(project,
+             'fetch --depth=(clone-depth) origin ' +
+                 (project.revision if _is_sha(project.revision) else \
+                     'refs/heads/' + project.revision))
+
+    else:
+        _inf(project, 'Fetching changes for (name-and-path)')
+
+        # If 'clone-depth' is not specified, fetch all branches on the
+        # remote. This gives a more usable repository.
+        _git(project, 'fetch origin')
+
+    # Create/update the 'manifest-rev' branch
+    _git(project,
+         'update-ref refs/heads/(manifest-rev-branch) ' +
+             (project.revision if _is_sha(project.revision) else
+                 'remotes/origin/' + project.revision))
+
+    if not exists:
+        # If we just initialized the repository, check out 'manifest-rev' in a
+        # detached HEAD state.
+        #
+        # Otherwise, the initial state would have nothing checked out, and HEAD
+        # would point to a non-existent refs/heads/master branch (that would
+        # get created if the user makes an initial commit). That state causes
+        # e.g. 'west rebase' to fail, and might look confusing.
+        #
+        # (The --detach flag is strictly redundant here, because the
+        # refs/heads/<branch> form already detaches HEAD, but it avoids a
+        # spammy detached HEAD warning from Git.)
+        _git(project, 'checkout --detach refs/heads/(manifest-rev-branch)')
+
+    return exists
+
+
+def _rebase(project):
+    _inf(project, 'Rebasing (name-and-path) to (manifest-rev-branch)')
+    _git(project, 'rebase (manifest-rev-branch)')
+
+
+def _cloned(project):
+    # Returns True if the project's path is a directory that looks
+    # like the top-level directory of a Git repository, and False
+    # otherwise.
+
+    def handle(result):
+        log.dbg('project', project.name,
+                'is {}cloned'.format('' if result else 'not '),
+                level=log.VERBOSE_EXTREME)
+        return result
+
+    if not os.path.isdir(project.abspath):
+        return handle(False)
+
+    # --is-inside-work-tree doesn't require that the directory is the top-level
+    # directory of a Git repository. Use --show-cdup instead, which prints an
+    # empty string (i.e., just a newline, which we strip) for the top-level
+    # directory.
+    res = _git(project, 'rev-parse --show-cdup', capture_stdout=True,
+               check=False)
+
+    return handle(not (res.returncode or res.stdout))
+
+
+def _branches(project):
+    # Returns a sorted list of all local branches in 'project'
+
+    # refname:lstrip=-1 isn't available before Git 2.8 (introduced by commit
+    # 'tag: do not show ambiguous tag names as "tags/foo"'). Strip
+    # 'refs/heads/' manually instead.
+    return [ref[len('refs/heads/'):] for ref in
+            _git(project,
+                 'for-each-ref --sort=refname --format=%(refname) refs/heads',
+                 capture_stdout=True).stdout.split('\n')]
+
+
+def _create_branch(project, branch):
+    if _has_branch(project, branch):
+        _inf(project, "Branch '{}' already exists in (name-and-path)"
+                      .format(branch))
+    else:
+        _inf(project, "Creating branch '{}' in (name-and-path)"
+                      .format(branch))
+        _git(project, 'branch --quiet --track {} (manifest-rev-branch)'
+                      .format(branch))
+
+
+def _has_branch(project, branch):
+    return _git(project, 'show-ref --quiet --verify refs/heads/' + branch,
+                check=False).returncode == 0
+
+
+def _checkout(project, branch):
+    _inf(project, "Checking out branch '{}' in (name-and-path)".format(branch))
+    _git(project, 'checkout ' + branch)
+
+
+def _special_project(name):
+    # Returns a Project instance for one of the special repositories in west/,
+    # so that we can reuse the project-related functions for them
+
+    return Project(
+        name,
+        'dummy URL for {} repository'.format(name),
+        'master',
+        os.path.join('west', name.lower()),  # Path
+        os.path.join(util.west_dir(), name.lower()),  # Absolute path
+        None  # Clone depth
+    )
+
+
+def _update(update_west, update_manifest):
+    # 'try' is a keyword
+    def attempt(project, cmd):
+        res = _git(project, cmd, capture_stdout=True, check=False)
+        if res.returncode:
+            # The Git command's stderr isn't redirected and will also be
+            # available
+            _die(project, _FAILED_UPDATE_MSG.format(cmd))
+        return res.stdout
+
+    projects = []
+    if update_west:
+        projects.append(_special_project('West'))
+    if update_manifest:
+        projects.append(_special_project('manifest'))
+
+    for project in projects:
+        _dbg(project, 'Updating (name-and-path)', level=log.VERBOSE_NORMAL)
+
+        # Fetch changes from upstream
+        attempt(project, 'fetch')
+
+        # Get the SHA of the last commit in common with the upstream branch
+        merge_base = attempt(project, 'merge-base HEAD remotes/origin/master')
+
+        # Get the current SHA of the upstream branch
+        head_sha = attempt(project, 'show-ref --hash remotes/origin/master')
+
+        # If they differ, we need to rebase
+        if merge_base != head_sha:
+            attempt(project, 'rebase remotes/origin/master')
+
+            _inf(project, 'Updated (rebased) (name-and-path) to the '
+                          'latest version')
+
+            if project.name == 'west':
+                # Signal self-update, which will cause a restart. This is a bit
+                # nicer than doing the restart here, as callers will have a
+                # chance to flush file buffers, etc.
+                raise WestUpdated()
+
+
+_FAILED_UPDATE_MSG = """
+Failed to update (name-and-path), while running command '{}'. Please fix the
+state of the repository, or pass --no-update to 'west fetch/pull' to skip
+updating the manifest and West for the duration of the command."""[1:]
+
+
+class WestUpdated(Exception):
+    '''Raised after West has updated its own source code'''
+
+
+def _is_sha(s):
+    try:
+        int(s, 16)
+    except ValueError:
+        return False
+
+    return len(s) == 40
+
+
+def _git_base(project, cmd, *, extra_args=(), capture_stdout=False,
+              check=True):
+    # Runs a git command in the West top directory. See _git_helper() for
+    # parameter documentation.
+    #
+    # Returns a CompletedProcess instance (see below).
+
+    return _git_helper(project, cmd, extra_args, util.west_topdir(),
+                       capture_stdout, check)
+
+
+def _git(project, cmd, *, extra_args=(), capture_stdout=False, check=True):
+    # Runs a git command within a particular project. See _git_helper() for
+    # parameter documentation.
+    #
+    # Returns a CompletedProcess instance (see below).
+
+    return _git_helper(project, cmd, extra_args, project.abspath,
+                       capture_stdout, check)
+
+
+def _git_helper(project, cmd, extra_args, cwd, capture_stdout, check):
+    # Runs a git command.
+    #
+    # project:
+    #   The Project instance for the project, derived from the manifest file.
+    #
+    # cmd:
+    #   String with git arguments. Supports some "(foo)" shorthands. See below.
+    #
+    # extra_args:
+    #   List of additional arguments to pass to the git command (e.g. from the
+    #   user).
+    #
+    # cwd:
+    #   Directory to switch to first (None = current directory)
+    #
+    # capture_stdout:
+    #   True if stdout should be captured into the returned
+    #   subprocess.CompletedProcess instance instead of being printed.
+    #
+    #   We never capture stderr, to prevent error messages from being eaten.
+    #
+    # check:
+    #   True if an error should be raised if the git command finishes with a
+    #   non-zero return code.
+    #
+    # Returns a subprocess.CompletedProcess instance.
+
+    # TODO: Run once somewhere?
+    if shutil.which('git') is None:
+        log.die('Git is not installed or cannot be found')
+
+    args = (('git',) +
+            tuple(_expand_shorthands(project, arg) for arg in cmd.split()) +
+            tuple(extra_args))
+    cmd_str = util.quote_sh_list(args)
+
+    log.dbg("running '{}'".format(cmd_str), 'in', cwd, level=log.VERBOSE_VERY)
+    popen = subprocess.Popen(
+        args, stdout=subprocess.PIPE if capture_stdout else None, cwd=cwd)
+
+    stdout, _ = popen.communicate()
+
+    dbg_msg = "'{}' in {} finished with exit status {}" \
+              .format(cmd_str, cwd, popen.returncode)
+    if capture_stdout:
+        dbg_msg += " and wrote {} to stdout".format(stdout)
+    log.dbg(dbg_msg, level=log.VERBOSE_VERY)
+
+    if check and popen.returncode:
+        _die(project, "Command '{}' failed for (name-and-path)"
+                      .format(cmd_str))
+
+    if capture_stdout:
+        # Manual UTF-8 decoding and universal newlines. Before Python 3.6,
+        # Popen doesn't seem to allow using universal newlines mode (which
+        # enables decoding) with a specific encoding (because the encoding=
+        # parameter is missing).
+        #
+        # Also strip all trailing newlines as convenience. The splitlines()
+        # already means we lose a final '\n' anyway.
+        stdout = "\n".join(stdout.decode('utf-8').splitlines()).rstrip("\n")
+
+    return CompletedProcess(popen.args, popen.returncode, stdout)
+
+
+def _expand_shorthands(project, s):
+    # Expands project-related shorthands in 's' to their values,
+    # returning the expanded string
+
+    return s.replace('(name)', project.name) \
+            .replace('(name-and-path)',
+                     '{} ({})'.format(
+                         project.name, os.path.join(project.path, ""))) \
+            .replace('(url)', project.url) \
+            .replace('(path)', project.path) \
+            .replace('(abspath)', project.abspath) \
+            .replace('(revision)', project.revision) \
+            .replace('(manifest-rev-branch)', _MANIFEST_REV_BRANCH) \
+            .replace('(clone-depth)', str(project.clone_depth))
+
+
+def _inf(project, msg):
+    # Print '=== msg' (to clearly separate it from Git output). Supports the
+    # same (foo) shorthands as the git commands.
+    #
+    # Prints the message in green if stdout is a terminal, to clearly separate
+    # it from command (usually Git) output.
+
+    log.inf('=== ' + _expand_shorthands(project, msg), colorize=True)
+
+
+def _wrn(project, msg):
+    # Warn with 'msg'. Supports the same (foo) shorthands as the git commands.
+
+    log.wrn(_expand_shorthands(project, msg))
+
+
+def _dbg(project, msg, level):
+    # Like _wrn(), for debug messages
+
+    log.dbg(_expand_shorthands(project, msg), level=level)
+
+
+def _die(project, msg):
+    # Like _wrn(), for dying
+
+    log.die(_expand_shorthands(project, msg))
+
+
+# subprocess.CompletedProcess-alike, used instead of the real deal for Python
+# 3.4 compatibility, and with two small differences:
+#
+# - Trailing newlines are stripped from stdout
+#
+# - The 'stderr' attribute is omitted, because we never capture stderr
+CompletedProcess = collections.namedtuple(
+    'CompletedProcess', 'args returncode stdout')

scripts/meta/west/commands/run_common.py

@@ -10,12 +10,13 @@ from os import getcwd, path
 from subprocess import CalledProcessError
 import textwrap
 
-from .. import cmake
+import cmake
-from .. import log
+import log
-from .. import util
+import util
-from ..runner import get_runner_cls, ZephyrBinaryRunner
+from build import DEFAULT_BUILD_DIR, is_zephyr_build
-from ..runner.core import RunnerConfig
+from runners import get_runner_cls, ZephyrBinaryRunner
-from . import CommandContextError
+from runners.core import RunnerConfig
+from commands import CommandContextError
 
 # Context-sensitive help indentation.
 # Don't change this, or output from argparse won't match up.
@@ -37,8 +38,10 @@ def add_parser_common(parser_adder, command):
 
     group.add_argument('-d', '--build-dir',
                        help='''Build directory to obtain runner information
-                       from; default is the current working directory.''')
+                       from. If not given, this command tries to use build/
-    group.add_argument('-c', '--cmake-cache', default=cmake.DEFAULT_CACHE,
+                       and then the current working directory, in that
+                       order.''')
+    group.add_argument('-c', '--cmake-cache',
                        help='''Path to CMake cache file containing runner
                        configuration (this is generated by the Zephyr
                        build system when compiling binaries);
@@ -127,13 +130,32 @@ def _override_config_from_namespace(cfg, namespace):
                 setattr(cfg, var, val)
 
 
+def _build_dir(args, die_if_none=True):
+    # Get the build directory for the given argument list and environment.
+    if args.build_dir:
+        return args.build_dir
+
+    cwd = getcwd()
+    default = path.join(cwd, DEFAULT_BUILD_DIR)
+    if is_zephyr_build(default):
+        return default
+    elif is_zephyr_build(cwd):
+        return cwd
+    elif die_if_none:
+        log.die('--build-dir was not given, and neither {} '
+                'nor {} are zephyr build directories.'.
+                format(default, cwd))
+    else:
+        return None
+
+
 def do_run_common(command, args, runner_args, cached_runner_var):
     if args.context:
         _dump_context(command, args, runner_args, cached_runner_var)
         return
 
     command_name = command.name
-    build_dir = args.build_dir or getcwd()
+    build_dir = _build_dir(args)
 
     if not args.skip_rebuild:
         try:
@@ -153,7 +175,7 @@ def do_run_common(command, args, runner_args, cached_runner_var):
     # line override. Get the ZephyrBinaryRunner class by name, and
     # make sure it supports the command.
 
-    cache_file = path.join(build_dir, args.cmake_cache)
+    cache_file = path.join(build_dir, args.cmake_cache or cmake.DEFAULT_CACHE)
     cache = cmake.CMakeCache(cache_file)
     board = cache['CACHED_BOARD']
     available = cache.get_list('ZEPHYR_RUNNERS')
@@ -218,35 +240,33 @@ def do_run_common(command, args, runner_args, cached_runner_var):
 #
 
 def _dump_context(command, args, runner_args, cached_runner_var):
-    build_dir = args.build_dir or getcwd()
+    build_dir = _build_dir(args, die_if_none=False)
 
-    # If the cache is a file, try to ensure build artifacts are up to
+    # Try to figure out the CMake cache file based on the build
-    # date. If that doesn't work, still try to print information on a
+    # directory or an explicit argument.
-    # best-effort basis.
+    if build_dir is not None:
-    cache_file = path.abspath(path.join(build_dir, args.cmake_cache))
+        cache_file = path.abspath(
-    cache = None
+            path.join(build_dir, args.cmake_cache or cmake.DEFAULT_CACHE))
-
+    elif args.cmake_cache:
-    if path.isfile(cache_file):
+        cache_file = path.abspath(args.cmake_cache)
-        have_cache_file = True
     else:
-        have_cache_file = False
+        cache_file = None
-        if args.build_dir:
-            msg = textwrap.dedent('''\
-            CMake cache {}: no such file or directory, --build-dir {}
-            is invalid'''.format(cache_file, args.build_dir))
-            log.die('\n'.join(textwrap.wrap(msg, initial_indent='',
-                                            subsequent_indent=INDENT,
-                                            break_on_hyphens=False)))
-        else:
-            msg = textwrap.dedent('''\
-            No cache file {} found; is this a build directory?
-            (Use --build-dir to set one if not, otherwise, output will be
-            limited.)'''.format(cache_file))
-            log.wrn('\n'.join(textwrap.wrap(msg, initial_indent='',
-                                            subsequent_indent=INDENT,
-                                            break_on_hyphens=False)))
 
-    if have_cache_file and not args.skip_rebuild:
+    # Load the cache itself, if possible.
+    if cache_file is None:
+        log.wrn('No build directory (--build-dir) or CMake cache '
+                '(--cache-file) given or found; output will be limited')
+        cache = None
+    else:
+        try:
+            cache = cmake.CMakeCache(cache_file)
+        except Exception:
+            log.die('Cannot load cache {}.'.format(cache_file))
+
+    # If we have a build directory, try to ensure build artifacts are
+    # up to date. If that doesn't work, still try to print information
+    # on a best-effort basis.
+    if build_dir and not args.skip_rebuild:
         try:
             cmake.run_build(build_dir)
         except CalledProcessError:
@@ -255,18 +275,12 @@ def _dump_context(command, args, runner_args, cached_runner_var):
                 msg += 'Is {} the right --build-dir?'.format(args.build_dir)
             else:
                 msg += textwrap.dedent('''\
-                Use --build-dir (-d) to specify a build directory; the default
+                Use --build-dir (-d) to specify a build directory; the one
-                is the current directory, {}.'''.format(build_dir))
+                used was {}.'''.format(build_dir))
             log.die('\n'.join(textwrap.wrap(msg, initial_indent='',
                                             subsequent_indent=INDENT,
                                             break_on_hyphens=False)))
 
-    if have_cache_file:
-        try:
-            cache = cmake.CMakeCache(cache_file)
-        except Exception:
-            log.die('Cannot load cache {}.'.format(cache_file))
-
     if cache is None:
         _dump_no_context_info(command, args)
         if not args.runner:
@@ -287,19 +301,24 @@ def _dump_context(command, args, runner_args, cached_runner_var):
     default_runner = cache.get(cached_runner_var)
     cfg = cached_runner_config(build_dir, cache)
 
-    log.inf('All Zephyr runners which support {}:'.format(command.name))
+    log.inf('All Zephyr runners which support {}:'.format(command.name),
+            colorize=True)
     for line in util.wrap(', '.join(all_cls.keys()), INDENT):
         log.inf(line)
-    log.inf('(Not all may work with this build, see available runners below.)')
+    log.inf('(Not all may work with this build, see available runners below.)',
+            colorize=True)
 
     if cache is None:
         log.warn('Missing or invalid CMake cache {}; there is no context.',
                  'Use --build-dir to specify the build directory.')
         return
 
-    log.inf('Build directory:', build_dir)
+    log.inf('Build directory:', colorize=True)
-    log.inf('Board:', board)
+    log.inf(INDENT + build_dir)
-    log.inf('CMake cache:', cache_file)
+    log.inf('Board:', colorize=True)
+    log.inf(INDENT + board)
+    log.inf('CMake cache:', colorize=True)
+    log.inf(INDENT + cache_file)
 
     if not available:
         # Bail with a message if no runners are available.
@@ -307,33 +326,39 @@ def _dump_context(command, args, runner_args, cached_runner_var):
                'Consult the documentation for instructions on how to run '
                'binaries on this target.').format(board)
         for line in util.wrap(msg, ''):
-            log.inf(line)
+            log.inf(line, colorize=True)
         return
 
-    log.inf('Available {} runners:'.format(command.name), ', '.join(available))
+    log.inf('Available {} runners:'.format(command.name), colorize=True)
-    log.inf('Additional options for available', command.name, 'runners:')
+    log.inf(INDENT + ', '.join(available))
+    log.inf('Additional options for available', command.name, 'runners:',
+            colorize=True)
     for runner in available:
         _dump_runner_opt_help(runner, all_cls[runner])
-    log.inf('Default {} runner: {}'.format(command.name, default_runner))
+    log.inf('Default {} runner:'.format(command.name), colorize=True)
+    log.inf(INDENT + default_runner)
     _dump_runner_config(cfg, '', INDENT)
-    log.inf('Runner-specific information:')
+    log.inf('Runner-specific information:', colorize=True)
     for runner in available:
-        log.inf('{}{}:'.format(INDENT, runner))
+        log.inf('{}{}:'.format(INDENT, runner), colorize=True)
         _dump_runner_cached_opts(cache, runner, INDENT * 2, INDENT * 3)
         _dump_runner_caps(available_cls[runner], INDENT * 2)
 
     if len(available) > 1:
-        log.inf('(Add -r RUNNER to just print information about one runner.)')
+        log.inf('(Add -r RUNNER to just print information about one runner.)',
+                colorize=True)
 
 
 def _dump_no_context_info(command, args):
     all_cls = {cls.name(): cls for cls in ZephyrBinaryRunner.get_runners() if
                command.name in cls.capabilities().commands}
-    log.inf('All Zephyr runners which support {}:'.format(command.name))
+    log.inf('All Zephyr runners which support {}:'.format(command.name),
+            colorize=True)
     for line in util.wrap(', '.join(all_cls.keys()), INDENT):
         log.inf(line)
     if not args.runner:
-        log.inf('Add -r RUNNER to print more information about any runner.')
+        log.inf('Add -r RUNNER to print more information about any runner.',
+                colorize=True)
 
 
 def _dump_one_runner_info(cache, args, build_dir, indent):
@@ -348,10 +373,14 @@ def _dump_one_runner_info(cache, args, build_dir, indent):
     available = runner in cache.get_list('ZEPHYR_RUNNERS')
     cfg = cached_runner_config(build_dir, cache)
 
-    log.inf('Build directory:', build_dir)
+    log.inf('Build directory:', colorize=True)
-    log.inf('Board:', cache['CACHED_BOARD'])
+    log.inf(INDENT + build_dir)
-    log.inf('CMake cache:', cache.cache_file)
+    log.inf('Board:', colorize=True)
-    log.inf(runner, 'is available:', 'yes' if available else 'no')
+    log.inf(INDENT + cache['CACHED_BOARD'])
+    log.inf('CMake cache:', colorize=True)
+    log.inf(INDENT + cache.cache_file)
+    log.inf(runner, 'is available:', 'yes' if available else 'no',
+            colorize=True)
     _dump_runner_opt_help(runner, cls)
     _dump_runner_config(cfg, '', indent)
     if available:
@@ -362,7 +391,7 @@ def _dump_one_runner_info(cache, args, build_dir, indent):
 
 
 def _dump_runner_caps(cls, base_indent):
-    log.inf('{}Capabilities:'.format(base_indent))
+    log.inf('{}Capabilities:'.format(base_indent), colorize=True)
     log.inf('{}{}'.format(base_indent + INDENT, cls.capabilities()))
 
 
@@ -379,15 +408,20 @@ def _dump_runner_opt_help(runner, cls):
         if len(actions) == 1 and actions[0].dest == 'command':
             # This is the lone positional argument. Skip it.
             continue
-        formatter.start_section('{} option help'.format(runner))
+        formatter.start_section('REMOVE ME')
         formatter.add_text(group.description)
         formatter.add_arguments(actions)
         formatter.end_section()
-    log.inf(formatter.format_help())
+    # Get the runner help, with the "REMOVE ME" string gone
+    runner_help = '\n'.join(formatter.format_help().splitlines()[1:])
+
+    log.inf('{} options:'.format(runner), colorize=True)
+    log.inf(runner_help)
 
 
 def _dump_runner_config(cfg, initial_indent, subsequent_indent):
-    log.inf('{}Cached common runner configuration:'.format(initial_indent))
+    log.inf('{}Cached common runner configuration:'.format(initial_indent),
+            colorize=True)
     for var in cfg.__slots__:
         log.inf('{}--{}={}'.format(subsequent_indent, var, getattr(cfg, var)))
 
@@ -397,8 +431,8 @@ def _dump_runner_cached_opts(cache, runner, initial_indent, subsequent_indent):
     if not runner_args:
         return
 
-    log.inf('{}Cached runner-specific options:'.format(
+    log.inf('{}Cached runner-specific options:'.format(initial_indent),
-        initial_indent))
+            colorize=True)
     for arg in runner_args:
         log.inf('{}{}'.format(subsequent_indent, arg))
 

scripts/meta/west/commands/schema.yml

@@ -0,0 +1,135 @@
+## A pykwalify schema for basic validation of the structure of a
+## manifest YAML file.  (Full validation would require additional work,
+## e.g. to validate that remote URLs obey the URL format specified in
+## rfc1738.)
+##
+## This schema has similar semantics to the repo XML format:
+##
+## https://gerrit.googlesource.com/git-repo/+/master/docs/manifest-format.txt
+##
+## However, the features don't map 1:1.
+
+# The top-level manifest is a map. The only top-level element is
+# 'manifest'. All other elements are contained within it. This allows
+# us a bit of future-proofing.
+type: map
+mapping:
+  manifest:
+    required: true
+    type: map
+    mapping:
+      # The "defaults" key specifies some default values used in the
+      # rest of the manifest.
+      #
+      # The value is a map with the following keys:
+      #
+      # - remote: if given, this is the default remote in each project
+      # - revision: if given, this is the default revision to check
+      #   out of each project
+      #
+      # See below for more information about remotes and projects.
+      #
+      # Examples:
+      #
+      # default:
+      #   remote: zephyrproject-rtos
+      #   revision: master
+      defaults:
+        required: false
+        type: map
+        mapping:
+          remote:
+            required: false
+            type: str
+          revision:
+            required: false
+            type: str
+
+      # The "remotes" key specifies a sequence of remotes, each of
+      # which has a name and a fetch URL.
+      #
+      # These work like repo remotes, in that they specify a URL
+      # prefix which remote-specific Git repositories hang off of.
+      # (This saves typing and makes it easier to move things around
+      # when most repositories are on the same server or GitHub
+      # organization.)
+      #
+      # Example:
+      #
+      #   remotes:
+      #     - name: zephyrproject-rtos
+      #       url: https://github.com/zephyrproject-rtos
+      #     - name: developer-fork
+      #       url: https://github.com/a-developer
+      remotes:
+        required: true
+        type: seq
+        sequence:
+          - type: map
+            mapping:
+              name:
+                required: true
+                type: str
+              url:
+                required: true
+                type: str
+
+      # The "projects" key specifies a sequence of "projects",
+      # i.e. Git repositories. These work like repo projects, in that
+      # each project has a name, a remote, and optional additional
+      # metadata.
+      #
+      # Each project is a map with the following keys:
+      #
+      # - name: Mandatory, the name of the git repository. The clone
+      #   URL is formed by remote + '/' + name
+      # - remote: Optional, the name of the remote to pull it from.
+      #   If the remote is missing, the remote'key in the top-level
+      #   defaults key is used instead. If both are missing, it's an error.
+      # - revision: Optional, the name of the revision to check out.
+      #   If not given, the value from the default element will be used.
+      #   If both are missing, then the default is 'master'.
+      # - path: Where to clone the repository locally. If missing,
+      #   it's cloned at top level in a directory given by its name.
+      # - clone-depth: if given, it is a number which creates a shallow
+      #   history in the cloned repository limited to the given number
+      #   of commits.
+      #
+      # Example, using default and non-default remotes:
+      #
+      #   projects:
+      #     # Uses default remote (zephyrproject-rtos), so clone URL is:
+      #     #   https://github.com/zephyrproject-rtos/zephyr
+      #     - name: zephyr
+      #     # Manually specified remote; clone URL is:
+      #     #   https://github.com/a-developer/west
+      #     - name: west
+      #       remote: developer-fork
+      #     # Manually specified remote, clone URL is:
+      #     #   https://github.com/zephyrproject-rtos/some-vendor-hal
+      #     # Local clone path (relative to installation root) is:
+      #     #   ext/hal/some-vendor
+      #     - name: some-vendor-hal
+      #       remote: zephyrproject-rtos
+      #       path: ext/hal/some-vendor
+      projects:
+        required: true
+        type: seq
+        sequence:
+          - type: map
+            mapping:
+              name:
+                required: true
+                type: str
+              remote:
+                required: false
+                type: str
+              revision:
+                required: false
+                type: text        # SHAs could be only numbers
+              path:
+                required: false
+                type: str
+              clone-depth:
+                required: false
+                type: int

scripts/meta/west/log.py

@@ -6,6 +6,7 @@
 
 Provides common methods for logging messages to display to the user.'''
 
+import colorama
 import sys
 
 VERBOSE_NONE = 0
@@ -40,27 +41,45 @@ def dbg(*args, level=VERBOSE_NORMAL):
     print(*args)
 
 
-def inf(*args):
+def inf(*args, colorize=False):
-    '''Print an informational message.'''
+    '''Print an informational message.
+
+    colorize (default: False):
+      If True, the message is printed in bright green if stdout is a terminal.
+    '''
+    # This approach colorizes any sep= and end= text too, as expected.
+    #
+    # colorama automatically strips the ANSI escapes when stdout isn't a
+    # terminal (by wrapping sys.stdout).
+    if colorize:
+        print(colorama.Fore.LIGHTGREEN_EX, end='')
+
     print(*args)
 
+    if colorize:
+        # The final flush=True avoids issues with unrelated output from
+        # commands (usually Git) becoming green, due to the final attribute
+        # reset ANSI escape getting line-buffered.
+        print(colorama.Style.RESET_ALL, end='', flush=True)
+
 
 def wrn(*args):
     '''Print a warning.'''
-    print('warning:', end=' ', file=sys.stderr, flush=False)
+    print(colorama.Fore.LIGHTRED_EX + 'WARNING: ', end='', file=sys.stderr)
     print(*args, file=sys.stderr)
+    print(colorama.Style.RESET_ALL, end='', file=sys.stderr, flush=True)
 
 
 def err(*args, fatal=False):
     '''Print an error.'''
-    if fatal:
+    print(colorama.Fore.LIGHTRED_EX
-        print('fatal', end=' ', file=sys.stderr, flush=False)
+              + ('FATAL ERROR: ' if fatal else 'ERROR: '),
-    print('error:', end=' ', file=sys.stderr, flush=False)
+          end='', file=sys.stderr)
     print(*args, file=sys.stderr)
+    print(colorama.Style.RESET_ALL, end='', file=sys.stderr, flush=True)
 
 
 def die(*args, exit_code=1):
     '''Print a fatal error, and abort with the given exit code.'''
-    print('fatal error:', end=' ', file=sys.stderr, flush=False)
+    err(*args, fatal=True)
-    print(*args, file=sys.stderr)
     sys.exit(exit_code)

scripts/meta/west/main.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # Copyright 2018 Open Source Foundries Limited.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -7,20 +9,52 @@
 
 
 import argparse
+import colorama
 from functools import partial
 import os
 import sys
-from subprocess import CalledProcessError
+from subprocess import CalledProcessError, check_output, DEVNULL
 
-from . import log
+import log
-from .cmd import CommandContextError
+from commands import CommandContextError
-from .cmd.flash import Flash
+from commands.build import Build
-from .cmd.debug import Debug, DebugServer
+from commands.flash import Flash
-from .util import quote_sh_list
+from commands.debug import Debug, DebugServer, Attach
+from commands.project import ListProjects, Fetch, Pull, Rebase, Branch, \
+                             Checkout, Diff, Status, Update, ForAll, \
+                             WestUpdated
+from util import quote_sh_list, in_multirepo_install
 
+IN_MULTIREPO_INSTALL = in_multirepo_install(__file__)
 
-COMMANDS = (Flash(), Debug(), DebugServer())
+BUILD_FLASH_COMMANDS = [
-'''Supported top-level commands.'''
+    Build(),
+    Flash(),
+    Debug(),
+    DebugServer(),
+    Attach(),
+]
+
+PROJECT_COMMANDS = [
+    ListProjects(),
+    Fetch(),
+    Pull(),
+    Rebase(),
+    Branch(),
+    Checkout(),
+    Diff(),
+    Status(),
+    Update(),
+    ForAll(),
+]
+
+# Built-in commands in this West. For compatibility with monorepo
+# installations of West within the Zephyr tree, we only expose the
+# project commands if this is a multirepo installation.
+COMMANDS = BUILD_FLASH_COMMANDS
+
+if IN_MULTIREPO_INSTALL:
+    COMMANDS += PROJECT_COMMANDS
 
 
 class InvalidWestContext(RuntimeError):
@@ -43,7 +77,38 @@ def validate_context(args, unknown):
             args.zephyr_base = os.environ['ZEPHYR_BASE']
 
 
+def print_version_info():
+    # The bootstrapper will print its own version, as well as that of
+    # the west repository itself, then exit. So if this file is being
+    # asked to print the version, it's because it's being run
+    # directly, and not via the bootstrapper.
+    #
+    # Rather than play tricks like invoking "pip show west" (which
+    # assumes the bootstrapper was installed via pip, the common but
+    # not universal case), refuse the temptation to make guesses and
+    # print an honest answer.
+    log.inf('West bootstrapper version: N/A, not run via bootstrapper')
+
+    # The running west installation.
+    if IN_MULTIREPO_INSTALL:
+        try:
+            desc = check_output(['git', 'describe', '--tags'],
+                                stderr=DEVNULL,
+                                cwd=os.path.dirname(__file__))
+            west_version = desc.decode(sys.getdefaultencoding()).strip()
+        except CalledProcessError as e:
+            west_version = 'unknown'
+    else:
+        west_version = 'N/A, monorepo installation'
+    west_src_west = os.path.dirname(__file__)
+    print('West repository version: {} ({})'.
+          format(west_version,
+                 os.path.dirname(os.path.dirname(west_src_west))))
+
+
 def parse_args(argv):
+    # The prog='west' override avoids the absolute path of the main.py script
+    # showing up when West is run via the wrapper
     west_parser = argparse.ArgumentParser(
         prog='west', description='The Zephyr RTOS meta-tool.',
         epilog='Run "west <command> -h" for help on each command.')
@@ -54,6 +119,7 @@ def parse_args(argv):
     west_parser.add_argument('-v', '--verbose', default=0, action='count',
                              help='''Display verbose output. May be given
                              multiple times to increase verbosity.''')
+    west_parser.add_argument('-V', '--version', action='store_true')
     subparser_gen = west_parser.add_subparsers(title='commands',
                                                dest='command')
 
@@ -63,6 +129,10 @@ def parse_args(argv):
 
     args, unknown = west_parser.parse_known_args(args=argv)
 
+    if args.version:
+        print_version_info()
+        sys.exit(0)
+
     # Set up logging verbosity before doing anything else, so
     # e.g. verbose messages related to argument handling errors
     # work properly.
@@ -84,6 +154,10 @@ def parse_args(argv):
 
 
 def main(argv=None):
+    # Makes ANSI color escapes work on Windows, and strips them when
+    # stdout/stderr isn't a terminal
+    colorama.init()
+
     if argv is None:
         argv = sys.argv[1:]
     args, unknown = parse_args(argv)
@@ -92,6 +166,10 @@ def main(argv=None):
         args.command)
     try:
         args.handler(args, unknown)
+    except WestUpdated:
+        # West has been automatically updated. Restart ourselves to run the
+        # latest version, with the same arguments that we were given.
+        os.execv(sys.executable, [sys.executable] + sys.argv)
     except KeyboardInterrupt:
         sys.exit(0)
     except CalledProcessError as cpe:
@@ -110,3 +188,6 @@ def main(argv=None):
             raise
         else:
             log.inf(for_stack_trace)
+
+if __name__ == "__main__":
+    main()

scripts/meta/west/runners/__init__.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-from .core import ZephyrBinaryRunner
+from runners.core import ZephyrBinaryRunner
 
 # We import these here to ensure the ZephyrBinaryRunner subclasses are
 # defined; otherwise, ZephyrBinaryRunner.create_for_shell_script()
@@ -10,19 +10,19 @@ from .core import ZephyrBinaryRunner
 
 # Explicitly silence the unused import warning.
 # flake8: noqa: F401
-from . import arc
+from runners import arc
-from . import bossac
+from runners import bossac
-from . import dfu
+from runners import dfu
-from . import esp32
+from runners import esp32
-from . import jlink
+from runners import jlink
-from . import nios2
+from runners import nios2
-from . import nrfjprog
+from runners import nrfjprog
-from . import nsim
+from runners import nsim
-from . import openocd
+from runners import openocd
-from . import pyocd
+from runners import pyocd
-from . import qemu
+from runners import qemu
-from . import xtensa
+from runners import xtensa
-from . import intel_s1000
+from runners import intel_s1000
 
 def get_runner_cls(runner):
     '''Get a runner's class object, given its name.'''

scripts/meta/west/runners/arc.py

@@ -7,7 +7,7 @@
 
 from os import path
 
-from .core import ZephyrBinaryRunner
+from runners.core import ZephyrBinaryRunner
 
 DEFAULT_ARC_TCL_PORT = 6333
 DEFAULT_ARC_TELNET_PORT = 4444

scripts/meta/west/runners/bossac.py

@@ -6,7 +6,7 @@
 
 import platform
 
-from .core import ZephyrBinaryRunner, RunnerCaps
+from runners.core import ZephyrBinaryRunner, RunnerCaps
 
 DEFAULT_BOSSAC_PORT = '/dev/ttyACM0'
 

scripts/meta/west/runners/core.py

@@ -18,8 +18,8 @@ import platform
 import signal
 import subprocess
 
-from .. import log
+import log
-from ..util import quote_sh_list
+from util import quote_sh_list
 
 # Turn on to enable just printing the commands that would be run,
 # without actually running them. This can break runners that are expecting
@@ -163,7 +163,7 @@ class RunnerCaps:
     Available capabilities:
 
     - commands: set of supported commands; default is {'flash',
-      'debug', 'debugserver'}.
+      'debug', 'debugserver', 'attach'}.
 
     - flash_addr: whether the runner supports flashing to an
       arbitrary address. Default is False. If true, the runner
@@ -171,7 +171,7 @@ class RunnerCaps:
     '''
 
     def __init__(self,
-                 commands={'flash', 'debug', 'debugserver'},
+                 commands={'flash', 'debug', 'debugserver', 'attach'},
                  flash_addr=False):
         self.commands = commands
         self.flash_addr = bool(flash_addr)
@@ -256,15 +256,21 @@ class ZephyrBinaryRunner(abc.ABC):
     - 'flash': flash a previously configured binary to the board,
       start execution on the target, then return.
 
-    - 'debug': connect to the board via a debugging protocol, then
+    - 'debug': connect to the board via a debugging protocol, program
-      drop the user into a debugger interface with symbol tables
+      the flash, then drop the user into a debugger interface with
-      loaded from the current binary, and block until it exits.
+      symbol tables loaded from the current binary, and block until it
+      exits.
 
     - 'debugserver': connect via a board-specific debugging protocol,
       then reset and halt the target. Ensure the user is now able to
       connect to a debug server with symbol tables loaded from the
       binary.
 
+    - 'attach': connect to the board via a debugging protocol, then drop
+      the user into a debugger interface with symbol tables loaded from
+      the current binary, and block until it exits. Unlike 'debug', this
+      command does not program the flash.
+
     This class provides an API for these commands. Every runner has a
     name (like 'pyocd'), and declares commands it can handle (like
     'flash'). Zephyr boards (like 'nrf52_pca10040') declare compatible
@@ -391,7 +397,7 @@ class ZephyrBinaryRunner(abc.ABC):
             return default
 
     def run(self, command, **kwargs):
-        '''Runs command ('flash', 'debug', 'debugserver').
+        '''Runs command ('flash', 'debug', 'debugserver', 'attach').
 
         This is the main entry point to this runner.'''
         caps = self.capabilities()

scripts/meta/west/runners/dfu.py

@@ -9,8 +9,8 @@ import os
 import sys
 import time
 
-from .. import log
+import log
-from .core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration
+from runners.core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration
 
 
 DfuSeConfig = namedtuple('DfuSeConfig', ['address', 'options'])

scripts/meta/west/runners/esp32.py

@@ -6,8 +6,8 @@
 
 from os import path
 
-from .. import log
+import log
-from .core import ZephyrBinaryRunner, RunnerCaps
+from runners.core import ZephyrBinaryRunner, RunnerCaps
 
 
 class Esp32BinaryRunner(ZephyrBinaryRunner):

scripts/meta/west/runners/intel_s1000.py

@@ -8,8 +8,8 @@ from os import path
 import time
 import subprocess
 
-from .. import log
+import log
-from .core import ZephyrBinaryRunner
+from runners.core import ZephyrBinaryRunner
 
 DEFAULT_XT_GDB_PORT = 20000
 

scripts/meta/west/runners/jlink.py

@@ -7,8 +7,8 @@
 import os
 import tempfile
 
-from .. import log
+import log
-from .core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration
+from runners.core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration
 
 DEFAULT_JLINK_GDB_PORT = 2331
 
@@ -42,7 +42,8 @@ class JLinkBinaryRunner(ZephyrBinaryRunner):
 
     @classmethod
     def capabilities(cls):
-        return RunnerCaps(flash_addr=True)
+        return RunnerCaps(commands={'flash', 'debug', 'debugserver', 'attach'},
+                          flash_addr=True)
 
     @classmethod
     def do_add_parser(cls, parser):
@@ -104,10 +105,11 @@ class JLinkBinaryRunner(ZephyrBinaryRunner):
             client_cmd = (self.gdb_cmd +
                           self.tui_arg +
                           [self.elf_name] +
-                          ['-ex', 'target remote :{}'.format(self.gdb_port),
+                          ['-ex', 'target remote :{}'.format(self.gdb_port)])
-                           '-ex', 'monitor halt',
+            if command == 'debug':
-                           '-ex', 'monitor reset',
+                client_cmd += ['-ex', 'monitor halt',
-                           '-ex', 'load'])
+                               '-ex', 'monitor reset',
+                               '-ex', 'load']
             self.print_gdbserver_message()
             self.run_server_and_client(server_cmd, client_cmd)
 

scripts/meta/west/runners/nios2.py

@@ -4,8 +4,8 @@
 
 '''Runner for NIOS II, based on quartus-flash.py and GDB.'''
 
-from .. import log
+import log
-from .core import ZephyrBinaryRunner, NetworkPortHelper
+from runners.core import ZephyrBinaryRunner, NetworkPortHelper
 
 
 class Nios2BinaryRunner(ZephyrBinaryRunner):

scripts/meta/west/runners/nrfjprog.py

@@ -6,8 +6,8 @@
 
 import sys
 
-from .. import log
+import log
-from .core import ZephyrBinaryRunner, RunnerCaps
+from runners.core import ZephyrBinaryRunner, RunnerCaps
 
 
 class NrfJprogBinaryRunner(ZephyrBinaryRunner):

scripts/meta/west/runners/nsim.py

@@ -7,7 +7,7 @@
 
 from os import path
 
-from .core import ZephyrBinaryRunner
+from runners.core import ZephyrBinaryRunner
 
 DEFAULT_ARC_GDB_PORT = 3333
 DEFAULT_PROPS_FILE = 'nsim.props'

scripts/meta/west/runners/openocd.py

@@ -6,7 +6,7 @@
 
 from os import path
 
-from .core import ZephyrBinaryRunner
+from runners.core import ZephyrBinaryRunner
 
 DEFAULT_OPENOCD_TCL_PORT = 6333
 DEFAULT_OPENOCD_TELNET_PORT = 4444

scripts/meta/west/runners/pyocd.py

@@ -6,8 +6,8 @@
 
 import os
 import sys
-from .core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration
+from runners.core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration
-from .. import log
+import log
 
 DEFAULT_PYOCD_GDB_PORT = 3333
 
@@ -51,7 +51,8 @@ class PyOcdBinaryRunner(ZephyrBinaryRunner):
 
     @classmethod
     def capabilities(cls):
-        return RunnerCaps(flash_addr=True)
+        return RunnerCaps(commands={'flash', 'debug', 'debugserver', 'attach'},
+                          flash_addr=True)
 
     @classmethod
     def do_add_parser(cls, parser):
@@ -140,8 +141,10 @@ class PyOcdBinaryRunner(ZephyrBinaryRunner):
             client_cmd = (self.gdb_cmd +
                           self.tui_args +
                           [self.elf_name] +
-                          ['-ex', 'target remote :{}'.format(self.gdb_port),
+                          ['-ex', 'target remote :{}'.format(self.gdb_port)])
-                           '-ex', 'load',
+            if command == 'debug':
-                           '-ex', 'monitor reset halt'])
+                client_cmd += ['-ex', 'load',
+                               '-ex', 'monitor reset halt']
+
             self.print_gdbserver_message()
             self.run_server_and_client(server_cmd, client_cmd)

scripts/meta/west/runners/qemu.py

@@ -4,7 +4,7 @@
 
 '''Runner stub for QEMU.'''
 
-from .core import ZephyrBinaryRunner, RunnerCaps
+from runners.core import ZephyrBinaryRunner, RunnerCaps
 
 
 class QemuBinaryRunner(ZephyrBinaryRunner):

scripts/meta/west/runners/xtensa.py

@@ -6,7 +6,7 @@
 
 from os import path
 
-from .core import ZephyrBinaryRunner, RunnerCaps
+from runners.core import ZephyrBinaryRunner, RunnerCaps
 
 
 class XtensaBinaryRunner(ZephyrBinaryRunner):

scripts/meta/west/util.py

@@ -5,6 +5,7 @@
 '''Miscellaneous utilities used by west
 '''
 
+import os
 import shlex
 import textwrap
 
@@ -20,3 +21,60 @@ def wrap(text, indent):
     '''Convenience routine for wrapping text to a consistent indent.'''
     return textwrap.wrap(text, initial_indent=indent,
                          subsequent_indent=indent)
+
+
+class WestNotFound(RuntimeError):
+    '''Neither the current directory nor any parent has a West installation.'''
+
+
+def west_dir(start=None):
+    '''Returns the absolute path of the west/ top level directory.
+
+    Starts the search from the start directory, and goes to its
+    parents. If the start directory is not specified, the current
+    directory is used.
+
+    Raises WestNotFound if no west top-level directory is found.
+    '''
+    return os.path.join(west_topdir(start), 'west')
+
+
+def west_topdir(start=None):
+    '''
+    Like west_dir(), but returns the path to the parent directory of the west/
+    directory instead, where project repositories are stored
+    '''
+    # If you change this function, make sure to update the bootstrap
+    # script's find_west_topdir().
+
+    if start is None:
+        cur_dir = os.getcwd()
+    else:
+        cur_dir = start
+
+    while True:
+        if os.path.isfile(os.path.join(cur_dir, 'west', '.west_topdir')):
+            return cur_dir
+
+        parent_dir = os.path.dirname(cur_dir)
+        if cur_dir == parent_dir:
+            # At the root
+            raise WestNotFound('Could not find a West installation '
+                               'in this or any parent directory')
+        cur_dir = parent_dir
+
+
+def in_multirepo_install(start=None):
+    '''Returns True iff the path ``start`` is in a multi-repo installation.
+
+    If start is not given, it defaults to the current working directory.
+
+    This is equivalent to checking if west_dir() raises an exception
+    when given the same start kwarg.
+    '''
+    try:
+        west_topdir(start)
+        result = True
+    except WestNotFound:
+        result = False
+    return result

scripts/west

@@ -1,5 +1,6 @@
 #!/bin/sh
 
-# UNIX operating system entry point to the west tool.
+# Zephyr meta-tool (west) launcher alias, which keeps
-export "PYTHONPATH=${PYTHONPATH:+${PYTHONPATH}:}$ZEPHYR_BASE/scripts/meta"
+# monorepo Zephyr installations' 'make flash' etc. working.
-python3 -m west $@
+here=$(readlink -f $(dirname $0))
+python3 "$here/west-launcher.py" $@

scripts/west-launcher.py

@@ -0,0 +1,103 @@
+# Zephyr launcher which is interoperable with:
+#
+# 1. "mono-repo" Zephyr installations that have 'make flash'
+#    etc. supplied by a copy of some west code in scripts/meta.
+#
+# 2. "multi-repo" Zephyr installations where west is provided in a
+#    separate Git repository elsewhere.
+#
+# This is basically a copy of the "wrapper" functionality in the west
+# bootstrap script for the multi-repo case, plus a fallback onto the
+# copy in scripts/meta/west for mono-repo installs.
+
+import os
+import subprocess
+import sys
+
+if sys.version_info < (3,):
+    sys.exit('fatal error: you are running Python 2')
+
+# Top-level west directory, containing west itself and the manifest.
+WEST_DIR = 'west'
+# Subdirectory to check out the west source repository into.
+WEST = 'west'
+# File inside of WEST_DIR which marks it as the top level of the
+# Zephyr project installation.
+#
+# (The WEST_DIR name is not distinct enough to use when searching for
+# the top level; other directories named "west" may exist elsewhere,
+# e.g. zephyr/doc/west.)
+WEST_MARKER = '.west_topdir'
+
+
+class WestError(RuntimeError):
+    pass
+
+
+class WestNotFound(WestError):
+    '''Neither the current directory nor any parent has a West installation.'''
+
+
+def find_west_topdir(start):
+    '''Find the top-level installation directory, starting at ``start``.
+
+    If none is found, raises WestNotFound.'''
+    cur_dir = start
+
+    while True:
+        if os.path.isfile(os.path.join(cur_dir, WEST_DIR, WEST_MARKER)):
+            return cur_dir
+
+        parent_dir = os.path.dirname(cur_dir)
+        if cur_dir == parent_dir:
+            # At the root
+            raise WestNotFound('Could not find a West installation '
+                               'in this or any parent directory')
+        cur_dir = parent_dir
+
+
+def append_to_pythonpath(directory):
+    pp = os.environ.get('PYTHONPATH')
+    os.environ['PYTHONPATH'] = ':'.join(([pp] if pp else []) + [directory])
+
+
+def wrap(topdir, argv):
+    # Replace the wrapper process with the "real" west
+
+    # sys.argv[1:] strips the argv[0] of the wrapper script itself
+    west_git_repo = os.path.join(topdir, WEST_DIR, WEST)
+    argv = ([sys.executable,
+             os.path.join(west_git_repo, 'src', 'west', 'main.py')] +
+            argv)
+
+    try:
+        append_to_pythonpath(os.path.join(west_git_repo, 'src'))
+        subprocess.check_call(argv)
+    except subprocess.CalledProcessError as e:
+        sys.exit(e.returncode)
+
+
+def run_scripts_meta_west():
+    try:
+        subprocess.check_call([sys.executable,
+                               os.path.join(os.environ['ZEPHYR_BASE'],
+                                            'scripts', 'meta', 'west',
+                                            'main.py')] + sys.argv[1:])
+    except subprocess.CalledProcessError as e:
+        sys.exit(e.returncode)
+
+
+def main():
+    try:
+        topdir = find_west_topdir(__file__)
+    except WestNotFound:
+        topdir = None
+
+    if topdir is not None:
+        wrap(topdir, sys.argv[1:])
+    else:
+        run_scripts_meta_west()
+
+
+if __name__ == '__main__':
+    main()

scripts/west-win.py

@@ -1,11 +0,0 @@
-# Windows-specific launcher alias for west (west wind?).
-
-import os
-import sys
-
-zephyr_base = os.environ['ZEPHYR_BASE']
-sys.path.append(os.path.join(zephyr_base, 'scripts', 'meta'))
-
-from west.main import main  # noqa E402 (silence flake8 warning)
-
-main(sys.argv[1:])

zephyr-env.cmd

@@ -5,5 +5,13 @@ if exist "%userprofile%\zephyrrc.cmd" (
 	call "%userprofile%\zephyrrc.cmd"
 )
 
-rem Zephyr meta-tool (west) launcher alias
+rem Zephyr meta-tool (west) launcher alias, which keeps monorepo
-doskey west=py -3 %ZEPHYR_BASE%\scripts\west-win.py $*
+rem Zephyr installations' 'make flash' etc. working.  See
+rem https://www.python.org/dev/peps/pep-0486/ for details on the
+rem virtualenv-related pieces. (We need to implement this manually
+rem because Zephyr's minimum supported Python version is 3.4.)
+if defined VIRTUAL_ENV (
+	doskey west=python %ZEPHYR_BASE%\scripts\west-launcher.py $*
+) else (
+	doskey west=py -3 %ZEPHYR_BASE%\scripts\west-launcher.py $*
+)