From 2b3d9df1d72ab0cc02614d2240d311080a74d6b8 Mon Sep 17 00:00:00 2001 From: Marti Bolivar Date: Mon, 22 Jul 2019 11:00:59 -0600 Subject: [PATCH] west: require v0.6.0 or higher The main change is the elimination of the bootstrapper, a design flaw / misfeature. Update the documentation to be compatible with the 0.6.x releases as well. This has to be done atomically, as there were incompatible changes. Make use of the versionchanged and versionadded directives to begin keeping track of how these APIs are evolving. (Note that west 0.6.0 will remain compatible with the extension commands in Zephyr v1.14 LTS as long as that is still alive. This change is targeted towards Zephyr 2.0 users.) This requires a bump in the shippable container and allows us to simplify the west_commands test procedure. Signed-off-by: Marti Bolivar --- .shippable.yml | 2 +- cmake/host-tools.cmake | 2 +- doc/CMakeLists.txt | 5 + doc/conf.py | 18 +-- doc/getting_started/index.rst | 6 +- doc/guides/west/build-flash-debug.rst | 2 +- doc/guides/west/index.rst | 2 +- doc/guides/west/manifest.rst | 5 + doc/guides/west/release-notes.rst | 95 +++++++++++++ doc/guides/west/repo-tool.rst | 83 ++---------- doc/guides/west/west-apis.rst | 183 ++++++++++++++++++++++---- scripts/ci/run_ci.sh | 3 +- scripts/west_commands/README.txt | 4 +- west.yml | 4 - 14 files changed, 290 insertions(+), 124 deletions(-) diff --git a/.shippable.yml b/.shippable.yml index 3481c20469c..e9714d73728 100644 --- a/.shippable.yml +++ b/.shippable.yml @@ -21,7 +21,7 @@ build: - ${SHIPPABLE_BUILD_DIR}/ccache pre_ci_boot: image_name: zephyrprojectrtos/ci - image_tag: v0.7 + image_tag: v0.8.0 pull: true options: "-e HOME=/home/buildslave --privileged=true --tty --net=bridge --user buildslave" diff --git a/cmake/host-tools.cmake b/cmake/host-tools.cmake index e2b205aab74..3c52b4c6bdb 100644 --- a/cmake/host-tools.cmake +++ b/cmake/host-tools.cmake @@ -12,7 +12,7 @@ if(${WEST} STREQUAL WEST-NOTFOUND) else() # If west is found, make sure its version matches the minimum # required one. - set(MIN_WEST_VERSION 0.5.6) + set(MIN_WEST_VERSION 0.6.0) execute_process( COMMAND ${WEST} --version diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 8defe83c071..b87e152cc4e 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -15,6 +15,11 @@ endif() file(TO_CMAKE_PATH "$ENV{ZEPHYR_BASE}" ZEPHYR_BASE) message(STATUS "Zephyr base: ${ZEPHYR_BASE}") +if(DEFINED WEST) + message(STATUS "West: ${WEST}") +else() + message(STATUS "West: not found") +endif() find_package(PythonInterp 3.4) set(DOXYGEN_SKIP_DOT True) diff --git a/doc/conf.py b/doc/conf.py index 546e7d35255..c45de0883bb 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -32,22 +32,10 @@ sys.path.insert(0, os.path.join(ZEPHYR_BASE, 'doc', 'extensions')) # for autodoc directives on runners.xyz. sys.path.insert(0, os.path.join(ZEPHYR_BASE, 'scripts', 'west_commands')) -west_found = False - try: - desc = check_output(['west', 'list', '-f{abspath}', 'west'], - stderr=DEVNULL, - cwd=os.path.dirname(__file__)) - west_path = desc.decode(sys.getdefaultencoding()).strip() - # Add west, to be able to pull in its API docs. - sys.path.append(os.path.join(west_path, 'src')) - west_found = True -except FileNotFoundError as e: - # west not installed - pass -except CalledProcessError as e: - # west not able to list itself - pass + import west as west_found +except ImportError: + west_found = False # -- General configuration ------------------------------------------------ diff --git a/doc/getting_started/index.rst b/doc/getting_started/index.rst index 1f0ed2d98c3..e40b1b8cf57 100644 --- a/doc/getting_started/index.rst +++ b/doc/getting_started/index.rst @@ -60,10 +60,10 @@ Zephyr's multi-purpose :ref:`west ` tool lets you easily get the Zephyr project repositories. (While it's easiest to develop with Zephyr using west, we also have :ref:`documentation for doing without it `.) -Bootstrap west -============== +Install west +============ -First, install the ``west`` binary and bootstrapper using ``pip3``: +First, install ``west`` using ``pip3``: .. code-block:: console diff --git a/doc/guides/west/build-flash-debug.rst b/doc/guides/west/build-flash-debug.rst index c3cd2d2390e..3599f1f9d6a 100644 --- a/doc/guides/west/build-flash-debug.rst +++ b/doc/guides/west/build-flash-debug.rst @@ -466,7 +466,7 @@ individual runner implementations are in other submodules, such as Hacking ******* -This section contains documentation the ``runners.core`` module used by the +This section documents the ``runners.core`` module used by the flash and debug commands. This is the core abstraction used to implement support for these features. diff --git a/doc/guides/west/index.rst b/doc/guides/west/index.rst index 48a38510315..4f27fd14d4c 100644 --- a/doc/guides/west/index.rst +++ b/doc/guides/west/index.rst @@ -27,7 +27,7 @@ You can run ``west --help`` (or ``west -h`` for short) to get top-level help for available west commands, and ``west -h`` for detailed help on each command. -The following pages document west's ``v0.5.x`` releases, and provide additional +The following pages document west's ``v0.6.x`` releases, and provide additional context about the tool. .. toctree:: diff --git a/doc/guides/west/manifest.rst b/doc/guides/west/manifest.rst index 5a4b14ea7c8..59d15e43be3 100644 --- a/doc/guides/west/manifest.rst +++ b/doc/guides/west/manifest.rst @@ -206,6 +206,11 @@ described next. - ``remote``: The name of the project's remote. If not given, the ``remote`` value in the ``defaults`` subsection is tried next. If both are missing, the manifest is invalid. +- ``repo-path``: Optional. If given, this is concatenated on to the remote's + ``url-base`` instead of the project's ``name`` to form its fetch URL. +- ``url``: Optional. If given, this is the project's complete fetch URL. + It cannot be combined with either ``remote`` or ``repo-path``. Note that this + overrides any default remote. - ``revision``: Optional. The current project revision used by ``west update``. If not given, the value from the ``defaults`` subsection will be used if present. If both are missing, ``master`` is used. A project revision can be diff --git a/doc/guides/west/release-notes.rst b/doc/guides/west/release-notes.rst index c72dbf43f60..cb4ebc7ec1b 100644 --- a/doc/guides/west/release-notes.rst +++ b/doc/guides/west/release-notes.rst @@ -1,6 +1,97 @@ West Release Notes ################## +v0.6.x +****** + +- No separate bootstrapper + + In west v0.5.x, the program was split into two components, a bootstrapper and + a per-installation clone. See `Multiple Repository Management in the v1.14 + documentation`_ for more details. + + This is similar to how Google's Repo tool works, and lets west iterate quickly + at first. It caused confusion, however, and west is now stable enough to be + distributed entirely as one piece via PyPI. + + From v0.6.x onwards, all of the core west commands and helper classes are + part of the west package distributed via PyPI. This eliminates complexity + and makes it possible to import west modules from anywhere in the system, + not just extension commands. +- The ``selfupdate`` command still exists for backwards compatibility, but + now simply exits after printing an error message. +- Manifest syntax changes + + - A west manifest file's ``projects`` elements can now specify their fetch + URLs directly, like so: + + .. code-block:: yaml + + manifest: + projects: + - name: example-project-name + url: https://github.com/example/example-project + + Project elements with ``url`` attributes set in this way may not also have + ``remote`` attributes. + - Project names must be unique: this restriction is needed to support future + work, but was not possible in west v0.5.x because distinct projects may + have URLs with the same final pathname component, like so: + + .. code-block:: yaml + + manifest: + remotes: + - name: remote-1 + url-base: https://github.com/remote-1 + - name: remote-2 + url-base: https://github.com/remote-2 + projects: + - name: project + remote: remote-1 + path: remote-1-project + - name: project + remote: remote-2 + path: remote-2-project + + These manifests can now be written with projects that use ``url`` + instead of ``remote``, like so: + + .. code-block:: yaml + + manifest: + projects: + - name: remote-1-project + url: https://github.com/remote-1/project + - name: remote-2-project + url: https://github.com/remote-2/project + +- The ``west list`` command now supports a ``{sha}`` format string key + +- The default format string for ``west list`` was changed to ``"{name:12} + {path:28} {revision:40} {url}"``. + +- The command ``west manifest --validate`` can now be run to load and validate + the current manifest file, among other error-handling fixes related to + manifest parsing. + +- Incompatible API changes were made to west's APIs. Further changes are + expected until API stability is declared in west v1.0. + + - The ``west.manifest.Project`` constructor's ``remote`` and ``defaults`` + positional arguments are now kwargs. A new ``url`` kwarg was also added; if + given, the ``Project`` URL is set to that value, and the ``remote`` kwarg + is ignored. + + - ``west.manifest.MANIFEST_SECTIONS`` was removed. There is only one section + now, namely ``manifest``. The *sections* kwargs in the + ``west.manifest.Manifest`` factory methods and constructor were also + removed. + + - The ``west.manifest.SpecialProject`` class was removed. Use + ``west.manifest.ManifestProject`` instead. + + v0.5.x ****** @@ -21,5 +112,9 @@ Versions Before v0.5.x Tags in the west repository before v0.5.x are prototypes which are of historical interest only. +.. _Multiple Repository Management in the v1.14 documentation: + https://docs.zephyrproject.org/1.14.0/guides/west/repo-tool.html + .. _west v0.5.x documentation: https://docs.zephyrproject.org/1.14.0/guides/west/index.html + diff --git a/doc/guides/west/repo-tool.rst b/doc/guides/west/repo-tool.rst index bd5d0be7fd5..e5350133dde 100644 --- a/doc/guides/west/repo-tool.rst +++ b/doc/guides/west/repo-tool.rst @@ -26,16 +26,14 @@ Git repositories installed under a common parent directory, which we call a `_. A west installation is the result of running the ``west init`` command, which -is described in more detail below. This command can either create a new -installation, or convert a standalone "mono-repo" zephyr repository into a full -west installation. For upstream Zephyr, the installation looks like this: +is described in more detail below. For upstream Zephyr, the installation looks +like this: .. code-block:: none zephyrproject ├── .west - │ ├── config - │ └── west + │ └── config ├── zephyr │ ├── west.yml │ └── [... other files ...] @@ -48,9 +46,7 @@ west installation. For upstream Zephyr, the installation looks like this: Above, :file:`zephyrproject` is the name of the west installation's root directory. This name is just an example -- it could be anything, like ``z``, ``my-zephyr-installation``, etc. The file :file:`.west/config` is the -installation's :ref:`local configuration file `. The directory -:file:`.west/west` is a clone of the west repository itself; more details on -why that is currently needed are given in the next section. +installation's :ref:`local configuration file `. Every west installation contains exactly one *manifest repository*, which is a Git repository containing a file named :file:`west.yml`, which is the *west @@ -152,62 +148,12 @@ The following three source code topologies supported by west: West Structure ************** -West is currently split in two: +West's code is distributed via PyPI in a `namespace package`_ named ``west``. +See :ref:`west-apis` for API documentation. -* Bootstrapper: Installed by ``pip3 install west``, which provides the ``west`` - binary and the ``west init`` command. -* Per-installation clone: this is the west repository cloned into each - installation, which provides the built-in commands. - -.. note:: - - This "bootstrapper" / "everything else" separation is similar to the model - used by Google's ``repo`` tool, but unfortunately in retrospect was not a - good strategy for west. - - In future versions, the ``west`` binary and all built-in commands (including - ``init``) will be installed by ``pip3 install west``. Besides eliminating - complexity, this will also make it possible to use :ref:`West's APIs - ` from any Python file, not just extension - commands. - - Updating west will still be possible manually, e.g. with ``pip3 - install --upgrade west``. If necessary, it will also still be possible to - use different versions of west on the same computer through Python virtual - environments. - -Bootstrapper -============ - -The bootstrapper module is distributed using `PyPI`_ and installed using -:file:`pip3`. A launcher named ``west`` is placed by :file:`pip3` in the user's -``PATH``. This the only entry point to west. It implements a single command: -``west init``. This command needs to be run first to use the rest of -functionality included in ``west``, by creating a west installation. The -command ``west init`` does the following: - -* Clones west itself in a :file:`.west/west` folder in the installation. -* Clones the manifest repository in the folder specified by the manifest file's - ``self.path`` section. -* Creates an initial local configuration file. - -Once ``west init`` has been run, the bootstrapper will delegate the handling of -any west commands other than ``init`` to the cloned west repository. This means -that there is a single bootstrapper instance installed at any time (unless you -use virtual environments), which can then be used to initialize as many -installations as needed, each of which can have a different version of west. - -.. _west-struct-installation: - -Per-Installation Clone -====================== - -A west installation, as described above, contains a clone of the west -repository in :file:`.west/west`. This is where the built-in command -implementations are currently provided. The rest of :ref:`West's APIs -` are also currently provided to extension commands by this -repository. So that west can update itself, the built-in ``west update`` and -``west selfupdate`` commands fetch and update the :file:`.west/west` repository. +This distribution also includes a launcher executable, also named ``west``. When +west is installed, the launcher is placed by :file:`pip3` somewhere in the +user's ``PATH``. This is the command-line entry point. The ``manifest-rev`` branch *************************** @@ -266,7 +212,8 @@ important to understand. 1. If you already have a local clone of the zephyr repository and want to create a west installation around it, you can use the ``-l`` switch to - pass its path to west, as in: ``west init -l path/to/zephyr``. + pass its path to west, as in: ``west init -l path/to/zephyr``. This is + the only reason to use ``-l``. 2. Otherwise, omit ``-l`` to create a new installation from a remote manifest repository. You can give the manifest URL using the ``-m`` switch, and its @@ -291,9 +238,6 @@ important to understand. the ``--rebase`` / ``-r`` and ``--keep-descendants`` / ``-k`` options for ways to influence this. - By default, ``west update`` also updates the west repository in the - installation. To prevent this, use ``--exclude-west``. - .. _west-multi-repo-misc: Miscellaneous Commands @@ -324,10 +268,11 @@ discussed here. forall -c 'git --options'``. Note that ``west forall`` can be used to run any command, though, not just Git commands. -- ``west selfupdate``: Updates the west repository in the installation. - .. _PyPI: https://pypi.org/project/west/ .. _Zephyr issue #6770: https://github.com/zephyrproject-rtos/zephyr/issues/6770 + +.. _namespace package: + https://www.python.org/dev/peps/pep-0420/ diff --git a/doc/guides/west/west-apis.rst b/doc/guides/west/west-apis.rst index 24113ee2b5e..070f9892d38 100644 --- a/doc/guides/west/west-apis.rst +++ b/doc/guides/west/west-apis.rst @@ -1,32 +1,26 @@ :orphan: .. _west-apis: +.. _west-apis-west: -West and Extension APIs -####################### +West APIs +######### This page documents the Python APIs provided by :ref:`west `, as well as some additional APIs used by the :ref:`west extensions ` in the zephyr repository. +.. warning:: + + These APIs should be considered unstable until west version 1.0 (see `west + #38`_). + + **Contents**: .. contents:: :local: -.. _west-apis-west: - -West -**** - -This section contains documentation for west's APIs. - -.. warning:: - - These APIs should be considered unstable until west version 1.0. Further, - until `west #38`_ is closed, these modules can only be imported from - extension command files (and within west itself, of course). - .. NOTE: documentation authors: 1. keep these sorted by package/module name. @@ -35,23 +29,111 @@ This section contains documentation for west's APIs. .. _west-apis-commands: west.commands -============= +************* -.. automodule:: west.commands - :members: WestCommand, CommandError, CommandContextError, ExtensionCommandError +.. module:: west.commands + +All built-in and extension commands are implemented as subclasses of the +:py:class:`WestCommand` class defined here. Some exception types are also +provided. + +.. py:class:: west.commands.WestCommand + + .. automethod:: __init__ + + .. versionadded:: 0.6 + The *requires_installation* parameter. + + Methods: + + .. automethod:: run + + .. versionchanged:: 0.6 + The *topdir* argument was added. + + .. automethod:: add_parser + + All subclasses must provide the following abstract methods, which are used + to implement the above: + + .. automethod:: do_add_parser + + .. automethod:: do_run + + Instance attributes: + + .. py:attribute:: name + + As passed to the constructor. + + .. py:attribute:: help + + As passed to the constructor. + + .. py:attribute:: description + + As passed to the constructor. + + .. py:attribute:: accepts_unknown_args + + As passed to the constructor. + + .. py:attribute:: requires_installation + + As passed to the constructor. + + .. py:attribute:: parser + + The argument parser created by calling ``WestCommand.add_parser()``. + +.. autoclass:: west.commands.CommandError + :show-inheritance: + + .. py:attribute:: returncode + + Recommended program exit code for this error. + +.. autoclass:: west.commands.CommandContextError + :show-inheritance: + +.. autoclass:: west.commands.ExtensionCommandError + :show-inheritance: + + .. py:method:: ExtensionCommandError.__init__(hint=None, **kwargs) + + If *hint* is given, it is a string indicating the cause of the problem. + All other kwargs are passed to the super constructor. + + .. py:attribute:: hint + + As passed to the constructor. .. _west-apis-configuration: west.configuration -================== +****************** .. automodule:: west.configuration - :members: ConfigFile, read_config, update_config + +.. autoclass:: west.configuration.ConfigFile + +.. autofunction:: west.configuration.read_config + +.. versionchanged:: 0.6 + Errors due to an inability to find a local configuration file are ignored. + +.. autofunction:: west.configuration.update_config + +.. py:data:: west.configuration.config + + Module-global ConfigParser instance for the current configuration. This + should be initialized with :py:func:`west.configuration.read_config` before + being read. .. _west-apis-log: west.log -======== +******** .. automodule:: west.log :members: set_verbosity, VERBOSE_NONE, VERBOSE_NORMAL, VERBOSE_VERY, VERBOSE_EXTREME, dbg, inf, wrn, err, die @@ -59,18 +141,69 @@ west.log .. _west-apis-manifest: west.manifest -============= +************* .. automodule:: west.manifest - :members: manifest_path, Manifest, Defaults, Remote, Project, SpecialProject, MalformedManifest, MalformedConfig, MANIFEST_SECTIONS, MANIFEST_PROJECT_INDEX, MANIFEST_REV_BRANCH, QUAL_MANIFEST_REV_BRANCH + +.. autodata:: MANIFEST_PROJECT_INDEX + +.. autodata:: MANIFEST_REV_BRANCH + +.. autodata:: QUAL_MANIFEST_REV_BRANCH + +.. autofunction:: west.manifest.manifest_path + +.. autoclass:: west.manifest.Manifest + + .. automethod:: from_file + + .. automethod:: from_data + + .. automethod:: __init__ + + .. automethod:: get_remote + + .. automethod:: as_frozen_dict + +.. autoclass:: west.manifest.Defaults + :members: + :member-order: groupwise + +.. autoclass:: west.manifest.Remote + :members: + :member-order: groupwise + +.. autoclass:: west.manifest.Project + :members: + :member-order: groupwise + +.. autoclass:: west.manifest.ManifestProject + :members: + :member-order: groupwise + +.. versionadded:: 0.6 + +.. autoclass:: west.manifest.MalformedManifest + :show-inheritance: + +.. autoclass:: west.manifest.MalformedConfig + :show-inheritance: .. _west-apis-util: west.util -========= +********* + +.. canon_path(), escapes_directory(), etc. intentionally not documented here. .. automodule:: west.util - :members: west_dir, west_topdir, WestNotFound + +.. autofunction:: west.util.west_dir + +.. autofunction:: west.util.west_topdir + +.. autoclass:: west.util.WestNotFound + :show-inheritance: .. _west #38: https://github.com/zephyrproject-rtos/west/issues/38 diff --git a/scripts/ci/run_ci.sh b/scripts/ci/run_ci.sh index ac37f9ee8a6..539cf14b209 100755 --- a/scripts/ci/run_ci.sh +++ b/scripts/ci/run_ci.sh @@ -271,8 +271,7 @@ if [ -n "$MAIN_CI" ]; then # run pytest could go here too. PYTEST=$(type -p pytest-3 || echo "pytest") mkdir -p $(dirname ${WEST_COMMANDS_RESULTS_FILE}) - WEST_SRC=$(west list --format='{abspath}' west)/src - PYTHONPATH=./scripts/west_commands:$WEST_SRC "${PYTEST}" \ + PYTHONPATH=./scripts/west_commands "${PYTEST}" \ --junitxml=${WEST_COMMANDS_RESULTS_FILE} \ ./scripts/west_commands/tests else diff --git a/scripts/west_commands/README.txt b/scripts/west_commands/README.txt index a2ff5f036e7..fc73123d55a 100644 --- a/scripts/west_commands/README.txt +++ b/scripts/west_commands/README.txt @@ -11,12 +11,12 @@ itself. When extending this code, please keep the unit tests (in tests/) up to date. You can run the tests with this command from this directory: -$ PYTHONPATH=$(west list --format="{abspath}" west)/src:$PWD py.test +$ PYTHONPATH=$PWD py.test Windows users will need to find the path to .west/west/src in their Zephyr installation, then run something like this: -> cmd /C "set PYTHONPATH=path\to\.west\west\src:path\to\zephyr\scripts\west_commands && py.test" +> cmd /C "set PYTHONPATH=path\to\zephyr\scripts\west_commands && py.test" Note that these tests are run as part of Zephyr's CI when submitting an upstream pull request, and pull requests which break the tests diff --git a/west.yml b/west.yml index 98b1c517b36..05e1eb51a3e 100644 --- a/west.yml +++ b/west.yml @@ -14,10 +14,6 @@ # new Zephyr installation. See the west documentation for more # information. -west: - url: https://github.com/zephyrproject-rtos/west - revision: v0.5.7 - manifest: defaults: remote: upstream