From 64ec6ee3a33235a0c4a70d7c05ed1d040fd7264d Mon Sep 17 00:00:00 2001 From: Torsten Rasmussen Date: Thu, 4 Nov 2021 14:28:21 +0100 Subject: [PATCH] scripts: build meta file workspace status extension The PR #39382 raised a discussion on build reproducibility and knowledge of west projects being out of sync with the west manifest. Similar to how `git submodules` will report the working tree dirty if any of the submodules HEAD points to a SHA different than the one recorded in the super project. Based on this discussion this commit extends the Zephyr build meta file with overall workspace status included in the meta file. It adds the following meta info to the build meta file: > workspace: > dirty: false / true > extra: false / true > off: false / true A project using west and having an extra Zephyr module loaded not controlled using git and a west project at a SHA different than the SHA referenced by the manifest can look like: zephyr: path: /.../zephyr revision: 863600cd0e3c0a271e86629c5089821e5e4380cc modules: - name: mcuboot path: /.../bootloader/mcuboot revision: c61538748ead773ea75a551a7beee299228bdcaf - name: local_module path: /.../local_module revision: null west: manifest: /.../zephyr/west.yml projects: - path: /.../zephyr revision: 863600cd0e3c0a271e86629c5089821e5e4380cc - path: /.../bootloader/mcuboot revision: c61538748ead773ea75a551a7beee299228bdcaf-off - path: /.../tools/net-tools revision: f49bd1354616fae4093bf36e5eaee43c51a55127 workspace: dirty: false extra: true 'off': true And without west: zephyr: path: /.../zephyr revision: 863600cd0e3c0a271e86629c5089821e5e4380cc modules: - name: hal_nordic path: /.../modules/hal/nordic revision: a6e5299041f152da5ae0ab17b2e44e088bb96d6d - name: local_module path: /.../local_module revision: null workspace: dirty: false extra: true Signed-off-by: Torsten Rasmussen --- Kconfig.zephyr | 10 +++++- scripts/zephyr_module.py | 69 +++++++++++++++++++++++++++++----------- 2 files changed, 60 insertions(+), 19 deletions(-) diff --git a/Kconfig.zephyr b/Kconfig.zephyr index a8ef40185b4..c4ff4a7ea72 100644 --- a/Kconfig.zephyr +++ b/Kconfig.zephyr @@ -512,7 +512,15 @@ config BUILD_OUTPUT_META Create a build meta file in the build directory containing lists of: - Zephyr: path and revision (if git repo) - Zephyr modules: name, path, and revision (if git repo) - - West projects: path and revision + - West: + - manifest: path and revision + - projects: path and revision + - Workspace: + - dirty: one or more repositories are marked dirty + - extra: extra Zephyr modules are manually included in the build + - off: the SHA of one or more west projects are not what the manifest + defined when `west update` was run the last time (`manifest-rev`). + The off state is only present if a west workspace is found. File extension is .meta endmenu diff --git a/scripts/zephyr_module.py b/scripts/zephyr_module.py index fd46580c9ee..55672f8a565 100755 --- a/scripts/zephyr_module.py +++ b/scripts/zephyr_module.py @@ -246,7 +246,7 @@ def process_twister(module, meta): return out -def process_meta(zephyr_base, west_projects, modules): +def process_meta(zephyr_base, west_projects, modules, extra_modules=None): # Process zephyr_base, projects, and modules and create a dictionary # with meta information for each input. # @@ -257,7 +257,11 @@ def process_meta(zephyr_base, west_projects, modules): # # returns the dictionary with said lists - meta = {'zephyr': None, 'modules': None, 'west': None} + meta = {'zephyr': None, 'modules': None, 'workspace': None} + + workspace_dirty = False + workspace_extra = extra_modules is not None + workspace_off = False def git_revision(path): rc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'], @@ -282,33 +286,59 @@ def process_meta(zephyr_base, west_projects, modules): stderr=None, cwd=path).wait() if rc: - return revision + '-dirty' - return revision - return None + return revision + '-dirty', True + return revision, False + return None, False - meta_project = {'path': zephyr_base, - 'revision': git_revision(zephyr_base)} - meta['zephyr'] = meta_project + zephyr_revision, zephyr_dirty = git_revision(zephyr_base) + zephyr_project = {'path': zephyr_base, + 'revision': zephyr_revision} + meta['zephyr'] = zephyr_project + meta['workspace'] = {} + workspace_dirty |= zephyr_dirty if west_projects is not None: + from west.manifest import MANIFEST_REV_BRANCH + projects = west_projects['projects'] meta_projects = [] - for project in west_projects['projects']: - project_path = PurePath(project).as_posix() + + # Special treatment of manifest project. + manifest_path = PurePath(projects[0].posixpath).as_posix() + manifest_revision, manifest_dirty = git_revision(manifest_path) + workspace_dirty |= manifest_dirty + manifest_project = {'path': manifest_path, + 'revision': manifest_revision} + meta_projects.append(manifest_project) + + for project in projects[1:]: + project_path = PurePath(project.posixpath).as_posix() + revision, dirty = git_revision(project_path) + workspace_dirty |= dirty + if project.sha(MANIFEST_REV_BRANCH) != revision: + revision += '-off' + workspace_off = True meta_project = {'path': project_path, - 'revision': git_revision(project_path)} + 'revision': revision} meta_projects.append(meta_project) - meta['west'] = {'manifest': west_projects['manifest'], - 'projects': meta_projects} + + meta.update({'west': {'manifest': west_projects['manifest'], + 'projects': meta_projects}}) + meta['workspace'].update({'off': workspace_off}) meta_projects = [] for module in modules: module_path = PurePath(module.project).as_posix() + revision, dirty = git_revision(module_path) + workspace_dirty |= dirty meta_project = {'name': module.meta['name'], 'path': module_path, - 'revision': git_revision(module_path)} + 'revision': revision} meta_projects.append(meta_project) meta['modules'] = meta_projects + meta['workspace'].update({'dirty': workspace_dirty, + 'extra': workspace_extra}) + return meta @@ -325,10 +355,10 @@ def west_projects(): try: manifest = Manifest.from_file() if version.parse(WestVersion) >= version.parse('0.9.0'): - projects = [p.posixpath for p in manifest.get_projects([]) + projects = [p for p in manifest.get_projects([]) if manifest.is_active(p)] else: - projects = [p.posixpath for p in manifest.get_projects([])] + projects = manifest.get_projects([]) manifest_file = manifest.path return {'manifest': manifest_file, 'projects': projects} except WestNotFound: @@ -438,7 +468,8 @@ def main(): if args.modules is None: west_proj = west_projects() modules = parse_modules(args.zephyr_base, - west_proj['projects'] if west_proj else None, + [p.posixpath for p in west_proj['projects']] + if west_proj else None, args.extra_modules) else: modules = parse_modules(args.zephyr_base, args.modules, @@ -477,7 +508,9 @@ def main(): fp.write(twister) if args.meta_out: - meta = process_meta(args.zephyr_base, west_proj, modules) + meta = process_meta(args.zephyr_base, west_proj, modules, + args.extra_modules) + with open(args.meta_out, 'w', encoding="utf-8") as fp: fp.write(yaml.dump(meta))