scripts: build meta file workspace status extension

The PR #39382 raised a discussion on build reproducibility and knowledge
of west projects being out of sync with the west manifest.

Similar to how `git submodules` will report the working tree dirty if
any of the submodules HEAD points to a SHA different than the one
recorded in the super project.

Based on this discussion this commit extends the Zephyr build meta file
with overall workspace status included in the meta file.

It adds the following meta info to the build meta file:
> workspace:
>  dirty: false / true
>  extra: false / true
>  off:   false / true

A project using west and having an extra Zephyr module loaded not
controlled using git and a west project at a SHA different than the
SHA referenced by the manifest can look like:
zephyr:
  path: /.../zephyr
  revision: 863600cd0e
modules:
- name: mcuboot
  path: /.../bootloader/mcuboot
  revision: c61538748ead773ea75a551a7beee299228bdcaf
- name: local_module
  path: /.../local_module
  revision: null
west:
  manifest: /.../zephyr/west.yml
  projects:
  - path: /.../zephyr
    revision: 863600cd0e
  - path: /.../bootloader/mcuboot
    revision: c61538748ead773ea75a551a7beee299228bdcaf-off
  - path: /.../tools/net-tools
    revision: f49bd1354616fae4093bf36e5eaee43c51a55127
workspace:
  dirty: false
  extra: true
  'off': true

And without west:
zephyr:
  path: /.../zephyr
  revision: 863600cd0e
modules:
- name: hal_nordic
  path: /.../modules/hal/nordic
  revision: a6e5299041f152da5ae0ab17b2e44e088bb96d6d
- name: local_module
  path: /.../local_module
  revision: null
workspace:
  dirty: false
  extra: true

Signed-off-by: Torsten Rasmussen <Torsten.Rasmussen@nordicsemi.no>
This commit is contained in:
Torsten Rasmussen 2021-11-04 14:28:21 +01:00 committed by Carles Cufí
commit 64ec6ee3a3
2 changed files with 60 additions and 19 deletions

View file

@ -512,7 +512,15 @@ config BUILD_OUTPUT_META
Create a build meta file in the build directory containing lists of:
- Zephyr: path and revision (if git repo)
- Zephyr modules: name, path, and revision (if git repo)
- West projects: path and revision
- West:
- manifest: path and revision
- projects: path and revision
- Workspace:
- dirty: one or more repositories are marked dirty
- extra: extra Zephyr modules are manually included in the build
- off: the SHA of one or more west projects are not what the manifest
defined when `west update` was run the last time (`manifest-rev`).
The off state is only present if a west workspace is found.
File extension is .meta
endmenu

View file

@ -246,7 +246,7 @@ def process_twister(module, meta):
return out
def process_meta(zephyr_base, west_projects, modules):
def process_meta(zephyr_base, west_projects, modules, extra_modules=None):
# Process zephyr_base, projects, and modules and create a dictionary
# with meta information for each input.
#
@ -257,7 +257,11 @@ def process_meta(zephyr_base, west_projects, modules):
#
# returns the dictionary with said lists
meta = {'zephyr': None, 'modules': None, 'west': None}
meta = {'zephyr': None, 'modules': None, 'workspace': None}
workspace_dirty = False
workspace_extra = extra_modules is not None
workspace_off = False
def git_revision(path):
rc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'],
@ -282,33 +286,59 @@ def process_meta(zephyr_base, west_projects, modules):
stderr=None,
cwd=path).wait()
if rc:
return revision + '-dirty'
return revision
return None
return revision + '-dirty', True
return revision, False
return None, False
meta_project = {'path': zephyr_base,
'revision': git_revision(zephyr_base)}
meta['zephyr'] = meta_project
zephyr_revision, zephyr_dirty = git_revision(zephyr_base)
zephyr_project = {'path': zephyr_base,
'revision': zephyr_revision}
meta['zephyr'] = zephyr_project
meta['workspace'] = {}
workspace_dirty |= zephyr_dirty
if west_projects is not None:
from west.manifest import MANIFEST_REV_BRANCH
projects = west_projects['projects']
meta_projects = []
for project in west_projects['projects']:
project_path = PurePath(project).as_posix()
# Special treatment of manifest project.
manifest_path = PurePath(projects[0].posixpath).as_posix()
manifest_revision, manifest_dirty = git_revision(manifest_path)
workspace_dirty |= manifest_dirty
manifest_project = {'path': manifest_path,
'revision': manifest_revision}
meta_projects.append(manifest_project)
for project in projects[1:]:
project_path = PurePath(project.posixpath).as_posix()
revision, dirty = git_revision(project_path)
workspace_dirty |= dirty
if project.sha(MANIFEST_REV_BRANCH) != revision:
revision += '-off'
workspace_off = True
meta_project = {'path': project_path,
'revision': git_revision(project_path)}
'revision': revision}
meta_projects.append(meta_project)
meta['west'] = {'manifest': west_projects['manifest'],
'projects': meta_projects}
meta.update({'west': {'manifest': west_projects['manifest'],
'projects': meta_projects}})
meta['workspace'].update({'off': workspace_off})
meta_projects = []
for module in modules:
module_path = PurePath(module.project).as_posix()
revision, dirty = git_revision(module_path)
workspace_dirty |= dirty
meta_project = {'name': module.meta['name'],
'path': module_path,
'revision': git_revision(module_path)}
'revision': revision}
meta_projects.append(meta_project)
meta['modules'] = meta_projects
meta['workspace'].update({'dirty': workspace_dirty,
'extra': workspace_extra})
return meta
@ -325,10 +355,10 @@ def west_projects():
try:
manifest = Manifest.from_file()
if version.parse(WestVersion) >= version.parse('0.9.0'):
projects = [p.posixpath for p in manifest.get_projects([])
projects = [p for p in manifest.get_projects([])
if manifest.is_active(p)]
else:
projects = [p.posixpath for p in manifest.get_projects([])]
projects = manifest.get_projects([])
manifest_file = manifest.path
return {'manifest': manifest_file, 'projects': projects}
except WestNotFound:
@ -438,7 +468,8 @@ def main():
if args.modules is None:
west_proj = west_projects()
modules = parse_modules(args.zephyr_base,
west_proj['projects'] if west_proj else None,
[p.posixpath for p in west_proj['projects']]
if west_proj else None,
args.extra_modules)
else:
modules = parse_modules(args.zephyr_base, args.modules,
@ -477,7 +508,9 @@ def main():
fp.write(twister)
if args.meta_out:
meta = process_meta(args.zephyr_base, west_proj, modules)
meta = process_meta(args.zephyr_base, west_proj, modules,
args.extra_modules)
with open(args.meta_out, 'w', encoding="utf-8") as fp:
fp.write(yaml.dump(meta))