scripts: zephyr_module: Add URL, version to SPDX
Improve the SPDX with the current values: - URL: extracted from `git remote`. If more than one remote, URL is not set. - Version: extracted from `git rev-parse` (commit id). - PURL and CPE for Zephyr: generated from URL and version. For zephyr, the tag is extracted, if present, and replace the commit id for the version field. Since official modules does not have tags, tags are not yet extracted for modules. To track vulnerabilities from modules dependencies, a new SBOM, `modules-deps.spdx` was created. It contains the `external-references` provided by the modules. It allows to easily track vulnerabilities from these external dependencies. Signed-off-by: Thomas Gagneret <thomas.gagneret@hexploy.com>
This commit is contained in:
parent
eaa903d852
commit
0d05318c96
8 changed files with 397 additions and 76 deletions
|
@ -1710,9 +1710,8 @@ if(CONFIG_BUILD_OUTPUT_BIN AND CONFIG_BUILD_OUTPUT_UF2)
|
|||
set(BYPRODUCT_KERNEL_UF2_NAME "${PROJECT_BINARY_DIR}/${KERNEL_UF2_NAME}" CACHE FILEPATH "Kernel uf2 file" FORCE)
|
||||
endif()
|
||||
|
||||
set(KERNEL_META_PATH ${PROJECT_BINARY_DIR}/${KERNEL_META_NAME} CACHE INTERNAL "")
|
||||
if(CONFIG_BUILD_OUTPUT_META)
|
||||
set(KERNEL_META_PATH ${PROJECT_BINARY_DIR}/${KERNEL_META_NAME} CACHE INTERNAL "")
|
||||
|
||||
list(APPEND
|
||||
post_build_commands
|
||||
COMMAND ${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/zephyr_module.py
|
||||
|
@ -1726,6 +1725,9 @@ if(CONFIG_BUILD_OUTPUT_META)
|
|||
post_build_byproducts
|
||||
${KERNEL_META_PATH}
|
||||
)
|
||||
else(CONFIG_BUILD_OUTPUT_META)
|
||||
# Prevent spdx to use invalid data
|
||||
file(REMOVE ${KERNEL_META_PATH})
|
||||
endif()
|
||||
|
||||
# Cleanup intermediate files
|
||||
|
|
|
@ -569,6 +569,45 @@ Build files located in a ``MODULE_EXT_ROOT`` can be described as:
|
|||
This allows control of the build inclusion to be described externally to the
|
||||
Zephyr module.
|
||||
|
||||
.. _modules-vulnerability-monitoring:
|
||||
|
||||
Vulnerability monitoring
|
||||
========================
|
||||
|
||||
The module description file :file:`zephyr/module.yml` can be used to improve vulnerability monitoring.
|
||||
|
||||
If your module needs to track vulnerabilities using an external reference
|
||||
(e.g your module is forked from another repository), you can use the ``security`` section.
|
||||
It contains the field ``external-references`` that contains a list of references that needs to
|
||||
be monitored for your module. The supported formats are:
|
||||
|
||||
- CPE (Common Platform Enumeration)
|
||||
- PURL (Package URL)
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
security:
|
||||
external-references:
|
||||
- <module-related-cpe>
|
||||
- <an-other-module-related-cpe>
|
||||
- <module-related-purl>
|
||||
|
||||
A real life example for `mbedTLS` module could look like this:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
security:
|
||||
external-references:
|
||||
- cpe:2.3:a:arm:mbed_tls:3.5.2:*:*:*:*:*:*:*
|
||||
- pkg:github/Mbed-TLS/mbedtls@V3.5.2
|
||||
|
||||
.. note::
|
||||
CPE field must follow the CPE 2.3 schema provided by `NVD
|
||||
<https://csrc.nist.gov/projects/security-content-automation-protocol/specifications/cpe>`_.
|
||||
PURL field must follow the PURL specification provided by `Github
|
||||
<https://github.com/package-url/purl-spec/blob/master/PURL-SPECIFICATION.rst>`_.
|
||||
|
||||
|
||||
Build system integration
|
||||
========================
|
||||
|
||||
|
|
|
@ -110,6 +110,8 @@ This generates the following SPDX bill-of-materials (BOM) documents in
|
|||
- :file:`app.spdx`: BOM for the application source files used for the build
|
||||
- :file:`zephyr.spdx`: BOM for the specific Zephyr source code files used for the build
|
||||
- :file:`build.spdx`: BOM for the built output files
|
||||
- :file:`modules-deps.spdx`: BOM for modules dependencies. Check
|
||||
:ref:`modules <modules-vulnerability-monitoring>` for more details.
|
||||
|
||||
Each file in the bill-of-materials is scanned, so that its hashes (SHA256 and
|
||||
SHA1) can be recorded, along with any detected licenses if an
|
||||
|
|
|
@ -71,6 +71,18 @@ class PackageConfig:
|
|||
# primary package purpose (ex. "LIBRARY", "APPLICATION", etc.)
|
||||
self.primaryPurpose = ""
|
||||
|
||||
# package URL
|
||||
self.url = ""
|
||||
|
||||
# package version
|
||||
self.version = ""
|
||||
|
||||
# package revision
|
||||
self.revision = ""
|
||||
|
||||
# package external references
|
||||
self.externalReferences = []
|
||||
|
||||
# the Package's declared license
|
||||
self.declaredLicense = "NOASSERTION"
|
||||
|
||||
|
|
|
@ -121,4 +121,10 @@ def makeSPDX(cfg):
|
|||
log.err("SPDX writer failed for build document; bailing")
|
||||
return False
|
||||
|
||||
# write modules document
|
||||
writeSPDX(os.path.join(cfg.spdxDir, "modules-deps.spdx"), w.docModulesExtRefs)
|
||||
if not retval:
|
||||
log.err("SPDX writer failed for modules-deps document; bailing")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
import os
|
||||
import yaml
|
||||
import re
|
||||
|
||||
from west import log
|
||||
from west.util import west_topdir, WestNotFound
|
||||
|
@ -47,6 +48,7 @@ class Walker:
|
|||
self.docZephyr = None
|
||||
self.docApp = None
|
||||
self.docSDK = None
|
||||
self.docModulesExtRefs = None
|
||||
|
||||
# dict of absolute file path => the Document that owns that file
|
||||
self.allFileLinks = {}
|
||||
|
@ -69,6 +71,40 @@ class Walker:
|
|||
# SDK install path from parsed CMake cache
|
||||
self.sdkPath = ""
|
||||
|
||||
def _build_purl(self, url, version=None):
|
||||
if not url:
|
||||
return None
|
||||
|
||||
purl = None
|
||||
# This is designed to match repository with the following url pattern:
|
||||
# '<protocol><base_url>/<namespace>/<package>
|
||||
COMMON_GIT_URL_REGEX=r'((git@|http(s)?:\/\/)(?P<base_url>[\w\.@]+)(\/|:))(?P<namespace>[\w,\-,\_]+)\/(?P<package>[\w,\-,\_]+)(.git){0,1}((\/){0,1})$'
|
||||
|
||||
match = re.fullmatch(COMMON_GIT_URL_REGEX, url)
|
||||
if match:
|
||||
purl = f'pkg:{match.group("base_url")}/{match.group("namespace")}/{match.group("package")}'
|
||||
|
||||
if purl and (version or len(version) > 0):
|
||||
purl += f'@{version}'
|
||||
|
||||
return purl
|
||||
|
||||
def _normalize_module_name(self, module_name):
|
||||
# Replace "_" by "-" since it's not allowed in spdx ID
|
||||
return module_name.replace("_", "-")
|
||||
|
||||
def _add_describe_relationship(self, doc, cfgpackage):
|
||||
# create DESCRIBES relationship data
|
||||
rd = RelationshipData()
|
||||
rd.ownerType = RelationshipDataElementType.DOCUMENT
|
||||
rd.ownerDocument = doc
|
||||
rd.otherType = RelationshipDataElementType.PACKAGEID
|
||||
rd.otherPackageID = cfgpackage.spdxID
|
||||
rd.rlnType = "DESCRIBES"
|
||||
|
||||
# add it to pending relationships queue
|
||||
self.pendingRelationships.append(rd)
|
||||
|
||||
# primary entry point
|
||||
def makeDocuments(self):
|
||||
# parse CMake cache file and get compiler path
|
||||
|
@ -163,16 +199,7 @@ class Walker:
|
|||
pkgApp = Package(cfgPackageApp, self.docApp)
|
||||
self.docApp.pkgs[pkgApp.cfg.spdxID] = pkgApp
|
||||
|
||||
# create DESCRIBES relationship data
|
||||
rd = RelationshipData()
|
||||
rd.ownerType = RelationshipDataElementType.DOCUMENT
|
||||
rd.ownerDocument = self.docApp
|
||||
rd.otherType = RelationshipDataElementType.PACKAGEID
|
||||
rd.otherPackageID = cfgPackageApp.spdxID
|
||||
rd.rlnType = "DESCRIBES"
|
||||
|
||||
# add it to pending relationships queue
|
||||
self.pendingRelationships.append(rd)
|
||||
self._add_describe_relationship(self.docApp, cfgPackageApp)
|
||||
|
||||
def setupBuildDocument(self):
|
||||
# set up build document
|
||||
|
@ -196,7 +223,7 @@ class Walker:
|
|||
# add it to pending relationships queue
|
||||
self.pendingRelationships.append(rd)
|
||||
|
||||
def setupZephyrDocument(self, modules):
|
||||
def setupZephyrDocument(self, zephyr, modules):
|
||||
# set up zephyr document
|
||||
cfgZephyr = DocumentConfig()
|
||||
cfgZephyr.name = "zephyr-sources"
|
||||
|
@ -217,40 +244,68 @@ class Walker:
|
|||
cfgPackageZephyr.spdxID = "SPDXRef-zephyr-sources"
|
||||
cfgPackageZephyr.relativeBaseDir = relativeBaseDir
|
||||
|
||||
zephyr_url = zephyr.get("remote", "")
|
||||
if zephyr_url:
|
||||
cfgPackageZephyr.url = zephyr_url
|
||||
|
||||
if zephyr.get("revision"):
|
||||
cfgPackageZephyr.revision = zephyr.get("revision")
|
||||
|
||||
purl = None
|
||||
zephyr_tags = zephyr.get("tags", "")
|
||||
if zephyr_tags:
|
||||
# Find tag vX.Y.Z
|
||||
for tag in zephyr_tags:
|
||||
version = re.fullmatch(r'^v(?P<version>\d+\.\d+\.\d+)$', tag)
|
||||
purl = self._build_purl(zephyr_url, tag)
|
||||
|
||||
if purl:
|
||||
cfgPackageZephyr.externalReferences.append(purl)
|
||||
|
||||
# Extract version from tag once
|
||||
if cfgPackageZephyr.version == "" and version:
|
||||
cfgPackageZephyr.version = version.group('version')
|
||||
|
||||
if len(cfgPackageZephyr.version) > 0:
|
||||
cpe = f'cpe:2.3:o:zephyrproject:zephyr:{cfgPackageZephyr.version}:-:*:*:*:*:*:*'
|
||||
cfgPackageZephyr.externalReferences.append(cpe)
|
||||
|
||||
pkgZephyr = Package(cfgPackageZephyr, self.docZephyr)
|
||||
self.docZephyr.pkgs[pkgZephyr.cfg.spdxID] = pkgZephyr
|
||||
|
||||
self._add_describe_relationship(self.docZephyr, cfgPackageZephyr)
|
||||
|
||||
for module in modules:
|
||||
module_name = module.get("name", None)
|
||||
module_path = module.get("path", None)
|
||||
module_url = module.get("remote", None)
|
||||
module_revision = module.get("revision", None)
|
||||
|
||||
if not module_name:
|
||||
log.err(f"cannot find module name in meta file; bailing")
|
||||
return False
|
||||
|
||||
# Replace "_" by "-" since it's not allowed in spdx ID
|
||||
module_name = module_name.replace("_", "-")
|
||||
module_name = self._normalize_module_name(module_name)
|
||||
|
||||
# set up zephyr sources package
|
||||
cfgPackageZephyrModule = PackageConfig()
|
||||
cfgPackageZephyrModule.name = module_name
|
||||
cfgPackageZephyrModule.name = module_name + "-sources"
|
||||
cfgPackageZephyrModule.spdxID = "SPDXRef-" + module_name + "-sources"
|
||||
cfgPackageZephyrModule.relativeBaseDir = module_path
|
||||
cfgPackageZephyrModule.primaryPurpose = "SOURCE"
|
||||
|
||||
if module_revision:
|
||||
cfgPackageZephyrModule.revision = module_revision
|
||||
|
||||
if module_url:
|
||||
cfgPackageZephyrModule.url = module_url
|
||||
|
||||
pkgZephyrModule = Package(cfgPackageZephyrModule, self.docZephyr)
|
||||
self.docZephyr.pkgs[pkgZephyrModule.cfg.spdxID] = pkgZephyrModule
|
||||
|
||||
# create DESCRIBES relationship data
|
||||
rd = RelationshipData()
|
||||
rd.ownerType = RelationshipDataElementType.DOCUMENT
|
||||
rd.ownerDocument = self.docZephyr
|
||||
rd.otherType = RelationshipDataElementType.PACKAGEID
|
||||
rd.otherPackageID = cfgPackageZephyr.spdxID
|
||||
rd.rlnType = "DESCRIBES"
|
||||
self._add_describe_relationship(self.docZephyr, cfgPackageZephyrModule)
|
||||
|
||||
# add it to pending relationships queue
|
||||
self.pendingRelationships.append(rd)
|
||||
return True
|
||||
|
||||
def setupSDKDocument(self):
|
||||
# set up SDK document
|
||||
|
@ -280,6 +335,42 @@ class Walker:
|
|||
# add it to pending relationships queue
|
||||
self.pendingRelationships.append(rd)
|
||||
|
||||
def setupModulesDocument(self, modules):
|
||||
# set up zephyr document
|
||||
cfgModuleExtRef = DocumentConfig()
|
||||
cfgModuleExtRef.name = "modules-deps"
|
||||
cfgModuleExtRef.namespace = self.cfg.namespacePrefix + "/modules-deps"
|
||||
cfgModuleExtRef.docRefID = "DocumentRef-modules-deps"
|
||||
self.docModulesExtRefs = Document(cfgModuleExtRef)
|
||||
|
||||
for module in modules:
|
||||
module_name = module.get("name", None)
|
||||
module_security = module.get("security", None)
|
||||
|
||||
if not module_name:
|
||||
log.err(f"cannot find module name in meta file; bailing")
|
||||
return False
|
||||
|
||||
module_name = self._normalize_module_name(module_name)
|
||||
|
||||
module_ext_ref = []
|
||||
if module_security:
|
||||
module_ext_ref = module_security.get("external-references")
|
||||
|
||||
# set up zephyr sources package
|
||||
cfgPackageModuleExtRef = PackageConfig()
|
||||
cfgPackageModuleExtRef.name = module_name + "-deps"
|
||||
cfgPackageModuleExtRef.spdxID = "SPDXRef-" + module_name + "-deps"
|
||||
|
||||
for ref in module_ext_ref:
|
||||
cfgPackageModuleExtRef.externalReferences.append(ref)
|
||||
|
||||
pkgModule = Package(cfgPackageModuleExtRef, self.docModulesExtRefs)
|
||||
self.docModulesExtRefs.pkgs[pkgModule.cfg.spdxID] = pkgModule
|
||||
|
||||
self._add_describe_relationship(self.docModulesExtRefs, cfgPackageModuleExtRef)
|
||||
|
||||
|
||||
# set up Documents before beginning
|
||||
def setupDocuments(self):
|
||||
log.dbg("setting up placeholder documents")
|
||||
|
@ -289,7 +380,8 @@ class Walker:
|
|||
try:
|
||||
with open(self.metaFile) as file:
|
||||
content = yaml.load(file.read(), yaml.SafeLoader)
|
||||
self.setupZephyrDocument(content["modules"])
|
||||
if not self.setupZephyrDocument(content["zephyr"], content["modules"]):
|
||||
return False
|
||||
except (FileNotFoundError, yaml.YAMLError):
|
||||
log.err(f"cannot find a valid zephyr_meta.yml required for SPDX generation; bailing")
|
||||
return False
|
||||
|
@ -299,6 +391,8 @@ class Walker:
|
|||
if self.cfg.includeSDK:
|
||||
self.setupSDKDocument()
|
||||
|
||||
self.setupModulesDocument(content["modules"])
|
||||
|
||||
return True
|
||||
|
||||
# walk through targets and gather information
|
||||
|
|
|
@ -8,6 +8,15 @@ from west import log
|
|||
|
||||
from zspdx.util import getHashes
|
||||
|
||||
import re
|
||||
|
||||
CPE23TYPE_REGEX = (
|
||||
r'^cpe:2\.3:[aho\*\-](:(((\?*|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!"#$$%&\'\(\)\+,\/:;<=>@\[\]\^'
|
||||
r"`\{\|}~]))+(\?*|\*?))|[\*\-])){5}(:(([a-zA-Z]{2,3}(-([a-zA-Z]{2}|[0-9]{3}))?)|[\*\-]))(:(((\?*"
|
||||
r'|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!"#$$%&\'\(\)\+,\/:;<=>@\[\]\^`\{\|}~]))+(\?*|\*?))|[\*\-])){4}$'
|
||||
)
|
||||
PURL_REGEX = r"^pkg:.+(\/.+)?\/.+(@.+)?(\?.+)?(#.+)?$"
|
||||
|
||||
# Output tag-value SPDX 2.3 content for the given Relationship object.
|
||||
# Arguments:
|
||||
# 1) f: file handle for SPDX document
|
||||
|
@ -42,6 +51,14 @@ FileChecksum: SHA1: {bf.sha1}
|
|||
writeRelationshipSPDX(f, rln)
|
||||
f.write("\n")
|
||||
|
||||
def generateDowloadUrl(url, revision):
|
||||
# Only git is supported
|
||||
# walker.py only parse revision if it's from git repositiory
|
||||
if len(revision) == 0:
|
||||
return url
|
||||
|
||||
return f'git+{url}@{revision}'
|
||||
|
||||
# Output tag-value SPDX 2.3 content for the given Package object.
|
||||
# Arguments:
|
||||
# 1) f: file handle for SPDX document
|
||||
|
@ -51,7 +68,6 @@ def writePackageSPDX(f, pkg):
|
|||
|
||||
PackageName: {pkg.cfg.name}
|
||||
SPDXID: {pkg.cfg.spdxID}
|
||||
PackageDownloadLocation: NOASSERTION
|
||||
PackageLicenseConcluded: {pkg.concludedLicense}
|
||||
""")
|
||||
f.write(f"""PackageLicenseDeclared: {pkg.cfg.declaredLicense}
|
||||
|
@ -61,6 +77,25 @@ PackageCopyrightText: {pkg.cfg.copyrightText}
|
|||
if pkg.cfg.primaryPurpose != "":
|
||||
f.write(f"PrimaryPackagePurpose: {pkg.cfg.primaryPurpose}\n")
|
||||
|
||||
if len(pkg.cfg.url) > 0:
|
||||
downloadUrl = generateDowloadUrl(pkg.cfg.url, pkg.cfg.revision)
|
||||
f.write(f"PackageDownloadLocation: {downloadUrl}\n")
|
||||
else:
|
||||
f.write("PackageDownloadLocation: NOASSERTION\n")
|
||||
|
||||
if len(pkg.cfg.version) > 0:
|
||||
f.write(f"PackageVersion: {pkg.cfg.version}\n")
|
||||
elif len(pkg.cfg.revision) > 0:
|
||||
f.write(f"PackageVersion: {pkg.cfg.revision}\n")
|
||||
|
||||
for ref in pkg.cfg.externalReferences:
|
||||
if re.fullmatch(CPE23TYPE_REGEX, ref):
|
||||
f.write(f"ExternalRef: SECURITY cpe23Type {ref}\n")
|
||||
elif re.fullmatch(PURL_REGEX, ref):
|
||||
f.write(f"ExternalRef: PACKAGE_MANAGER purl {ref}\n")
|
||||
else:
|
||||
log.wrn(f"Unknown external reference ({ref})")
|
||||
|
||||
# flag whether files analyzed / any files present
|
||||
if len(pkg.files) > 0:
|
||||
if len(pkg.licenseInfoFromFiles) > 0:
|
||||
|
|
|
@ -152,6 +152,15 @@ mapping:
|
|||
doc-url:
|
||||
required: false
|
||||
type: str
|
||||
security:
|
||||
required: false
|
||||
type: map
|
||||
mapping:
|
||||
external-references:
|
||||
required: false
|
||||
type: seq
|
||||
sequence:
|
||||
- type: str
|
||||
'''
|
||||
|
||||
MODULE_YML_PATH = PurePath('zephyr/module.yml')
|
||||
|
@ -408,24 +417,7 @@ def process_twister(module, meta):
|
|||
return out
|
||||
|
||||
|
||||
def process_meta(zephyr_base, west_projs, modules, extra_modules=None,
|
||||
propagate_state=False):
|
||||
# Process zephyr_base, projects, and modules and create a dictionary
|
||||
# with meta information for each input.
|
||||
#
|
||||
# The dictionary will contain meta info in the following lists:
|
||||
# - zephyr: path and revision
|
||||
# - modules: name, path, and revision
|
||||
# - west-projects: path and revision
|
||||
#
|
||||
# returns the dictionary with said lists
|
||||
|
||||
meta = {'zephyr': None, 'modules': None, 'workspace': None}
|
||||
|
||||
workspace_dirty = False
|
||||
workspace_extra = extra_modules is not None
|
||||
workspace_off = False
|
||||
|
||||
def _create_meta_project(project_path):
|
||||
def git_revision(path):
|
||||
rc = subprocess.Popen(['git', 'rev-parse', '--is-inside-work-tree'],
|
||||
stdout=subprocess.PIPE,
|
||||
|
@ -453,77 +445,213 @@ def process_meta(zephyr_base, west_projs, modules, extra_modules=None,
|
|||
return revision, False
|
||||
return None, False
|
||||
|
||||
zephyr_revision, zephyr_dirty = git_revision(zephyr_base)
|
||||
zephyr_project = {'path': zephyr_base,
|
||||
'revision': zephyr_revision}
|
||||
def git_remote(path):
|
||||
popen = subprocess.Popen(['git', 'remote'],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
cwd=path)
|
||||
stdout, stderr = popen.communicate()
|
||||
stdout = stdout.decode('utf-8')
|
||||
|
||||
remotes_name = []
|
||||
if not (popen.returncode or stderr):
|
||||
remotes_name = stdout.rstrip().split('\n')
|
||||
|
||||
remote_url = None
|
||||
|
||||
# If more than one remote, do not return any remote
|
||||
if len(remotes_name) == 1:
|
||||
remote = remotes_name[0]
|
||||
popen = subprocess.Popen(['git', 'remote', 'get-url', remote],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
cwd=path)
|
||||
stdout, stderr = popen.communicate()
|
||||
stdout = stdout.decode('utf-8')
|
||||
|
||||
if not (popen.returncode or stderr):
|
||||
remote_url = stdout.rstrip()
|
||||
|
||||
return remote_url
|
||||
|
||||
def git_tags(path, revision):
|
||||
if not revision or len(revision) == 0:
|
||||
return None
|
||||
|
||||
popen = subprocess.Popen(['git', '-P', 'tag', '--points-at', revision],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
cwd=path)
|
||||
stdout, stderr = popen.communicate()
|
||||
stdout = stdout.decode('utf-8')
|
||||
|
||||
tags = None
|
||||
if not (popen.returncode or stderr):
|
||||
tags = stdout.rstrip().splitlines()
|
||||
|
||||
return tags
|
||||
|
||||
workspace_dirty = False
|
||||
path = PurePath(project_path).as_posix()
|
||||
|
||||
revision, dirty = git_revision(path)
|
||||
workspace_dirty |= dirty
|
||||
remote = git_remote(path)
|
||||
tags = git_tags(path, revision)
|
||||
|
||||
meta_project = {'path': path,
|
||||
'revision': revision}
|
||||
|
||||
if remote:
|
||||
meta_project['remote'] = remote
|
||||
|
||||
if tags:
|
||||
meta_project['tags'] = tags
|
||||
|
||||
return meta_project, workspace_dirty
|
||||
|
||||
|
||||
def _get_meta_project(meta_projects_list, project_path):
|
||||
projects = [ prj for prj in meta_projects_list[1:] if prj["path"] == project_path ]
|
||||
|
||||
return projects[0] if len(projects) == 1 else None
|
||||
|
||||
|
||||
def process_meta(zephyr_base, west_projs, modules, extra_modules=None,
|
||||
propagate_state=False):
|
||||
# Process zephyr_base, projects, and modules and create a dictionary
|
||||
# with meta information for each input.
|
||||
#
|
||||
# The dictionary will contain meta info in the following lists:
|
||||
# - zephyr: path and revision
|
||||
# - modules: name, path, and revision
|
||||
# - west-projects: path and revision
|
||||
#
|
||||
# returns the dictionary with said lists
|
||||
|
||||
meta = {'zephyr': None, 'modules': None, 'workspace': None}
|
||||
|
||||
zephyr_project, zephyr_dirty = _create_meta_project(zephyr_base)
|
||||
zephyr_off = zephyr_project.get("remote") is None
|
||||
|
||||
workspace_dirty = zephyr_dirty
|
||||
workspace_extra = extra_modules is not None
|
||||
workspace_off = zephyr_off
|
||||
|
||||
if zephyr_off:
|
||||
zephyr_project['revision'] += '-off'
|
||||
|
||||
meta['zephyr'] = zephyr_project
|
||||
meta['workspace'] = {}
|
||||
workspace_dirty |= zephyr_dirty
|
||||
|
||||
if west_projs is not None:
|
||||
from west.manifest import MANIFEST_REV_BRANCH
|
||||
projects = west_projs['projects']
|
||||
meta_projects = []
|
||||
|
||||
# Special treatment of manifest project.
|
||||
manifest_proj_path = PurePath(projects[0].posixpath).as_posix()
|
||||
manifest_revision, manifest_dirty = git_revision(manifest_proj_path)
|
||||
workspace_dirty |= manifest_dirty
|
||||
manifest_project = {'path': manifest_proj_path,
|
||||
'revision': manifest_revision}
|
||||
meta_projects.append(manifest_project)
|
||||
manifest_path = projects[0].posixpath
|
||||
|
||||
# Special treatment of manifest project
|
||||
# Git information (remote/revision) are not provided by west for the Manifest (west.yml)
|
||||
# To mitigate this, we check if we don't use the manifest from the zephyr repository or an other project.
|
||||
# If it's from zephyr, reuse zephyr information
|
||||
# If it's from an other project, ignore it, it will be added later
|
||||
# If it's not found, we extract data manually (remote/revision) from the directory
|
||||
|
||||
manifest_project = None
|
||||
manifest_dirty = False
|
||||
manifest_off = False
|
||||
|
||||
if zephyr_base == manifest_path:
|
||||
manifest_project = zephyr_project
|
||||
manifest_dirty = zephyr_dirty
|
||||
manifest_off = zephyr_off
|
||||
elif not [ prj for prj in projects[1:] if prj.posixpath == manifest_path ]:
|
||||
manifest_project, manifest_dirty = _create_meta_project(
|
||||
projects[0].posixpath)
|
||||
manifest_off = manifest_project.get("remote") is None
|
||||
if manifest_off:
|
||||
manifest_project["revision"] += "-off"
|
||||
|
||||
if manifest_project:
|
||||
workspace_off |= manifest_off
|
||||
workspace_dirty |= manifest_dirty
|
||||
meta_projects.append(manifest_project)
|
||||
|
||||
# Iterates on all projects except the first one (manifest)
|
||||
for project in projects[1:]:
|
||||
project_path = PurePath(project.posixpath).as_posix()
|
||||
revision, dirty = git_revision(project_path)
|
||||
meta_project, dirty = _create_meta_project(project.posixpath)
|
||||
workspace_dirty |= dirty
|
||||
if project.sha(MANIFEST_REV_BRANCH) != revision:
|
||||
revision += '-off'
|
||||
workspace_off = True
|
||||
meta_project = {'path': project_path,
|
||||
'revision': revision}
|
||||
meta_projects.append(meta_project)
|
||||
|
||||
off = False
|
||||
if not meta_project.get("remote") or project.sha(MANIFEST_REV_BRANCH) != meta_project['revision'].removesuffix("-dirty"):
|
||||
off = True
|
||||
if not meta_project.get('remote') or project.url != meta_project['remote']:
|
||||
# Force manifest URL and set commit as 'off'
|
||||
meta_project['url'] = project.url
|
||||
off = True
|
||||
|
||||
if off:
|
||||
meta_project['revision'] += '-off'
|
||||
workspace_off |= off
|
||||
|
||||
# If manifest is in project, updates related variables
|
||||
if project.posixpath == manifest_path:
|
||||
manifest_dirty |= dirty
|
||||
manifest_off |= off
|
||||
manifest_project = meta_project
|
||||
|
||||
meta.update({'west': {'manifest': west_projs['manifest_path'],
|
||||
'projects': meta_projects}})
|
||||
meta['workspace'].update({'off': workspace_off})
|
||||
|
||||
meta_projects = []
|
||||
# Iterates on all modules
|
||||
meta_modules = []
|
||||
for module in modules:
|
||||
module_path = PurePath(module.project).as_posix()
|
||||
revision, dirty = git_revision(module_path)
|
||||
workspace_dirty |= dirty
|
||||
meta_project = {'name': module.meta['name'],
|
||||
'path': module_path,
|
||||
'revision': revision}
|
||||
meta_projects.append(meta_project)
|
||||
meta['modules'] = meta_projects
|
||||
# Check if modules is not in projects
|
||||
# It allows to have the "-off" flag since `modules` variable` does not provide URL/remote
|
||||
meta_module = _get_meta_project(meta_projects, module.project)
|
||||
|
||||
if not meta_module:
|
||||
meta_module, dirty = _create_meta_project(module.project)
|
||||
workspace_dirty |= dirty
|
||||
|
||||
meta_module['name'] = module.meta.get('name')
|
||||
|
||||
if module.meta.get('security'):
|
||||
meta_module['security'] = module.meta.get('security')
|
||||
meta_modules.append(meta_module)
|
||||
|
||||
meta['modules'] = meta_modules
|
||||
|
||||
meta['workspace'].update({'dirty': workspace_dirty,
|
||||
'extra': workspace_extra})
|
||||
|
||||
if propagate_state:
|
||||
zephyr_revision = zephyr_project['revision']
|
||||
if workspace_dirty and not zephyr_dirty:
|
||||
zephyr_revision += '-dirty'
|
||||
if workspace_extra:
|
||||
zephyr_revision += '-extra'
|
||||
if workspace_off:
|
||||
if workspace_off and not zephyr_off:
|
||||
zephyr_revision += '-off'
|
||||
zephyr_project.update({'revision': zephyr_revision})
|
||||
|
||||
if west_projs is not None:
|
||||
manifest_revision = manifest_project['revision']
|
||||
if workspace_dirty and not manifest_dirty:
|
||||
manifest_revision += '-dirty'
|
||||
if workspace_extra:
|
||||
manifest_revision += '-extra'
|
||||
if workspace_off:
|
||||
if workspace_off and not manifest_off:
|
||||
manifest_revision += '-off'
|
||||
manifest_project.update({'revision': manifest_revision})
|
||||
|
||||
return meta
|
||||
|
||||
|
||||
def west_projects(manifest = None):
|
||||
def west_projects(manifest=None):
|
||||
manifest_path = None
|
||||
projects = []
|
||||
# West is imported here, as it is optional
|
||||
|
@ -691,7 +819,8 @@ def main():
|
|||
for module in modules:
|
||||
kconfig += process_kconfig(module.project, module.meta)
|
||||
cmake += process_cmake(module.project, module.meta)
|
||||
sysbuild_kconfig += process_sysbuildkconfig(module.project, module.meta)
|
||||
sysbuild_kconfig += process_sysbuildkconfig(
|
||||
module.project, module.meta)
|
||||
sysbuild_cmake += process_sysbuildcmake(module.project, module.meta)
|
||||
settings += process_settings(module.project, module.meta)
|
||||
twister += process_twister(module.project, module.meta)
|
||||
|
@ -735,6 +864,8 @@ def main():
|
|||
args.extra_modules, args.meta_state_propagate)
|
||||
|
||||
with open(args.meta_out, 'w', encoding="utf-8") as fp:
|
||||
# Ignore references and insert data instead
|
||||
yaml.Dumper.ignore_aliases = lambda self, data: True
|
||||
fp.write(yaml.dump(meta))
|
||||
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue