diff --git a/scripts/requirements-base.txt b/scripts/requirements-base.txt index a232e829253..3938fa7b349 100644 --- a/scripts/requirements-base.txt +++ b/scripts/requirements-base.txt @@ -17,10 +17,12 @@ pykwalify canopen packaging progress +patool psutil pylink-square pyserial requests +semver # for ram/rom reports anytree diff --git a/scripts/west-commands.yml b/scripts/west-commands.yml index 24bbddb9845..6b7835e7562 100644 --- a/scripts/west-commands.yml +++ b/scripts/west-commands.yml @@ -76,3 +76,8 @@ west-commands: - name: simulate class: Simulate help: simulate board + - file: scripts/west_commands/sdk.py + commands: + - name: sdk + class: Sdk + help: manage Zephyr SDK diff --git a/scripts/west_commands/sdk.py b/scripts/west_commands/sdk.py new file mode 100755 index 00000000000..396edc92282 --- /dev/null +++ b/scripts/west_commands/sdk.py @@ -0,0 +1,574 @@ +# Copyright (c) 2024 TOKITA Hiroshi +# +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import hashlib +import os +import patoolib +import platform +import re +import requests +import semver +import shutil +import subprocess +import tempfile +import textwrap +import zcmake +from pathlib import Path + +from west.commands import WestCommand + + +class Sdk(WestCommand): + def __init__(self): + super().__init__( + "sdk", + "manage Zephyr SDK", + "List and Install Zephyr SDK", + ) + + def do_add_parser(self, parser_adder): + parser = parser_adder.add_parser( + self.name, + help=self.help, + description=self.description, + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent( + """ + Listing SDKs: + + Run 'west sdk' or 'west sdk list' to list installed SDKs. + See 'west sdk list --help' for details. + + + Installing SDK: + + Run 'west sdk install' to install Zephyr SDK. + See 'west sdk install --help' for details. + """ + ), + ) + + subparsers_gen = parser.add_subparsers( + metavar="", + dest="subcommand", + help="select a subcommand. If omitted, treat it as the 'list' selected.", + ) + + subparsers_gen.add_parser( + "list", + help="list installed Zephyr SDKs", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent( + """ + Listing SDKs: + + Run 'west sdk' or 'west sdk list' command information about available SDKs is displayed. + """ + ), + ) + + install_args_parser = subparsers_gen.add_parser( + "install", + help="install Zephyr SDK", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent( + """ + Installing SDK: + + Run 'west sdk install' to install Zephyr SDK. + + Set --version option to install a specific version of the SDK. + If not specified, the install version is detected from "${ZEPHYR_BASE}/SDK_VERSION file. + SDKs older than 0.14.1 are not supported. + + You can specify the installation directory with --install-dir or --install-base. + If the specified version of the SDK is already installed, + the already installed SDK will be used regardless of the settings of + --install-dir and --install-base. + + Typically, Zephyr SDK archives contain only one directory named zephyr-sdk- + at the top level. + The SDK archive is extracted to the home directory if both --install-dir and --install-base + are not specified. + In this case, SDK will install into ${HOME}/zephyr-sdk-. + If --install-base is specified, the archive will be extracted under the specified path. + In this case, SDK will install into /zephyr-sdk- . + If --install-dir is specified, the directory contained in the archive will be renamed + and placed to the specified location. + + --interactive, --toolchains, --no-toolchains and --no-hosttools options + specify the behavior of the installer. Please see the description of each option. + + --personal-access-token specifies the GitHub personal access token. + This helps to relax the limits on the number of REST API calls. + + --api-url specifies the REST API endpoint for GitHub releases information + when installing the SDK from a different GitHub repository. + """ + ), + ) + + install_args_parser.add_argument( + "--version", + default=None, + nargs="?", + metavar="SDK_VER", + help="version of the Zephyr SDK to install. " + "If not specified, the install version is detected from " + "${ZEPHYR_BASE}/SDK_VERSION file.", + ) + install_args_parser.add_argument( + "-b", + "--install-base", + default=None, + metavar="BASE", + help="Base directory to SDK install. " + "The subdirectory created by extracting the archive in will be the SDK installation directory. " + "For example, -b /foo/bar will install the SDK in `/foo/bar/zephyr-sdk-'." + ) + install_args_parser.add_argument( + "-d", + "--install-dir", + default=None, + metavar="DIR", + help="SDK install destination directory. " + "The SDK will be installed on the specified path. " + "The directory contained in the archive will be renamed and installed for the specified directory. " + "For example, if you specify -b /foo/bar/baz, The archive's zephyr-sdk- directory will be renamed baz and placed under /foo/bar. " + "If this option is specified, the --install-base option is ignored. " + ) + install_args_parser.add_argument( + "-i", + "--interactive", + action="store_true", + help="launches installer in interactive mode. " + "--toolchains, --no-toolchains and --no-hosttools will be ignored if this option is enabled.", + ) + install_args_parser.add_argument( + "-t", + "--toolchains", + metavar="toolchain_name", + nargs="+", + help="toolchain(s) to install (e.g. 'arm-zephyr-eabi'). " + "If this option is not given, toolchains for all architectures will be installed.", + ) + install_args_parser.add_argument( + "-T", + "--no-toolchains", + action="store_true", + help="do not install toolchains. " + "--toolchains will be ignored if this option is enabled.", + ) + install_args_parser.add_argument( + "-H", + "--no-hosttools", + action="store_true", + help="do not install host-tools.", + ) + install_args_parser.add_argument( + "--personal-access-token", help="GitHub personal access token." + ) + install_args_parser.add_argument( + "--api-url", + default="https://api.github.com/repos/zephyrproject-rtos/sdk-ng/releases", + help="GitHub releases API endpoint used to look for Zephyr SDKs.", + ) + + return parser + + def os_arch_name(self): + system = platform.system() + machine = platform.machine() + + if system == "Linux": + osname = "linux" + elif system == "Darwin": + osname = "macos" + elif system == "Windows": + osname = "windows" + else: + self.die(f"Unsupported system: {system}") + + if machine in ["aarch64", "arm64"]: + arch = "aarch64" + elif machine in ["x86_64", "AMD64"]: + arch = "x86_64" + else: + self.die(f"Unsupported machine: {machine}") + + return (osname, arch) + + def detect_version(self, args): + if args.version: + version = args.version + else: + if os.environ["ZEPHYR_BASE"]: + zephyr_base = Path(os.environ["ZEPHYR_BASE"]) + else: + zephyr_base = Path(__file__).parents[2] + + sdk_version_file = zephyr_base / "SDK_VERSION" + + if not sdk_version_file.exists(): + self.die(f"{str(sdk_version_file)} does not exist.") + + with open(sdk_version_file) as f: + version = f.readlines()[0].strip() + self.inf( + f"Found '{str(sdk_version_file)}', installing version {version}." + ) + + try: + semver.Version.parse(version) + except Exception: + self.die(f"Invalid version format: {version}") + + if semver.compare(version, "0.14.1") < 0: + self.die(f"Versions older than v0.14.1 are not supported.") + + return version + + def fetch_releases(self, url, req_headers): + """fetch releases data via GitHub REST API""" + + releases = [] + page = 1 + + while True: + params = {"page": page, "per_page": 100} + resp = requests.get(url, headers=req_headers, params=params) + if resp.status_code != 200: + raise Exception(f"Failed to fetch: {resp.status_code}, {resp.text}") + + data = resp.json() + if not data: + break + + releases.extend(data) + page += 1 + + return releases + + def minimal_sdk_filename(self, release): + (osname, arch) = self.os_arch_name() + version = re.sub(r"^v", "", release["tag_name"]) + + if semver.compare(version, "0.16.0") < 0: + if osname == "windows": + ext = ".zip" + else: + ext = ".tar.gz" + else: + if osname == "windows": + ext = ".7z" + else: + ext = ".tar.xz" + + return f"zephyr-sdk-{version}_{osname}-{arch}_minimal{ext}" + + def minimal_sdk_sha256(self, sha256_list, release): + name = self.minimal_sdk_filename(release) + tuples = [(re.split(r"\s+", t)) for t in sha256_list.splitlines()] + hashtable = {t[1]: t[0] for t in tuples} + + return hashtable[name] + + def minimal_sdk_url(self, release): + name = self.minimal_sdk_filename(release) + assets = release.get("assets", []) + minimal_sdk_asset = next(filter(lambda x: x["name"] == name, assets)) + + return minimal_sdk_asset["browser_download_url"] + + def sha256_sum_url(self, release): + assets = release.get("assets", []) + minimal_sdk_asset = next(filter(lambda x: x["name"] == "sha256.sum", assets)) + + return minimal_sdk_asset["browser_download_url"] + + def download_and_extract(self, base_dir, dir_name, target_release, req_headers): + self.inf("Fetching sha256...") + sha256_url = self.sha256_sum_url(target_release) + resp = requests.get(sha256_url, headers=req_headers, stream=True) + if resp.status_code != 200: + raise Exception(f"Failed to download {sha256_url}: {resp.status_code}") + + sha256 = self.minimal_sdk_sha256(resp.content.decode("UTF-8"), target_release) + + archive_url = self.minimal_sdk_url(target_release) + self.inf(f"Downloading {archive_url}...") + resp = requests.get(archive_url, headers=req_headers, stream=True) + if resp.status_code != 200: + raise Exception(f"Failed to download {archive_url}: {resp.status_code}") + + try: + Path(base_dir).mkdir(parents=True, exist_ok=True) + + with tempfile.TemporaryDirectory(dir=base_dir) as tempdir: + # download archive file + filename = Path(tempdir) / re.sub(r"^.*/", "", archive_url) + file = open(filename, mode="wb") + total_length = int(resp.headers["Content-Length"]) + count = 0 + + for chunk in resp.iter_content(chunk_size=8192): + file.write(chunk) + count = count + len(chunk) + self.inf(f"\r {count}/{total_length}", end="") + self.inf() + self.inf(f"Downloaded: {file.name}") + file.close() + + # check sha256 hash + with open(file.name, "rb") as sha256file: + digest = hashlib.sha256(sha256file.read()).hexdigest() + if sha256 != digest: + raise Exception(f"sha256 mismatched: {sha256}:{digest}") + + # extract archive file + self.inf(f"Extract: {file.name}") + patoolib.extract_archive(file.name, outdir=tempdir) + + # We expect that only the zephyr-sdk-x.y.z directory will be present in the archive. + extracted_dirs = [d for d in Path(tempdir).iterdir() if d.is_dir()] + if len(extracted_dirs) != 1: + raise Exception("Unexpected archive format") + + # move to destination dir + if dir_name: + dest_dir = Path(base_dir / dir_name) + else: + dest_dir = Path(base_dir / extracted_dirs[0].name) + + Path(dest_dir).parent.mkdir(parents=True, exist_ok=True) + + self.inf(f"Move: {str(extracted_dirs[0])} to {dest_dir}.") + shutil.move(extracted_dirs[0], dest_dir) + + return dest_dir + except PermissionError as pe: + self.die(pe) + + def run_setup(self, args, sdk_dir): + if "Windows" == platform.system(): + setup = Path(sdk_dir) / "setup.cmd" + optsep = "/" + else: + setup = Path(sdk_dir) / "setup.sh" + optsep = "-" + + # Associate installed SDK so that it can be found. + cmds_cmake_pkg = [str(setup), f"{optsep}c"] + self.dbg("Run: ", cmds_cmake_pkg) + result = subprocess.run(cmds_cmake_pkg) + if result.returncode != 0: + self.die(f"command \"{' '.join(cmds_cmake_pkg)}\" failed") + + cmds = [str(setup)] + + if not args.interactive and not args.no_toolchains: + if not args.toolchains: + cmds.extend([f"{optsep}t", "all"]) + else: + for tc in args.toolchains: + cmds.extend([f"{optsep}t", tc]) + + if not args.interactive and not args.no_hosttools: + cmds.extend([f"{optsep}h"]) + + if args.interactive or len(cmds) != 1: + self.dbg("Run: ", cmds) + result = subprocess.run(cmds) + if result.returncode != 0: + self.die(f"command \"{' '.join(cmds)}\" failed") + + def install_sdk(self, args, user_args): + version = self.detect_version(args) + (osname, arch) = self.os_arch_name() + + if args.personal_access_token: + req_headers = { + "Authorization": f"Bearer {args.personal_access_token}", + } + else: + req_headers = {} + + self.inf("Fetching Zephyr SDK list...") + releases = self.fetch_releases(args.api_url, req_headers) + self.dbg("releases: ", "\n".join([x["tag_name"] for x in releases])) + + # checking version + def check_semver(version): + try: + semver.Version.parse(version) + return True + except Exception: + return False + + available_versions = [ + re.sub(r"^v", "", x["tag_name"]) + for x in releases + if check_semver(re.sub(r"^v", "", x["tag_name"])) + ] + + if not version in available_versions: + self.die( + f"Unavailable SDK version: {version}." + + "Please select from the list below:\n" + + "\n".join(available_versions) + ) + + target_release = [x for x in releases if x["tag_name"] == f"v{version}"][0] + + # checking toolchains parameters + assets = target_release["assets"] + self.dbg("assets: ", "\n".join([x["browser_download_url"] for x in assets])) + + prefix = f"toolchain_{osname}-{arch}_" + available_toolchains = [ + re.sub(r"\..*", "", x["name"].replace(prefix, "")) + for x in assets + if x["name"].startswith(prefix) + ] + + if args.toolchains: + for tc in args.toolchains: + if not tc in available_toolchains: + self.die( + f"toolchain {tc} is not available.\n" + + "Please select from the list below:\n" + + "\n".join(available_toolchains) + ) + + installed_info = [v for (k, v) in self.fetch_sdk_info().items() if k == version] + if len(installed_info) == 0: + if args.install_dir: + base_dir = Path(args.install_dir).parent + dir_name = Path(args.install_dir).name + elif args.install_base: + base_dir = Path(args.install_base) + dir_name = None + else: + base_dir = Path("~").expanduser() + dir_name = None + + sdk_dir = self.download_and_extract( + base_dir, dir_name, target_release, req_headers + ) + else: + sdk_dir = Path(installed_info[0]["path"]) + self.inf( + f"Zephyr SDK version {version} is already installed at {str(sdk_dir)}. Using it." + ) + + self.run_setup(args, sdk_dir) + + def fetch_sdk_info(self): + sdk_lines = [] + try: + cmds = [ + "-P", + str(Path(__file__).parent / "sdk" / "listsdk.cmake"), + ] + + output = zcmake.run_cmake(cmds, capture_output=True) + if output: + # remove '-- Zephyr-sdk,' leader + sdk_lines = [l[15:] for l in output if l.startswith("-- Zephyr-sdk,")] + else: + sdk_lines = [] + + except Exception as e: + self.die(e) + + def parse_sdk_entry(line): + class SdkEntry: + def __init__(self): + self.version = None + self.path = None + + info = SdkEntry() + for ent in line.split(","): + kv = ent.split("=") + if kv[0].strip() == "ver": + info.version = kv[1].strip() + elif kv[0].strip() == "dir": + info.path = kv[1].strip() + + return info + + sdk_info = {} + for sdk_ent in [parse_sdk_entry(l) for l in reversed(sdk_lines)]: + entry = {} + + ver = None + sdk_path = Path(sdk_ent.path) + sdk_version_path = sdk_path / "sdk_version" + if sdk_version_path.exists(): + with open(str(sdk_version_path)) as f: + ver = f.readline().strip() + else: + continue + + entry["path"] = sdk_path + + if (sdk_path / "sysroots").exists(): + entry["hosttools"] = "installed" + + # Identify toolchain directory by the existence of /bin/-gcc + if "Windows" == platform.system(): + gcc_postfix = "-gcc.exe" + else: + gcc_postfix = "-gcc" + + toolchains = [ + tc.name + for tc in sdk_path.iterdir() + if (sdk_path / tc / "bin" / (tc.name + gcc_postfix)).exists() + ] + + if len(toolchains) > 0: + entry["toolchains"] = toolchains + + if ver: + sdk_info[ver] = entry + + return sdk_info + + def list_sdk(self): + sdk_info = self.fetch_sdk_info() + + if len(sdk_info) == 0: + self.die("No Zephyr SDK installed.") + + for k, v in sdk_info.items(): + self.inf(f"{k}:") + self.inf(f" path: {v['path']}") + if "hosttools" in v: + self.inf(f" hosttools: {v['hosttools']}") + if "toolchains" in v: + self.inf(" installed-toolchains:") + for tc in v["toolchains"]: + self.inf(f" - {tc}") + + # Since version 0.15.2, the sdk_toolchains file is included, + # so we can get information about available toolchains from there. + if (Path(v["path"]) / "sdk_toolchains").exists(): + with open(Path(v["path"]) / "sdk_toolchains") as f: + all_tcs = [l.strip() for l in f.readlines()] + + self.inf(" available-toolchains:") + for tc in all_tcs: + if tc not in v["toolchains"]: + self.inf(f" - {tc}") + + self.inf() + + def do_run(self, args, user_args): + self.dbg("args: ", args) + if args.subcommand == "install": + self.install_sdk(args, user_args) + elif args.subcommand == "list" or not args.subcommand: + self.list_sdk() diff --git a/scripts/west_commands/sdk/listsdk.cmake b/scripts/west_commands/sdk/listsdk.cmake new file mode 100644 index 00000000000..e084c59e863 --- /dev/null +++ b/scripts/west_commands/sdk/listsdk.cmake @@ -0,0 +1,9 @@ +# Copyright (c) 2024 TOKITA Hiroshi +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) + +set(ZEPHYR_BASE $ENV{ZEPHYR_BASE} CACHE PATH "Zephyr base") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${ZEPHYR_BASE}/cmake/modules) + +find_package(Zephyr-sdk COMPONENTS LIST)