tests: mcuboot: pytest: Add image swap test with mcumgr

Added application based on SMP Server Sample. Application is built together
with MCUboot using sysbuild and is flashed onto device in one step.
Tests are automated with pytest - new harness of Twister.
The image for upgrade is prepared using west sign command
then is uploaded by mcumgr into device and tested.
Automated scenarios to test upgrade (image upload, test, revert, confirm),
to test downgrade prevention mechanism and to test upgrade with image,
that is signed with an invalid key.

Signed-off-by: Grzegorz Chwierut <grzegorz.chwierut@nordicsemi.no>

scripts/pylib/pytest-twister-harness/src/twister_harness/helpers/shell.py

@@ -8,6 +8,9 @@ import logging
 import re
 import time
 
+from dataclasses import dataclass, field
+from inspect import signature
+
 from twister_harness.device.device_adapter import DeviceAdapter
 from twister_harness.exceptions import TwisterHarnessTimeoutException
 
@@ -78,3 +81,51 @@ class Shell:
             ])
         )
         return list(filter(lambda l: not regex_filter.search(l), command_lines))
+
+
+@dataclass
+class ShellMCUbootArea:
+    name: str
+    version: str
+    image_size: str
+    magic: str = 'unset'
+    swap_type: str = 'none'
+    copy_done: str = 'unset'
+    image_ok: str = 'unset'
+
+    @classmethod
+    def from_kwargs(cls, **kwargs) -> ShellMCUbootArea:
+        cls_fields = {field for field in signature(cls).parameters}
+        native_args = {}
+        for name, val in kwargs.items():
+            if name in cls_fields:
+                native_args[name] = val
+        return cls(**native_args)
+
+
+@dataclass
+class ShellMCUbootCommandParsed:
+    """
+    Helper class to keep data from `mcuboot` shell command.
+    """
+    areas: list[ShellMCUbootArea] = field(default_factory=list)
+
+    @classmethod
+    def create_from_cmd_output(cls, cmd_output: list[str]) -> ShellMCUbootCommandParsed:
+        """
+        Factory to create class from the output of `mcuboot` shell command.
+        """
+        areas: list[dict] = []
+        re_area = re.compile(r'(.+ area.*):\s*$')
+        re_key = re.compile(r'(?P<key>.+):(?P<val>.+)')
+        for line in cmd_output:
+            if m := re_area.search(line):
+                areas.append({'name': m.group(1)})
+            elif areas:
+                if m := re_key.search(line):
+                    areas[-1][m.group('key').strip().replace(' ', '_')] = m.group('val').strip()
+        data_areas: list[ShellMCUbootArea] = []
+        for area in areas:
+            data_areas.append(ShellMCUbootArea.from_kwargs(**area))
+
+        return cls(data_areas)

scripts/pylib/pytest-twister-harness/tests/helpers/shell_mcuboot_command_parser_test.py

@@ -0,0 +1,56 @@
+# Copyright (c) 2023 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import textwrap
+
+from twister_harness.helpers.shell import ShellMCUbootCommandParsed, ShellMCUbootArea
+
+
+def test_if_mcuboot_command_output_is_parsed_two_areas() -> None:
+    cmd_output = textwrap.dedent("""
+        \x1b[1;32muart:~$ \x1b[mmcuboot
+        swap type: revert
+        confirmed: 0
+        primary area (1):
+        version: 0.0.2+0
+        image size: 68240
+        magic: good
+        swap type: test
+        copy done: set
+        image ok: unset
+        secondary area (3):
+        version: 0.0.0+0
+        image size: 68240
+        magic: unset
+        swap type: none
+        copy done: unset
+        image ok: unset
+        \x1b[1;32muart:~$ \x1b[m
+    """)
+    mcuboot_parsed = ShellMCUbootCommandParsed.create_from_cmd_output(cmd_output.splitlines())
+    assert isinstance(mcuboot_parsed, ShellMCUbootCommandParsed)
+    assert isinstance(mcuboot_parsed.areas[0], ShellMCUbootArea)
+    assert len(mcuboot_parsed.areas) == 2
+    assert mcuboot_parsed.areas[0].version == '0.0.2+0'
+    assert mcuboot_parsed.areas[0].swap_type == 'test'
+
+
+def test_if_mcuboot_command_output_is_parsed_with_failed_area() -> None:
+    cmd_output = textwrap.dedent("""
+        \x1b[1;32muart:~$ \x1b[mmcuboot
+        swap type: revert
+        confirmed: 0
+        primary area (1):
+        version: 1.1.1+1
+        image size: 68240
+        magic: good
+        swap type: test
+        copy done: set
+        image ok: unset
+        failed to read secondary area (1) header: -5
+        \x1b[1;32muart:~$ \x1b[m
+    """)
+    mcuboot_parsed = ShellMCUbootCommandParsed.create_from_cmd_output(cmd_output.splitlines())
+    assert len(mcuboot_parsed.areas) == 1
+    assert mcuboot_parsed.areas[0].version == '1.1.1+1'

tests/boot/with_mcumgr/CMakeLists.txt

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20.0)
+
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(with_mcumgr)
+
+FILE(GLOB app_sources src/*.c)
+target_sources(app PRIVATE ${app_sources})

tests/boot/with_mcumgr/README.rst

@@ -0,0 +1,30 @@
+Upgrade testing with MCUmgr
+###########################
+
+This application is based on :ref:`smp_svr_sample`. It is built
+using **sysbuild**. Tests are automated with pytest, a new harness of Twister
+(more information can be found here :ref:`integration-with-pytest`)
+
+.. note::
+   Pytest uses the MCUmgr fixture which requires the ``mcumgr`` available
+   in the system PATH.
+   More information about MCUmgr can be found here :ref:`mcu_mgr`.
+
+To run tests with Twister on ``nrf52840dk_nrf52840`` platform,
+use following command:
+
+.. code-block:: console
+
+    ./zephyr/scripts/twister -vv --west-flash --enable-slow -T zephyr/tests/boot/with_mcumgr \
+    -p nrf52840dk_nrf52840 --device-testing --device-serial /dev/ttyACM0
+
+.. note::
+   Twister requires ``--west-flash`` flag enabled (without additional parameters
+   like ``erase``) to use sysbuild.
+
+Test scripts can be found in ``pytest`` directory. To list available
+scenarios with described procedures, one can use a pytest command:
+
+.. code-block:: console
+
+    pytest zephyr/tests/boot/with_mcumgr/pytest --collect-only -v

tests/boot/with_mcumgr/prj.conf

@@ -0,0 +1,22 @@
+# Enable MCUmgr and dependencies.
+CONFIG_NET_BUF=y
+CONFIG_ZCBOR=y
+CONFIG_CRC=y
+CONFIG_MCUMGR=y
+CONFIG_STREAM_FLASH=y
+CONFIG_FLASH_MAP=y
+
+# Enable the shell MCUmgr transport.
+CONFIG_BASE64=y
+CONFIG_SHELL=y
+CONFIG_SHELL_BACKEND_SERIAL=y
+CONFIG_MCUMGR_TRANSPORT_SHELL=y
+
+# Enable most core commands.
+CONFIG_FLASH=y
+CONFIG_IMG_MANAGER=y
+CONFIG_MCUMGR_GRP_IMG=y
+CONFIG_MCUMGR_GRP_OS=y
+
+# mcumgr-cli application doesn't accepts log in the channel it uses
+CONFIG_SHELL_LOG_BACKEND=n

tests/boot/with_mcumgr/pytest/test_downgrade_prevention.py

@@ -0,0 +1,62 @@
+# Copyright (c) 2023 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import logging
+
+from pathlib import Path
+from twister_harness import DeviceAdapter, Shell, MCUmgr
+from utils import (
+    find_in_config,
+    match_lines,
+    match_no_lines,
+    check_with_shell_command,
+    check_with_mcumgr_command,
+)
+from test_upgrade import create_signed_image, PROJECT_NAME
+
+
+logger = logging.getLogger(__name__)
+
+
+def test_downgrade_prevention(dut: DeviceAdapter, shell: Shell, mcumgr: MCUmgr):
+    """
+    Verify that the application is not downgraded
+    1) Device flashed with MCUboot and an application that contains SMP server.
+       Image version is 1.1.1+1
+    2) Prepare an update of an application containing the SMP server, where
+       image version is 0.0.0 (lower than version of the original app)
+    3) Upload the application update to slot 1 using mcumgr
+    4) Flag the application update in slot 1 as 'pending' by using mcumgr 'test'
+    5) Restart the device, verify that downgrade prevention mechanism
+       blocked the image swap
+    6) Verify that the original application is booted (version 1.1.1)
+    """
+    origin_version = find_in_config(
+        Path(dut.device_config.build_dir) / PROJECT_NAME / 'zephyr' / '.config',
+        'CONFIG_MCUBOOT_IMGTOOL_SIGN_VERSION'
+    )
+    check_with_shell_command(shell, origin_version)
+    assert origin_version != '0.0.0+0'
+
+    logger.info('Prepare upgrade image with lower version')
+    image_to_test = create_signed_image(dut.device_config.build_dir, '0.0.0+0')
+
+    logger.info('Upload image with mcumgr')
+    dut.disconnect()
+    mcumgr.image_upload(image_to_test)
+
+    logger.info('Test uploaded APP image')
+    second_hash = mcumgr.get_hash_to_test()
+    mcumgr.image_test(second_hash)
+    mcumgr.reset_device()
+
+    dut.connect()
+    output = dut.readlines_until('Launching primary slot application')
+    match_no_lines(output, ['Starting swap using move algorithm'])
+    match_lines(output, ['erased due to downgrade prevention'])
+    logger.info('Verify that the original APP is booted')
+    check_with_shell_command(shell, origin_version)
+    dut.disconnect()
+    check_with_mcumgr_command(mcumgr, origin_version)

tests/boot/with_mcumgr/pytest/test_upgrade.py

@@ -0,0 +1,204 @@
+# Copyright (c) 2023 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import pytest
+import logging
+
+from pathlib import Path
+from twister_harness import DeviceAdapter, Shell, MCUmgr
+from west_sign_wrapper import west_sign_with_imgtool
+from utils import (
+    find_in_config,
+    match_lines,
+    match_no_lines,
+    check_with_shell_command,
+    check_with_mcumgr_command,
+)
+
+logger = logging.getLogger(__name__)
+PROJECT_NAME = 'with_mcumgr'
+
+
+def create_signed_image(build_dir: Path, version: str) -> Path:
+    image_to_test = Path(build_dir) / 'test_{}.signed.bin'.format(
+        version.replace('.', '_').replace('+', '_'))
+    origin_key_file = find_in_config(
+        Path(build_dir) / 'mcuboot' / 'zephyr' / '.config',
+        'CONFIG_BOOT_SIGNATURE_KEY_FILE'
+    )
+    west_sign_with_imgtool(
+        build_dir=Path(build_dir) / PROJECT_NAME,
+        output_bin=image_to_test,
+        key_file=Path(origin_key_file),
+        version=version
+    )
+    assert image_to_test.is_file()
+    return image_to_test
+
+
+def test_upgrade_with_confirm(dut: DeviceAdapter, shell: Shell, mcumgr: MCUmgr):
+    """
+    Verify that the application can be updated
+    1) Device flashed with MCUboot and an application that contains SMP server
+    2) Prepare an update of an application containing the SMP server
+    3) Upload the application update to slot 1 using mcumgr
+    4) Flag the application update in slot 1 as 'pending' by using mcumgr 'test'
+    5) Restart the device, verify that swapping process is initiated
+    6) Verify that the updated application is booted
+    7) Confirm the image using mcumgr
+    8) Restart the device, and verify that the new application is still booted
+    """
+    origin_version = find_in_config(
+        Path(dut.device_config.build_dir) / PROJECT_NAME / 'zephyr' / '.config',
+        'CONFIG_MCUBOOT_IMGTOOL_SIGN_VERSION'
+    )
+    check_with_shell_command(shell, origin_version)
+
+    logger.info('Prepare upgrade image')
+    new_version = '0.0.2+0'
+    image_to_test = create_signed_image(dut.device_config.build_dir, new_version)
+
+    logger.info('Upload image with mcumgr')
+    dut.disconnect()
+    mcumgr.image_upload(image_to_test)
+
+    logger.info('Test uploaded APP image')
+    second_hash = mcumgr.get_hash_to_test()
+    mcumgr.image_test(second_hash)
+    mcumgr.reset_device()
+
+    dut.connect()
+    output = dut.readlines_until('Launching primary slot application')
+    match_lines(output, [
+        'Swap type: test',
+        'Starting swap using move algorithm'
+    ])
+    logger.info('Verify new APP is booted')
+    check_with_shell_command(shell, new_version, swap_type='test')
+    dut.disconnect()
+    check_with_mcumgr_command(mcumgr, new_version)
+
+    logger.info('Confirm the image')
+    mcumgr.image_confirm(second_hash)
+    mcumgr.reset_device()
+
+    dut.connect()
+    output = dut.readlines_until('Launching primary slot application')
+    match_no_lines(output, [
+        'Starting swap using move algorithm'
+    ])
+    logger.info('Verify new APP is still booted')
+    check_with_shell_command(shell, new_version)
+
+
+def test_upgrade_with_revert(dut: DeviceAdapter, shell: Shell, mcumgr: MCUmgr):
+    """
+    Verify that MCUboot will roll back an image that is not confirmed
+    1) Device flashed with MCUboot and an application that contains SMP server
+    2) Prepare an update of an application containing the SMP server
+    3) Upload the application update to slot 1 using mcumgr
+    4) Flag the application update in slot 1 as 'pending' by using mcumgr 'test'
+    5) Restart the device, verify that swapping process is initiated
+    6) Verify that the updated application is booted
+    7) Reset the device without confirming the image
+    8) Verify that MCUboot reverts update
+    """
+    origin_version = find_in_config(
+        Path(dut.device_config.build_dir) / PROJECT_NAME / 'zephyr' / '.config',
+        'CONFIG_MCUBOOT_IMGTOOL_SIGN_VERSION'
+    )
+    check_with_shell_command(shell, origin_version)
+
+    logger.info('Prepare upgrade image')
+    new_version = '0.0.3+0'
+    image_to_test = create_signed_image(dut.device_config.build_dir, new_version)
+
+    logger.info('Upload image with mcumgr')
+    dut.disconnect()
+    mcumgr.image_upload(image_to_test)
+
+    logger.info('Test uploaded APP image')
+    second_hash = mcumgr.get_hash_to_test()
+    mcumgr.image_test(second_hash)
+    mcumgr.reset_device()
+
+    dut.connect()
+    output = dut.readlines_until('Launching primary slot application')
+    match_lines(output, [
+        'Swap type: test',
+        'Starting swap using move algorithm'
+    ])
+    logger.info('Verify new APP is booted')
+    check_with_shell_command(shell, new_version, swap_type='test')
+    dut.disconnect()
+    check_with_mcumgr_command(mcumgr, new_version)
+
+    logger.info('Revert images')
+    mcumgr.reset_device()
+
+    dut.connect()
+    output = dut.readlines_until('Launching primary slot application')
+    match_lines(output, [
+        'Swap type: revert',
+        'Starting swap using move algorithm'
+    ])
+    logger.info('Verify that MCUboot reverts update')
+    check_with_shell_command(shell, origin_version)
+
+
+@pytest.mark.parametrize(
+    'key_file', [None, 'root-ec-p256.pem'],
+    ids=[
+        'no_key',
+        'invalid_key'
+    ])
+def test_upgrade_signature(dut: DeviceAdapter, shell: Shell, mcumgr: MCUmgr, key_file):
+    """
+    Verify that the application is not updated when app is not signed or signed with invalid key
+    1) Device flashed with MCUboot and an application that contains SMP server
+    2) Prepare an update of an application containing the SMP server that has
+       been signed:
+       a) without any key
+       b) with a different key than MCUboot was compiled with
+    3) Upload the application update to slot 1 using mcumgr
+    4) Flag the application update in slot 1 as 'pending' by using mcumgr 'test'
+    5) Restart the device, verify that swap is not started
+    """
+    if key_file:
+        origin_key_file = find_in_config(
+            Path(dut.device_config.build_dir) / 'mcuboot' / 'zephyr' / '.config',
+            'CONFIG_BOOT_SIGNATURE_KEY_FILE'
+        ).strip('"\'')
+        key_file = Path(origin_key_file).parent / key_file
+        assert key_file.is_file()
+        assert not key_file.samefile(origin_key_file)
+        image_to_test = image_to_test = Path(dut.device_config.build_dir) / 'test_invalid_key.bin'
+        logger.info('Sign second image with an invalid key')
+    else:
+        image_to_test = image_to_test = Path(dut.device_config.build_dir) / 'test_no_key.bin'
+        logger.info('Sign second imagewith no key')
+
+    west_sign_with_imgtool(
+        build_dir=Path(dut.device_config.build_dir) / PROJECT_NAME,
+        output_bin=image_to_test,
+        key_file=key_file,
+        version='0.0.3+4'  # must differ from the origin version, if not then hash is not updated
+    )
+    assert image_to_test.is_file()
+
+    logger.info('Upload image with mcumgr')
+    dut.disconnect()
+    mcumgr.image_upload(image_to_test)
+
+    logger.info('Test uploaded APP image')
+    second_hash = mcumgr.get_hash_to_test()
+    mcumgr.image_test(second_hash)
+    mcumgr.reset_device()
+
+    logger.info('Verify that swap is not started')
+    dut.connect()
+    output = dut.readlines_until('Launching primary slot application')
+    match_no_lines(output, ['Starting swap using move algorithm'])
+    match_lines(output, ['Image in the secondary slot is not valid'])

tests/boot/with_mcumgr/pytest/utils.py

@@ -0,0 +1,50 @@
+# Copyright (c) 2023 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import logging
+import re
+
+from pathlib import Path
+from twister_harness import Shell, MCUmgr
+from twister_harness.helpers.shell import ShellMCUbootCommandParsed
+
+
+logger = logging.getLogger(__name__)
+
+
+def find_in_config(config_file: Path | str, config_key: str) -> str:
+    re_key = re.compile(rf'{config_key}=(.+)')
+    with open(config_file) as f:
+        lines = f.readlines()
+    for line in lines:
+        if m := re_key.match(line):
+            logger.debug('Found matching key: %s' % line.strip())
+            return m.group(1).strip('"\'')
+    return ''
+
+
+def match_lines(output_lines: list[str], searched_lines: list[str]) -> None:
+    """Check all lines exist in the output"""
+    for sl in searched_lines:
+        assert any(sl in line for line in output_lines)
+
+
+def match_no_lines(output_lines: list[str], searched_lines: list[str]) -> None:
+    """Check lines not found in the output"""
+    for sl in searched_lines:
+        assert all(sl not in line for line in output_lines)
+
+
+def check_with_shell_command(shell: Shell, version: str, swap_type: str | None = None) -> None:
+    mcuboot_areas = ShellMCUbootCommandParsed.create_from_cmd_output(shell.exec_command('mcuboot'))
+    assert mcuboot_areas.areas[0].version == version
+    if swap_type:
+        assert mcuboot_areas.areas[0].swap_type == swap_type
+
+
+def check_with_mcumgr_command(mcumgr: MCUmgr, version: str) -> None:
+    image_list = mcumgr.get_image_list()
+    # version displayed by MCUmgr does not print +0 and changes + to '.' for non-zero values
+    assert image_list[0].version == version.replace('+0', '').replace('+', '.')

tests/boot/with_mcumgr/pytest/west_sign_wrapper.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2023 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import logging
+import shlex
+
+from subprocess import check_output
+from pathlib import Path
+
+
+logger = logging.getLogger(__name__)
+
+
+def west_sign_with_imgtool(
+        build_dir: Path,
+        output_bin: Path | None = None,
+        key_file: Path | None = None,
+        version: str | None = None,
+        timeout: int = 10
+):
+    """Wrapper method for `west sign -t imgtool` comamnd"""
+    command = [
+        'west', 'sign',
+        '-t', 'imgtool',
+        '--no-hex',
+        '--build-dir', str(build_dir)
+    ]
+    if output_bin:
+        command.extend(['--sbin', str(output_bin)])
+
+    command_extra_args = []
+    if key_file:
+        command_extra_args.extend(['--key', str(key_file)])
+    if version:
+        command_extra_args.extend(['--version', version])
+
+    if command_extra_args:
+        command.append('--')
+        command.extend(command_extra_args)
+
+    logger.info(f"CMD: {shlex.join(command)}")
+    output = check_output(command, text=True, timeout=timeout)
+    logger.debug('OUT: %s' % output)

tests/boot/with_mcumgr/src/main.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright (c) 2023 Nordic Semiconductor ASA
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <zephyr/kernel.h>
+
+/* Main entry point */
+int main(void)
+{
+	printk("Launching primary slot application on %s\n", CONFIG_BOARD);
+	return 0;
+}

tests/boot/with_mcumgr/sysbuild.conf

@@ -0,0 +1 @@
+SB_CONFIG_BOOTLOADER_MCUBOOT=y

tests/boot/with_mcumgr/sysbuild/mcuboot.conf

@@ -0,0 +1 @@
+CONFIG_MCUBOOT_LOG_LEVEL_INF=y

tests/boot/with_mcumgr/testcase.yaml

@@ -0,0 +1,34 @@
+common:
+  sysbuild: true
+  platform_allow:
+    - nrf52840dk_nrf52840
+    - nrf5340dk_nrf5340_cpuapp
+    - nrf9160dk_nrf9160
+  integration_platforms:
+    - nrf52840dk_nrf52840
+  timeout: 600
+  slow: true
+tests:
+  boot.with_mcumgr.test_upgrade:
+    tags:
+      - pytest
+      - mcuboot
+      - mcumgr
+    harness: pytest
+    harness_config:
+      pytest_root:
+        - "pytest/test_upgrade.py"
+
+  boot.with_mcumgr.test_downgrade_prevention:
+    tags:
+      - pytest
+      - mcuboot
+      - mcumgr
+    harness: pytest
+    harness_config:
+      pytest_root:
+        - "pytest/test_downgrade_prevention.py"
+    extra_args:
+      - mcuboot_CONFIG_MCUBOOT_DOWNGRADE_PREVENTION=y
+    extra_configs:
+      - CONFIG_MCUBOOT_IMGTOOL_SIGN_VERSION="1.1.1+1"