scripts: run_ci.sh: add pytest-based testing for west commands

Incorporate these into run_ci.sh the same way that btsim results were
done. This adds a dependency on pytest.

Signed-off-by: Marti Bolivar <marti@foundries.io>
This commit is contained in:
Marti Bolivar 2019-01-24 21:17:46 -07:00 committed by Carles Cufí
commit 63841df184
6 changed files with 479 additions and 0 deletions

View file

@ -25,6 +25,7 @@ SANITYCHECK_OPTIONS_RETRY_2="${SANITYCHECK_OPTIONS} --only-failed --outdir=out-3
export BSIM_OUT_PATH="${BSIM_OUT_PATH:-/opt/bsim/}"
export BSIM_COMPONENTS_PATH="${BSIM_OUT_PATH}/components/"
BSIM_BT_TEST_RESULTS_FILE="./bsim_bt_out/bsim_results.xml"
WEST_COMMANDS_RESULTS_FILE="./pytest_out/west_commands.xml"
MATRIX_BUILDS=1
MATRIX=1
@ -161,6 +162,11 @@ function on_complete() {
cp ${BSIM_BT_TEST_RESULTS_FILE} shippable/testresults/;
fi;
if [ -e ${WEST_COMMANDS_RESULTS_FILE} ]; then
echo "Copy ${WEST_COMMANDS_RESULTS_FILE}"
cp ${WEST_COMMANDS_RESULTS_FILE} shippable/testresults;
fi;
if [ "$MATRIX" = "1" ]; then
echo "Handle coverage data..."
handle_coverage
@ -227,6 +233,20 @@ if [ -n "$MAIN_CI" ]; then
echo "Skipping BT simulator tests"
fi
if [ "$MATRIX" = "1" ]; then
# Run pytest-based testing for Python in matrix
# builder 1. For now, this is just done for the west
# extension commands, but additional directories which
# run pytest could go here too.
mkdir -p $(dirname ${WEST_COMMANDS_RESULTS_FILE})
WEST_SRC=$(west list --format='{abspath}' west)/src
PYTHONPATH=./scripts/west_commands:$WEST_SRC pytest \
--junitxml=${WEST_COMMANDS_RESULTS_FILE} \
./scripts/west_commands/tests
else
echo "Skipping west command tests"
fi
# In a pull-request see if we have changed any tests or board definitions
if [ -n "${PULL_REQUEST_NR}" ]; then
get_tests_to_run

View file

@ -18,3 +18,4 @@ windows-curses; sys_platform == "win32"
colorama
Pillow
intelhex
pytest

View file

@ -0,0 +1,26 @@
# Copyright (c) 2018 Foundries.io
#
# SPDX-License-Identifier: Apache-2.0
'''Common fixtures for use testing the runner package.'''
import pytest
from runners.core import RunnerConfig
RC_BUILD_DIR = '/test/build-dir'
RC_BOARD_DIR = '/test/zephyr/boards/test-arch/test-board'
RC_KERNEL_ELF = 'test-zephyr.elf'
RC_KERNEL_HEX = 'test-zephyr.hex'
RC_KERNEL_BIN = 'test-zephyr.bin'
RC_GDB = 'test-none-gdb'
RC_OPENOCD = 'test-openocd'
RC_OPENOCD_SEARCH = '/test/openocd/search'
@pytest.fixture
def runner_config():
'''Fixture which provides a runners.core.RunnerConfig.'''
return RunnerConfig(RC_BUILD_DIR, RC_BOARD_DIR, RC_KERNEL_ELF,
RC_KERNEL_HEX, RC_KERNEL_BIN, gdb=RC_GDB,
openocd=RC_OPENOCD, openocd_search=RC_OPENOCD_SEARCH)

View file

@ -0,0 +1,15 @@
from runners.core import ZephyrBinaryRunner
def test_runner_imports():
# Ensure that all runner modules are imported and returned by
# get_runners().
#
# This is just a basic sanity check against errors introduced by
# tree-wide refactorings for runners that don't have their own
# test suites.
runner_names = set(r.name() for r in ZephyrBinaryRunner.get_runners())
expected = set(('arc-nsim', 'bossac', 'dfu-util', 'em-starterkit', 'esp32',
'jlink', 'nios2', 'nrfjprog', 'openocd', 'pyocd',
'qemu', 'xtensa', 'intel_s1000', 'blackmagicprobe'))
assert runner_names == expected

View file

@ -0,0 +1,206 @@
# Copyright (c) 2018 Foundries.io
#
# SPDX-License-Identifier: Apache-2.0
import argparse
from unittest.mock import patch, call
import pytest
from runners.nrfjprog import NrfJprogBinaryRunner
from conftest import RC_KERNEL_HEX
#
# Test values
#
TEST_DEF_SNR = 'test-default-serial-number' # for mocking user input
TEST_OVR_SNR = 'test-override-serial-number'
#
# Expected results.
#
# This dictionary maps different configurations to the commands we expect to be
# executed for them. Verification is done by mocking the check_call() method,
# which is used to run the commands.
#
# The key naming scheme is <F><SR><SN><E>, where:
#
# - F: family, 1 for 'NRF51' or 2 for 'NRF52'
# - SR: soft reset, Y for yes, N for pin reset
# - SNR: serial number override, Y for yes, N for 'use default'
# - E: full chip erase, Y for yes, N for sector / sector and UICR only
#
EXPECTED_COMMANDS = {
# NRF51:
'1NNN':
(['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF51', '--snr', TEST_DEF_SNR, '--sectorerase'], # noqa: E501
['nrfjprog', '--pinreset', '-f', 'NRF51', '--snr', TEST_DEF_SNR]),
'1NNY':
(['nrfjprog', '--eraseall', '-f', 'NRF51', '--snr', TEST_DEF_SNR],
['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF51', '--snr', TEST_DEF_SNR], # noqa: E501
['nrfjprog', '--pinreset', '-f', 'NRF51', '--snr', TEST_DEF_SNR]),
'1NYN':
(['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF51', '--snr', TEST_OVR_SNR, '--sectorerase'], # noqa: E501
['nrfjprog', '--pinreset', '-f', 'NRF51', '--snr', TEST_OVR_SNR]),
'1NYY':
(['nrfjprog', '--eraseall', '-f', 'NRF51', '--snr', TEST_OVR_SNR],
['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF51', '--snr', TEST_OVR_SNR], # noqa: E501
['nrfjprog', '--pinreset', '-f', 'NRF51', '--snr', TEST_OVR_SNR]),
'1YNN':
(['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF51', '--snr', TEST_DEF_SNR, '--sectorerase'], # noqa: E501
['nrfjprog', '--reset', '-f', 'NRF51', '--snr', TEST_DEF_SNR]),
'1YNY':
(['nrfjprog', '--eraseall', '-f', 'NRF51', '--snr', TEST_DEF_SNR],
['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF51', '--snr', TEST_DEF_SNR], # noqa: E501
['nrfjprog', '--reset', '-f', 'NRF51', '--snr', TEST_DEF_SNR]),
'1YYN':
(['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF51', '--snr', TEST_OVR_SNR, '--sectorerase'], # noqa: E501
['nrfjprog', '--reset', '-f', 'NRF51', '--snr', TEST_OVR_SNR]),
'1YYY':
(['nrfjprog', '--eraseall', '-f', 'NRF51', '--snr', TEST_OVR_SNR],
['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF51', '--snr', TEST_OVR_SNR], # noqa: E501
['nrfjprog', '--reset', '-f', 'NRF51', '--snr', TEST_OVR_SNR]),
# NRF52:
'2NNN':
(['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF52', '--snr', TEST_DEF_SNR, '--sectoranduicrerase'], # noqa: E501
['nrfjprog', '--pinresetenable', '-f', 'NRF52', '--snr', TEST_DEF_SNR],
['nrfjprog', '--pinreset', '-f', 'NRF52', '--snr', TEST_DEF_SNR]),
'2NNY':
(['nrfjprog', '--eraseall', '-f', 'NRF52', '--snr', TEST_DEF_SNR],
['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF52', '--snr', TEST_DEF_SNR], # noqa: E501
['nrfjprog', '--pinresetenable', '-f', 'NRF52', '--snr', TEST_DEF_SNR],
['nrfjprog', '--pinreset', '-f', 'NRF52', '--snr', TEST_DEF_SNR]),
'2NYN':
(['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF52', '--snr', TEST_OVR_SNR, '--sectoranduicrerase'], # noqa: E501
['nrfjprog', '--pinresetenable', '-f', 'NRF52', '--snr', TEST_OVR_SNR],
['nrfjprog', '--pinreset', '-f', 'NRF52', '--snr', TEST_OVR_SNR]),
'2NYY':
(['nrfjprog', '--eraseall', '-f', 'NRF52', '--snr', TEST_OVR_SNR],
['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF52', '--snr', TEST_OVR_SNR], # noqa: E501
['nrfjprog', '--pinresetenable', '-f', 'NRF52', '--snr', TEST_OVR_SNR],
['nrfjprog', '--pinreset', '-f', 'NRF52', '--snr', TEST_OVR_SNR]),
'2YNN':
(['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF52', '--snr', TEST_DEF_SNR, '--sectoranduicrerase'], # noqa: E501
['nrfjprog', '--reset', '-f', 'NRF52', '--snr', TEST_DEF_SNR]),
'2YNY':
(['nrfjprog', '--eraseall', '-f', 'NRF52', '--snr', TEST_DEF_SNR],
['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF52', '--snr', TEST_DEF_SNR], # noqa: E501
['nrfjprog', '--reset', '-f', 'NRF52', '--snr', TEST_DEF_SNR]),
'2YYN':
(['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF52', '--snr', TEST_OVR_SNR, '--sectoranduicrerase'], # noqa: E501
['nrfjprog', '--reset', '-f', 'NRF52', '--snr', TEST_OVR_SNR]),
'2YYY':
(['nrfjprog', '--eraseall', '-f', 'NRF52', '--snr', TEST_OVR_SNR],
['nrfjprog', '--program', RC_KERNEL_HEX, '-f', 'NRF52', '--snr', TEST_OVR_SNR], # noqa: E501
['nrfjprog', '--reset', '-f', 'NRF52', '--snr', TEST_OVR_SNR]),
}
def expected_commands(family, softreset, snr, erase):
'''Expected NrfJprogBinaryRunner results given parameters.
Returns a factory function which expects the following arguments:
- family: string, 'NRF51' or 'NRF52'
- softreset: boolean, controls whether soft reset is performed
- snr: string serial number of board, or None
- erase: boolean, whether to do a full chip erase or not
'''
expected_key = '{}{}{}{}'.format(
'1' if family == 'NRF51' else '2',
'Y' if softreset else 'N',
'Y' if snr else 'N',
'Y' if erase else 'N')
return EXPECTED_COMMANDS[expected_key]
#
# Test cases
#
TEST_CASES = [(f, sr, snr, e)
for f in ('NRF51', 'NRF52')
for sr in (False, True)
for snr in (TEST_OVR_SNR, None)
for e in (False, True)]
def get_board_snr_patch():
return TEST_DEF_SNR
def id_fn(test_case):
ret = ''
for x in test_case:
if x in ('NRF51', 'NRF52'):
ret += x[-1:]
else:
ret += 'Y' if x else 'N'
return ret
@pytest.mark.parametrize('test_case', TEST_CASES, ids=id_fn)
@patch('runners.nrfjprog.NrfJprogBinaryRunner.get_board_snr_from_user',
side_effect=get_board_snr_patch)
@patch('runners.nrfjprog.NrfJprogBinaryRunner.check_call')
def test_nrfjprog_init(cc, get_snr, test_case, runner_config):
family, softreset, snr, erase = test_case
runner = NrfJprogBinaryRunner(runner_config, family, softreset, snr,
erase=erase)
runner.run('flash')
assert cc.call_args_list == [call(x) for x in
expected_commands(*test_case)]
if snr is None:
get_snr.assert_called_once_with()
else:
get_snr.assert_not_called()
@pytest.mark.parametrize('test_case', TEST_CASES, ids=id_fn)
@patch('runners.nrfjprog.NrfJprogBinaryRunner.get_board_snr_from_user',
side_effect=get_board_snr_patch)
@patch('runners.nrfjprog.NrfJprogBinaryRunner.check_call')
def test_nrfjprog_create(cc, get_snr, test_case, runner_config):
family, softreset, snr, erase = test_case
args = ['--nrf-family', family]
if softreset:
args.append('--softreset')
if snr is not None:
args.extend(['--snr', snr])
if erase:
args.append('--erase')
parser = argparse.ArgumentParser()
NrfJprogBinaryRunner.add_parser(parser)
arg_namespace = parser.parse_args(args)
runner = NrfJprogBinaryRunner.create(runner_config, arg_namespace)
runner.run('flash')
assert cc.call_args_list == [call(x) for x in
expected_commands(*test_case)]
if snr is None:
get_snr.assert_called_once_with()
else:
get_snr.assert_not_called()

View file

@ -0,0 +1,211 @@
# Copyright (c) 2018 Foundries.io
#
# SPDX-License-Identifier: Apache-2.0
import argparse
from unittest.mock import patch
import pytest
from runners.pyocd import PyOcdBinaryRunner
from conftest import RC_BUILD_DIR, RC_GDB, RC_KERNEL_HEX, RC_KERNEL_ELF
#
# Test values to provide as constructor arguments and command line
# parameters, to verify they're respected.
#
TEST_TOOL = 'test-tool'
TEST_ADDR = 0xadd
TEST_BOARD_ID = 'test-board-id'
TEST_FREQUENCY = 'test-frequency'
TEST_DAPARG = 'test-daparg'
TEST_TARGET = 'test-target'
TEST_FLASHTOOL_OPTS = ['--test-flashtool', 'args']
TEST_SERVER = 'test-pyocd-gdbserver'
TEST_PORT = 1
TEST_ALL_KWARGS = {
'flashtool': TEST_TOOL,
'flash_addr': TEST_ADDR,
'flashtool_opts': TEST_FLASHTOOL_OPTS,
'gdbserver': TEST_SERVER,
'gdb_port': TEST_PORT,
'tui': False,
'board_id': TEST_BOARD_ID,
'frequency': TEST_FREQUENCY,
'daparg': TEST_DAPARG,
}
TEST_DEF_KWARGS = {}
TEST_ALL_PARAMS = (['--target', TEST_TARGET,
'--daparg', TEST_DAPARG,
'--flashtool', TEST_TOOL] +
['--flashtool-opt={}'.format(o) for o in
TEST_FLASHTOOL_OPTS] +
['--gdbserver', TEST_SERVER,
'--gdb-port', str(TEST_PORT),
'--board-id', TEST_BOARD_ID,
'--frequency', str(TEST_FREQUENCY)])
TEST_DEF_PARAMS = ['--target', TEST_TARGET]
#
# Expected results.
#
# These record expected argument lists for system calls made by the
# pyocd runner using its check_call() and run_server_and_client()
# methods.
#
# They are shared between tests that create runners directly and
# tests that construct runners from parsed command-line arguments, to
# ensure that results are consistent.
#
FLASH_ALL_EXPECTED_CALL = ([TEST_TOOL,
'-a', hex(TEST_ADDR), '-da', TEST_DAPARG,
'-t', TEST_TARGET, '-b', TEST_BOARD_ID,
'-f', TEST_FREQUENCY] +
TEST_FLASHTOOL_OPTS +
[RC_KERNEL_HEX])
FLASH_DEF_EXPECTED_CALL = ['pyocd-flashtool', '-t', TEST_TARGET, RC_KERNEL_HEX]
DEBUG_ALL_EXPECTED_SERVER = [TEST_SERVER,
'-da', TEST_DAPARG,
'-p', str(TEST_PORT),
'-t', TEST_TARGET,
'-b', TEST_BOARD_ID,
'-f', TEST_FREQUENCY]
DEBUG_ALL_EXPECTED_CLIENT = [RC_GDB, RC_KERNEL_ELF,
'-ex', 'target remote :{}'.format(TEST_PORT),
'-ex', 'monitor halt',
'-ex', 'monitor reset',
'-ex', 'load']
DEBUG_DEF_EXPECTED_SERVER = ['pyocd-gdbserver',
'-p', '3333',
'-t', TEST_TARGET]
DEBUG_DEF_EXPECTED_CLIENT = [RC_GDB, RC_KERNEL_ELF,
'-ex', 'target remote :3333',
'-ex', 'monitor halt',
'-ex', 'monitor reset',
'-ex', 'load']
DEBUGSERVER_ALL_EXPECTED_CALL = [TEST_SERVER,
'-da', TEST_DAPARG,
'-p', str(TEST_PORT),
'-t', TEST_TARGET,
'-b', TEST_BOARD_ID,
'-f', TEST_FREQUENCY]
DEBUGSERVER_DEF_EXPECTED_CALL = ['pyocd-gdbserver',
'-p', '3333',
'-t', TEST_TARGET]
#
# Fixtures
#
@pytest.fixture
def pyocd(runner_config, tmpdir):
'''PyOcdBinaryRunner from constructor kwargs or command line parameters'''
# This factory takes either a dict of kwargs to pass to the
# constructor, or a list of command-line arguments to parse and
# use with the create() method.
def _factory(args):
# Ensure kernel binaries exist (as empty files, so commands
# which use them must be patched out).
tmpdir.ensure(RC_KERNEL_HEX)
tmpdir.ensure(RC_KERNEL_ELF)
tmpdir.chdir()
if isinstance(args, dict):
return PyOcdBinaryRunner(runner_config, TEST_TARGET, **args)
elif isinstance(args, list):
parser = argparse.ArgumentParser()
PyOcdBinaryRunner.add_parser(parser)
arg_namespace = parser.parse_args(args)
return PyOcdBinaryRunner.create(runner_config, arg_namespace)
return _factory
#
# Test cases for runners created by constructor.
#
@pytest.mark.parametrize('pyocd_args,expected', [
(TEST_ALL_KWARGS, FLASH_ALL_EXPECTED_CALL),
(TEST_DEF_KWARGS, FLASH_DEF_EXPECTED_CALL)
])
@patch('runners.pyocd.PyOcdBinaryRunner.check_call')
def test_flash(cc, pyocd_args, expected, pyocd):
pyocd(pyocd_args).run('flash')
cc.assert_called_once_with(expected)
@pytest.mark.parametrize('pyocd_args,expectedv', [
(TEST_ALL_KWARGS, (DEBUG_ALL_EXPECTED_SERVER, DEBUG_ALL_EXPECTED_CLIENT)),
(TEST_DEF_KWARGS, (DEBUG_DEF_EXPECTED_SERVER, DEBUG_DEF_EXPECTED_CLIENT))
])
@patch('runners.pyocd.PyOcdBinaryRunner.run_server_and_client')
def test_debug(rsc, pyocd_args, expectedv, pyocd):
pyocd(pyocd_args).run('debug')
rsc.assert_called_once_with(*expectedv)
@pytest.mark.parametrize('pyocd_args,expected', [
(TEST_ALL_KWARGS, DEBUGSERVER_ALL_EXPECTED_CALL),
(TEST_DEF_KWARGS, DEBUGSERVER_DEF_EXPECTED_CALL)
])
@patch('runners.pyocd.PyOcdBinaryRunner.check_call')
def test_debugserver(cc, pyocd_args, expected, pyocd):
pyocd(pyocd_args).run('debugserver')
cc.assert_called_once_with(expected)
#
# Test cases for runners created via command line arguments.
#
# (Unlike the constructor tests, these require additional patching to mock and
# verify runners.core.BuildConfiguration usage.)
#
@pytest.mark.parametrize('pyocd_args,flash_addr,expected', [
(TEST_ALL_PARAMS, TEST_ADDR, FLASH_ALL_EXPECTED_CALL),
(TEST_DEF_PARAMS, 0x0, FLASH_DEF_EXPECTED_CALL)
])
@patch('runners.pyocd.BuildConfiguration')
@patch('runners.pyocd.PyOcdBinaryRunner.check_call')
def test_flash_args(cc, bc, pyocd_args, flash_addr, expected, pyocd):
with patch.object(PyOcdBinaryRunner, 'get_flash_address',
return_value=flash_addr):
pyocd(pyocd_args).run('flash')
bc.assert_called_once_with(RC_BUILD_DIR)
cc.assert_called_once_with(expected)
@pytest.mark.parametrize('pyocd_args, expectedv', [
(TEST_ALL_PARAMS, (DEBUG_ALL_EXPECTED_SERVER, DEBUG_ALL_EXPECTED_CLIENT)),
(TEST_DEF_PARAMS, (DEBUG_DEF_EXPECTED_SERVER, DEBUG_DEF_EXPECTED_CLIENT)),
])
@patch('runners.pyocd.BuildConfiguration')
@patch('runners.pyocd.PyOcdBinaryRunner.run_server_and_client')
def test_debug_args(rsc, bc, pyocd_args, expectedv, pyocd):
pyocd(pyocd_args).run('debug')
bc.assert_called_once_with(RC_BUILD_DIR)
rsc.assert_called_once_with(*expectedv)
@pytest.mark.parametrize('pyocd_args, expected', [
(TEST_ALL_PARAMS, DEBUGSERVER_ALL_EXPECTED_CALL),
(TEST_DEF_PARAMS, DEBUGSERVER_DEF_EXPECTED_CALL),
])
@patch('runners.pyocd.BuildConfiguration')
@patch('runners.pyocd.PyOcdBinaryRunner.check_call')
def test_debugserver_args(cc, bc, pyocd_args, expected, pyocd):
pyocd(pyocd_args).run('debugserver')
bc.assert_called_once_with(RC_BUILD_DIR)
cc.assert_called_once_with(expected)