michaelh/zephyr
1
0
Fork 0
Code Releases Activity

scripts: pytest: add tests to new adapters API

Browse source 
According to new adapters API add new tests which verify their
correctness. Introduce some minor naming changes to describe better
their role.

Signed-off-by: Piotr Golyzniak <piotr.golyzniak@nordicsemi.no>
This commit is contained in:
Piotr Golyzniak 2023-08-07 11:02:37 +02:00 • committed by Anas Nashif
commit 36e2b8e2b2
3 changed files with 123 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

137
scripts/pylib/pytest-twister-harness/tests/device/binary_adapter_test.py
View file

@ -2,9 +2,12 @@
#
# SPDX-License-Identifier: Apache-2.0
import logging
import os
import subprocess
import time
from pathlib import Path
from typing import Generator
from unittest import mock
import pytest
@ -18,22 +21,40 @@ from twister_harness.exceptions import TwisterHarnessException, TwisterHarnessTi
from twister_harness.twister_harness_config import DeviceConfig
@pytest.fixture
def script_path(resources: Path) -> str:
return str(resources.joinpath('mock_script.py'))
@pytest.fixture(name='device')
def fixture_adapter(tmp_path) -> NativeSimulatorAdapter:
return NativeSimulatorAdapter(DeviceConfig(build_dir=tmp_path, type='native'))
def fixture_device_adapter(tmp_path: Path) -> Generator[NativeSimulatorAdapter, None, None]:
build_dir = tmp_path / 'build_dir'
os.mkdir(build_dir)
device = NativeSimulatorAdapter(DeviceConfig(build_dir=build_dir, type='native', base_timeout=5.0))
try:
yield device
finally:
device.close() # to make sure all running processes are closed
def test_if_simulator_adapter_runs_without_errors(
resources: Path, device: NativeSimulatorAdapter
) -> None:
@pytest.fixture(name='launched_device')
def fixture_launched_device_adapter(
device: NativeSimulatorAdapter, script_path: str
) -> Generator[NativeSimulatorAdapter, None, None]:
device.command = ['python3', script_path]
try:
device.launch()
yield device
finally:
device.close() # to make sure all running processes are closed
def test_if_binary_adapter_runs_without_errors(launched_device: NativeSimulatorAdapter) -> None:
"""
Run script which prints text line by line and ends without errors.
Verify if subprocess was ended without errors, and without timeout.
"""
script_path = resources.joinpath('mock_script.py')
# patching original command by mock_script.py to simulate same behaviour as zephyr.exe
device.command = ['python3', str(script_path)]
device.launch()
device = launched_device
lines = device.readlines_until(regex='Returns with code')
device.close()
assert 'Readability counts.' in lines
@ -43,13 +64,12 @@ def test_if_simulator_adapter_runs_without_errors(
assert file_lines[-2:] == lines[-2:]
def test_if_simulator_adapter_finishes_after_timeout_while_there_is_no_data_from_subprocess(
resources: Path, device: NativeSimulatorAdapter
def test_if_binary_adapter_finishes_after_timeout_while_there_is_no_data_from_subprocess(
device: NativeSimulatorAdapter, script_path: str
) -> None:
"""Test if thread finishes after timeout when there is no data on stdout, but subprocess is still running"""
script_path = resources.joinpath('mock_script.py')
device.base_timeout = 1.0
device.command = ['python3', str(script_path), '--long-sleep', '--sleep=5']
device.base_timeout = 0.3
device.command = ['python3', script_path, '--long-sleep', '--sleep=5']
device.launch()
with pytest.raises(TwisterHarnessTimeoutException, match='Read from device timeout occurred'):
device.readlines_until(regex='Returns with code')
@ -61,7 +81,7 @@ def test_if_simulator_adapter_finishes_after_timeout_while_there_is_no_data_from
assert 'End of script' not in file_lines, 'Script has not been terminated before end'
def test_if_simulator_adapter_raises_exception_empty_command(device: NativeSimulatorAdapter) -> None:
def test_if_binary_adapter_raises_exception_empty_command(device: NativeSimulatorAdapter) -> None:
device.command = []
exception_msg = 'Run command is empty, please verify if it was generated properly.'
with pytest.raises(TwisterHarnessException, match=exception_msg):
@ -69,7 +89,7 @@ def test_if_simulator_adapter_raises_exception_empty_command(device: NativeSimul
@mock.patch('subprocess.Popen', side_effect=subprocess.SubprocessError(1, 'Exception message'))
def test_if_simulator_adapter_raises_exception_when_subprocess_raised_subprocess_error(
def test_if_binary_adapter_raises_exception_when_subprocess_raised_subprocess_error(
patched_popen, device: NativeSimulatorAdapter
) -> None:
device.command = ['echo', 'TEST']
@ -78,7 +98,7 @@ def test_if_simulator_adapter_raises_exception_when_subprocess_raised_subprocess
@mock.patch('subprocess.Popen', side_effect=FileNotFoundError(1, 'File not found', 'fake_file.txt'))
def test_if_simulator_adapter_raises_exception_file_not_found(
def test_if_binary_adapter_raises_exception_file_not_found(
patched_popen, device: NativeSimulatorAdapter
) -> None:
device.command = ['echo', 'TEST']
@ -87,25 +107,90 @@ def test_if_simulator_adapter_raises_exception_file_not_found(
@mock.patch('subprocess.Popen', side_effect=Exception(1, 'Raised other exception'))
def test_if_simulator_adapter_raises_exception_when_subprocess_raised_an_error(
patched_run, device: NativeSimulatorAdapter
def test_if_binary_adapter_raises_exception_when_subprocess_raised_an_error(
patched_popen, device: NativeSimulatorAdapter
) -> None:
device.command = ['echo', 'TEST']
with pytest.raises(TwisterHarnessException, match='Raised other exception'):
device._flash_and_run()
def test_if_native_simulator_adapter_get_command_returns_proper_string(
device: NativeSimulatorAdapter, resources: Path
def test_if_binary_adapter_connect_disconnect_print_warnings_properly(
caplog: pytest.LogCaptureFixture, launched_device: NativeSimulatorAdapter
) -> None:
device.device_config.build_dir = resources
device = launched_device
assert device._device_connected.is_set() and device.is_device_connected()
caplog.set_level(logging.DEBUG)
device.connect()
warning_msg = 'Device already connected'
assert warning_msg in caplog.text
for record in caplog.records:
if record.message == warning_msg:
assert record.levelname == 'DEBUG'
break
device.disconnect()
assert not device._device_connected.is_set() and not device.is_device_connected()
device.disconnect()
warning_msg = 'Device already disconnected'
assert warning_msg in caplog.text
for record in caplog.records:
if record.message == warning_msg:
assert record.levelname == 'DEBUG'
break
def test_if_binary_adapter_raise_exc_during_connect_read_and_write_after_close(
launched_device: NativeSimulatorAdapter
) -> None:
device = launched_device
assert device._device_run.is_set() and device.is_device_running()
device.close()
assert not device._device_run.is_set() and not device.is_device_running()
with pytest.raises(TwisterHarnessException, match='Cannot connect to not working device'):
device.connect()
with pytest.raises(TwisterHarnessException, match='No connection to the device'):
device.write(b'')
device.clear_buffer()
with pytest.raises(TwisterHarnessException, match='No connection to the device and no more data to read.'):
device.readline()
def test_if_binary_adapter_raise_exc_during_read_and_write_after_close(
launched_device: NativeSimulatorAdapter
) -> None:
device = launched_device
device.disconnect()
with pytest.raises(TwisterHarnessException, match='No connection to the device'):
device.write(b'')
device.clear_buffer()
with pytest.raises(TwisterHarnessException, match='No connection to the device and no more data to read.'):
device.readline()
def test_if_binary_adapter_is_able_to_read_leftovers_after_disconnect_or_close(
device: NativeSimulatorAdapter, script_path: str
) -> None:
device.command = ['python3', script_path, '--sleep=0.05']
device.launch()
device.readlines_until(regex='Beautiful is better than ugly.')
time.sleep(0.1)
device.disconnect()
assert len(device.readlines()) > 0
device.connect()
device.readlines_until(regex='Flat is better than nested.')
time.sleep(0.1)
device.close()
assert len(device.readlines()) > 0
def test_if_native_binary_adapter_get_command_returns_proper_string(device: NativeSimulatorAdapter) -> None:
device.generate_command()
assert isinstance(device.command, list)
assert device.command == [str(resources.joinpath('zephyr', 'zephyr.exe'))]
assert device.command == [str(device.device_config.build_dir / 'zephyr' / 'zephyr.exe')]
@mock.patch('shutil.which', return_value='west')
def test_if_custom_simulator_adapter_get_command_returns_proper_string(patched_which, tmp_path: Path) -> None:
def test_if_custom_binary_adapter_get_command_returns_proper_string(patched_which, tmp_path: Path) -> None:
device = CustomSimulatorAdapter(DeviceConfig(build_dir=tmp_path, type='custom'))
device.generate_command()
assert isinstance(device.command, list)
@ -113,13 +198,13 @@ def test_if_custom_simulator_adapter_get_command_returns_proper_string(patched_w
@mock.patch('shutil.which', return_value=None)
def test_if_custom_simulator_adapter_raise_exception_when_west_not_found(patched_which, tmp_path: Path) -> None:
def test_if_custom_binary_adapter_raise_exception_when_west_not_found(patched_which, tmp_path: Path) -> None:
device = CustomSimulatorAdapter(DeviceConfig(build_dir=tmp_path, type='custom'))
with pytest.raises(TwisterHarnessException, match='west not found'):
device.generate_command()
def test_if_unit_simulator_adapter_get_command_returns_proper_string(tmp_path: Path) -> None:
def test_if_unit_binary_adapter_get_command_returns_proper_string(tmp_path: Path) -> None:
device = UnitSimulatorAdapter(DeviceConfig(build_dir=tmp_path, type='unit'))
device.generate_command()
assert isinstance(device.command, list)

21
scripts/pylib/pytest-twister-harness/tests/device/hardware_adapter_test.py
View file

@ -19,10 +19,11 @@ def fixture_adapter(tmp_path) -> HardwareAdapter:
os.mkdir(build_dir)
device_config = DeviceConfig(
type='hardware',
runner='runner',
build_dir=build_dir,
runner='runner',
platform='platform',
id='p_id',
base_timeout=5.0,
)
return HardwareAdapter(device_config)
@ -35,7 +36,7 @@ def test_if_hardware_adapter_raise_exception_when_west_not_found(patched_which,
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_1(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.generate_command()
assert isinstance(device.command, list)
assert device.command == ['west', 'flash', '--skip-rebuild', '--build-dir', 'build', '--runner', 'runner']
@ -43,7 +44,7 @@ def test_if_get_command_returns_proper_string_1(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_2(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'pyocd'
device.generate_command()
assert isinstance(device.command, list)
@ -54,7 +55,7 @@ def test_if_get_command_returns_proper_string_2(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_3(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'nrfjprog'
device.generate_command()
assert isinstance(device.command, list)
@ -65,7 +66,7 @@ def test_if_get_command_returns_proper_string_3(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_4(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'openocd'
device.device_config.product = 'STM32 STLink'
device.generate_command()
@ -78,7 +79,7 @@ def test_if_get_command_returns_proper_string_4(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_5(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'openocd'
device.device_config.product = 'EDBG CMSIS-DAP'
device.generate_command()
@ -91,7 +92,7 @@ def test_if_get_command_returns_proper_string_5(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_6(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'jlink'
device.generate_command()
assert isinstance(device.command, list)
@ -103,7 +104,7 @@ def test_if_get_command_returns_proper_string_6(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_7(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'stm32cubeprogrammer'
device.generate_command()
assert isinstance(device.command, list)
@ -115,7 +116,7 @@ def test_if_get_command_returns_proper_string_7(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_8(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'openocd'
device.device_config.product = 'STLINK-V3'
device.generate_command()
@ -130,7 +131,7 @@ def test_if_get_command_returns_proper_string_8(patched_which, device: HardwareA
def test_if_get_command_returns_proper_string_with_west_flash_extra_args(
patched_which, device: HardwareAdapter
) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.west_flash_extra_args = ['--board-id=foobar', '--erase']
device.device_config.runner = 'pyocd'
device.device_config.id = ''

2
scripts/pylib/pytest-twister-harness/tests/device/qemu_adapter_test.py
View file

@ -18,7 +18,7 @@ from twister_harness.twister_harness_config import DeviceConfig
def fixture_device_adapter(tmp_path) -> Generator[QemuAdapter, None, None]:
build_dir = tmp_path / 'build_dir'
os.mkdir(build_dir)
device = QemuAdapter(DeviceConfig(build_dir=build_dir, type='qemu'))
device = QemuAdapter(DeviceConfig(build_dir=build_dir, type='qemu', base_timeout=5.0))
try:
yield device
finally: