scripts: pytest: add tests to new adapters API

According to new adapters API add new tests which verify their
correctness. Introduce some minor naming changes to describe better
their role.

Signed-off-by: Piotr Golyzniak <piotr.golyzniak@nordicsemi.no>
This commit is contained in:
Piotr Golyzniak 2023-08-07 11:02:37 +02:00 committed by Anas Nashif
commit 36e2b8e2b2
3 changed files with 123 additions and 37 deletions

View file

@ -2,9 +2,12 @@
#
# SPDX-License-Identifier: Apache-2.0
import logging
import os
import subprocess
import time
from pathlib import Path
from typing import Generator
from unittest import mock
import pytest
@ -18,22 +21,40 @@ from twister_harness.exceptions import TwisterHarnessException, TwisterHarnessTi
from twister_harness.twister_harness_config import DeviceConfig
@pytest.fixture
def script_path(resources: Path) -> str:
return str(resources.joinpath('mock_script.py'))
@pytest.fixture(name='device')
def fixture_adapter(tmp_path) -> NativeSimulatorAdapter:
return NativeSimulatorAdapter(DeviceConfig(build_dir=tmp_path, type='native'))
def fixture_device_adapter(tmp_path: Path) -> Generator[NativeSimulatorAdapter, None, None]:
build_dir = tmp_path / 'build_dir'
os.mkdir(build_dir)
device = NativeSimulatorAdapter(DeviceConfig(build_dir=build_dir, type='native', base_timeout=5.0))
try:
yield device
finally:
device.close() # to make sure all running processes are closed
def test_if_simulator_adapter_runs_without_errors(
resources: Path, device: NativeSimulatorAdapter
) -> None:
@pytest.fixture(name='launched_device')
def fixture_launched_device_adapter(
device: NativeSimulatorAdapter, script_path: str
) -> Generator[NativeSimulatorAdapter, None, None]:
device.command = ['python3', script_path]
try:
device.launch()
yield device
finally:
device.close() # to make sure all running processes are closed
def test_if_binary_adapter_runs_without_errors(launched_device: NativeSimulatorAdapter) -> None:
"""
Run script which prints text line by line and ends without errors.
Verify if subprocess was ended without errors, and without timeout.
"""
script_path = resources.joinpath('mock_script.py')
# patching original command by mock_script.py to simulate same behaviour as zephyr.exe
device.command = ['python3', str(script_path)]
device.launch()
device = launched_device
lines = device.readlines_until(regex='Returns with code')
device.close()
assert 'Readability counts.' in lines
@ -43,13 +64,12 @@ def test_if_simulator_adapter_runs_without_errors(
assert file_lines[-2:] == lines[-2:]
def test_if_simulator_adapter_finishes_after_timeout_while_there_is_no_data_from_subprocess(
resources: Path, device: NativeSimulatorAdapter
def test_if_binary_adapter_finishes_after_timeout_while_there_is_no_data_from_subprocess(
device: NativeSimulatorAdapter, script_path: str
) -> None:
"""Test if thread finishes after timeout when there is no data on stdout, but subprocess is still running"""
script_path = resources.joinpath('mock_script.py')
device.base_timeout = 1.0
device.command = ['python3', str(script_path), '--long-sleep', '--sleep=5']
device.base_timeout = 0.3
device.command = ['python3', script_path, '--long-sleep', '--sleep=5']
device.launch()
with pytest.raises(TwisterHarnessTimeoutException, match='Read from device timeout occurred'):
device.readlines_until(regex='Returns with code')
@ -61,7 +81,7 @@ def test_if_simulator_adapter_finishes_after_timeout_while_there_is_no_data_from
assert 'End of script' not in file_lines, 'Script has not been terminated before end'
def test_if_simulator_adapter_raises_exception_empty_command(device: NativeSimulatorAdapter) -> None:
def test_if_binary_adapter_raises_exception_empty_command(device: NativeSimulatorAdapter) -> None:
device.command = []
exception_msg = 'Run command is empty, please verify if it was generated properly.'
with pytest.raises(TwisterHarnessException, match=exception_msg):
@ -69,7 +89,7 @@ def test_if_simulator_adapter_raises_exception_empty_command(device: NativeSimul
@mock.patch('subprocess.Popen', side_effect=subprocess.SubprocessError(1, 'Exception message'))
def test_if_simulator_adapter_raises_exception_when_subprocess_raised_subprocess_error(
def test_if_binary_adapter_raises_exception_when_subprocess_raised_subprocess_error(
patched_popen, device: NativeSimulatorAdapter
) -> None:
device.command = ['echo', 'TEST']
@ -78,7 +98,7 @@ def test_if_simulator_adapter_raises_exception_when_subprocess_raised_subprocess
@mock.patch('subprocess.Popen', side_effect=FileNotFoundError(1, 'File not found', 'fake_file.txt'))
def test_if_simulator_adapter_raises_exception_file_not_found(
def test_if_binary_adapter_raises_exception_file_not_found(
patched_popen, device: NativeSimulatorAdapter
) -> None:
device.command = ['echo', 'TEST']
@ -87,25 +107,90 @@ def test_if_simulator_adapter_raises_exception_file_not_found(
@mock.patch('subprocess.Popen', side_effect=Exception(1, 'Raised other exception'))
def test_if_simulator_adapter_raises_exception_when_subprocess_raised_an_error(
patched_run, device: NativeSimulatorAdapter
def test_if_binary_adapter_raises_exception_when_subprocess_raised_an_error(
patched_popen, device: NativeSimulatorAdapter
) -> None:
device.command = ['echo', 'TEST']
with pytest.raises(TwisterHarnessException, match='Raised other exception'):
device._flash_and_run()
def test_if_native_simulator_adapter_get_command_returns_proper_string(
device: NativeSimulatorAdapter, resources: Path
def test_if_binary_adapter_connect_disconnect_print_warnings_properly(
caplog: pytest.LogCaptureFixture, launched_device: NativeSimulatorAdapter
) -> None:
device.device_config.build_dir = resources
device = launched_device
assert device._device_connected.is_set() and device.is_device_connected()
caplog.set_level(logging.DEBUG)
device.connect()
warning_msg = 'Device already connected'
assert warning_msg in caplog.text
for record in caplog.records:
if record.message == warning_msg:
assert record.levelname == 'DEBUG'
break
device.disconnect()
assert not device._device_connected.is_set() and not device.is_device_connected()
device.disconnect()
warning_msg = 'Device already disconnected'
assert warning_msg in caplog.text
for record in caplog.records:
if record.message == warning_msg:
assert record.levelname == 'DEBUG'
break
def test_if_binary_adapter_raise_exc_during_connect_read_and_write_after_close(
launched_device: NativeSimulatorAdapter
) -> None:
device = launched_device
assert device._device_run.is_set() and device.is_device_running()
device.close()
assert not device._device_run.is_set() and not device.is_device_running()
with pytest.raises(TwisterHarnessException, match='Cannot connect to not working device'):
device.connect()
with pytest.raises(TwisterHarnessException, match='No connection to the device'):
device.write(b'')
device.clear_buffer()
with pytest.raises(TwisterHarnessException, match='No connection to the device and no more data to read.'):
device.readline()
def test_if_binary_adapter_raise_exc_during_read_and_write_after_close(
launched_device: NativeSimulatorAdapter
) -> None:
device = launched_device
device.disconnect()
with pytest.raises(TwisterHarnessException, match='No connection to the device'):
device.write(b'')
device.clear_buffer()
with pytest.raises(TwisterHarnessException, match='No connection to the device and no more data to read.'):
device.readline()
def test_if_binary_adapter_is_able_to_read_leftovers_after_disconnect_or_close(
device: NativeSimulatorAdapter, script_path: str
) -> None:
device.command = ['python3', script_path, '--sleep=0.05']
device.launch()
device.readlines_until(regex='Beautiful is better than ugly.')
time.sleep(0.1)
device.disconnect()
assert len(device.readlines()) > 0
device.connect()
device.readlines_until(regex='Flat is better than nested.')
time.sleep(0.1)
device.close()
assert len(device.readlines()) > 0
def test_if_native_binary_adapter_get_command_returns_proper_string(device: NativeSimulatorAdapter) -> None:
device.generate_command()
assert isinstance(device.command, list)
assert device.command == [str(resources.joinpath('zephyr', 'zephyr.exe'))]
assert device.command == [str(device.device_config.build_dir / 'zephyr' / 'zephyr.exe')]
@mock.patch('shutil.which', return_value='west')
def test_if_custom_simulator_adapter_get_command_returns_proper_string(patched_which, tmp_path: Path) -> None:
def test_if_custom_binary_adapter_get_command_returns_proper_string(patched_which, tmp_path: Path) -> None:
device = CustomSimulatorAdapter(DeviceConfig(build_dir=tmp_path, type='custom'))
device.generate_command()
assert isinstance(device.command, list)
@ -113,13 +198,13 @@ def test_if_custom_simulator_adapter_get_command_returns_proper_string(patched_w
@mock.patch('shutil.which', return_value=None)
def test_if_custom_simulator_adapter_raise_exception_when_west_not_found(patched_which, tmp_path: Path) -> None:
def test_if_custom_binary_adapter_raise_exception_when_west_not_found(patched_which, tmp_path: Path) -> None:
device = CustomSimulatorAdapter(DeviceConfig(build_dir=tmp_path, type='custom'))
with pytest.raises(TwisterHarnessException, match='west not found'):
device.generate_command()
def test_if_unit_simulator_adapter_get_command_returns_proper_string(tmp_path: Path) -> None:
def test_if_unit_binary_adapter_get_command_returns_proper_string(tmp_path: Path) -> None:
device = UnitSimulatorAdapter(DeviceConfig(build_dir=tmp_path, type='unit'))
device.generate_command()
assert isinstance(device.command, list)

View file

@ -19,10 +19,11 @@ def fixture_adapter(tmp_path) -> HardwareAdapter:
os.mkdir(build_dir)
device_config = DeviceConfig(
type='hardware',
runner='runner',
build_dir=build_dir,
runner='runner',
platform='platform',
id='p_id',
base_timeout=5.0,
)
return HardwareAdapter(device_config)
@ -35,7 +36,7 @@ def test_if_hardware_adapter_raise_exception_when_west_not_found(patched_which,
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_1(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.generate_command()
assert isinstance(device.command, list)
assert device.command == ['west', 'flash', '--skip-rebuild', '--build-dir', 'build', '--runner', 'runner']
@ -43,7 +44,7 @@ def test_if_get_command_returns_proper_string_1(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_2(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'pyocd'
device.generate_command()
assert isinstance(device.command, list)
@ -54,7 +55,7 @@ def test_if_get_command_returns_proper_string_2(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_3(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'nrfjprog'
device.generate_command()
assert isinstance(device.command, list)
@ -65,7 +66,7 @@ def test_if_get_command_returns_proper_string_3(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_4(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'openocd'
device.device_config.product = 'STM32 STLink'
device.generate_command()
@ -78,7 +79,7 @@ def test_if_get_command_returns_proper_string_4(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_5(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'openocd'
device.device_config.product = 'EDBG CMSIS-DAP'
device.generate_command()
@ -91,7 +92,7 @@ def test_if_get_command_returns_proper_string_5(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_6(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'jlink'
device.generate_command()
assert isinstance(device.command, list)
@ -103,7 +104,7 @@ def test_if_get_command_returns_proper_string_6(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_7(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'stm32cubeprogrammer'
device.generate_command()
assert isinstance(device.command, list)
@ -115,7 +116,7 @@ def test_if_get_command_returns_proper_string_7(patched_which, device: HardwareA
@mock.patch('shutil.which', return_value='west')
def test_if_get_command_returns_proper_string_8(patched_which, device: HardwareAdapter) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.runner = 'openocd'
device.device_config.product = 'STLINK-V3'
device.generate_command()
@ -130,7 +131,7 @@ def test_if_get_command_returns_proper_string_8(patched_which, device: HardwareA
def test_if_get_command_returns_proper_string_with_west_flash_extra_args(
patched_which, device: HardwareAdapter
) -> None:
device.device_config.build_dir = 'build'
device.device_config.build_dir = Path('build')
device.device_config.west_flash_extra_args = ['--board-id=foobar', '--erase']
device.device_config.runner = 'pyocd'
device.device_config.id = ''

View file

@ -18,7 +18,7 @@ from twister_harness.twister_harness_config import DeviceConfig
def fixture_device_adapter(tmp_path) -> Generator[QemuAdapter, None, None]:
build_dir = tmp_path / 'build_dir'
os.mkdir(build_dir)
device = QemuAdapter(DeviceConfig(build_dir=build_dir, type='qemu'))
device = QemuAdapter(DeviceConfig(build_dir=build_dir, type='qemu', base_timeout=5.0))
try:
yield device
finally: