michaelh/zephyr
1
0
Fork 0
Code Releases Activity

scripts: tests: Blackbox test expansion - filter

Browse source 
Adds tests related to filter flags:
--arch
--vendor

Signed-off-by: Kamil Paszkiet <kamilx.paszkiet@intel.com>
This commit is contained in:
Kamil Paszkiet 2024-03-21 08:28:13 +01:00 • committed by Fabio Baltieri
commit fff833eda4
5 changed files with 164 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

8
scripts/tests/twister_blackbox/test_data/tests/no_filter/dummy/CMakeLists.txt Normal file
View file

@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
cmake_minimum_required(VERSION 3.20.0)
find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
project(integration)
FILE(GLOB app_sources src/*.c)
target_sources(app PRIVATE ${app_sources})

1
scripts/tests/twister_blackbox/test_data/tests/no_filter/dummy/prj.conf Normal file
View file

@ -0,0 +1 @@
CONFIG_ZTEST=y

26
scripts/tests/twister_blackbox/test_data/tests/no_filter/dummy/src/main.c Normal file
View file

@ -0,0 +1,26 @@
/*
* Copyright (c) 2024 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <zephyr/ztest.h>
ZTEST_SUITE(a1_1_tests, NULL, NULL, NULL, NULL, NULL);
/**
* @brief Test Asserts
*
* This test verifies various assert macros provided by ztest.
*
*/
ZTEST(a1_1_tests, test_assert)
{
zassert_true(1, "1 was false");
zassert_false(0, "0 was true");
zassert_is_null(NULL, "NULL was not NULL");
zassert_not_null("foo", "\"foo\" was NULL");
zassert_equal(1, 1, "1 was not equal to 1");
zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL");
}

3
scripts/tests/twister_blackbox/test_data/tests/no_filter/dummy/test_data.yaml Normal file
View file

@ -0,0 +1,3 @@
tests:
no_filter.dummy:
tags: no_filter

126
scripts/tests/twister_blackbox/test_filter.py
View file

@ -12,12 +12,72 @@ import os
import pytest
import sys
import json
import re
from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock
from twisterlib.testplan import TestPlan
class TestFilter:
TESTDATA_1 = [
(
'x86',
[
r'(it8xxx2_evb).*?(SKIPPED: Command line testsuite arch filter)',
r'(frdm_k64f).*?(SKIPPED: Command line testsuite arch filter)',
r'(DEBUG\s+- adding qemu_x86)',
r'(DEBUG\s+- adding intel_ish_5_4_1)'
],
),
(
'arm',
[
r'(it8xxx2_evb).*?(SKIPPED: Command line testsuite arch filter)',
r'(qemu_x86).*?(SKIPPED: Command line testsuite arch filter)',
r'(intel_ish_5_4_1).*?(SKIPPED: Command line testsuite arch filter)',
r'(DEBUG\s+- adding frdm_k64f)'
]
),
(
'riscv',
[
r'(qemu_x86).*?(SKIPPED: Command line testsuite arch filter)',
r'(frdm_k64f).*?(SKIPPED: Command line testsuite arch filter)',
r'(intel_ish_5_4_1).*?(SKIPPED: Command line testsuite arch filter)',
r'(DEBUG\s+- adding it8xxx2_evb)'
]
)
]
TESTDATA_2 = [
(
'nxp',
[
r'(it8xxx2_evb).*?(SKIPPED: Not a selected vendor platform)',
r'(intel_ish_5_4_1).*?(SKIPPED: Not a selected vendor platform)',
r'(qemu_x86).*?(SKIPPED: Not a selected vendor platform)',
r'(DEBUG\s+- adding frdm_k64f)'
],
),
(
'intel',
[
r'(it8xxx2_evb).*?(SKIPPED: Not a selected vendor platform)',
r'(qemu_x86).*?(SKIPPED: Not a selected vendor platform)',
r'(frdm_k64f).*?(SKIPPED: Not a selected vendor platform)',
r'(DEBUG\s+- adding intel_ish_5_4_1)'
]
),
(
'ite',
[
r'(qemu_x86).*?(SKIPPED: Not a selected vendor platform)',
r'(frdm_k64f).*?(SKIPPED: Not a selected vendor platform)',
r'(intel_ish_5_4_1).*?(SKIPPED: Not a selected vendor platform)',
r'(DEBUG\s+- adding it8xxx2_evb)'
]
)
]
@classmethod
def setup_class(cls):
apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
@ -122,3 +182,69 @@ class TestFilter:
assert str(sys_exit.value) == '0'
assert len(filtered_j) == 3
@pytest.mark.parametrize(
'arch, expected',
TESTDATA_1,
ids=[
'arch x86',
'arch arm',
'arch riscv'
],
)
@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
def test_arch(self, capfd, out_path, arch, expected):
path = os.path.join(TEST_DATA, 'tests', 'no_filter')
test_platforms = ['qemu_x86', 'intel_ish_5_4_1', 'frdm_k64f', 'it8xxx2_evb']
args = ['--outdir', out_path, '-T', path, '-vv'] + \
['--arch', arch] + \
[val for pair in zip(
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
out, err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
assert str(sys_exit.value) == '0'
for line in expected:
assert re.search(line, err)
@pytest.mark.parametrize(
'vendor, expected',
TESTDATA_2,
ids=[
'vendor nxp',
'vendor intel',
'vendor ite'
],
)
@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
def test_vendor(self, capfd, out_path, vendor, expected):
path = os.path.join(TEST_DATA, 'tests', 'no_filter')
test_platforms = ['qemu_x86', 'intel_ish_5_4_1', 'frdm_k64f', 'it8xxx2_evb']
args = ['--outdir', out_path, '-T', path, '-vv'] + \
['--vendor', vendor] + \
[val for pair in zip(
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
out, err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
for line in expected:
assert re.search(line, err)
assert str(sys_exit.value) == '0'