024bd41efb
To build LLEXT images using the xt-clang toolchain from Cadence linker flags have to be set similar to other toolchains. Add the missing cmake files. Signed-off-by: Guennadi Liakhovetski <guennadi.liakhovetski@linux.intel.com>
1837 lines
57 KiB
Python
1837 lines
57 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright (c) 2020 Intel Corporation
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
'''
|
|
This test file contains testsuites for testsuite.py module of twister
|
|
'''
|
|
import sys
|
|
import os
|
|
import mock
|
|
import pytest
|
|
|
|
from contextlib import nullcontext
|
|
|
|
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
|
|
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
|
|
|
|
from twisterlib.testplan import TestPlan, change_skip_to_error_if_integration
|
|
from twisterlib.testinstance import TestInstance
|
|
from twisterlib.testsuite import TestSuite
|
|
from twisterlib.platform import Platform
|
|
from twisterlib.quarantine import Quarantine
|
|
from twisterlib.error import TwisterRuntimeError
|
|
|
|
|
|
def test_testplan_add_testsuites_short(class_testplan):
|
|
""" Testing add_testcase function of Testsuite class in twister """
|
|
# Test 1: Check the list of testsuites after calling add testsuites function is as expected
|
|
class_testplan.SAMPLE_FILENAME = 'test_sample_app.yaml'
|
|
class_testplan.TESTSUITE_FILENAME = 'test_data.yaml'
|
|
class_testplan.add_testsuites()
|
|
|
|
tests_rel_dir = 'scripts/tests/twister/test_data/testsuites/tests/'
|
|
expected_testsuites = ['test_b.check_1',
|
|
'test_b.check_2',
|
|
'test_c.check_1',
|
|
'test_c.check_2',
|
|
'test_a.check_1',
|
|
'test_a.check_2',
|
|
'test_d.check_1',
|
|
'test_e.check_1',
|
|
'sample_test.app',
|
|
'test_config.main']
|
|
testsuite_list = []
|
|
for key in sorted(class_testplan.testsuites.keys()):
|
|
testsuite_list.append(os.path.basename(os.path.normpath(key)))
|
|
assert sorted(testsuite_list) == sorted(expected_testsuites)
|
|
|
|
# Test 2 : Assert Testcase name is expected & all testsuites values are testcase class objects
|
|
suite = class_testplan.testsuites.get(tests_rel_dir + 'test_a/test_a.check_1')
|
|
assert suite.name == tests_rel_dir + 'test_a/test_a.check_1'
|
|
assert all(isinstance(n, TestSuite) for n in class_testplan.testsuites.values())
|
|
|
|
@pytest.mark.parametrize("board_root_dir", [("board_config_file_not_exist"), ("board_config")])
|
|
def test_add_configurations_short(test_data, class_env, board_root_dir):
|
|
""" Testing add_configurations function of TestPlan class in Twister
|
|
Test : Asserting on default platforms list
|
|
"""
|
|
class_env.board_roots = [os.path.abspath(test_data + board_root_dir)]
|
|
plan = TestPlan(class_env)
|
|
plan.parse_configuration(config_file=class_env.test_config)
|
|
if board_root_dir == "board_config":
|
|
plan.add_configurations()
|
|
assert sorted(plan.default_platforms) == sorted(['demo_board_1', 'demo_board_3'])
|
|
elif board_root_dir == "board_config_file_not_exist":
|
|
plan.add_configurations()
|
|
assert sorted(plan.default_platforms) != sorted(['demo_board_1'])
|
|
|
|
|
|
def test_get_all_testsuites_short(class_testplan, all_testsuites_dict):
|
|
""" Testing get_all_testsuites function of TestPlan class in Twister """
|
|
plan = class_testplan
|
|
plan.testsuites = all_testsuites_dict
|
|
expected_tests = ['sample_test.app', 'test_a.check_1.1a',
|
|
'test_a.check_1.1c',
|
|
'test_a.check_1.2a', 'test_a.check_1.2b',
|
|
'test_a.check_1.Unit_1c', 'test_a.check_1.unit_1a',
|
|
'test_a.check_1.unit_1b', 'test_a.check_2.1a',
|
|
'test_a.check_2.1c', 'test_a.check_2.2a',
|
|
'test_a.check_2.2b', 'test_a.check_2.Unit_1c',
|
|
'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b',
|
|
'test_b.check_1', 'test_b.check_2', 'test_c.check_1',
|
|
'test_c.check_2', 'test_d.check_1.unit_1a',
|
|
'test_d.check_1.unit_1b',
|
|
'test_e.check_1.1a', 'test_e.check_1.1b',
|
|
'test_config.main']
|
|
|
|
assert sorted(plan.get_all_tests()) == sorted(expected_tests)
|
|
|
|
def test_get_platforms_short(class_testplan, platforms_list):
|
|
""" Testing get_platforms function of TestPlan class in Twister """
|
|
plan = class_testplan
|
|
plan.platforms = platforms_list
|
|
platform = plan.get_platform("demo_board_1")
|
|
assert isinstance(platform, Platform)
|
|
assert platform.name == "demo_board_1"
|
|
|
|
TESTDATA_PART1 = [
|
|
("toolchain_allow", ['gcc'], None, None, "Not in testsuite toolchain allow list"),
|
|
("platform_allow", ['demo_board_1'], None, None, "Not in testsuite platform allow list"),
|
|
("toolchain_exclude", ['zephyr'], None, None, "In test case toolchain exclude"),
|
|
("platform_exclude", ['demo_board_2'], None, None, "In test case platform exclude"),
|
|
("arch_exclude", ['x86'], None, None, "In test case arch exclude"),
|
|
("arch_allow", ['arm'], None, None, "Not in test case arch allow list"),
|
|
("skip", True, None, None, "Skip filter"),
|
|
("tags", set(['sensor', 'bluetooth']), "ignore_tags", ['bluetooth'], "Excluded tags per platform (exclude_tags)"),
|
|
("min_flash", "2024", "flash", "1024", "Not enough FLASH"),
|
|
("min_ram", "500", "ram", "256", "Not enough RAM"),
|
|
("None", "None", "env", ['BSIM_OUT_PATH', 'demo_env'], "Environment (BSIM_OUT_PATH, demo_env) not satisfied"),
|
|
("build_on_all", True, None, None, "Platform is excluded on command line."),
|
|
("build_on_all", True, "level", "foobar", "Unknown test level 'foobar'"),
|
|
(None, None, "supported_toolchains", ['gcc', 'xcc', 'xt-clang'], "Not supported by the toolchain"),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("tc_attribute, tc_value, plat_attribute, plat_value, expected_discards",
|
|
TESTDATA_PART1)
|
|
def test_apply_filters_part1(class_testplan, all_testsuites_dict, platforms_list,
|
|
tc_attribute, tc_value, plat_attribute, plat_value, expected_discards):
|
|
""" Testing apply_filters function of TestPlan class in Twister
|
|
Part 1: Response of apply_filters function have
|
|
appropriate values according to the filters
|
|
"""
|
|
plan = class_testplan
|
|
if tc_attribute is None and plat_attribute is None:
|
|
plan.apply_filters()
|
|
|
|
plan.platforms = platforms_list
|
|
plan.platform_names = [p.name for p in platforms_list]
|
|
plan.testsuites = all_testsuites_dict
|
|
for plat in plan.platforms:
|
|
if plat_attribute == "ignore_tags":
|
|
plat.ignore_tags = plat_value
|
|
if plat_attribute == "flash":
|
|
plat.flash = plat_value
|
|
if plat_attribute == "ram":
|
|
plat.ram = plat_value
|
|
if plat_attribute == "env":
|
|
plat.env = plat_value
|
|
plat.env_satisfied = False
|
|
if plat_attribute == "supported_toolchains":
|
|
plat.supported_toolchains = plat_value
|
|
for _, testcase in plan.testsuites.items():
|
|
if tc_attribute == "toolchain_allow":
|
|
testcase.toolchain_allow = tc_value
|
|
if tc_attribute == "platform_allow":
|
|
testcase.platform_allow = tc_value
|
|
if tc_attribute == "toolchain_exclude":
|
|
testcase.toolchain_exclude = tc_value
|
|
if tc_attribute == "platform_exclude":
|
|
testcase.platform_exclude = tc_value
|
|
if tc_attribute == "arch_exclude":
|
|
testcase.arch_exclude = tc_value
|
|
if tc_attribute == "arch_allow":
|
|
testcase.arch_allow = tc_value
|
|
if tc_attribute == "skip":
|
|
testcase.skip = tc_value
|
|
if tc_attribute == "tags":
|
|
testcase.tags = tc_value
|
|
if tc_attribute == "min_flash":
|
|
testcase.min_flash = tc_value
|
|
if tc_attribute == "min_ram":
|
|
testcase.min_ram = tc_value
|
|
|
|
if plat_attribute == "level":
|
|
plan.options.level = plat_value
|
|
|
|
if tc_attribute == "build_on_all":
|
|
for _, testcase in plan.testsuites.items():
|
|
testcase.build_on_all = tc_value
|
|
plan.apply_filters(exclude_platform=['demo_board_1'])
|
|
elif plat_attribute == "supported_toolchains":
|
|
plan.apply_filters(force_toolchain=False,
|
|
exclude_platform=['demo_board_1'],
|
|
platform=['demo_board_2'])
|
|
elif tc_attribute is None and plat_attribute is None:
|
|
plan.apply_filters()
|
|
else:
|
|
plan.apply_filters(exclude_platform=['demo_board_1'],
|
|
platform=['demo_board_2'])
|
|
|
|
filtered_instances = list(filter(lambda item: item.status == "filtered", plan.instances.values()))
|
|
for d in filtered_instances:
|
|
assert d.reason == expected_discards
|
|
|
|
TESTDATA_PART2 = [
|
|
("runnable", "True", "Not runnable on device"),
|
|
("exclude_tag", ['test_a'], "Command line testsuite exclude filter"),
|
|
("run_individual_tests", ['scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1'], "TestSuite name filter"),
|
|
("arch", ['arm_test'], "Command line testsuite arch filter"),
|
|
("tag", ['test_d'], "Command line testsuite tag filter")
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("extra_filter, extra_filter_value, expected_discards", TESTDATA_PART2)
|
|
def test_apply_filters_part2(class_testplan, all_testsuites_dict,
|
|
platforms_list, extra_filter, extra_filter_value, expected_discards):
|
|
""" Testing apply_filters function of TestPlan class in Twister
|
|
Part 2 : Response of apply_filters function (discard dictionary) have
|
|
appropriate values according to the filters
|
|
"""
|
|
|
|
class_testplan.platforms = platforms_list
|
|
class_testplan.platform_names = [p.name for p in platforms_list]
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
kwargs = {
|
|
extra_filter : extra_filter_value,
|
|
"exclude_platform" : [
|
|
'demo_board_1'
|
|
],
|
|
"platform" : [
|
|
'demo_board_2'
|
|
]
|
|
}
|
|
class_testplan.apply_filters(**kwargs)
|
|
filtered_instances = list(filter(lambda item: item.status == "filtered", class_testplan.instances.values()))
|
|
for d in filtered_instances:
|
|
assert d.reason == expected_discards
|
|
|
|
|
|
TESTDATA_PART3 = [
|
|
(20, 20, -1, 0),
|
|
(-2, -1, 10, 20),
|
|
(0, 0, 0, 0)
|
|
]
|
|
|
|
@pytest.mark.parametrize("tc_min_flash, plat_flash, tc_min_ram, plat_ram",
|
|
TESTDATA_PART3)
|
|
def test_apply_filters_part3(class_testplan, all_testsuites_dict, platforms_list,
|
|
tc_min_flash, plat_flash, tc_min_ram, plat_ram):
|
|
""" Testing apply_filters function of TestPlan class in Twister
|
|
Part 3 : Testing edge cases for ram and flash values of platforms & testsuites
|
|
"""
|
|
class_testplan.platforms = platforms_list
|
|
class_testplan.platform_names = [p.name for p in platforms_list]
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
|
|
for plat in class_testplan.platforms:
|
|
plat.flash = plat_flash
|
|
plat.ram = plat_ram
|
|
for _, testcase in class_testplan.testsuites.items():
|
|
testcase.min_ram = tc_min_ram
|
|
testcase.min_flash = tc_min_flash
|
|
class_testplan.apply_filters(exclude_platform=['demo_board_1'],
|
|
platform=['demo_board_2'])
|
|
|
|
filtered_instances = list(filter(lambda item: item.status == "filtered", class_testplan.instances.values()))
|
|
assert not filtered_instances
|
|
|
|
def test_add_instances_short(tmp_path, class_env, all_testsuites_dict, platforms_list):
|
|
""" Testing add_instances() function of TestPlan class in Twister
|
|
Test 1: instances dictionary keys have expected values (Platform Name + Testcase Name)
|
|
Test 2: Values of 'instances' dictionary in Testsuite class are an
|
|
instance of 'TestInstance' class
|
|
Test 3: Values of 'instances' dictionary have expected values.
|
|
"""
|
|
class_env.outdir = tmp_path
|
|
plan = TestPlan(class_env)
|
|
plan.platforms = platforms_list
|
|
platform = plan.get_platform("demo_board_2")
|
|
instance_list = []
|
|
for _, testcase in all_testsuites_dict.items():
|
|
instance = TestInstance(testcase, platform, class_env.outdir)
|
|
instance_list.append(instance)
|
|
plan.add_instances(instance_list)
|
|
assert list(plan.instances.keys()) == \
|
|
[platform.name + '/' + s for s in list(all_testsuites_dict.keys())]
|
|
assert all(isinstance(n, TestInstance) for n in list(plan.instances.values()))
|
|
assert list(plan.instances.values()) == instance_list
|
|
|
|
|
|
QUARANTINE_BASIC = {
|
|
'demo_board_1/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3'
|
|
}
|
|
|
|
QUARANTINE_WITH_REGEXP = {
|
|
'demo_board_2/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'a2 and c2 on x86',
|
|
'demo_board_1/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
|
'demo_board_2/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
|
|
'demo_board_2/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'a2 and c2 on x86'
|
|
}
|
|
|
|
QUARANTINE_PLATFORM = {
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_2' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_e/test_e.check_1' : 'all on board_3',
|
|
'demo_board_3/scripts/tests/twister/test_data/testsuites/tests/test_config/test_config.main' : 'all on board_3'
|
|
}
|
|
|
|
QUARANTINE_MULTIFILES = {
|
|
**QUARANTINE_BASIC,
|
|
**QUARANTINE_WITH_REGEXP
|
|
}
|
|
|
|
@pytest.mark.parametrize(
|
|
("quarantine_files, quarantine_verify, expected_val"),
|
|
[
|
|
(['basic.yaml'], False, QUARANTINE_BASIC),
|
|
(['with_regexp.yaml'], False, QUARANTINE_WITH_REGEXP),
|
|
(['with_regexp.yaml'], True, QUARANTINE_WITH_REGEXP),
|
|
(['platform.yaml'], False, QUARANTINE_PLATFORM),
|
|
(['basic.yaml', 'with_regexp.yaml'], False, QUARANTINE_MULTIFILES),
|
|
(['empty.yaml'], False, {})
|
|
],
|
|
ids=[
|
|
'basic',
|
|
'with_regexp',
|
|
'quarantine_verify',
|
|
'platform',
|
|
'multifiles',
|
|
'empty'
|
|
])
|
|
def test_quarantine_short(class_testplan, platforms_list, test_data,
|
|
quarantine_files, quarantine_verify, expected_val):
|
|
""" Testing quarantine feature in Twister
|
|
"""
|
|
class_testplan.options.all = True
|
|
class_testplan.platforms = platforms_list
|
|
class_testplan.platform_names = [p.name for p in platforms_list]
|
|
class_testplan.TESTSUITE_FILENAME = 'test_data.yaml'
|
|
class_testplan.add_testsuites()
|
|
|
|
quarantine_list = [
|
|
os.path.join(test_data, 'quarantines', quarantine_file) for quarantine_file in quarantine_files
|
|
]
|
|
class_testplan.quarantine = Quarantine(quarantine_list)
|
|
class_testplan.options.quarantine_verify = quarantine_verify
|
|
class_testplan.apply_filters()
|
|
|
|
for testname, instance in class_testplan.instances.items():
|
|
if quarantine_verify:
|
|
if testname in expected_val:
|
|
assert not instance.status
|
|
else:
|
|
assert instance.status == 'filtered'
|
|
assert instance.reason == "Not under quarantine"
|
|
else:
|
|
if testname in expected_val:
|
|
assert instance.status == 'filtered'
|
|
assert instance.reason == "Quarantine: " + expected_val[testname]
|
|
else:
|
|
assert not instance.status
|
|
|
|
|
|
TESTDATA_PART4 = [
|
|
(os.path.join('test_d', 'test_d.check_1'), ['dummy'],
|
|
None, 'Snippet not supported'),
|
|
(os.path.join('test_c', 'test_c.check_1'), ['cdc-acm-console'],
|
|
0, None),
|
|
(os.path.join('test_d', 'test_d.check_1'), ['dummy', 'cdc-acm-console'],
|
|
2, 'Snippet not supported'),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'testpath, required_snippets, expected_filtered_len, expected_filtered_reason',
|
|
TESTDATA_PART4,
|
|
ids=['app', 'global', 'multiple']
|
|
)
|
|
def test_required_snippets_short(
|
|
class_testplan,
|
|
all_testsuites_dict,
|
|
platforms_list,
|
|
testpath,
|
|
required_snippets,
|
|
expected_filtered_len,
|
|
expected_filtered_reason
|
|
):
|
|
""" Testing required_snippets function of TestPlan class in Twister """
|
|
plan = class_testplan
|
|
testpath = os.path.join('scripts', 'tests', 'twister', 'test_data',
|
|
'testsuites', 'tests', testpath)
|
|
testsuite = class_testplan.testsuites.get(testpath)
|
|
plan.platforms = platforms_list
|
|
plan.platform_names = [p.name for p in platforms_list]
|
|
plan.testsuites = {testpath: testsuite}
|
|
|
|
print(plan.testsuites)
|
|
|
|
for _, testcase in plan.testsuites.items():
|
|
testcase.exclude_platform = []
|
|
testcase.required_snippets = required_snippets
|
|
testcase.build_on_all = True
|
|
|
|
plan.apply_filters()
|
|
|
|
filtered_instances = list(
|
|
filter(lambda item: item.status == "filtered", plan.instances.values())
|
|
)
|
|
if expected_filtered_len is not None:
|
|
assert len(filtered_instances) == expected_filtered_len
|
|
if expected_filtered_reason is not None:
|
|
for d in filtered_instances:
|
|
assert d.reason == expected_filtered_reason
|
|
|
|
|
|
def test_testplan_get_level():
|
|
testplan = TestPlan(env=mock.Mock())
|
|
lvl1 = mock.Mock()
|
|
lvl1.name = 'a lvl'
|
|
lvl2 = mock.Mock()
|
|
lvl2.name = 'a lvl'
|
|
lvl3 = mock.Mock()
|
|
lvl3.name = 'other lvl'
|
|
testplan.levels.append(lvl1)
|
|
testplan.levels.append(lvl2)
|
|
testplan.levels.append(lvl3)
|
|
|
|
name = 'a lvl'
|
|
|
|
res = testplan.get_level(name)
|
|
assert res == lvl1
|
|
|
|
res = testplan.get_level(name)
|
|
assert res == lvl1
|
|
|
|
lvl_missed = mock.Mock()
|
|
lvl_missed.name = 'missed lvl'
|
|
res = testplan.get_level('missed_lvl')
|
|
assert res is None
|
|
|
|
testplan.levels.remove(lvl1)
|
|
testplan.levels.remove(lvl2)
|
|
|
|
res = testplan.get_level(name)
|
|
assert res is None
|
|
|
|
|
|
TESTDATA_1 = [
|
|
('', {}),
|
|
(
|
|
"""\
|
|
levels:
|
|
- name: lvl1
|
|
adds:
|
|
- sc1
|
|
- sc2
|
|
inherits: []
|
|
- name: lvl2
|
|
adds:
|
|
- sc1-1
|
|
- sc1-2
|
|
inherits: [lvl1]
|
|
""",
|
|
{
|
|
'lvl1': ['sc1', 'sc2'],
|
|
'lvl2': ['sc1-1', 'sc1-2', 'sc1', 'sc2']
|
|
}
|
|
),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'config_yaml, expected_scenarios',
|
|
TESTDATA_1,
|
|
ids=['no config', 'valid config']
|
|
)
|
|
def test_testplan_parse_configuration(tmp_path, config_yaml, expected_scenarios):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.scenarios = ['sc1', 'sc1-1', 'sc1-2', 'sc2']
|
|
|
|
tmp_config_file = tmp_path / 'config_file.yaml'
|
|
if config_yaml:
|
|
tmp_config_file.write_text(config_yaml)
|
|
|
|
with pytest.raises(TwisterRuntimeError) if not config_yaml else nullcontext():
|
|
testplan.parse_configuration(tmp_config_file)
|
|
|
|
if not testplan.levels:
|
|
assert expected_scenarios == {}
|
|
for level in testplan.levels:
|
|
assert sorted(level.scenarios) == sorted(expected_scenarios[level.name])
|
|
|
|
|
|
TESTDATA_2 = [
|
|
([], [], False),
|
|
(['ts1.tc3'], [], True),
|
|
(['ts2.tc2'], ['- ts2'], False),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'sub_tests, expected_outs, expect_error',
|
|
TESTDATA_2,
|
|
ids=['no subtests', 'subtests not found', 'valid subtests']
|
|
)
|
|
def test_testplan_find_subtests(
|
|
capfd,
|
|
sub_tests,
|
|
expected_outs,
|
|
expect_error
|
|
):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.options = mock.Mock(sub_test=sub_tests)
|
|
testplan.run_individual_testsuite = []
|
|
testplan.testsuites = {
|
|
'ts1': mock.Mock(
|
|
testcases=[
|
|
mock.Mock(),
|
|
mock.Mock(),
|
|
]
|
|
),
|
|
'ts2': mock.Mock(
|
|
testcases=[
|
|
mock.Mock(),
|
|
mock.Mock(),
|
|
mock.Mock(),
|
|
]
|
|
)
|
|
}
|
|
testplan.testsuites['ts1'].name = 'ts1'
|
|
testplan.testsuites['ts1'].testcases[0].name = 'ts1.tc1'
|
|
testplan.testsuites['ts1'].testcases[1].name = 'ts1.tc2'
|
|
testplan.testsuites['ts2'].name = 'ts2'
|
|
testplan.testsuites['ts2'].testcases[0].name = 'ts2.tc1'
|
|
testplan.testsuites['ts2'].testcases[1].name = 'ts2.tc2'
|
|
testplan.testsuites['ts2'].testcases[2].name = 'ts2.tc3'
|
|
|
|
with pytest.raises(TwisterRuntimeError) if expect_error else nullcontext():
|
|
testplan.find_subtests()
|
|
|
|
out, err = capfd.readouterr()
|
|
sys.stdout.write(out)
|
|
sys.stdout.write(err)
|
|
|
|
assert all([printout in out for printout in expected_outs])
|
|
|
|
|
|
TESTDATA_3 = [
|
|
(0, 0, [], False, [], TwisterRuntimeError, []),
|
|
(1, 1, [], False, [], TwisterRuntimeError, []),
|
|
(1, 0, [], True, [], TwisterRuntimeError, ['No quarantine list given to be verified']),
|
|
# (1, 0, ['qfile.yaml'], False, ['# empty'], None, ['Quarantine file qfile.yaml is empty']),
|
|
(1, 0, ['qfile.yaml'], False, ['- platforms:\n - demo_board_3\n comment: "board_3"'], None, []),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'added_testsuite_count, load_errors, ql, qv, ql_data, exception, expected_logs',
|
|
TESTDATA_3,
|
|
ids=['no tests', 'load errors', 'quarantine verify without quarantine list',
|
|
# 'empty quarantine file',
|
|
'valid quarantine file']
|
|
)
|
|
def test_testplan_discover(
|
|
tmp_path,
|
|
caplog,
|
|
added_testsuite_count,
|
|
load_errors,
|
|
ql,
|
|
qv,
|
|
ql_data,
|
|
exception,
|
|
expected_logs
|
|
):
|
|
for qf, data in zip(ql, ql_data):
|
|
tmp_qf = tmp_path / qf
|
|
tmp_qf.write_text(data)
|
|
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.options = mock.Mock(
|
|
test='ts1',
|
|
quarantine_list=[tmp_path / qf for qf in ql],
|
|
quarantine_verify=qv,
|
|
)
|
|
testplan.testsuites = {
|
|
'ts1': mock.Mock(id=1),
|
|
'ts2': mock.Mock(id=2),
|
|
}
|
|
testplan.run_individual_testsuite = 'ts0'
|
|
testplan.load_errors = load_errors
|
|
testplan.add_testsuites = mock.Mock(return_value=added_testsuite_count)
|
|
testplan.find_subtests = mock.Mock()
|
|
testplan.report_duplicates = mock.Mock()
|
|
testplan.parse_configuration = mock.Mock()
|
|
testplan.add_configurations = mock.Mock()
|
|
|
|
with pytest.raises(exception) if exception else nullcontext():
|
|
testplan.discover()
|
|
|
|
testplan.add_testsuites.assert_called_once_with(testsuite_filter='ts1')
|
|
assert all([log in caplog.text for log in expected_logs])
|
|
|
|
|
|
TESTDATA_4 = [
|
|
(None, None, None, None, '00',
|
|
TwisterRuntimeError, [], []),
|
|
(None, True, None, None, '6/4',
|
|
TwisterRuntimeError, set(['t-p3', 't-p4', 't-p1', 't-p2']), []),
|
|
(None, None, 'load_tests.json', None, '0/4',
|
|
TwisterRuntimeError, set(['lt-p1', 'lt-p3', 'lt-p4', 'lt-p2']), []),
|
|
('suffix', None, None, True, '2/4',
|
|
None, set(['ts-p4', 'ts-p2', 'ts-p3']), [2, 4]),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'report_suffix, only_failed, load_tests, test_only, subset,' \
|
|
' exception, expected_selected_platforms, expected_generate_subset_args',
|
|
TESTDATA_4,
|
|
ids=['apply_filters only', 'only failed', 'load tests', 'test only']
|
|
)
|
|
def test_testplan_load(
|
|
tmp_path,
|
|
report_suffix,
|
|
only_failed,
|
|
load_tests,
|
|
test_only,
|
|
subset,
|
|
exception,
|
|
expected_selected_platforms,
|
|
expected_generate_subset_args
|
|
):
|
|
twister_json = """\
|
|
{
|
|
"testsuites": [
|
|
{
|
|
"name": "ts1",
|
|
"platform": "t-p1",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts1",
|
|
"platform": "t-p2",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts2",
|
|
"platform": "t-p3",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts2",
|
|
"platform": "t-p4",
|
|
"testcases": []
|
|
}
|
|
]
|
|
}
|
|
"""
|
|
twister_file = tmp_path / 'twister.json'
|
|
twister_file.write_text(twister_json)
|
|
|
|
twister_suffix_json = """\
|
|
{
|
|
"testsuites": [
|
|
{
|
|
"name": "ts1",
|
|
"platform": "ts-p1",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts1",
|
|
"platform": "ts-p2",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts2",
|
|
"platform": "ts-p3",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts2",
|
|
"platform": "ts-p4",
|
|
"testcases": []
|
|
}
|
|
]
|
|
}
|
|
"""
|
|
twister_suffix_file = tmp_path / 'twister_suffix.json'
|
|
twister_suffix_file.write_text(twister_suffix_json)
|
|
|
|
load_tests_json = """\
|
|
{
|
|
"testsuites": [
|
|
{
|
|
"name": "ts1",
|
|
"platform": "lt-p1",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts1",
|
|
"platform": "lt-p2",
|
|
"testcases": []
|
|
},
|
|
{
|
|
"name": "ts2",
|
|
"platform": "lt-p3",
|
|
\"testcases": []
|
|
},
|
|
{
|
|
"name": "ts2",
|
|
"platform": "lt-p4",
|
|
"testcases": []
|
|
}
|
|
]
|
|
}
|
|
"""
|
|
load_tests_file = tmp_path / 'load_tests.json'
|
|
load_tests_file.write_text(load_tests_json)
|
|
|
|
testplan = TestPlan(env=mock.Mock(outdir=tmp_path))
|
|
testplan.testsuites = {
|
|
'ts1': mock.Mock(testcases=[], extra_configs=[]),
|
|
'ts2': mock.Mock(testcases=[], extra_configs=[]),
|
|
}
|
|
testplan.testsuites['ts1'].name = 'ts1'
|
|
testplan.testsuites['ts2'].name = 'ts2'
|
|
testplan.options = mock.Mock(
|
|
report_summary=None,
|
|
outdir=tmp_path,
|
|
report_suffix=report_suffix,
|
|
only_failed=only_failed,
|
|
load_tests=tmp_path / load_tests if load_tests else None,
|
|
test_only=test_only,
|
|
exclude_platform=['t-p0', 't-p1',
|
|
'ts-p0', 'ts-p1',
|
|
'lt-p0', 'lt-p1'],
|
|
platform=['t-p1', 't-p2', 't-p3', 't-p4',
|
|
'ts-p1', 'ts-p2', 'ts-p3', 'ts-p4',
|
|
'lt-p1', 'lt-p2', 'lt-p3', 'lt-p4'],
|
|
subset=subset
|
|
)
|
|
testplan.platforms=[mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(),
|
|
mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(),
|
|
mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock()]
|
|
testplan.platforms[0].name = 't-p1'
|
|
testplan.platforms[1].name = 't-p2'
|
|
testplan.platforms[2].name = 't-p3'
|
|
testplan.platforms[3].name = 't-p4'
|
|
testplan.platforms[4].name = 'ts-p1'
|
|
testplan.platforms[5].name = 'ts-p2'
|
|
testplan.platforms[6].name = 'ts-p3'
|
|
testplan.platforms[7].name = 'ts-p4'
|
|
testplan.platforms[8].name = 'lt-p1'
|
|
testplan.platforms[9].name = 'lt-p2'
|
|
testplan.platforms[10].name = 'lt-p3'
|
|
testplan.platforms[11].name = 'lt-p4'
|
|
testplan.platforms[0].normalized_name = 't-p1'
|
|
testplan.platforms[1].normalized_name = 't-p2'
|
|
testplan.platforms[2].normalized_name = 't-p3'
|
|
testplan.platforms[3].normalized_name = 't-p4'
|
|
testplan.platforms[4].normalized_name = 'ts-p1'
|
|
testplan.platforms[5].normalized_name = 'ts-p2'
|
|
testplan.platforms[6].normalized_name = 'ts-p3'
|
|
testplan.platforms[7].normalized_name = 'ts-p4'
|
|
testplan.platforms[8].normalized_name = 'lt-p1'
|
|
testplan.platforms[9].normalized_name = 'lt-p2'
|
|
testplan.platforms[10].normalized_name = 'lt-p3'
|
|
testplan.platforms[11].normalized_name = 'lt-p4'
|
|
testplan.generate_subset = mock.Mock()
|
|
testplan.apply_filters = mock.Mock()
|
|
|
|
with mock.patch('twisterlib.testinstance.TestInstance.create_overlay', mock.Mock()), \
|
|
pytest.raises(exception) if exception else nullcontext():
|
|
testplan.load()
|
|
|
|
assert testplan.selected_platforms == expected_selected_platforms
|
|
if expected_generate_subset_args:
|
|
testplan.generate_subset.assert_called_once_with(*expected_generate_subset_args)
|
|
else:
|
|
testplan.generate_subset.assert_not_called()
|
|
|
|
|
|
TESTDATA_5 = [
|
|
(False, False, None, 1, 2,
|
|
['plat1/testA', 'plat1/testB', 'plat1/testC',
|
|
'plat3/testA', 'plat3/testB', 'plat3/testC']),
|
|
(False, False, None, 1, 5,
|
|
['plat1/testA',
|
|
'plat3/testA', 'plat3/testB', 'plat3/testC']),
|
|
(False, False, None, 2, 2,
|
|
['plat2/testA', 'plat2/testB']),
|
|
(True, False, None, 1, 2,
|
|
['plat1/testA', 'plat2/testA', 'plat1/testB',
|
|
'plat3/testA', 'plat3/testB', 'plat3/testC']),
|
|
(True, False, None, 2, 2,
|
|
['plat2/testB', 'plat1/testC']),
|
|
(True, True, 123, 1, 2,
|
|
['plat2/testA', 'plat2/testB', 'plat1/testC',
|
|
'plat3/testB', 'plat3/testA', 'plat3/testC']),
|
|
(True, True, 123, 2, 2,
|
|
['plat1/testB', 'plat1/testA']),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'device_testing, shuffle, seed, subset, sets, expected_subset',
|
|
TESTDATA_5,
|
|
ids=['subset 1', 'subset 1 out of 5', 'subset 2',
|
|
'device testing, subset 1', 'device testing, subset 2',
|
|
'device testing, shuffle with seed, subset 1',
|
|
'device testing, shuffle with seed, subset 2']
|
|
)
|
|
def test_testplan_generate_subset(
|
|
device_testing,
|
|
shuffle,
|
|
seed,
|
|
subset,
|
|
sets,
|
|
expected_subset
|
|
):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.options = mock.Mock(
|
|
device_testing=device_testing,
|
|
shuffle_tests=shuffle,
|
|
shuffle_tests_seed=seed
|
|
)
|
|
testplan.instances = {
|
|
'plat1/testA': mock.Mock(status=None),
|
|
'plat1/testB': mock.Mock(status=None),
|
|
'plat1/testC': mock.Mock(status=None),
|
|
'plat2/testA': mock.Mock(status=None),
|
|
'plat2/testB': mock.Mock(status=None),
|
|
'plat3/testA': mock.Mock(status='skipped'),
|
|
'plat3/testB': mock.Mock(status='skipped'),
|
|
'plat3/testC': mock.Mock(status='error'),
|
|
}
|
|
|
|
testplan.generate_subset(subset, sets)
|
|
|
|
assert [instance for instance in testplan.instances.keys()] == \
|
|
expected_subset
|
|
|
|
|
|
def test_testplan_handle_modules():
|
|
testplan = TestPlan(env=mock.Mock())
|
|
|
|
modules = [mock.Mock(meta={'name': 'name1'}),
|
|
mock.Mock(meta={'name': 'name2'})]
|
|
|
|
with mock.patch('twisterlib.testplan.parse_modules', return_value=modules):
|
|
testplan.handle_modules()
|
|
|
|
assert testplan.modules == ['name1', 'name2']
|
|
|
|
|
|
TESTDATA_6 = [
|
|
(True, False, False, 0, 'report_test_tree'),
|
|
(True, True, False, 0, 'report_test_tree'),
|
|
(True, False, True, 0, 'report_test_tree'),
|
|
(True, True, True, 0, 'report_test_tree'),
|
|
(False, True, False, 0, 'report_test_list'),
|
|
(False, True, True, 0, 'report_test_list'),
|
|
(False, False, True, 0, 'report_tag_list'),
|
|
(False, False, False, 1, None),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'test_tree, list_tests, list_tags, expected_res, expected_method',
|
|
TESTDATA_6,
|
|
ids=['test tree', 'test tree + test list', 'test tree + tag list',
|
|
'test tree + test list + tag list', 'test list',
|
|
'test list + tag list', 'tag list', 'no report']
|
|
)
|
|
def test_testplan_report(
|
|
test_tree,
|
|
list_tests,
|
|
list_tags,
|
|
expected_res,
|
|
expected_method
|
|
):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.report_test_tree = mock.Mock()
|
|
testplan.report_test_list = mock.Mock()
|
|
testplan.report_tag_list = mock.Mock()
|
|
|
|
testplan.options = mock.Mock(
|
|
test_tree=test_tree,
|
|
list_tests=list_tests,
|
|
list_tags=list_tags,
|
|
)
|
|
|
|
res = testplan.report()
|
|
|
|
assert res == expected_res
|
|
|
|
methods = ['report_test_tree', 'report_test_list', 'report_tag_list']
|
|
if expected_method:
|
|
methods.remove(expected_method)
|
|
getattr(testplan, expected_method).assert_called_once()
|
|
for method in methods:
|
|
getattr(testplan, method).assert_not_called()
|
|
|
|
|
|
TESTDATA_7 = [
|
|
(
|
|
[
|
|
mock.Mock(
|
|
yamlfile='a.yaml',
|
|
scenarios=['scenario1', 'scenario2']
|
|
),
|
|
mock.Mock(
|
|
yamlfile='b.yaml',
|
|
scenarios=['scenario1']
|
|
)
|
|
],
|
|
TwisterRuntimeError,
|
|
'Duplicated test scenarios found:\n' \
|
|
'- scenario1 found in:\n' \
|
|
' - a.yaml\n' \
|
|
' - b.yaml\n',
|
|
[]
|
|
),
|
|
(
|
|
[
|
|
mock.Mock(
|
|
yamlfile='a.yaml',
|
|
scenarios=['scenario.a.1', 'scenario.a.2']
|
|
),
|
|
mock.Mock(
|
|
yamlfile='b.yaml',
|
|
scenarios=['scenario.b.1']
|
|
)
|
|
],
|
|
None,
|
|
None,
|
|
['No duplicates found.']
|
|
),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'testsuites, expected_error, error_msg, expected_logs',
|
|
TESTDATA_7,
|
|
ids=['a duplicate', 'no duplicates']
|
|
)
|
|
def test_testplan_report_duplicates(
|
|
capfd,
|
|
caplog,
|
|
testsuites,
|
|
expected_error,
|
|
error_msg,
|
|
expected_logs
|
|
):
|
|
def mock_get(name):
|
|
return list(filter(lambda x: name in x.scenarios, testsuites))
|
|
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.scenarios = [scenario for testsuite in testsuites \
|
|
for scenario in testsuite.scenarios]
|
|
testplan.get_testsuite = mock.Mock(side_effect=mock_get)
|
|
|
|
with pytest.raises(expected_error) if expected_error is not None else \
|
|
nullcontext() as err:
|
|
testplan.report_duplicates()
|
|
|
|
if expected_error:
|
|
assert str(err._excinfo[1]) == error_msg
|
|
|
|
assert all([log in caplog.text for log in expected_logs])
|
|
|
|
|
|
def test_testplan_report_tag_list(capfd):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.testsuites = {
|
|
'testsuite0': mock.Mock(tags=set(['tag1', 'tag2'])),
|
|
'testsuite1': mock.Mock(tags=set(['tag1', 'tag2', 'tag3'])),
|
|
'testsuite2': mock.Mock(tags=set(['tag1', 'tag3'])),
|
|
'testsuite3': mock.Mock(tags=set(['tag']))
|
|
}
|
|
|
|
testplan.report_tag_list()
|
|
|
|
out,err = capfd.readouterr()
|
|
sys.stdout.write(out)
|
|
sys.stderr.write(err)
|
|
|
|
assert '- tag' in out
|
|
assert '- tag1' in out
|
|
assert '- tag2' in out
|
|
assert '- tag3' in out
|
|
|
|
|
|
def test_testplan_report_test_tree(capfd):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.get_tests_list = mock.Mock(
|
|
return_value=['1.dummy.case.1', '1.dummy.case.2',
|
|
'2.dummy.case.1', '2.dummy.case.2',
|
|
'3.dummy.case.1', '3.dummy.case.2',
|
|
'4.dummy.case.1', '4.dummy.case.2',
|
|
'5.dummy.case.1', '5.dummy.case.2',
|
|
'sample.group1.case1', 'sample.group1.case2',
|
|
'sample.group2.case', 'sample.group3.case1',
|
|
'sample.group3.case2', 'sample.group3.case3']
|
|
)
|
|
|
|
testplan.report_test_tree()
|
|
|
|
out,err = capfd.readouterr()
|
|
sys.stdout.write(out)
|
|
sys.stderr.write(err)
|
|
|
|
expected = """
|
|
Testsuite
|
|
├── Samples
|
|
│ ├── group1
|
|
│ │ ├── sample.group1.case1
|
|
│ │ └── sample.group1.case2
|
|
│ ├── group2
|
|
│ │ └── sample.group2.case
|
|
│ └── group3
|
|
│ ├── sample.group3.case1
|
|
│ ├── sample.group3.case2
|
|
│ └── sample.group3.case3
|
|
└── Tests
|
|
├── 1
|
|
│ └── dummy
|
|
│ ├── 1.dummy.case.1
|
|
│ └── 1.dummy.case.2
|
|
├── 2
|
|
│ └── dummy
|
|
│ ├── 2.dummy.case.1
|
|
│ └── 2.dummy.case.2
|
|
├── 3
|
|
│ └── dummy
|
|
│ ├── 3.dummy.case.1
|
|
│ └── 3.dummy.case.2
|
|
├── 4
|
|
│ └── dummy
|
|
│ ├── 4.dummy.case.1
|
|
│ └── 4.dummy.case.2
|
|
└── 5
|
|
└── dummy
|
|
├── 5.dummy.case.1
|
|
└── 5.dummy.case.2
|
|
"""
|
|
expected = expected[1:]
|
|
|
|
assert expected in out
|
|
|
|
|
|
def test_testplan_report_test_list(capfd):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.get_tests_list = mock.Mock(
|
|
return_value=['4.dummy.case.1', '4.dummy.case.2',
|
|
'3.dummy.case.2', '2.dummy.case.2',
|
|
'1.dummy.case.1', '1.dummy.case.2',
|
|
'3.dummy.case.1', '2.dummy.case.1',
|
|
'5.dummy.case.1', '5.dummy.case.2']
|
|
)
|
|
|
|
testplan.report_test_list()
|
|
|
|
out,err = capfd.readouterr()
|
|
sys.stdout.write(out)
|
|
sys.stderr.write(err)
|
|
|
|
assert ' - 1.dummy.case.1\n' \
|
|
' - 1.dummy.case.2\n' \
|
|
' - 2.dummy.case.1\n' \
|
|
' - 2.dummy.case.2\n' \
|
|
' - 3.dummy.case.1\n' \
|
|
' - 3.dummy.case.2\n' \
|
|
' - 4.dummy.case.1\n' \
|
|
' - 4.dummy.case.2\n' \
|
|
' - 5.dummy.case.1\n' \
|
|
' - 5.dummy.case.2\n' \
|
|
'10 total.' in out
|
|
|
|
|
|
def test_testplan_info(capfd):
|
|
TestPlan.info('dummy text')
|
|
|
|
out, err = capfd.readouterr()
|
|
sys.stdout.write(out)
|
|
sys.stderr.write(err)
|
|
|
|
assert 'dummy text\n' in out
|
|
|
|
|
|
TESTDATA_8 = [
|
|
(False, False, ['p1e2', 'p2', 'p3', 'p3@B'], ['p2']),
|
|
(False, True, None, None),
|
|
(True, False, ['p1e2', 'p2', 'p3', 'p3@B'], ['p3']),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'override_default_platforms, create_duplicate, expected_platform_names, expected_defaults',
|
|
TESTDATA_8,
|
|
ids=['no override defaults', 'create duplicate', 'override defaults']
|
|
)
|
|
def test_testplan_add_configurations(
|
|
tmp_path,
|
|
override_default_platforms,
|
|
create_duplicate,
|
|
expected_platform_names,
|
|
expected_defaults
|
|
):
|
|
# tmp_path
|
|
# └ boards <- board root
|
|
# ├ x86
|
|
# │ ├ p1
|
|
# │ | ├ p1e1.yaml
|
|
# │ | └ p1e2.yaml
|
|
# │ └ p2
|
|
# │ ├ p2.yaml
|
|
# │ └ p2-1.yaml <- duplicate
|
|
# │ └ p2-2.yaml <- load error
|
|
# └ arm
|
|
# └ p3
|
|
# ├ p3.yaml
|
|
# └ p3_B.conf
|
|
|
|
tmp_board_root_dir = tmp_path / 'boards'
|
|
tmp_board_root_dir.mkdir()
|
|
|
|
tmp_arch1_dir = tmp_board_root_dir / 'x86'
|
|
tmp_arch1_dir.mkdir()
|
|
|
|
tmp_p1_dir = tmp_arch1_dir / 'p1'
|
|
tmp_p1_dir.mkdir()
|
|
|
|
p1e1_bs_yaml = """\
|
|
boards:
|
|
|
|
- name: ple1
|
|
vendor: zephyr
|
|
socs:
|
|
- name: unit_testing
|
|
- name: ple2
|
|
vendor: zephyr
|
|
socs:
|
|
- name: unit_testing
|
|
"""
|
|
p1e1_yamlfile = tmp_p1_dir / 'board.yml'
|
|
p1e1_yamlfile.write_text(p1e1_bs_yaml)
|
|
|
|
p1e1_yaml = """\
|
|
identifier: p1e1
|
|
name: Platform 1 Edition 1
|
|
type: native
|
|
arch: x86
|
|
vendor: vendor1
|
|
toolchain:
|
|
- zephyr
|
|
twister: False
|
|
"""
|
|
p1e1_yamlfile = tmp_p1_dir / 'p1e1.yaml'
|
|
p1e1_yamlfile.write_text(p1e1_yaml)
|
|
|
|
p1e2_yaml = """\
|
|
identifier: p1e2
|
|
name: Platform 1 Edition 2
|
|
type: native
|
|
arch: x86
|
|
vendor: vendor1
|
|
toolchain:
|
|
- zephyr
|
|
"""
|
|
p1e2_yamlfile = tmp_p1_dir / 'p1e2.yaml'
|
|
p1e2_yamlfile.write_text(p1e2_yaml)
|
|
|
|
tmp_p2_dir = tmp_arch1_dir / 'p2'
|
|
tmp_p2_dir.mkdir()
|
|
|
|
p2_bs_yaml = """\
|
|
boards:
|
|
|
|
- name: p2
|
|
vendor: zephyr
|
|
socs:
|
|
- name: unit_testing
|
|
- name: p2_2
|
|
vendor: zephyr
|
|
socs:
|
|
- name: unit_testing
|
|
"""
|
|
p2_yamlfile = tmp_p2_dir / 'board.yml'
|
|
p2_yamlfile.write_text(p2_bs_yaml)
|
|
|
|
p2_yaml = """\
|
|
identifier: p2
|
|
name: Platform 2
|
|
type: sim
|
|
arch: x86
|
|
vendor: vendor2
|
|
toolchain:
|
|
- zephyr
|
|
testing:
|
|
default: True
|
|
"""
|
|
p2_yamlfile = tmp_p2_dir / 'p2.yaml'
|
|
p2_yamlfile.write_text(p2_yaml)
|
|
|
|
if create_duplicate:
|
|
p2_yamlfile = tmp_p2_dir / 'p2-1.yaml'
|
|
p2_yamlfile.write_text(p2_yaml)
|
|
|
|
p2_2_yaml = """\
|
|
testing:
|
|
ć#@%!#!#^#@%@:1.0
|
|
identifier: p2_2
|
|
name: Platform 2 2
|
|
type: sim
|
|
arch: x86
|
|
vendor: vendor2
|
|
toolchain:
|
|
- zephyr
|
|
"""
|
|
p2_2_yamlfile = tmp_p2_dir / 'p2-2.yaml'
|
|
p2_2_yamlfile.write_text(p2_2_yaml)
|
|
|
|
tmp_arch2_dir = tmp_board_root_dir / 'arm'
|
|
tmp_arch2_dir.mkdir()
|
|
|
|
tmp_p3_dir = tmp_arch2_dir / 'p3'
|
|
tmp_p3_dir.mkdir()
|
|
|
|
p3_bs_yaml = """\
|
|
boards:
|
|
|
|
- name: p3
|
|
vendor: zephyr
|
|
socs:
|
|
- name: unit_testing
|
|
"""
|
|
p3_yamlfile = tmp_p3_dir / 'board.yml'
|
|
p3_yamlfile.write_text(p3_bs_yaml)
|
|
|
|
p3_yaml = """\
|
|
identifier: p3
|
|
name: Platform 3
|
|
type: unit
|
|
arch: arm
|
|
vendor: vendor3
|
|
toolchain:
|
|
- zephyr
|
|
"""
|
|
p3_yamlfile = tmp_p3_dir / 'p3.yaml'
|
|
p3_yamlfile.write_text(p3_yaml)
|
|
p3_yamlfile = tmp_p3_dir / 'p3_B.conf'
|
|
p3_yamlfile.write_text('')
|
|
|
|
env = mock.Mock(board_roots=[tmp_board_root_dir])
|
|
|
|
testplan = TestPlan(env=env)
|
|
|
|
testplan.test_config = {
|
|
'platforms': {
|
|
'override_default_platforms': override_default_platforms,
|
|
'default_platforms': ['p3', 'p1e1']
|
|
}
|
|
}
|
|
|
|
with pytest.raises(Exception) if create_duplicate else nullcontext():
|
|
testplan.add_configurations()
|
|
|
|
if expected_defaults is not None:
|
|
assert sorted(expected_defaults) == sorted(testplan.default_platforms)
|
|
if expected_platform_names is not None:
|
|
assert sorted(expected_platform_names) == sorted(testplan.platform_names)
|
|
|
|
|
|
def test_testplan_get_all_tests():
|
|
testplan = TestPlan(env=mock.Mock())
|
|
tc1 = mock.Mock()
|
|
tc1.name = 'tc1'
|
|
tc2 = mock.Mock()
|
|
tc2.name = 'tc2'
|
|
tc3 = mock.Mock()
|
|
tc3.name = 'tc3'
|
|
tc4 = mock.Mock()
|
|
tc4.name = 'tc4'
|
|
tc5 = mock.Mock()
|
|
tc5.name = 'tc5'
|
|
ts1 = mock.Mock(testcases=[tc1, tc2])
|
|
ts2 = mock.Mock(testcases=[tc3, tc4, tc5])
|
|
testplan.testsuites = {
|
|
'ts1': ts1,
|
|
'ts2': ts2
|
|
}
|
|
|
|
res = testplan.get_all_tests()
|
|
|
|
assert sorted(res) == ['tc1', 'tc2', 'tc3', 'tc4', 'tc5']
|
|
|
|
|
|
TESTDATA_9 = [
|
|
([], False, 7),
|
|
([], True, 5),
|
|
(['good_test/dummy.common.1', 'good_test/dummy.common.2', 'good_test/dummy.common.3'], False, 3),
|
|
(['good_test/dummy.common.1', 'good_test/dummy.common.2', 'good_test/dummy.common.3'], True, 0),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'testsuite_filter, use_alt_root, expected_suite_count',
|
|
TESTDATA_9,
|
|
ids=['no testsuite filter', 'no testsuite filter, alt root',
|
|
'testsuite filter', 'testsuite filter, alt root']
|
|
)
|
|
def test_testplan_add_testsuites(tmp_path, testsuite_filter, use_alt_root, expected_suite_count):
|
|
# tmp_path
|
|
# ├ tests <- test root
|
|
# │ ├ good_test
|
|
# │ │ └ testcase.yaml
|
|
# │ ├ wrong_test
|
|
# │ │ └ testcase.yaml
|
|
# │ ├ good_sample
|
|
# │ │ └ sample.yaml
|
|
# │ └ others
|
|
# │ └ other.txt
|
|
# └ other_tests <- alternate test root
|
|
# └ good_test
|
|
# └ testcase.yaml
|
|
tmp_test_root_dir = tmp_path / 'tests'
|
|
tmp_test_root_dir.mkdir()
|
|
|
|
tmp_good_test_dir = tmp_test_root_dir / 'good_test'
|
|
tmp_good_test_dir.mkdir()
|
|
testcase_yaml_1 = """\
|
|
tests:
|
|
dummy.common.1:
|
|
build_on_all: true
|
|
dummy.common.2:
|
|
build_on_all: true
|
|
dummy.common.3:
|
|
build_on_all: true
|
|
dummy.special:
|
|
build_on_all: false
|
|
"""
|
|
testfile_1 = tmp_good_test_dir / 'testcase.yaml'
|
|
testfile_1.write_text(testcase_yaml_1)
|
|
|
|
tmp_bad_test_dir = tmp_test_root_dir / 'wrong_test'
|
|
tmp_bad_test_dir.mkdir()
|
|
testcase_yaml_2 = """\
|
|
tests:
|
|
wrong:
|
|
yaml: {]}
|
|
"""
|
|
testfile_2 = tmp_bad_test_dir / 'testcase.yaml'
|
|
testfile_2.write_text(testcase_yaml_2)
|
|
|
|
tmp_good_sample_dir = tmp_test_root_dir / 'good_sample'
|
|
tmp_good_sample_dir.mkdir()
|
|
samplecase_yaml_1 = """\
|
|
tests:
|
|
sample.dummy.common.1:
|
|
tags:
|
|
- samples
|
|
sample.dummy.common.2:
|
|
tags:
|
|
- samples
|
|
sample.dummy.special.1:
|
|
tags:
|
|
- samples
|
|
"""
|
|
samplefile_1 = tmp_good_sample_dir / 'sample.yaml'
|
|
samplefile_1.write_text(samplecase_yaml_1)
|
|
|
|
tmp_other_dir = tmp_test_root_dir / 'others'
|
|
tmp_other_dir.mkdir()
|
|
_ = tmp_other_dir / 'other.txt'
|
|
|
|
tmp_alt_test_root_dir = tmp_path / 'other_tests'
|
|
tmp_alt_test_root_dir.mkdir()
|
|
|
|
tmp_alt_good_test_dir = tmp_alt_test_root_dir / 'good_test'
|
|
tmp_alt_good_test_dir.mkdir()
|
|
testcase_yaml_3 = """\
|
|
tests:
|
|
dummy.alt.1:
|
|
build_on_all: true
|
|
dummy.alt.2:
|
|
build_on_all: true
|
|
"""
|
|
testfile_3 = tmp_alt_good_test_dir / 'testcase.yaml'
|
|
testfile_3.write_text(testcase_yaml_3)
|
|
|
|
env = mock.Mock(
|
|
test_roots=[tmp_test_root_dir],
|
|
alt_config_root=[tmp_alt_test_root_dir] if use_alt_root else []
|
|
)
|
|
|
|
testplan = TestPlan(env=env)
|
|
|
|
res = testplan.add_testsuites(testsuite_filter)
|
|
|
|
assert res == expected_suite_count
|
|
|
|
|
|
def test_testplan_str():
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.name = 'my name'
|
|
|
|
res = testplan.__str__()
|
|
|
|
assert res == 'my name'
|
|
|
|
|
|
TESTDATA_10 = [
|
|
('a platform', True),
|
|
('other platform', False),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'name, expect_found',
|
|
TESTDATA_10,
|
|
ids=['platform exists', 'no platform']
|
|
)
|
|
def test_testplan_get_platform(name, expect_found):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
p1 = mock.Mock()
|
|
p1.name = 'some platform'
|
|
p2 = mock.Mock()
|
|
p2.name = 'a platform'
|
|
testplan.platforms = [p1, p2]
|
|
|
|
res = testplan.get_platform(name)
|
|
|
|
if expect_found:
|
|
assert res.name == name
|
|
else:
|
|
assert res is None
|
|
|
|
|
|
TESTDATA_11 = [
|
|
(True, 'runnable'),
|
|
(False, 'buildable'),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'device_testing, expected_tfilter',
|
|
TESTDATA_11,
|
|
ids=['device testing', 'no device testing']
|
|
)
|
|
def test_testplan_load_from_file(caplog, device_testing, expected_tfilter):
|
|
def get_platform(name):
|
|
p = mock.Mock()
|
|
p.name = name
|
|
p.normalized_name = name
|
|
return p
|
|
|
|
ts1tc1 = mock.Mock()
|
|
ts1tc1.name = 'TS1.tc1'
|
|
ts1 = mock.Mock(testcases=[ts1tc1])
|
|
ts1.name = 'TestSuite 1'
|
|
ts2 = mock.Mock(testcases=[])
|
|
ts2.name = 'TestSuite 2'
|
|
ts3tc1 = mock.Mock()
|
|
ts3tc1.name = 'TS3.tc1'
|
|
ts3tc2 = mock.Mock()
|
|
ts3tc2.name = 'TS3.tc2'
|
|
ts3 = mock.Mock(testcases=[ts3tc1, ts3tc2])
|
|
ts3.name = 'TestSuite 3'
|
|
ts4tc1 = mock.Mock()
|
|
ts4tc1.name = 'TS4.tc1'
|
|
ts4 = mock.Mock(testcases=[ts4tc1])
|
|
ts4.name = 'TestSuite 4'
|
|
ts5 = mock.Mock(testcases=[])
|
|
ts5.name = 'TestSuite 5'
|
|
|
|
testplan = TestPlan(env=mock.Mock(outdir=os.path.join('out', 'dir')))
|
|
testplan.options = mock.Mock(device_testing=device_testing, test_only=True, report_summary=None)
|
|
testplan.testsuites = {
|
|
'TestSuite 1': ts1,
|
|
'TestSuite 2': ts2,
|
|
'TestSuite 3': ts3,
|
|
'TestSuite 4': ts4,
|
|
'TestSuite 5': ts5
|
|
}
|
|
|
|
testplan.get_platform = mock.Mock(side_effect=get_platform)
|
|
|
|
testplan_data = """\
|
|
{
|
|
"testsuites": [
|
|
{
|
|
"name": "TestSuite 1",
|
|
"platform": "Platform 1",
|
|
"run_id": 1,
|
|
"execution_time": 60.00,
|
|
"used_ram": 4096,
|
|
"available_ram": 12278,
|
|
"used_rom": 1024,
|
|
"available_rom": 1047552,
|
|
"status": "passed",
|
|
"reason": "OK",
|
|
"testcases": [
|
|
{
|
|
"identifier": "TS1.tc1",
|
|
"status": "passed",
|
|
"reason": "passed",
|
|
"execution_time": 60.00,
|
|
"log": ""
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "TestSuite 2",
|
|
"platform": "Platform 1"
|
|
},
|
|
{
|
|
"name": "TestSuite 3",
|
|
"platform": "Platform 1",
|
|
"run_id": 1,
|
|
"execution_time": 360.00,
|
|
"used_ram": 4096,
|
|
"available_ram": 12278,
|
|
"used_rom": 1024,
|
|
"available_rom": 1047552,
|
|
"status": "error",
|
|
"reason": "File Not Found Error",
|
|
"testcases": [
|
|
{
|
|
"identifier": "TS3.tc1",
|
|
"status": "error",
|
|
"reason": "File Not Found Error.",
|
|
"execution_time": 360.00,
|
|
"log": "[ERROR]: File 'dummy.yaml' not found!\\nClosing..."
|
|
},
|
|
{
|
|
"identifier": "TS3.tc2"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "TestSuite 4",
|
|
"platform": "Platform 1",
|
|
"execution_time": 360.00,
|
|
"used_ram": 4096,
|
|
"available_ram": 12278,
|
|
"used_rom": 1024,
|
|
"available_rom": 1047552,
|
|
"status": "skipped",
|
|
"reason": "Not in requested test list.",
|
|
"testcases": [
|
|
{
|
|
"identifier": "TS4.tc1",
|
|
"status": "skipped",
|
|
"reason": "Not in requested test list.",
|
|
"execution_time": 360.00,
|
|
"log": "[INFO] Parsing..."
|
|
},
|
|
{
|
|
"identifier": "TS3.tc2"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "TestSuite 5",
|
|
"platform": "Platform 2"
|
|
}
|
|
]
|
|
}
|
|
"""
|
|
|
|
filter_platform = ['Platform 1']
|
|
|
|
check_runnable_mock = mock.Mock(return_value=True)
|
|
|
|
with mock.patch('builtins.open', mock.mock_open(read_data=testplan_data)), \
|
|
mock.patch('twisterlib.testinstance.TestInstance.check_runnable', check_runnable_mock), \
|
|
mock.patch('twisterlib.testinstance.TestInstance.create_overlay', mock.Mock()):
|
|
testplan.load_from_file('dummy.yaml', filter_platform)
|
|
|
|
expected_instances = {
|
|
'Platform 1/TestSuite 1': {
|
|
'metrics': {
|
|
'handler_time': 60.0,
|
|
'used_ram': 4096,
|
|
'used_rom': 1024,
|
|
'available_ram': 12278,
|
|
'available_rom': 1047552
|
|
},
|
|
'retries': 0,
|
|
'testcases': {
|
|
'TS1.tc1': {
|
|
'status': 'passed',
|
|
'reason': None,
|
|
'duration': 60.0,
|
|
'output': ''
|
|
}
|
|
}
|
|
},
|
|
'Platform 1/TestSuite 2': {
|
|
'metrics': {
|
|
'handler_time': 0,
|
|
'used_ram': 0,
|
|
'used_rom': 0,
|
|
'available_ram': 0,
|
|
'available_rom': 0
|
|
},
|
|
'retries': 0,
|
|
'testcases': []
|
|
},
|
|
'Platform 1/TestSuite 3': {
|
|
'metrics': {
|
|
'handler_time': 360.0,
|
|
'used_ram': 4096,
|
|
'used_rom': 1024,
|
|
'available_ram': 12278,
|
|
'available_rom': 1047552
|
|
},
|
|
'retries': 1,
|
|
'testcases': {
|
|
'TS3.tc1': {
|
|
'status': 'error',
|
|
'reason': None,
|
|
'duration': 360.0,
|
|
'output': '[ERROR]: File \'dummy.yaml\' not found!\nClosing...'
|
|
},
|
|
'TS3.tc2': {
|
|
'status': None,
|
|
'reason': None,
|
|
'duration': 0,
|
|
'output': ''
|
|
}
|
|
}
|
|
},
|
|
'Platform 1/TestSuite 4': {
|
|
'metrics': {
|
|
'handler_time': 360.0,
|
|
'used_ram': 4096,
|
|
'used_rom': 1024,
|
|
'available_ram': 12278,
|
|
'available_rom': 1047552
|
|
},
|
|
'retries': 0,
|
|
'testcases': {
|
|
'TS4.tc1': {
|
|
'status': 'skipped',
|
|
'reason': 'Not in requested test list.',
|
|
'duration': 360.0,
|
|
'output': '[INFO] Parsing...'
|
|
}
|
|
}
|
|
},
|
|
}
|
|
|
|
for n, i in testplan.instances.items():
|
|
assert expected_instances[n]['metrics'] == i.metrics
|
|
assert expected_instances[n]['retries'] == i.retries
|
|
for t in i.testcases:
|
|
assert expected_instances[n]['testcases'][str(t)]['status'] == t.status
|
|
assert expected_instances[n]['testcases'][str(t)]['reason'] == t.reason
|
|
assert expected_instances[n]['testcases'][str(t)]['duration'] == t.duration
|
|
assert expected_instances[n]['testcases'][str(t)]['output'] == t.output
|
|
|
|
check_runnable_mock.assert_called_with(mock.ANY, expected_tfilter, mock.ANY, mock.ANY)
|
|
|
|
expected_logs = [
|
|
'loading TestSuite 1...',
|
|
'loading TestSuite 2...',
|
|
'loading TestSuite 3...',
|
|
'loading TestSuite 4...',
|
|
]
|
|
assert all([log in caplog.text for log in expected_logs])
|
|
|
|
|
|
def test_testplan_add_instances():
|
|
testplan = TestPlan(env=mock.Mock())
|
|
instance1 = mock.Mock()
|
|
instance1.name = 'instance 1'
|
|
instance2 = mock.Mock()
|
|
instance2.name = 'instance 2'
|
|
instance_list = [instance1, instance2]
|
|
|
|
testplan.add_instances(instance_list)
|
|
|
|
assert testplan.instances == {
|
|
'instance 1': instance1,
|
|
'instance 2': instance2,
|
|
}
|
|
|
|
|
|
def test_testplan_get_testsuite():
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.testsuites = {
|
|
'testsuite0': mock.Mock(testcases=[mock.Mock(), mock.Mock()]),
|
|
'testsuite1': mock.Mock(testcases=[mock.Mock()]),
|
|
'testsuite2': mock.Mock(testcases=[mock.Mock(), mock.Mock()]),
|
|
'testsuite3': mock.Mock(testcases=[])
|
|
}
|
|
testplan.testsuites['testsuite0'].testcases[0].name = 'testcase name 0'
|
|
testplan.testsuites['testsuite0'].testcases[1].name = 'testcase name 1'
|
|
testplan.testsuites['testsuite1'].testcases[0].name = 'sample id'
|
|
testplan.testsuites['testsuite2'].testcases[0].name = 'dummy id'
|
|
testplan.testsuites['testsuite2'].testcases[1].name = 'sample id'
|
|
|
|
id = 'sample id'
|
|
|
|
res = testplan.get_testsuite(id)
|
|
|
|
assert len(res) == 2
|
|
assert testplan.testsuites['testsuite1'] in res
|
|
assert testplan.testsuites['testsuite2'] in res
|
|
|
|
|
|
def test_testplan_verify_platforms_existence(caplog):
|
|
testplan = TestPlan(env=mock.Mock())
|
|
testplan.platform_names = ['a platform', 'other platform']
|
|
|
|
platform_names = ['other platform', 'some platform']
|
|
log_info = 'PLATFORM ERROR'
|
|
|
|
with pytest.raises(SystemExit) as se:
|
|
testplan.verify_platforms_existence(platform_names, log_info)
|
|
|
|
assert str(se.value) == '2'
|
|
assert 'PLATFORM ERROR - unrecognized platform - some platform'
|
|
|
|
|
|
TESTDATA_12 = [
|
|
(True),
|
|
(False)
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'exists',
|
|
TESTDATA_12,
|
|
ids=['links dir exists', 'links dir does not exist']
|
|
)
|
|
def test_testplan_create_build_dir_links(exists):
|
|
outdir = os.path.join('out', 'dir')
|
|
instances_linked = []
|
|
|
|
def mock_link(links_dir_path, instance):
|
|
assert links_dir_path == os.path.join(outdir, 'twister_links')
|
|
instances_linked.append(instance)
|
|
|
|
instances = {
|
|
'inst0': mock.Mock(status='passed'),
|
|
'inst1': mock.Mock(status='skipped'),
|
|
'inst2': mock.Mock(status='error'),
|
|
}
|
|
expected_instances = [instances['inst0'], instances['inst2']]
|
|
|
|
testplan = TestPlan(env=mock.Mock(outdir=outdir))
|
|
testplan._create_build_dir_link = mock.Mock(side_effect=mock_link)
|
|
testplan.instances = instances
|
|
|
|
with mock.patch('os.path.exists', return_value=exists), \
|
|
mock.patch('os.mkdir', mock.Mock()) as mkdir_mock:
|
|
testplan.create_build_dir_links()
|
|
|
|
if not exists:
|
|
mkdir_mock.assert_called_once()
|
|
|
|
assert expected_instances == instances_linked
|
|
|
|
|
|
TESTDATA_13 = [
|
|
('nt'),
|
|
('Linux')
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'os_name',
|
|
TESTDATA_13,
|
|
)
|
|
def test_testplan_create_build_dir_link(os_name):
|
|
def mock_makedirs(path, exist_ok=False):
|
|
assert exist_ok
|
|
assert path == instance_build_dir
|
|
|
|
def mock_symlink(source, target):
|
|
assert source == instance_build_dir
|
|
assert target == os.path.join('links', 'path', 'test_0')
|
|
|
|
def mock_call(cmd, shell=False):
|
|
assert shell
|
|
assert cmd == ['mklink', '/J', os.path.join('links', 'path', 'test_0'),
|
|
instance_build_dir]
|
|
|
|
def mock_join(*paths):
|
|
slash = "\\" if os.name == 'nt' else "/"
|
|
return slash.join(paths)
|
|
|
|
with mock.patch('os.name', os_name), \
|
|
mock.patch('os.symlink', side_effect=mock_symlink), \
|
|
mock.patch('os.makedirs', side_effect=mock_makedirs), \
|
|
mock.patch('subprocess.call', side_effect=mock_call), \
|
|
mock.patch('os.path.join', side_effect=mock_join):
|
|
|
|
testplan = TestPlan(env=mock.Mock())
|
|
links_dir_path = os.path.join('links', 'path')
|
|
instance_build_dir = os.path.join('some', 'far', 'off', 'build', 'dir')
|
|
instance = mock.Mock(build_dir=instance_build_dir)
|
|
testplan._create_build_dir_link(links_dir_path, instance)
|
|
|
|
assert instance.build_dir == os.path.join('links', 'path', 'test_0')
|
|
assert testplan.link_dir_counter == 1
|
|
|
|
|
|
TESTDATA_14 = [
|
|
('bad platform', 'dummy reason', [],
|
|
'dummy status', 'dummy reason'),
|
|
('good platform', 'quarantined', [],
|
|
'error', 'quarantined but is one of the integration platforms'),
|
|
('good platform', 'dummy reason', [{'type': 'command line filter'}],
|
|
'dummy status', 'dummy reason'),
|
|
('good platform', 'dummy reason', [{'type': 'Skip filter'}],
|
|
'dummy status', 'dummy reason'),
|
|
('good platform', 'dummy reason', [{'type': 'platform key filter'}],
|
|
'dummy status', 'dummy reason'),
|
|
('good platform', 'dummy reason', [{'type': 'Toolchain filter'}],
|
|
'dummy status', 'dummy reason'),
|
|
('good platform', 'dummy reason', [{'type': 'Module filter'}],
|
|
'dummy status', 'dummy reason'),
|
|
('good platform', 'dummy reason', [{'type': 'testsuite filter'}],
|
|
'error', 'dummy reason but is one of the integration platforms'),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'platform_name, reason, filters,' \
|
|
' expected_status, expected_reason',
|
|
TESTDATA_14,
|
|
ids=['wrong platform', 'quarantined', 'command line filtered',
|
|
'skip filtered', 'platform key filtered', 'toolchain filtered',
|
|
'module filtered', 'skip to error change']
|
|
)
|
|
def test_change_skip_to_error_if_integration(
|
|
platform_name,
|
|
reason,
|
|
filters,
|
|
expected_status,
|
|
expected_reason
|
|
):
|
|
options = mock.Mock()
|
|
platform = mock.Mock()
|
|
platform.name = platform_name
|
|
testsuite = mock.Mock(integration_platforms=['good platform', 'a platform'])
|
|
instance = mock.Mock(
|
|
testsuite=testsuite,
|
|
platform=platform,
|
|
filters=filters,
|
|
status='dummy status',
|
|
reason=reason
|
|
)
|
|
|
|
change_skip_to_error_if_integration(options, instance)
|
|
|
|
assert instance.status == expected_status
|
|
assert instance.reason == expected_reason
|