twister: Add options deciding if paths be included in tests' names

Test suites names are not being handled uniformly for tests not
in zephyr tree. Their names depend on -T arg used in twister's CLI.
The newly added options allow to select if twister should add paths
to suite names. This is needed if test plans are to be used for tests
outside of zephyr tree.

Signed-off-by: Maciej Perkowski <Maciej.Perkowski@nordicsemi.no>
This commit is contained in:
Maciej Perkowski 2023-07-21 16:09:39 +02:00 committed by Carles Cufí
commit 066cc2c9d2
7 changed files with 81 additions and 10 deletions

View file

@ -442,6 +442,21 @@ structure in the main Zephyr tree: boards/<arch>/<board_name>/""")
help="Re-use the outdir before building. Will result in "
"faster compilation since builds will be incremental.")
parser.add_argument(
'--detailed-test-id', action='store_true',
help="Include paths to tests' locations in tests' names. Names will follow "
"PATH_TO_TEST/SCENARIO_NAME schema "
"e.g. samples/hello_world/sample.basic.helloworld")
parser.add_argument(
"--no-detailed-test-id", dest='detailed_test_id', action="store_false",
help="Don't put paths into tests' names. "
"With this arg a test name will be a scenario name "
"e.g. sample.basic.helloworld.")
# Include paths in names by default.
parser.set_defaults(detailed_test_id=True)
# To be removed in favor of --detailed-skipped-report
parser.add_argument(
"--no-skipped-report", action="store_true",

View file

@ -420,6 +420,7 @@ class Reporting:
def synopsis(self):
cnt = 0
example_instance = None
detailed_test_id = self.env.options.detailed_test_id
for instance in self.instances.values():
if instance.status not in ["passed", "filtered", "skipped"]:
cnt = cnt + 1
@ -435,11 +436,14 @@ class Reporting:
if cnt and example_instance:
logger.info("")
logger.info("To rerun the tests, call twister using the following commandline:")
logger.info("west twister -p <PLATFORM> -s <TEST ID>, for example:")
extra_parameters = '' if detailed_test_id else ' --no-detailed-test-id'
logger.info(f"west twister -p <PLATFORM> -s <TEST ID>{extra_parameters}, for example:")
logger.info("")
logger.info(f"west twister -p {example_instance.platform.name} -s {example_instance.testsuite.name}")
logger.info(f"west twister -p {example_instance.platform.name} -s {example_instance.testsuite.name}"
f"{extra_parameters}")
logger.info(f"or with west:")
logger.info(f"west build -p -b {example_instance.platform.name} -T {example_instance.testsuite.name}")
logger.info(f"west build -p -b {example_instance.platform.name} "
f"{example_instance.testsuite.source_dir_rel} -T {example_instance.testsuite.id}")
logger.info("-+" * 40)
def summary(self, results, unrecognized_sections, duration):

View file

@ -55,7 +55,12 @@ class TestInstance:
self.name = os.path.join(platform.name, testsuite.name)
self.run_id = self._get_run_id()
self.dut = None
self.build_dir = os.path.join(outdir, platform.name, testsuite.name)
if testsuite.detailed_test_id:
self.build_dir = os.path.join(outdir, platform.name, testsuite.name)
else:
# if suite is not in zephyr, keep only the part after ".." in reconstructed dir structure
source_dir_rel = testsuite.source_dir_rel.rsplit(os.pardir+os.path.sep, 1)[-1]
self.build_dir = os.path.join(outdir, platform.name, source_dir_rel, testsuite.name)
self.domains = None

View file

@ -530,7 +530,7 @@ class TestPlan:
for name in parsed_data.scenarios.keys():
suite_dict = parsed_data.get_scenario(name)
suite = TestSuite(root, suite_path, name, data=suite_dict)
suite = TestSuite(root, suite_path, name, data=suite_dict, detailed_test_id=self.options.detailed_test_id)
suite.add_subcases(suite_dict, subcases, ztest_suite_names)
if testsuite_filter:
if suite.name and suite.name in testsuite_filter:

View file

@ -370,7 +370,7 @@ class TestSuite(DisablePyTestCollectionMixin):
"""Class representing a test application
"""
def __init__(self, suite_root, suite_path, name, data=None):
def __init__(self, suite_root, suite_path, name, data=None, detailed_test_id=True):
"""TestSuite constructor.
This gets called by TestPlan as it finds and reads test yaml files.
@ -391,12 +391,14 @@ class TestSuite(DisablePyTestCollectionMixin):
"""
workdir = os.path.relpath(suite_path, suite_root)
self.name = self.get_unique(suite_root, workdir, name)
assert self.check_suite_name(name, suite_root, workdir)
self.detailed_test_id = detailed_test_id
self.name = self.get_unique(suite_root, workdir, name) if self.detailed_test_id else name
self.id = name
self.source_dir = suite_path
self.source_dir_rel = os.path.relpath(os.path.realpath(suite_path),
start=canonical_zephyr_base)
self.source_dir_rel = os.path.relpath(os.path.realpath(suite_path), start=canonical_zephyr_base)
self.yamlfile = suite_path
self.testcases = []
@ -449,10 +451,14 @@ class TestSuite(DisablePyTestCollectionMixin):
# workdir can be "."
unique = os.path.normpath(os.path.join(relative_ts_root, workdir, name))
return unique
@staticmethod
def check_suite_name(name, testsuite_root, workdir):
check = name.split(".")
if len(check) < 2:
raise TwisterException(f"""bad test name '{name}' in {testsuite_root}/{workdir}. \
Tests should reference the category and subsystem with a dot as a separator.
"""
)
return unique
return True

View file

@ -40,6 +40,7 @@ def gtest():
mock_platform.name = "mock_platform"
mock_testsuite = mock.Mock()
mock_testsuite.name = "mock_testsuite"
mock_testsuite.detailed_test_id = True
mock_testsuite.id = "id"
mock_testsuite.testcases = []
instance = TestInstance(testsuite=mock_testsuite, platform=mock_platform, outdir="")

View file

@ -846,3 +846,43 @@ def test_testcase_dunders():
assert case_lesser < case_greater
assert str(case_greater) == 'a greater name'
assert repr(case_greater) == '<TestCase a greater name with success>'
TESTDATA_11 = [
(
ZEPHYR_BASE + '/scripts/tests/twister/test_data/testsuites',
ZEPHYR_BASE + '/scripts/tests/twister/test_data/testsuites/tests/test_a',
'test_a.check_1',
'test_a.check_1'
),
(
ZEPHYR_BASE,
ZEPHYR_BASE,
'test_a.check_1',
'test_a.check_1'
),
(
ZEPHYR_BASE,
ZEPHYR_BASE + '/scripts/tests/twister/test_data/testsuites/test_b',
'test_b.check_1',
'test_b.check_1'
),
(
os.path.join(ZEPHYR_BASE, 'scripts/tests'),
os.path.join(ZEPHYR_BASE, 'scripts/tests'),
'test_b.check_1',
'test_b.check_1'
),
(
ZEPHYR_BASE,
ZEPHYR_BASE,
'test_a.check_1.check_2',
'test_a.check_1.check_2'
),
]
@pytest.mark.parametrize("testsuite_root, suite_path, name, expected", TESTDATA_11)
def test_get_no_detailed_test_id(testsuite_root, suite_path, name, expected):
'''Test to check if the name without path is given for each testsuite'''
suite = TestSuite(testsuite_root, suite_path, name, detailed_test_id=False)
print(suite.name)
assert suite.name == expected