From 066cc2c9d2ee86c35cb10e38bc7ba152c1560602 Mon Sep 17 00:00:00 2001 From: Maciej Perkowski Date: Fri, 21 Jul 2023 16:09:39 +0200 Subject: [PATCH] twister: Add options deciding if paths be included in tests' names Test suites names are not being handled uniformly for tests not in zephyr tree. Their names depend on -T arg used in twister's CLI. The newly added options allow to select if twister should add paths to suite names. This is needed if test plans are to be used for tests outside of zephyr tree. Signed-off-by: Maciej Perkowski --- .../pylib/twister/twisterlib/environment.py | 15 +++++++ scripts/pylib/twister/twisterlib/reports.py | 10 +++-- .../pylib/twister/twisterlib/testinstance.py | 7 +++- scripts/pylib/twister/twisterlib/testplan.py | 2 +- scripts/pylib/twister/twisterlib/testsuite.py | 16 +++++--- scripts/tests/twister/test_harness.py | 1 + scripts/tests/twister/test_testsuite.py | 40 +++++++++++++++++++ 7 files changed, 81 insertions(+), 10 deletions(-) diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index ecb5232364b..9c676ceb733 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -442,6 +442,21 @@ structure in the main Zephyr tree: boards///""") help="Re-use the outdir before building. Will result in " "faster compilation since builds will be incremental.") + parser.add_argument( + '--detailed-test-id', action='store_true', + help="Include paths to tests' locations in tests' names. Names will follow " + "PATH_TO_TEST/SCENARIO_NAME schema " + "e.g. samples/hello_world/sample.basic.helloworld") + + parser.add_argument( + "--no-detailed-test-id", dest='detailed_test_id', action="store_false", + help="Don't put paths into tests' names. " + "With this arg a test name will be a scenario name " + "e.g. sample.basic.helloworld.") + + # Include paths in names by default. + parser.set_defaults(detailed_test_id=True) + # To be removed in favor of --detailed-skipped-report parser.add_argument( "--no-skipped-report", action="store_true", diff --git a/scripts/pylib/twister/twisterlib/reports.py b/scripts/pylib/twister/twisterlib/reports.py index 0b34a5d309b..c1f160caad9 100644 --- a/scripts/pylib/twister/twisterlib/reports.py +++ b/scripts/pylib/twister/twisterlib/reports.py @@ -420,6 +420,7 @@ class Reporting: def synopsis(self): cnt = 0 example_instance = None + detailed_test_id = self.env.options.detailed_test_id for instance in self.instances.values(): if instance.status not in ["passed", "filtered", "skipped"]: cnt = cnt + 1 @@ -435,11 +436,14 @@ class Reporting: if cnt and example_instance: logger.info("") logger.info("To rerun the tests, call twister using the following commandline:") - logger.info("west twister -p -s , for example:") + extra_parameters = '' if detailed_test_id else ' --no-detailed-test-id' + logger.info(f"west twister -p -s {extra_parameters}, for example:") logger.info("") - logger.info(f"west twister -p {example_instance.platform.name} -s {example_instance.testsuite.name}") + logger.info(f"west twister -p {example_instance.platform.name} -s {example_instance.testsuite.name}" + f"{extra_parameters}") logger.info(f"or with west:") - logger.info(f"west build -p -b {example_instance.platform.name} -T {example_instance.testsuite.name}") + logger.info(f"west build -p -b {example_instance.platform.name} " + f"{example_instance.testsuite.source_dir_rel} -T {example_instance.testsuite.id}") logger.info("-+" * 40) def summary(self, results, unrecognized_sections, duration): diff --git a/scripts/pylib/twister/twisterlib/testinstance.py b/scripts/pylib/twister/twisterlib/testinstance.py index 666a8d68d03..958019b411a 100644 --- a/scripts/pylib/twister/twisterlib/testinstance.py +++ b/scripts/pylib/twister/twisterlib/testinstance.py @@ -55,7 +55,12 @@ class TestInstance: self.name = os.path.join(platform.name, testsuite.name) self.run_id = self._get_run_id() self.dut = None - self.build_dir = os.path.join(outdir, platform.name, testsuite.name) + if testsuite.detailed_test_id: + self.build_dir = os.path.join(outdir, platform.name, testsuite.name) + else: + # if suite is not in zephyr, keep only the part after ".." in reconstructed dir structure + source_dir_rel = testsuite.source_dir_rel.rsplit(os.pardir+os.path.sep, 1)[-1] + self.build_dir = os.path.join(outdir, platform.name, source_dir_rel, testsuite.name) self.domains = None diff --git a/scripts/pylib/twister/twisterlib/testplan.py b/scripts/pylib/twister/twisterlib/testplan.py index 6275785bdcd..905da2bc42e 100755 --- a/scripts/pylib/twister/twisterlib/testplan.py +++ b/scripts/pylib/twister/twisterlib/testplan.py @@ -530,7 +530,7 @@ class TestPlan: for name in parsed_data.scenarios.keys(): suite_dict = parsed_data.get_scenario(name) - suite = TestSuite(root, suite_path, name, data=suite_dict) + suite = TestSuite(root, suite_path, name, data=suite_dict, detailed_test_id=self.options.detailed_test_id) suite.add_subcases(suite_dict, subcases, ztest_suite_names) if testsuite_filter: if suite.name and suite.name in testsuite_filter: diff --git a/scripts/pylib/twister/twisterlib/testsuite.py b/scripts/pylib/twister/twisterlib/testsuite.py index 71dd9b493c9..ecb7c8b4000 100644 --- a/scripts/pylib/twister/twisterlib/testsuite.py +++ b/scripts/pylib/twister/twisterlib/testsuite.py @@ -370,7 +370,7 @@ class TestSuite(DisablePyTestCollectionMixin): """Class representing a test application """ - def __init__(self, suite_root, suite_path, name, data=None): + def __init__(self, suite_root, suite_path, name, data=None, detailed_test_id=True): """TestSuite constructor. This gets called by TestPlan as it finds and reads test yaml files. @@ -391,12 +391,14 @@ class TestSuite(DisablePyTestCollectionMixin): """ workdir = os.path.relpath(suite_path, suite_root) - self.name = self.get_unique(suite_root, workdir, name) + + assert self.check_suite_name(name, suite_root, workdir) + self.detailed_test_id = detailed_test_id + self.name = self.get_unique(suite_root, workdir, name) if self.detailed_test_id else name self.id = name self.source_dir = suite_path - self.source_dir_rel = os.path.relpath(os.path.realpath(suite_path), - start=canonical_zephyr_base) + self.source_dir_rel = os.path.relpath(os.path.realpath(suite_path), start=canonical_zephyr_base) self.yamlfile = suite_path self.testcases = [] @@ -449,10 +451,14 @@ class TestSuite(DisablePyTestCollectionMixin): # workdir can be "." unique = os.path.normpath(os.path.join(relative_ts_root, workdir, name)) + return unique + + @staticmethod + def check_suite_name(name, testsuite_root, workdir): check = name.split(".") if len(check) < 2: raise TwisterException(f"""bad test name '{name}' in {testsuite_root}/{workdir}. \ Tests should reference the category and subsystem with a dot as a separator. """ ) - return unique + return True diff --git a/scripts/tests/twister/test_harness.py b/scripts/tests/twister/test_harness.py index a33d6431ab4..1da2aed3f46 100644 --- a/scripts/tests/twister/test_harness.py +++ b/scripts/tests/twister/test_harness.py @@ -40,6 +40,7 @@ def gtest(): mock_platform.name = "mock_platform" mock_testsuite = mock.Mock() mock_testsuite.name = "mock_testsuite" + mock_testsuite.detailed_test_id = True mock_testsuite.id = "id" mock_testsuite.testcases = [] instance = TestInstance(testsuite=mock_testsuite, platform=mock_platform, outdir="") diff --git a/scripts/tests/twister/test_testsuite.py b/scripts/tests/twister/test_testsuite.py index f72330ef897..74402561fc1 100644 --- a/scripts/tests/twister/test_testsuite.py +++ b/scripts/tests/twister/test_testsuite.py @@ -846,3 +846,43 @@ def test_testcase_dunders(): assert case_lesser < case_greater assert str(case_greater) == 'a greater name' assert repr(case_greater) == '' + + +TESTDATA_11 = [ + ( + ZEPHYR_BASE + '/scripts/tests/twister/test_data/testsuites', + ZEPHYR_BASE + '/scripts/tests/twister/test_data/testsuites/tests/test_a', + 'test_a.check_1', + 'test_a.check_1' + ), + ( + ZEPHYR_BASE, + ZEPHYR_BASE, + 'test_a.check_1', + 'test_a.check_1' + ), + ( + ZEPHYR_BASE, + ZEPHYR_BASE + '/scripts/tests/twister/test_data/testsuites/test_b', + 'test_b.check_1', + 'test_b.check_1' + ), + ( + os.path.join(ZEPHYR_BASE, 'scripts/tests'), + os.path.join(ZEPHYR_BASE, 'scripts/tests'), + 'test_b.check_1', + 'test_b.check_1' + ), + ( + ZEPHYR_BASE, + ZEPHYR_BASE, + 'test_a.check_1.check_2', + 'test_a.check_1.check_2' + ), +] +@pytest.mark.parametrize("testsuite_root, suite_path, name, expected", TESTDATA_11) +def test_get_no_detailed_test_id(testsuite_root, suite_path, name, expected): + '''Test to check if the name without path is given for each testsuite''' + suite = TestSuite(testsuite_root, suite_path, name, detailed_test_id=False) + print(suite.name) + assert suite.name == expected