diff --git a/scripts/pylib/twister/twisterlib/config_parser.py b/scripts/pylib/twister/twisterlib/config_parser.py index add741e7de9..97fea2001cc 100644 --- a/scripts/pylib/twister/twisterlib/config_parser.py +++ b/scripts/pylib/twister/twisterlib/config_parser.py @@ -65,6 +65,7 @@ class TwisterConfigParser: "toolchain_exclude": {"type": "set"}, "toolchain_allow": {"type": "set"}, "filter": {"type": "str"}, + "levels": {"type": "list", "default": []}, "harness": {"type": "str", "default": "test"}, "harness_config": {"type": "map", "default": {}}, "seed": {"type": "int", "default": 0}, diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index d6a649d71ad..d83d4bab742 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -268,6 +268,13 @@ structure in the main Zephyr tree: boards///""") "Default to html. " "Valid options are html, xml, csv, txt, coveralls, sonarqube.") + parser.add_argument("--test-config", action="store", default=os.path.join(ZEPHYR_BASE, "tests", "test_config.yaml"), + help="Path to file with plans and test configurations.") + + parser.add_argument("--level", action="store", + help="Test level to be used. By default, no levels are used for filtering" + "and do the selection based on existing filters.") + parser.add_argument( "-D", "--all-deltas", action="store_true", help="Show all footprint deltas, positive or negative. Implies " @@ -744,6 +751,8 @@ class TwisterEnv: self.hwm = None + self.test_config = options.test_config + def discover(self): self.check_zephyr_version() self.get_toolchain() diff --git a/scripts/pylib/twister/twisterlib/testplan.py b/scripts/pylib/twister/twisterlib/testplan.py index a975573c18b..3794baecc1e 100755 --- a/scripts/pylib/twister/twisterlib/testplan.py +++ b/scripts/pylib/twister/twisterlib/testplan.py @@ -59,6 +59,11 @@ class Filters: SKIP = 'Skip filter' +class TestLevel: + name = None + levels = [] + scenarios = [] + class TestPlan: config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$') dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$') @@ -70,6 +75,8 @@ class TestPlan: os.path.join(ZEPHYR_BASE, "scripts", "schemas", "twister", "quarantine-schema.yaml")) + tc_schema_path = os.path.join(ZEPHYR_BASE, "scripts", "schemas", "twister", "test-config-schema.yaml") + SAMPLE_FILENAME = 'sample.yaml' TESTSUITE_FILENAME = 'testcase.yaml' @@ -90,12 +97,54 @@ class TestPlan: self.instances = dict() self.warnings = 0 + self.scenarios = [] + self.hwm = env.hwm # used during creating shorter build paths self.link_dir_counter = 0 self.modules = [] self.run_individual_testsuite = [] + self.levels = [] + self.test_config = {} + + + def get_level(self, name): + level = next((l for l in self.levels if l.name == name), None) + return level + + def parse_configuration(self, config_file): + if os.path.exists(config_file): + tc_schema = scl.yaml_load(self.tc_schema_path) + self.test_config = scl.yaml_load_verify(config_file, tc_schema) + else: + raise TwisterRuntimeError(f"File {config_file} not found.") + + levels = self.test_config.get('levels', []) + + # Do first pass on levels to get initial data. + for level in levels: + adds = [] + for s in level.get('adds', []): + r = re.compile(s) + adds.extend(list(filter(r.fullmatch, self.scenarios))) + + tl = TestLevel() + tl.name = level['name'] + tl.scenarios = adds + tl.levels = level.get('inherits', []) + self.levels.append(tl) + + # Go over levels again to resolve inheritance. + for level in levels: + inherit = level.get('inherits', []) + _level = self.get_level(level['name']) + if inherit: + for inherted_level in inherit: + _inherited = self.get_level(inherted_level) + _inherited_scenarios = _inherited.scenarios + level_scenarios = _level.scenarios + level_scenarios.extend(_inherited_scenarios) def find_subtests(self): sub_tests = self.options.sub_test @@ -122,6 +171,11 @@ class TestPlan: raise TwisterRuntimeError("No test cases found at the specified location...") self.find_subtests() + # get list of scenarios we have parsed into one list + for _, ts in self.testsuites.items(): + self.scenarios.append(ts.id) + + self.parse_configuration(config_file=self.env.test_config) self.add_configurations() if self.load_errors: @@ -251,10 +305,7 @@ class TestPlan: return 1 def report_duplicates(self): - all_identifiers = [] - for _, ts in self.testsuites.items(): - all_identifiers.append(ts.id) - dupes = [item for item, count in collections.Counter(all_identifiers).items() if count > 1] + dupes = [item for item, count in collections.Counter(self.scenarios).items() if count > 1] if dupes: print("Tests with duplicate identifiers:") for dupe in dupes: @@ -358,6 +409,7 @@ class TestPlan: logger.debug("Reading platform configuration files under %s..." % board_root) + platform_config = self.test_config.get('platforms', {}) for file in glob.glob(os.path.join(board_root, "*", "*", "*.yaml")): try: platform = Platform() @@ -365,37 +417,47 @@ class TestPlan: if platform.name in [p.name for p in self.platforms]: logger.error(f"Duplicate platform {platform.name} in {file}") raise Exception(f"Duplicate platform identifier {platform.name} found") - if platform.twister: - self.platforms.append(platform) + + if not platform.twister: + continue + + self.platforms.append(platform) + if not platform_config.get('override_default_platforms', False): if platform.default: + logger.debug(f"adding {platform.name} to default platforms") + self.default_platforms.append(platform.name) + else: + if platform.name in platform_config.get('default_platforms', []): + logger.debug(f"adding {platform.name} to default platforms") self.default_platforms.append(platform.name) - # support board@revision - # if there is already an existed _.yaml, then use it to - # load platform directly, otherwise, iterate the directory to - # get all valid board revision based on each _.conf. - if not "@" in platform.name: - tmp_dir = os.listdir(os.path.dirname(file)) - for item in tmp_dir: - # Need to make sure the revision matches - # the permitted patterns as described in - # cmake/modules/extensions.cmake. - revision_patterns = ["[A-Z]", - "[0-9]+", - "(0|[1-9][0-9]*)(_[0-9]+)*(_[0-9]+)*"] - for pattern in revision_patterns: - result = re.match(f"{platform.name}_(?P{pattern})\\.conf", item) - if result: - revision = result.group("revision") - yaml_file = f"{platform.name}_{revision}.yaml" - if yaml_file not in tmp_dir: - platform_revision = copy.deepcopy(platform) - revision = revision.replace("_", ".") - platform_revision.name = f"{platform.name}@{revision}" - platform_revision.default = False - self.platforms.append(platform_revision) + # support board@revision + # if there is already an existed _.yaml, then use it to + # load platform directly, otherwise, iterate the directory to + # get all valid board revision based on each _.conf. + if not "@" in platform.name: + tmp_dir = os.listdir(os.path.dirname(file)) + for item in tmp_dir: + # Need to make sure the revision matches + # the permitted patterns as described in + # cmake/modules/extensions.cmake. + revision_patterns = ["[A-Z]", + "[0-9]+", + "(0|[1-9][0-9]*)(_[0-9]+)*(_[0-9]+)*"] - break + for pattern in revision_patterns: + result = re.match(f"{platform.name}_(?P{pattern})\\.conf", item) + if result: + revision = result.group("revision") + yaml_file = f"{platform.name}_{revision}.yaml" + if yaml_file not in tmp_dir: + platform_revision = copy.deepcopy(platform) + revision = revision.replace("_", ".") + platform_revision.name = f"{platform.name}@{revision}" + platform_revision.default = False + self.platforms.append(platform_revision) + + break except RuntimeError as e: @@ -434,7 +496,6 @@ class TestPlan: try: parsed_data = TwisterConfigParser(suite_yaml_path, self.suite_schema) parsed_data.load() - subcases, ztest_suite_names = scan_testsuite_path(suite_path) for name in parsed_data.scenarios.keys(): @@ -552,7 +613,6 @@ class TestPlan: default_platforms = False emulation_platforms = False - if all_filter: logger.info("Selecting all possible platforms per test case") # When --all used, any --platform arguments ignored @@ -572,7 +632,7 @@ class TestPlan: elif arch_filter: platforms = list(filter(lambda p: p.arch in arch_filter, self.platforms)) elif default_platforms: - _platforms = list(filter(lambda p: p.default, self.platforms)) + _platforms = list(filter(lambda p: p.name in self.default_platforms, self.platforms)) platforms = [] # default platforms that can't be run are dropped from the list of # the default platforms list. Default platforms should always be @@ -586,13 +646,13 @@ class TestPlan: else: platforms = self.platforms + platform_config = self.test_config.get('platforms', {}) logger.info("Building initial testsuite list...") keyed_tests = {} for ts_name, ts in self.testsuites.items(): - - if ts.build_on_all and not platform_filter: + if ts.build_on_all and not platform_filter and platform_config.get('increased_platform_scope', True): platform_scope = self.platforms elif ts.integration_platforms and self.options.integration: self.verify_platforms_existence( @@ -606,15 +666,17 @@ class TestPlan: # If there isn't any overlap between the platform_allow list and the platform_scope # we set the scope to the platform_allow list - if ts.platform_allow and not platform_filter and not integration: + if ts.platform_allow and not platform_filter and not integration and platform_config.get('increased_platform_scope', True): self.verify_platforms_existence( ts.platform_allow, f"{ts_name} - platform_allow") a = set(platform_scope) b = set(filter(lambda item: item.name in ts.platform_allow, self.platforms)) c = a.intersection(b) if not c: - platform_scope = list(filter(lambda item: item.name in ts.platform_allow, \ + _platform_scope = list(filter(lambda item: item.name in ts.platform_allow, \ self.platforms)) + if len(_platform_scope) > 0: + platform_scope = _platform_scope[:1] # list of instances per testsuite, aka configurations. @@ -648,6 +710,12 @@ class TestPlan: if not set(ts.modules).issubset(set(self.modules)): instance.add_filter(f"one or more required modules not available: {','.join(ts.modules)}", Filters.TESTSUITE) + if self.options.level: + tl = self.get_level(self.options.level) + planned_scenarios = tl.scenarios + if ts.id not in planned_scenarios and not set(ts.levels).intersection(set(tl.levels)): + instance.add_filter("Not part of requested test plan", Filters.TESTSUITE) + if runnable and not instance.run: instance.add_filter("Not runnable on device", Filters.PLATFORM) @@ -779,7 +847,7 @@ class TestPlan: else: self.add_instances(instance_list) else: - instances = list(filter(lambda ts: ts.platform.default, instance_list)) + instances = list(filter(lambda ts: ts.platform.name in self.default_platforms, instance_list)) self.add_instances(instances) elif integration: instances = list(filter(lambda item: item.platform.name in ts.integration_platforms, instance_list)) diff --git a/scripts/schemas/twister/test-config-schema.yaml b/scripts/schemas/twister/test-config-schema.yaml new file mode 100644 index 00000000000..d8e29204e40 --- /dev/null +++ b/scripts/schemas/twister/test-config-schema.yaml @@ -0,0 +1,44 @@ +# +# Schema to validate a YAML file describing a Zephyr test configuration. +# + +type: map +mapping: + "platforms": + type: map + required: false + mapping: + "override_default_platforms": + type: bool + required: false + "increased_platform_scope": + type: bool + required: false + "default_platforms": + type: seq + required: false + sequence: + - type: str + "levels": + type: seq + required: false + sequence: + - type: map + required: false + mapping: + "name": + type: str + required: true + "description": + type: str + required: false + "adds": + type: seq + required: false + sequence: + - type: str + "inherits": + type: seq + required: false + sequence: + - type: str diff --git a/scripts/schemas/twister/testsuite-schema.yaml b/scripts/schemas/twister/testsuite-schema.yaml index f30dd3b446c..02ad2a497d6 100644 --- a/scripts/schemas/twister/testsuite-schema.yaml +++ b/scripts/schemas/twister/testsuite-schema.yaml @@ -64,6 +64,12 @@ mapping: "ignore_qemu_crash": type: bool required: false + "levels": + type: seq + required: false + sequence: + - type: str + enum: ["smoke", "unit", "integration", "acceptance", "system", "regression"] "testcases": type: seq required: false @@ -237,6 +243,12 @@ mapping: "filter": type: str required: false + "levels": + type: seq + required: false + sequence: + - type: str + enum: ["smoke", "unit", "integration", "acceptance", "system", "regression"] "integration_platforms": type: seq required: false diff --git a/scripts/tests/twister/conftest.py b/scripts/tests/twister/conftest.py index 630ac69160c..86f40c12bf3 100644 --- a/scripts/tests/twister/conftest.py +++ b/scripts/tests/twister/conftest.py @@ -38,8 +38,9 @@ def tesenv_obj(test_data, testsuites_dir, tmpdir_factory): parser = add_parse_arguments() options = parse_arguments(parser, []) env = TwisterEnv(options) - env.board_roots = [test_data +"board_config/1_level/2_level/"] - env.test_roots = [testsuites_dir + '/tests', testsuites_dir + '/samples'] + env.board_roots = [os.path.join(test_data, "board_config", "1_level", "2_level")] + env.test_roots = [os.path.join(testsuites_dir, 'tests', testsuites_dir, 'samples')] + env.test_config = os.path.join(test_data, "test_config.yaml") env.outdir = tmpdir_factory.mktemp("sanity_out_demo") return env @@ -52,6 +53,7 @@ def testplan_obj(test_data, class_env, testsuites_dir, tmpdir_factory): env.test_roots = [testsuites_dir + '/tests', testsuites_dir + '/samples'] env.outdir = tmpdir_factory.mktemp("sanity_out_demo") plan = TestPlan(env) + plan.parse_configuration(config_file=env.test_config) return plan @pytest.fixture(name='all_testsuites_dict') @@ -67,8 +69,9 @@ def testsuites_dict(class_testplan): def all_platforms_list(test_data, class_testplan): """ Pytest fixture to call add_configurations function of Testsuite class and return the Platforms list""" - class_testplan.env.board_roots = [os.path.abspath(test_data + "board_config")] + class_testplan.env.board_roots = [os.path.abspath(os.path.join(test_data, "board_config"))] plan = TestPlan(class_testplan.env) + plan.parse_configuration(config_file=class_testplan.env.test_config) plan.add_configurations() return plan.platforms diff --git a/scripts/tests/twister/test_data/test_config.yaml b/scripts/tests/twister/test_data/test_config.yaml new file mode 100644 index 00000000000..927c489221f --- /dev/null +++ b/scripts/tests/twister/test_data/test_config.yaml @@ -0,0 +1,19 @@ +platforms: + override_default_platforms: false + increased_platform_scope: true +levels: + - name: smoke + description: > + A plan to be used verifying basic zephyr features on hardware. + adds: + - kernel.threads.* + - kernel.timer.behavior + - arch.interrupt + - boards.* + - name: acceptance + description: > + More coverage + adds: + - kernel.* + - arch.interrupt + - boards.* diff --git a/scripts/tests/twister/test_testplan_class.py b/scripts/tests/twister/test_testplan_class.py index 6e28d306838..04afc96bd5b 100644 --- a/scripts/tests/twister/test_testplan_class.py +++ b/scripts/tests/twister/test_testplan_class.py @@ -54,6 +54,7 @@ def test_add_configurations(test_data, class_env, board_root_dir): """ class_env.board_roots = [os.path.abspath(test_data + board_root_dir)] plan = TestPlan(class_env) + plan.parse_configuration(config_file=class_env.test_config) if board_root_dir == "board_config": plan.add_configurations() assert sorted(plan.default_platforms) == sorted(['demo_board_1', 'demo_board_3']) @@ -62,9 +63,9 @@ def test_add_configurations(test_data, class_env, board_root_dir): assert sorted(plan.default_platforms) != sorted(['demo_board_1']) -def test_get_all_testsuites(class_env, all_testsuites_dict): +def test_get_all_testsuites(class_testplan, all_testsuites_dict): """ Testing get_all_testsuites function of TestPlan class in Twister """ - plan = TestPlan(class_env) + plan = class_testplan plan.testsuites = all_testsuites_dict expected_tests = ['sample_test.app', 'test_a.check_1.1a', 'test_a.check_1.1c', @@ -79,9 +80,9 @@ def test_get_all_testsuites(class_env, all_testsuites_dict): 'test_d.check_1.unit_1b', 'test_config.main'] assert sorted(plan.get_all_tests()) == sorted(expected_tests) -def test_get_platforms(class_env, platforms_list): +def test_get_platforms(class_testplan, platforms_list): """ Testing get_platforms function of TestPlan class in Twister """ - plan = TestPlan(class_env) + plan = class_testplan plan.platforms = platforms_list platform = plan.get_platform("demo_board_1") assert isinstance(platform, Platform) @@ -106,13 +107,13 @@ TESTDATA_PART1 = [ @pytest.mark.parametrize("tc_attribute, tc_value, plat_attribute, plat_value, expected_discards", TESTDATA_PART1) -def test_apply_filters_part1(class_env, all_testsuites_dict, platforms_list, +def test_apply_filters_part1(class_testplan, all_testsuites_dict, platforms_list, tc_attribute, tc_value, plat_attribute, plat_value, expected_discards): """ Testing apply_filters function of TestPlan class in Twister Part 1: Response of apply_filters function have appropriate values according to the filters """ - plan = TestPlan(class_env) + plan = class_testplan if tc_attribute is None and plat_attribute is None: plan.apply_filters() diff --git a/tests/test_config.yaml b/tests/test_config.yaml new file mode 100644 index 00000000000..927c489221f --- /dev/null +++ b/tests/test_config.yaml @@ -0,0 +1,19 @@ +platforms: + override_default_platforms: false + increased_platform_scope: true +levels: + - name: smoke + description: > + A plan to be used verifying basic zephyr features on hardware. + adds: + - kernel.threads.* + - kernel.timer.behavior + - arch.interrupt + - boards.* + - name: acceptance + description: > + More coverage + adds: + - kernel.* + - arch.interrupt + - boards.*