From 14d88f8425cb35be52c81d8b928f123b4e89984d Mon Sep 17 00:00:00 2001 From: Anas Nashif Date: Mon, 21 Nov 2022 15:17:21 -0500 Subject: [PATCH] twister: add support for levels and test configuration Add support test levels and the ability to assign a specific test to one or more levels. Using command line options of twister it is then possible to select a level and just execute the tests included in this level. Additionally, a test configuration allows definiing level dependencies and additional inclusion of tests into a specific level if the test itself does not have this information already. In the configuration file you can include complete components using regular expressions and you can specify which test level to import from the same file, making management of levels easier. To help with testing outside of upstream CI infrastructure, additional options are available in the configuration file, which can be hosted locally. As of now, those options are available: - Ability to ignore default platforms as defined in board definitions (Those are mostly emulation platforms used to run tests in upstream CI) - Option to specify your own list of default platforms overriding what upstream defines. - Ability to override build_onl_all options used in some testscases. This will treat tests or sample as any other just build for default platforms you specify in the configuation file or on the command line. Signed-off-by: Anas Nashif --- .../pylib/twister/twisterlib/config_parser.py | 1 + .../pylib/twister/twisterlib/environment.py | 9 ++ scripts/pylib/twister/twisterlib/testplan.py | 146 +++++++++++++----- .../schemas/twister/test-config-schema.yaml | 44 ++++++ scripts/schemas/twister/testsuite-schema.yaml | 12 ++ scripts/tests/twister/conftest.py | 9 +- .../tests/twister/test_data/test_config.yaml | 19 +++ scripts/tests/twister/test_testplan_class.py | 13 +- tests/test_config.yaml | 19 +++ 9 files changed, 224 insertions(+), 48 deletions(-) create mode 100644 scripts/schemas/twister/test-config-schema.yaml create mode 100644 scripts/tests/twister/test_data/test_config.yaml create mode 100644 tests/test_config.yaml diff --git a/scripts/pylib/twister/twisterlib/config_parser.py b/scripts/pylib/twister/twisterlib/config_parser.py index add741e7de9..97fea2001cc 100644 --- a/scripts/pylib/twister/twisterlib/config_parser.py +++ b/scripts/pylib/twister/twisterlib/config_parser.py @@ -65,6 +65,7 @@ class TwisterConfigParser: "toolchain_exclude": {"type": "set"}, "toolchain_allow": {"type": "set"}, "filter": {"type": "str"}, + "levels": {"type": "list", "default": []}, "harness": {"type": "str", "default": "test"}, "harness_config": {"type": "map", "default": {}}, "seed": {"type": "int", "default": 0}, diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index d6a649d71ad..d83d4bab742 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -268,6 +268,13 @@ structure in the main Zephyr tree: boards///""") "Default to html. " "Valid options are html, xml, csv, txt, coveralls, sonarqube.") + parser.add_argument("--test-config", action="store", default=os.path.join(ZEPHYR_BASE, "tests", "test_config.yaml"), + help="Path to file with plans and test configurations.") + + parser.add_argument("--level", action="store", + help="Test level to be used. By default, no levels are used for filtering" + "and do the selection based on existing filters.") + parser.add_argument( "-D", "--all-deltas", action="store_true", help="Show all footprint deltas, positive or negative. Implies " @@ -744,6 +751,8 @@ class TwisterEnv: self.hwm = None + self.test_config = options.test_config + def discover(self): self.check_zephyr_version() self.get_toolchain() diff --git a/scripts/pylib/twister/twisterlib/testplan.py b/scripts/pylib/twister/twisterlib/testplan.py index a975573c18b..3794baecc1e 100755 --- a/scripts/pylib/twister/twisterlib/testplan.py +++ b/scripts/pylib/twister/twisterlib/testplan.py @@ -59,6 +59,11 @@ class Filters: SKIP = 'Skip filter' +class TestLevel: + name = None + levels = [] + scenarios = [] + class TestPlan: config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$') dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$') @@ -70,6 +75,8 @@ class TestPlan: os.path.join(ZEPHYR_BASE, "scripts", "schemas", "twister", "quarantine-schema.yaml")) + tc_schema_path = os.path.join(ZEPHYR_BASE, "scripts", "schemas", "twister", "test-config-schema.yaml") + SAMPLE_FILENAME = 'sample.yaml' TESTSUITE_FILENAME = 'testcase.yaml' @@ -90,12 +97,54 @@ class TestPlan: self.instances = dict() self.warnings = 0 + self.scenarios = [] + self.hwm = env.hwm # used during creating shorter build paths self.link_dir_counter = 0 self.modules = [] self.run_individual_testsuite = [] + self.levels = [] + self.test_config = {} + + + def get_level(self, name): + level = next((l for l in self.levels if l.name == name), None) + return level + + def parse_configuration(self, config_file): + if os.path.exists(config_file): + tc_schema = scl.yaml_load(self.tc_schema_path) + self.test_config = scl.yaml_load_verify(config_file, tc_schema) + else: + raise TwisterRuntimeError(f"File {config_file} not found.") + + levels = self.test_config.get('levels', []) + + # Do first pass on levels to get initial data. + for level in levels: + adds = [] + for s in level.get('adds', []): + r = re.compile(s) + adds.extend(list(filter(r.fullmatch, self.scenarios))) + + tl = TestLevel() + tl.name = level['name'] + tl.scenarios = adds + tl.levels = level.get('inherits', []) + self.levels.append(tl) + + # Go over levels again to resolve inheritance. + for level in levels: + inherit = level.get('inherits', []) + _level = self.get_level(level['name']) + if inherit: + for inherted_level in inherit: + _inherited = self.get_level(inherted_level) + _inherited_scenarios = _inherited.scenarios + level_scenarios = _level.scenarios + level_scenarios.extend(_inherited_scenarios) def find_subtests(self): sub_tests = self.options.sub_test @@ -122,6 +171,11 @@ class TestPlan: raise TwisterRuntimeError("No test cases found at the specified location...") self.find_subtests() + # get list of scenarios we have parsed into one list + for _, ts in self.testsuites.items(): + self.scenarios.append(ts.id) + + self.parse_configuration(config_file=self.env.test_config) self.add_configurations() if self.load_errors: @@ -251,10 +305,7 @@ class TestPlan: return 1 def report_duplicates(self): - all_identifiers = [] - for _, ts in self.testsuites.items(): - all_identifiers.append(ts.id) - dupes = [item for item, count in collections.Counter(all_identifiers).items() if count > 1] + dupes = [item for item, count in collections.Counter(self.scenarios).items() if count > 1] if dupes: print("Tests with duplicate identifiers:") for dupe in dupes: @@ -358,6 +409,7 @@ class TestPlan: logger.debug("Reading platform configuration files under %s..." % board_root) + platform_config = self.test_config.get('platforms', {}) for file in glob.glob(os.path.join(board_root, "*", "*", "*.yaml")): try: platform = Platform() @@ -365,37 +417,47 @@ class TestPlan: if platform.name in [p.name for p in self.platforms]: logger.error(f"Duplicate platform {platform.name} in {file}") raise Exception(f"Duplicate platform identifier {platform.name} found") - if platform.twister: - self.platforms.append(platform) + + if not platform.twister: + continue + + self.platforms.append(platform) + if not platform_config.get('override_default_platforms', False): if platform.default: + logger.debug(f"adding {platform.name} to default platforms") + self.default_platforms.append(platform.name) + else: + if platform.name in platform_config.get('default_platforms', []): + logger.debug(f"adding {platform.name} to default platforms") self.default_platforms.append(platform.name) - # support board@revision - # if there is already an existed _.yaml, then use it to - # load platform directly, otherwise, iterate the directory to - # get all valid board revision based on each _.conf. - if not "@" in platform.name: - tmp_dir = os.listdir(os.path.dirname(file)) - for item in tmp_dir: - # Need to make sure the revision matches - # the permitted patterns as described in - # cmake/modules/extensions.cmake. - revision_patterns = ["[A-Z]", - "[0-9]+", - "(0|[1-9][0-9]*)(_[0-9]+)*(_[0-9]+)*"] - for pattern in revision_patterns: - result = re.match(f"{platform.name}_(?P{pattern})\\.conf", item) - if result: - revision = result.group("revision") - yaml_file = f"{platform.name}_{revision}.yaml" - if yaml_file not in tmp_dir: - platform_revision = copy.deepcopy(platform) - revision = revision.replace("_", ".") - platform_revision.name = f"{platform.name}@{revision}" - platform_revision.default = False - self.platforms.append(platform_revision) + # support board@revision + # if there is already an existed _.yaml, then use it to + # load platform directly, otherwise, iterate the directory to + # get all valid board revision based on each _.conf. + if not "@" in platform.name: + tmp_dir = os.listdir(os.path.dirname(file)) + for item in tmp_dir: + # Need to make sure the revision matches + # the permitted patterns as described in + # cmake/modules/extensions.cmake. + revision_patterns = ["[A-Z]", + "[0-9]+", + "(0|[1-9][0-9]*)(_[0-9]+)*(_[0-9]+)*"] - break + for pattern in revision_patterns: + result = re.match(f"{platform.name}_(?P{pattern})\\.conf", item) + if result: + revision = result.group("revision") + yaml_file = f"{platform.name}_{revision}.yaml" + if yaml_file not in tmp_dir: + platform_revision = copy.deepcopy(platform) + revision = revision.replace("_", ".") + platform_revision.name = f"{platform.name}@{revision}" + platform_revision.default = False + self.platforms.append(platform_revision) + + break except RuntimeError as e: @@ -434,7 +496,6 @@ class TestPlan: try: parsed_data = TwisterConfigParser(suite_yaml_path, self.suite_schema) parsed_data.load() - subcases, ztest_suite_names = scan_testsuite_path(suite_path) for name in parsed_data.scenarios.keys(): @@ -552,7 +613,6 @@ class TestPlan: default_platforms = False emulation_platforms = False - if all_filter: logger.info("Selecting all possible platforms per test case") # When --all used, any --platform arguments ignored @@ -572,7 +632,7 @@ class TestPlan: elif arch_filter: platforms = list(filter(lambda p: p.arch in arch_filter, self.platforms)) elif default_platforms: - _platforms = list(filter(lambda p: p.default, self.platforms)) + _platforms = list(filter(lambda p: p.name in self.default_platforms, self.platforms)) platforms = [] # default platforms that can't be run are dropped from the list of # the default platforms list. Default platforms should always be @@ -586,13 +646,13 @@ class TestPlan: else: platforms = self.platforms + platform_config = self.test_config.get('platforms', {}) logger.info("Building initial testsuite list...") keyed_tests = {} for ts_name, ts in self.testsuites.items(): - - if ts.build_on_all and not platform_filter: + if ts.build_on_all and not platform_filter and platform_config.get('increased_platform_scope', True): platform_scope = self.platforms elif ts.integration_platforms and self.options.integration: self.verify_platforms_existence( @@ -606,15 +666,17 @@ class TestPlan: # If there isn't any overlap between the platform_allow list and the platform_scope # we set the scope to the platform_allow list - if ts.platform_allow and not platform_filter and not integration: + if ts.platform_allow and not platform_filter and not integration and platform_config.get('increased_platform_scope', True): self.verify_platforms_existence( ts.platform_allow, f"{ts_name} - platform_allow") a = set(platform_scope) b = set(filter(lambda item: item.name in ts.platform_allow, self.platforms)) c = a.intersection(b) if not c: - platform_scope = list(filter(lambda item: item.name in ts.platform_allow, \ + _platform_scope = list(filter(lambda item: item.name in ts.platform_allow, \ self.platforms)) + if len(_platform_scope) > 0: + platform_scope = _platform_scope[:1] # list of instances per testsuite, aka configurations. @@ -648,6 +710,12 @@ class TestPlan: if not set(ts.modules).issubset(set(self.modules)): instance.add_filter(f"one or more required modules not available: {','.join(ts.modules)}", Filters.TESTSUITE) + if self.options.level: + tl = self.get_level(self.options.level) + planned_scenarios = tl.scenarios + if ts.id not in planned_scenarios and not set(ts.levels).intersection(set(tl.levels)): + instance.add_filter("Not part of requested test plan", Filters.TESTSUITE) + if runnable and not instance.run: instance.add_filter("Not runnable on device", Filters.PLATFORM) @@ -779,7 +847,7 @@ class TestPlan: else: self.add_instances(instance_list) else: - instances = list(filter(lambda ts: ts.platform.default, instance_list)) + instances = list(filter(lambda ts: ts.platform.name in self.default_platforms, instance_list)) self.add_instances(instances) elif integration: instances = list(filter(lambda item: item.platform.name in ts.integration_platforms, instance_list)) diff --git a/scripts/schemas/twister/test-config-schema.yaml b/scripts/schemas/twister/test-config-schema.yaml new file mode 100644 index 00000000000..d8e29204e40 --- /dev/null +++ b/scripts/schemas/twister/test-config-schema.yaml @@ -0,0 +1,44 @@ +# +# Schema to validate a YAML file describing a Zephyr test configuration. +# + +type: map +mapping: + "platforms": + type: map + required: false + mapping: + "override_default_platforms": + type: bool + required: false + "increased_platform_scope": + type: bool + required: false + "default_platforms": + type: seq + required: false + sequence: + - type: str + "levels": + type: seq + required: false + sequence: + - type: map + required: false + mapping: + "name": + type: str + required: true + "description": + type: str + required: false + "adds": + type: seq + required: false + sequence: + - type: str + "inherits": + type: seq + required: false + sequence: + - type: str diff --git a/scripts/schemas/twister/testsuite-schema.yaml b/scripts/schemas/twister/testsuite-schema.yaml index f30dd3b446c..02ad2a497d6 100644 --- a/scripts/schemas/twister/testsuite-schema.yaml +++ b/scripts/schemas/twister/testsuite-schema.yaml @@ -64,6 +64,12 @@ mapping: "ignore_qemu_crash": type: bool required: false + "levels": + type: seq + required: false + sequence: + - type: str + enum: ["smoke", "unit", "integration", "acceptance", "system", "regression"] "testcases": type: seq required: false @@ -237,6 +243,12 @@ mapping: "filter": type: str required: false + "levels": + type: seq + required: false + sequence: + - type: str + enum: ["smoke", "unit", "integration", "acceptance", "system", "regression"] "integration_platforms": type: seq required: false diff --git a/scripts/tests/twister/conftest.py b/scripts/tests/twister/conftest.py index 630ac69160c..86f40c12bf3 100644 --- a/scripts/tests/twister/conftest.py +++ b/scripts/tests/twister/conftest.py @@ -38,8 +38,9 @@ def tesenv_obj(test_data, testsuites_dir, tmpdir_factory): parser = add_parse_arguments() options = parse_arguments(parser, []) env = TwisterEnv(options) - env.board_roots = [test_data +"board_config/1_level/2_level/"] - env.test_roots = [testsuites_dir + '/tests', testsuites_dir + '/samples'] + env.board_roots = [os.path.join(test_data, "board_config", "1_level", "2_level")] + env.test_roots = [os.path.join(testsuites_dir, 'tests', testsuites_dir, 'samples')] + env.test_config = os.path.join(test_data, "test_config.yaml") env.outdir = tmpdir_factory.mktemp("sanity_out_demo") return env @@ -52,6 +53,7 @@ def testplan_obj(test_data, class_env, testsuites_dir, tmpdir_factory): env.test_roots = [testsuites_dir + '/tests', testsuites_dir + '/samples'] env.outdir = tmpdir_factory.mktemp("sanity_out_demo") plan = TestPlan(env) + plan.parse_configuration(config_file=env.test_config) return plan @pytest.fixture(name='all_testsuites_dict') @@ -67,8 +69,9 @@ def testsuites_dict(class_testplan): def all_platforms_list(test_data, class_testplan): """ Pytest fixture to call add_configurations function of Testsuite class and return the Platforms list""" - class_testplan.env.board_roots = [os.path.abspath(test_data + "board_config")] + class_testplan.env.board_roots = [os.path.abspath(os.path.join(test_data, "board_config"))] plan = TestPlan(class_testplan.env) + plan.parse_configuration(config_file=class_testplan.env.test_config) plan.add_configurations() return plan.platforms diff --git a/scripts/tests/twister/test_data/test_config.yaml b/scripts/tests/twister/test_data/test_config.yaml new file mode 100644 index 00000000000..927c489221f --- /dev/null +++ b/scripts/tests/twister/test_data/test_config.yaml @@ -0,0 +1,19 @@ +platforms: + override_default_platforms: false + increased_platform_scope: true +levels: + - name: smoke + description: > + A plan to be used verifying basic zephyr features on hardware. + adds: + - kernel.threads.* + - kernel.timer.behavior + - arch.interrupt + - boards.* + - name: acceptance + description: > + More coverage + adds: + - kernel.* + - arch.interrupt + - boards.* diff --git a/scripts/tests/twister/test_testplan_class.py b/scripts/tests/twister/test_testplan_class.py index 6e28d306838..04afc96bd5b 100644 --- a/scripts/tests/twister/test_testplan_class.py +++ b/scripts/tests/twister/test_testplan_class.py @@ -54,6 +54,7 @@ def test_add_configurations(test_data, class_env, board_root_dir): """ class_env.board_roots = [os.path.abspath(test_data + board_root_dir)] plan = TestPlan(class_env) + plan.parse_configuration(config_file=class_env.test_config) if board_root_dir == "board_config": plan.add_configurations() assert sorted(plan.default_platforms) == sorted(['demo_board_1', 'demo_board_3']) @@ -62,9 +63,9 @@ def test_add_configurations(test_data, class_env, board_root_dir): assert sorted(plan.default_platforms) != sorted(['demo_board_1']) -def test_get_all_testsuites(class_env, all_testsuites_dict): +def test_get_all_testsuites(class_testplan, all_testsuites_dict): """ Testing get_all_testsuites function of TestPlan class in Twister """ - plan = TestPlan(class_env) + plan = class_testplan plan.testsuites = all_testsuites_dict expected_tests = ['sample_test.app', 'test_a.check_1.1a', 'test_a.check_1.1c', @@ -79,9 +80,9 @@ def test_get_all_testsuites(class_env, all_testsuites_dict): 'test_d.check_1.unit_1b', 'test_config.main'] assert sorted(plan.get_all_tests()) == sorted(expected_tests) -def test_get_platforms(class_env, platforms_list): +def test_get_platforms(class_testplan, platforms_list): """ Testing get_platforms function of TestPlan class in Twister """ - plan = TestPlan(class_env) + plan = class_testplan plan.platforms = platforms_list platform = plan.get_platform("demo_board_1") assert isinstance(platform, Platform) @@ -106,13 +107,13 @@ TESTDATA_PART1 = [ @pytest.mark.parametrize("tc_attribute, tc_value, plat_attribute, plat_value, expected_discards", TESTDATA_PART1) -def test_apply_filters_part1(class_env, all_testsuites_dict, platforms_list, +def test_apply_filters_part1(class_testplan, all_testsuites_dict, platforms_list, tc_attribute, tc_value, plat_attribute, plat_value, expected_discards): """ Testing apply_filters function of TestPlan class in Twister Part 1: Response of apply_filters function have appropriate values according to the filters """ - plan = TestPlan(class_env) + plan = class_testplan if tc_attribute is None and plat_attribute is None: plan.apply_filters() diff --git a/tests/test_config.yaml b/tests/test_config.yaml new file mode 100644 index 00000000000..927c489221f --- /dev/null +++ b/tests/test_config.yaml @@ -0,0 +1,19 @@ +platforms: + override_default_platforms: false + increased_platform_scope: true +levels: + - name: smoke + description: > + A plan to be used verifying basic zephyr features on hardware. + adds: + - kernel.threads.* + - kernel.timer.behavior + - arch.interrupt + - boards.* + - name: acceptance + description: > + More coverage + adds: + - kernel.* + - arch.interrupt + - boards.*