diff --git a/scripts/pylib/twister/twister/testsuite.py b/scripts/pylib/twister/twister/testsuite.py index 6f7207b914a..08984312c2b 100644 --- a/scripts/pylib/twister/twister/testsuite.py +++ b/scripts/pylib/twister/twister/testsuite.py @@ -183,7 +183,7 @@ def _find_regular_ztest_testcases(search_area, suite_regex_matches, is_registere re.MULTILINE) search_start, search_end = \ - TestSuite._get_search_area_boundary(search_area, suite_regex_matches, is_registered_test_suite) + _get_search_area_boundary(search_area, suite_regex_matches, is_registered_test_suite) limited_search_area = search_area[search_start:search_end] testcase_names, warnings = \ _find_ztest_testcases(limited_search_area, testcase_regex) diff --git a/scripts/tests/twister/test_testinstance.py b/scripts/tests/twister/test_testinstance.py new file mode 100644 index 00000000000..c0315dde5b4 --- /dev/null +++ b/scripts/tests/twister/test_testinstance.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# pylint: disable=line-too-long +""" +Tests for testinstance class +""" + +import os +import sys +import pytest + +ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") +sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister")) +from twister.testinstance import TestInstance +from twister.error import BuildError, TwisterException +from twister.testsuite import TestSuite + + +TESTDATA_1 = [ + (False, False, "console", "na", "qemu", False, [], (False, True)), + (False, False, "console", "native", "qemu", False, [], (False, True)), + (True, False, "console", "native", "nsim", False, [], (True, False)), + (True, True, "console", "native", "renode", False, [], (True, False)), + (False, False, "sensor", "native", "", False, [], (True, False)), + (False, False, "sensor", "na", "", False, [], (True, False)), + (False, True, "sensor", "native", "", True, [], (True, False)), +] +@pytest.mark.parametrize("build_only, slow, harness, platform_type, platform_sim, device_testing,fixture, expected", TESTDATA_1) +def test_check_build_or_run(class_testplan, monkeypatch, all_testsuites_dict, platforms_list, build_only, slow, harness, platform_type, platform_sim, device_testing, fixture, expected): + """" Test to check the conditions for build_only and run scenarios + Scenario 1: Test when different parameters are passed, build_only and run are set correctly + Scenario 2: Test if build_only is enabled when the OS is Windows""" + + class_testplan.testsuites = all_testsuites_dict + testsuite = class_testplan.testsuites.get('scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1') + + + class_testplan.platforms = platforms_list + platform = class_testplan.get_platform("demo_board_2") + platform.type = platform_type + platform.simulation = platform_sim + testsuite.harness = harness + testsuite.build_only = build_only + testsuite.slow = slow + + testinstance = TestInstance(testsuite, platform, class_testplan.env.outdir) + run = testinstance.check_runnable(slow, device_testing, fixture) + _, r = expected + assert run == r + + monkeypatch.setattr("os.name", "nt") + run = testinstance.check_runnable() + assert not run + +TESTDATA_2 = [ + (True, True, True, ["demo_board_2"], "native", '\nCONFIG_COVERAGE=y\nCONFIG_COVERAGE_DUMP=y\nCONFIG_ASAN=y\nCONFIG_UBSAN=y'), + (True, False, True, ["demo_board_2"], "native", '\nCONFIG_COVERAGE=y\nCONFIG_COVERAGE_DUMP=y\nCONFIG_ASAN=y'), + (False, False, True, ["demo_board_2"], 'native', '\nCONFIG_COVERAGE=y\nCONFIG_COVERAGE_DUMP=y'), + (True, False, True, ["demo_board_2"], 'mcu', '\nCONFIG_COVERAGE=y\nCONFIG_COVERAGE_DUMP=y'), + (False, False, False, ["demo_board_2"], 'native', ''), + (False, False, True, ['demo_board_1'], 'native', ''), + (True, False, False, ["demo_board_2"], 'native', '\nCONFIG_ASAN=y'), + (False, True, False, ["demo_board_2"], 'native', '\nCONFIG_UBSAN=y'), +] + +@pytest.mark.parametrize("enable_asan, enable_ubsan, enable_coverage, coverage_platform, platform_type, expected_content", TESTDATA_2) +def test_create_overlay(class_testplan, all_testsuites_dict, platforms_list, enable_asan, enable_ubsan, enable_coverage, coverage_platform, platform_type, expected_content): + """Test correct content is written to testcase_extra.conf based on if conditions + TO DO: Add extra_configs to the input list""" + class_testplan.testsuites = all_testsuites_dict + testcase = class_testplan.testsuites.get('scripts/tests/twister/test_data/testsuites/samples/test_app/sample_test.app') + class_testplan.platforms = platforms_list + platform = class_testplan.get_platform("demo_board_2") + + testinstance = TestInstance(testcase, platform, class_testplan.env.outdir) + platform.type = platform_type + assert testinstance.create_overlay(platform, enable_asan, enable_ubsan, enable_coverage, coverage_platform) == expected_content + +def test_calculate_sizes(class_testplan, all_testsuites_dict, platforms_list): + """ Test Calculate sizes method for zephyr elf""" + class_testplan.testsuites = all_testsuites_dict + testcase = class_testplan.testsuites.get('scripts/tests/twister/test_data/testsuites/samples/test_app/sample_test.app') + class_testplan.platforms = platforms_list + platform = class_testplan.get_platform("demo_board_2") + testinstance = TestInstance(testcase, platform, class_testplan.env.outdir) + + with pytest.raises(BuildError): + assert testinstance.calculate_sizes() == "Missing/multiple output ELF binary" + +TESTDATA_3 = [ + (ZEPHYR_BASE + '/scripts/tests/twister/test_data/testsuites', ZEPHYR_BASE, '/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1', '/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1'), + (ZEPHYR_BASE, '.', 'test_a.check_1', 'test_a.check_1'), + (ZEPHYR_BASE, '/scripts/tests/twister/test_data/testsuites/test_b', 'test_b.check_1', '/scripts/tests/twister/test_data/testsuites/test_b/test_b.check_1'), + (os.path.join(ZEPHYR_BASE, '/scripts/tests'), '.', 'test_b.check_1', 'test_b.check_1'), + (os.path.join(ZEPHYR_BASE, '/scripts/tests'), '.', '.', '.'), + (ZEPHYR_BASE, '.', 'test_a.check_1.check_2', 'test_a.check_1.check_2'), +] +@pytest.mark.parametrize("testcase_root, workdir, name, expected", TESTDATA_3) +def test_get_unique(testcase_root, workdir, name, expected): + '''Test to check if the unique name is given for each testcase root and workdir''' + unique = TestSuite(testcase_root, workdir, name) + assert unique.name == expected + +TESTDATA_4 = [ + (ZEPHYR_BASE, '.', 'test_c', 'Tests should reference the category and subsystem with a dot as a separator.'), + (os.path.join(ZEPHYR_BASE, '/scripts/tests'), '.', '', 'Tests should reference the category and subsystem with a dot as a separator.'), +] +@pytest.mark.parametrize("testcase_root, workdir, name, exception", TESTDATA_4) +def test_get_unique_exception(testcase_root, workdir, name, exception): + '''Test to check if tests reference the category and subsystem with a dot as a separator''' + + with pytest.raises(TwisterException): + unique = TestSuite(testcase_root, workdir, name) + assert unique == exception diff --git a/scripts/tests/twister/test_testplan_class.py b/scripts/tests/twister/test_testplan_class.py index adb9c1cf236..0b3038dfeb7 100644 --- a/scripts/tests/twister/test_testplan_class.py +++ b/scripts/tests/twister/test_testplan_class.py @@ -13,9 +13,8 @@ import pytest ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister")) -from twister.testplan import TestPlan, ScanPathResult +from twister.testplan import TestPlan from twister.testinstance import TestInstance -from twister.error import BuildError, TwisterException from twister.testsuite import TestSuite from twister.platform import Platform @@ -256,3 +255,4 @@ def test_add_instances(test_data, class_env, all_testsuites_dict, platforms_list [platform.name + '/' + s for s in list(all_testsuites_dict.keys())] assert all(isinstance(n, TestInstance) for n in list(plan.instances.values())) assert list(plan.instances.values()) == instance_list + diff --git a/scripts/tests/twister/test_testsuite.py b/scripts/tests/twister/test_testsuite.py new file mode 100644 index 00000000000..b64cbd92e64 --- /dev/null +++ b/scripts/tests/twister/test_testsuite.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# pylint: disable=line-too-long +""" +Tests for testinstance class +""" + +import os +import sys +import pytest + +ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") +sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister")) + +from twister.testsuite import scan_file, ScanPathResult + +TESTDATA_5 = [ + ("testsuites/tests/test_ztest.c", + ScanPathResult( + warnings=None, + matches=['a', 'c', 'unit_a', + 'newline', + 'test_test_aa', + 'user', 'last'], + has_registered_test_suites=False, + has_run_registered_test_suites=False, + has_test_main=False, + ztest_suite_names = ["test_api"])), + ("testsuites/tests/test_a/test_ztest_error.c", + ScanPathResult( + warnings="Found a test that does not start with test_", + matches=['1a', '1c', '2a', '2b'], + has_registered_test_suites=False, + has_run_registered_test_suites=False, + has_test_main=True, + ztest_suite_names = ["feature1", "feature2"])), + ("testsuites/tests/test_a/test_ztest_error_1.c", + ScanPathResult( + warnings="found invalid #ifdef, #endif in ztest_test_suite()", + matches=['unit_1a', 'unit_1b', 'Unit_1c'], + has_registered_test_suites=False, + has_run_registered_test_suites=False, + has_test_main=False, + ztest_suite_names = ["feature3"])), + ("testsuites/tests/test_d/test_ztest_error_register_test_suite.c", + ScanPathResult( + warnings=None, matches=['unit_1a', 'unit_1b'], + has_registered_test_suites=True, + has_run_registered_test_suites=False, + has_test_main=False, + ztest_suite_names = ["feature4"])), +] + +@pytest.mark.parametrize("test_file, expected", TESTDATA_5) +def test_scan_file(test_data, test_file, class_env, expected: ScanPathResult): + '''Testing scan_file method with different ztest files for warnings and results''' + + result: ScanPathResult = scan_file(os.path.join(test_data, test_file)) + assert result == expected +