diff --git a/include/linker/common-ram.ld b/include/linker/common-ram.ld index 2914e5d5397..81949ea0daa 100644 --- a/include/linker/common-ram.ld +++ b/include/linker/common-ram.ld @@ -140,3 +140,7 @@ #ifdef CONFIG_USERSPACE _static_kernel_objects_end = .; #endif + +#if defined(CONFIG_ZTEST) + ITERABLE_SECTION_RAM(ztest_suite_node, 4) +#endif /* CONFIG_ZTEST */ diff --git a/scripts/pylib/twister/twisterlib.py b/scripts/pylib/twister/twisterlib.py index d6251348630..05907c7b187 100755 --- a/scripts/pylib/twister/twisterlib.py +++ b/scripts/pylib/twister/twisterlib.py @@ -3,7 +3,6 @@ # # Copyright (c) 2018 Intel Corporation # SPDX-License-Identifier: Apache-2.0 - import os import contextlib import string @@ -34,6 +33,7 @@ import platform import yaml import json from multiprocessing import Lock, Process, Value +from typing import List try: # Use the C LibYAML parser if available, rather than the Python parser. @@ -1576,6 +1576,41 @@ class DisablePyTestCollectionMixin(object): __test__ = False +class ScanPathResult: + """Result of the TestCase.scan_path function call. + + Attributes: + matches A list of test cases + warnings A string containing one or more + warnings to display + has_registered_test_suites Whether or not the path contained any + calls to the ztest_register_test_suite + macro. + has_run_registered_test_suites Whether or not the path contained at + least one call to + ztest_run_registered_test_suites. + """ + def __init__(self, + matches: List[str] = None, + warnings: str = None, + has_registered_test_suites: bool = False, + has_run_registered_test_suites: bool = False): + self.matches = matches + self.warnings = warnings + self.has_registered_test_suites = has_registered_test_suites + self.has_run_registered_test_suites = has_run_registered_test_suites + + def __eq__(self, other): + if not isinstance(other, ScanPathResult): + return False + return (sorted(self.matches) == sorted(other.matches) and + self.warnings == other.warnings and + (self.has_registered_test_suites == + other.has_registered_test_suites) and + (self.has_run_registered_test_suites == + other.has_run_registered_test_suites)) + + class TestCase(DisablePyTestCollectionMixin): """Class representing a test application """ @@ -1662,29 +1697,42 @@ Tests should reference the category and subsystem with a dot as a separator. # line--as we only search starting the end of this match br"^\s*ztest_test_suite\(\s*(?P[a-zA-Z0-9_]+)\s*,", re.MULTILINE) + registered_suite_regex = re.compile( + br"^\s*ztest_register_test_suite" + br"\(\s*(?P[a-zA-Z0-9_]+)\s*,", + re.MULTILINE) stc_regex = re.compile( - br"^\s*" # empy space at the beginning is ok + br"""^\s* # empy space at the beginning is ok # catch the case where it is declared in the same sentence, e.g: # # ztest_test_suite(mutex_complex, ztest_user_unit_test(TESTNAME)); - br"(?:ztest_test_suite\([a-zA-Z0-9_]+,\s*)?" + # ztest_register_test_suite(n, p, ztest_user_unit_test(TESTNAME), + (?:ztest_ + (?:test_suite\(|register_test_suite\([a-zA-Z0-9_]+\s*,\s*) + [a-zA-Z0-9_]+\s*,\s* + )? # Catch ztest[_user]_unit_test-[_setup_teardown](TESTNAME) - br"ztest_(?:1cpu_)?(?:user_)?unit_test(?:_setup_teardown)?" + ztest_(?:1cpu_)?(?:user_)?unit_test(?:_setup_teardown)? # Consume the argument that becomes the extra testcse - br"\(\s*" - br"(?P[a-zA-Z0-9_]+)" + \(\s*(?P[a-zA-Z0-9_]+) # _setup_teardown() variant has two extra arguments that we ignore - br"(?:\s*,\s*[a-zA-Z0-9_]+\s*,\s*[a-zA-Z0-9_]+)?" - br"\s*\)", + (?:\s*,\s*[a-zA-Z0-9_]+\s*,\s*[a-zA-Z0-9_]+)? + \s*\)""", # We don't check how it finishes; we don't care - re.MULTILINE) + re.MULTILINE | re.VERBOSE) suite_run_regex = re.compile( br"^\s*ztest_run_test_suite\((?P[a-zA-Z0-9_]+)\)", re.MULTILINE) + registered_suite_run_regex = re.compile( + br"^\s*ztest_run_registered_test_suites\(" + br"(\*+|&)?(?P[a-zA-Z0-9_]+)\)", + re.MULTILINE) achtung_regex = re.compile( br"(#ifdef|#endif)", re.MULTILINE) warnings = None + has_registered_test_suites = False + has_run_registered_test_suites = False with open(inf_name) as inf: if os.name == 'nt': @@ -1695,52 +1743,94 @@ Tests should reference the category and subsystem with a dot as a separator. with contextlib.closing(mmap.mmap(**mmap_args)) as main_c: suite_regex_match = suite_regex.search(main_c) - if not suite_regex_match: + registered_suite_regex_match = registered_suite_regex.search( + main_c) + + if registered_suite_regex_match: + has_registered_test_suites = True + if registered_suite_run_regex.search(main_c): + has_run_registered_test_suites = True + + if not suite_regex_match and not has_registered_test_suites: # can't find ztest_test_suite, maybe a client, because # it includes ztest.h - return None, None + return ScanPathResult( + matches=None, + warnings=None, + has_registered_test_suites=has_registered_test_suites, + has_run_registered_test_suites=has_run_registered_test_suites) suite_run_match = suite_run_regex.search(main_c) - if not suite_run_match: + if suite_regex_match and not suite_run_match: raise ValueError("can't find ztest_run_test_suite") + if suite_regex_match: + search_start = suite_regex_match.end() + else: + search_start = registered_suite_regex_match.end() + + if suite_run_match: + search_end = suite_run_match.start() + else: + search_end = re.compile(br"\);", re.MULTILINE) \ + .search(main_c, search_start) \ + .end() achtung_matches = re.findall( achtung_regex, - main_c[suite_regex_match.end():suite_run_match.start()]) + main_c[search_start:search_end]) if achtung_matches: warnings = "found invalid %s in ztest_test_suite()" \ % ", ".join(sorted({match.decode() for match in achtung_matches},reverse = True)) _matches = re.findall( stc_regex, - main_c[suite_regex_match.end():suite_run_match.start()]) + main_c[search_start:search_end]) for match in _matches: if not match.decode().startswith("test_"): warnings = "Found a test that does not start with test_" matches = [match.decode().replace("test_", "", 1) for match in _matches] - return matches, warnings + return ScanPathResult( + matches=matches, + warnings=warnings, + has_registered_test_suites=has_registered_test_suites, + has_run_registered_test_suites=has_run_registered_test_suites) def scan_path(self, path): subcases = [] + has_registered_test_suites = False + has_run_registered_test_suites = False for filename in glob.glob(os.path.join(path, "src", "*.c*")): try: - _subcases, warnings = self.scan_file(filename) - if warnings: - logger.error("%s: %s" % (filename, warnings)) - raise TwisterRuntimeError("%s: %s" % (filename, warnings)) - if _subcases: - subcases += _subcases + result: ScanPathResult = self.scan_file(filename) + if result.warnings: + logger.error("%s: %s" % (filename, result.warnings)) + raise TwisterRuntimeError( + "%s: %s" % (filename, result.warnings)) + if result.matches: + subcases += result.matches + if result.has_registered_test_suites: + has_registered_test_suites = True + if result.has_run_registered_test_suites: + has_run_registered_test_suites = True except ValueError as e: logger.error("%s: can't find: %s" % (filename, e)) for filename in glob.glob(os.path.join(path, "*.c")): try: - _subcases, warnings = self.scan_file(filename) - if warnings: - logger.error("%s: %s" % (filename, warnings)) - if _subcases: - subcases += _subcases + result: ScanPathResult = self.scan_file(filename) + if result.warnings: + logger.error("%s: %s" % (filename, result.warnings)) + if result.matches: + subcases += result.matches except ValueError as e: logger.error("%s: can't find: %s" % (filename, e)) + + if has_registered_test_suites and not has_run_registered_test_suites: + warning = \ + "Found call to 'ztest_register_test_suite()' but no "\ + "call to 'ztest_run_registered_test_suites()'" + logger.error(warning) + raise TwisterRuntimeError(warning) + return subcases def parse_subcases(self, test_path): diff --git a/scripts/tests/twister/test_data/testcases/tests/test_d/test_data.yaml b/scripts/tests/twister/test_data/testcases/tests/test_d/test_data.yaml new file mode 100644 index 00000000000..726a3b3a040 --- /dev/null +++ b/scripts/tests/twister/test_data/testcases/tests/test_d/test_data.yaml @@ -0,0 +1,4 @@ +tests: + test_d.check_1: + tags: test_d + build_only: True diff --git a/scripts/tests/twister/test_data/testcases/tests/test_d/test_ztest_error_register_test_suite.c b/scripts/tests/twister/test_data/testcases/tests/test_d/test_ztest_error_register_test_suite.c new file mode 100644 index 00000000000..55d8036da32 --- /dev/null +++ b/scripts/tests/twister/test_data/testcases/tests/test_d/test_ztest_error_register_test_suite.c @@ -0,0 +1,9 @@ +/* + * Copyright 2021 Google LLC + * + * SPDX-License-Identifier: Apache-2.0 + */ + +ztest_register_test_suite(feature4, NULL, + ztest_unit_test(test_unit_1a), + ztest_unit_test(test_unit_1b)); diff --git a/scripts/tests/twister/test_testinstance.py b/scripts/tests/twister/test_testinstance.py index 15c9a2e9eb5..07d89ec02e0 100644 --- a/scripts/tests/twister/test_testinstance.py +++ b/scripts/tests/twister/test_testinstance.py @@ -13,7 +13,8 @@ import pytest ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister")) -from twisterlib import TestInstance, BuildError, TestCase, TwisterException +from twisterlib import (TestInstance, BuildError, TestCase, TwisterException, + ScanPathResult) TESTDATA_1 = [ @@ -112,21 +113,45 @@ def test_get_unique_exception(testcase_root, workdir, name, exception): unique = TestCase(testcase_root, workdir, name) assert unique == exception + TESTDATA_5 = [ - ("testcases/tests/test_ztest.c", None, ['a', 'c', 'unit_a', 'newline', 'test_test_aa', 'user', 'last']), - ("testcases/tests/test_a/test_ztest_error.c", "Found a test that does not start with test_", ['1a', '1c', '2a', '2b']), - ("testcases/tests/test_a/test_ztest_error_1.c", "found invalid #ifdef, #endif in ztest_test_suite()", ['unit_1a', 'unit_1b', 'Unit_1c']), + ("testcases/tests/test_ztest.c", + ScanPathResult( + warnings=None, + matches=['a', 'c', 'unit_a', + 'newline', + 'test_test_aa', + 'user', 'last'], + has_registered_test_suites=False, + has_run_registered_test_suites=False)), + ("testcases/tests/test_a/test_ztest_error.c", + ScanPathResult( + warnings="Found a test that does not start with test_", + matches=['1a', '1c', '2a', '2b'], + has_registered_test_suites=False, + has_run_registered_test_suites=False)), + ("testcases/tests/test_a/test_ztest_error_1.c", + ScanPathResult( + warnings="found invalid #ifdef, #endif in ztest_test_suite()", + matches=['unit_1a', 'unit_1b', 'Unit_1c'], + has_registered_test_suites=False, + has_run_registered_test_suites=False)), + ("testcases/tests/test_d/test_ztest_error_register_test_suite.c", + ScanPathResult( + warnings=None, matches=['unit_1a', 'unit_1b'], + has_registered_test_suites=True, + has_run_registered_test_suites=False)), ] -@pytest.mark.parametrize("test_file, expected_warnings, expected_subcases", TESTDATA_5) -def test_scan_file(test_data, test_file, expected_warnings, expected_subcases): +@pytest.mark.parametrize("test_file, expected", TESTDATA_5) +def test_scan_file(test_data, test_file, expected: ScanPathResult): '''Testing scan_file method with different ztest files for warnings and results''' - testcase = TestCase("/scripts/tests/twister/test_data/testcases/tests", ".", "test_a.check_1") + testcase = TestCase("/scripts/tests/twister/test_data/testcases/tests", ".", + "test_a.check_1") - results, warnings = testcase.scan_file(os.path.join(test_data, test_file)) - assert sorted(results) == sorted(expected_subcases) - assert warnings == expected_warnings + result: ScanPathResult = testcase.scan_file(os.path.join(test_data, test_file)) + assert result == expected TESTDATA_6 = [ diff --git a/scripts/tests/twister/test_testsuite_class.py b/scripts/tests/twister/test_testsuite_class.py index 704322b8002..5c03011d1f5 100755 --- a/scripts/tests/twister/test_testsuite_class.py +++ b/scripts/tests/twister/test_testsuite_class.py @@ -30,6 +30,7 @@ def test_testsuite_add_testcases(class_testsuite): 'test_c.check_2', 'test_a.check_1', 'test_a.check_2', + 'test_d.check_1', 'sample_test.app'] testcase_list = [] for key in sorted(class_testsuite.testcases.keys()): @@ -58,9 +59,18 @@ def test_add_configurations(test_data, class_testsuite, board_root_dir): def test_get_all_testcases(class_testsuite, all_testcases_dict): """ Testing get_all_testcases function of TestSuite class in Twister """ class_testsuite.testcases = all_testcases_dict - expected_tests = ['sample_test.app', 'test_a.check_1.1a', 'test_a.check_1.1c', - 'test_a.check_1.2a', 'test_a.check_1.2b', 'test_a.check_1.Unit_1c', 'test_a.check_1.unit_1a', 'test_a.check_1.unit_1b', 'test_a.check_2.1a', 'test_a.check_2.1c', 'test_a.check_2.2a', 'test_a.check_2.2b', 'test_a.check_2.Unit_1c', 'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b', 'test_b.check_1', 'test_b.check_2', 'test_c.check_1', 'test_c.check_2'] - assert len(class_testsuite.get_all_tests()) == 19 + expected_tests = ['sample_test.app', 'test_a.check_1.1a', + 'test_a.check_1.1c', + 'test_a.check_1.2a', 'test_a.check_1.2b', + 'test_a.check_1.Unit_1c', 'test_a.check_1.unit_1a', + 'test_a.check_1.unit_1b', 'test_a.check_2.1a', + 'test_a.check_2.1c', 'test_a.check_2.2a', + 'test_a.check_2.2b', 'test_a.check_2.Unit_1c', + 'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b', + 'test_b.check_1', 'test_b.check_2', 'test_c.check_1', + 'test_c.check_2', 'test_d.check_1.unit_1a', + 'test_d.check_1.unit_1b'] + assert len(class_testsuite.get_all_tests()) == len(expected_tests) assert sorted(class_testsuite.get_all_tests()) == sorted(expected_tests) def test_get_platforms(class_testsuite, platforms_list): diff --git a/subsys/testsuite/include/ztest.ld b/subsys/testsuite/include/ztest.ld new file mode 100644 index 00000000000..7012c8ee064 --- /dev/null +++ b/subsys/testsuite/include/ztest.ld @@ -0,0 +1,16 @@ +/* + * Copyright 2021 Google LLC + * + * SPDX-License-Identifier: Apache-2.0 + */ + +SECTIONS +{ + .data.ztest_suite_node_area : ALIGN(4) + { + _ztest_suite_node_list_start = .; + KEEP(*(SORT_BY_NAME(._ztest_suite_node.static.*))) + _ztest_suite_node_list_end = .; + } +} +INSERT AFTER .data; diff --git a/subsys/testsuite/unittest.cmake b/subsys/testsuite/unittest.cmake index 3546ec5da63..c5427f56279 100644 --- a/subsys/testsuite/unittest.cmake +++ b/subsys/testsuite/unittest.cmake @@ -62,6 +62,10 @@ target_compile_options(testbinary PRIVATE $<$:${EXTRA_AFLAGS_AS_LIST}> ) +target_link_options(testbinary PRIVATE + -T "${ZEPHYR_BASE}/subsys/testsuite/include/ztest.ld" + ) + target_link_libraries(testbinary PRIVATE ${EXTRA_LDFLAGS_AS_LIST} ) diff --git a/subsys/testsuite/ztest/include/ztest_test.h b/subsys/testsuite/ztest/include/ztest_test.h index 8e8d8c14d30..a337e70adce 100644 --- a/subsys/testsuite/ztest/include/ztest_test.h +++ b/subsys/testsuite/ztest/include/ztest_test.h @@ -14,6 +14,8 @@ #define ZEPHYR_TESTSUITE_ZTEST_TEST_H_ #include +#include +#include #ifdef __cplusplus extern "C" { @@ -27,7 +29,80 @@ struct unit_test { uint32_t thread_options; }; -void z_ztest_run_test_suite(const char *name, struct unit_test *suite); +/** + * Stats about a ztest suite + */ +struct ztest_suite_stats { + /** The number of times that the suite ran */ + uint32_t run_count; + /** The number of times that the suite was skipped */ + uint32_t skip_count; + /** The number of times that the suite failed */ + uint32_t fail_count; +}; + +/** + * A single node of test suite. Each node should be added to a single linker section which will + * allow ztest_run_registered_test_suites() to iterate over the various nodes. + */ +struct ztest_suite_node { + /** The name of the test suite. */ + const char *name; + /** Pointer to the test suite. */ + struct unit_test *suite; + /** + * An optional predicate function to determine if the test should run. If NULL, then the + * test will only run once on the first attempt. + * + * @param state The current state of the test application. + * @return True if the suite should be run; false to skip. + */ + bool (*predicate)(const void *state); + /** Stats */ + struct ztest_suite_stats stats; +}; + +extern struct ztest_suite_node _ztest_suite_node_list_start[]; +extern struct ztest_suite_node _ztest_suite_node_list_end[]; + +/** + * Create and register a ztest suite. Using this macro creates a new test suite (using + * ztest_test_suite). It then creates a struct ztest_suite_node in a specific linker section. + * + * Tests can then be run by calling ztest_run_registered_test_suites(const void *state) by passing + * in the current state. See the documentation for ztest_run_registered_test_suites for more info. + * + * @param SUITE_NAME The name of the suite (see ztest_test_suite for more info) + * @param PREDICATE A function to test against the state and determine if the test should run. + * @param args Varargs placeholder for the remaining arguments passed for the unit tests. + */ +#define ztest_register_test_suite(SUITE_NAME, PREDICATE, args...) \ + ztest_test_suite(SUITE_NAME, ##args); \ + static STRUCT_SECTION_ITERABLE(ztest_suite_node, z_ztest_test_node_##SUITE_NAME) = { \ + .name = #SUITE_NAME, \ + .suite = _##SUITE_NAME, \ + .predicate = PREDICATE, \ + }; + +/** + * Run the registered unit tests which return true from their pragma function. + * + * @param state The current state of the machine as it relates to the test executable. + * @return The number of tests that ran. + */ +int ztest_run_registered_test_suites(const void *state); + +/** + * @brief Run a test suite. + * + * Internal implementation. Do not call directly. This will run the full test suite along with some + * checks for fast failures and initialization. + * + * @param name The name of the suite to run. + * @param suite Pointer to the first unit test. + * @return Negative value if the test suite never ran; otherwise, return the number of failures. + */ +int z_ztest_run_test_suite(const char *name, struct unit_test *suite); /** * @defgroup ztest_test Ztest testing macros diff --git a/subsys/testsuite/ztest/src/ztest.c b/subsys/testsuite/ztest/src/ztest.c index 4e1abbf73a2..0a98cd3b434 100644 --- a/subsys/testsuite/ztest/src/ztest.c +++ b/subsys/testsuite/ztest/src/ztest.c @@ -396,12 +396,12 @@ static int run_test(struct unit_test *test) #endif /* !KERNEL */ -void z_ztest_run_test_suite(const char *name, struct unit_test *suite) +int z_ztest_run_test_suite(const char *name, struct unit_test *suite) { int fail = 0; if (test_status < 0) { - return; + return test_status; } init_testing(); @@ -418,6 +418,8 @@ void z_ztest_run_test_suite(const char *name, struct unit_test *suite) TC_SUITE_END(name, (fail > 0 ? TC_FAIL : TC_PASS)); test_status = (test_status || fail) ? 1 : 0; + + return fail; } void end_report(void) @@ -433,6 +435,36 @@ void end_report(void) K_APPMEM_PARTITION_DEFINE(ztest_mem_partition); #endif +int ztest_run_registered_test_suites(const void *state) +{ + struct ztest_suite_node *ptr; + int count = 0; + + for (ptr = _ztest_suite_node_list_start; ptr < _ztest_suite_node_list_end; ++ptr) { + struct ztest_suite_stats *stats = &ptr->stats; + bool should_run = true; + + if (ptr->predicate != NULL) { + should_run = ptr->predicate(state); + } else { + /* If pragma is NULL, only run this test once. */ + should_run = stats->run_count == 0; + } + + if (should_run) { + int fail = z_ztest_run_test_suite(ptr->name, ptr->suite); + + count++; + stats->run_count++; + stats->fail_count += (fail != 0) ? 1 : 0; + } else { + stats->skip_count++; + } + } + + return count; +} + #ifndef KERNEL int main(void) { diff --git a/tests/ztest/register/CMakeLists.txt b/tests/ztest/register/CMakeLists.txt new file mode 100644 index 00000000000..a06945233f8 --- /dev/null +++ b/tests/ztest/register/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2021 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(register) + +FILE(GLOB app_sources src/*.c) +target_include_directories(app PRIVATE include) +target_sources(app PRIVATE ${app_sources}) diff --git a/tests/ztest/register/include/common.h b/tests/ztest/register/include/common.h new file mode 100644 index 00000000000..c518a5af5bc --- /dev/null +++ b/tests/ztest/register/include/common.h @@ -0,0 +1,23 @@ +/* + * Copyright 2021 Google LLC + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef TESTS_ZTEST_REGISTER_INCLUDE_COMMON_H_ +#define TESTS_ZTEST_REGISTER_INCLUDE_COMMON_H_ + +#include + +enum phase { + PHASE_VERIFY, + PHASE_NULL_PREDICATE_0, + PHASE_NULL_PREDICATE_1, + PHASE_STEPS_0, + PHASE_STEPS_1, +}; +struct global_test_state { + enum phase phase; +}; + +#endif /* TESTS_ZTEST_REGISTER_INCLUDE_COMMON_H_ */ diff --git a/tests/ztest/register/prj.conf b/tests/ztest/register/prj.conf new file mode 100644 index 00000000000..9467c292689 --- /dev/null +++ b/tests/ztest/register/prj.conf @@ -0,0 +1 @@ +CONFIG_ZTEST=y diff --git a/tests/ztest/register/src/main.c b/tests/ztest/register/src/main.c new file mode 100644 index 00000000000..9d727a2815a --- /dev/null +++ b/tests/ztest/register/src/main.c @@ -0,0 +1,198 @@ +/* + * Copyright 2021 Google LLC + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include "common.h" + +#define num_registered_suites (_ztest_suite_node_list_end - _ztest_suite_node_list_start) + +/** The current state of the test application. */ +static struct global_test_state global_state; + +/** + * Copies of the test entry's snapshot, used to test assertions. There's no good way to get the + * number of registered test suites at compile time so this is set to an arbitrary large size that + * should be enough even if the number of tests grows. There's a runtime check for this in + * test_verify_execution. + */ +static struct ztest_suite_stats stats_snapshot[128]; + +/** The results of a single execution */ +struct execution_results { + /** The test phase that was run */ + enum phase test_phase; + /** The number of tests that ran */ + int test_run_count; +} execution_results; + +/** + * Helper function used to find a test entry by name. + * + * @param name The name of the test entry. + * @return Pointer to the struct unit_test_node or NULL if not found. + */ +static struct ztest_suite_node *find_test_node(const char *name) +{ + struct ztest_suite_node *ptr; + + for (ptr = _ztest_suite_node_list_start; ptr != _ztest_suite_node_list_end; ++ptr) { + if (strcmp(ptr->name, name) == 0) { + return ptr; + } + } + + return NULL; +} + +/** + * @brief Find a snapshot in the stats_snapshot array + * + * Lookup a test case by name and find the matching ztest_suite_stats. + * + * @param name The name of the test entry. + * @return Pointer to the stats snapshot. + */ +static struct ztest_suite_stats *find_snapshot(const char *name) +{ + int index = find_test_node(name) - _ztest_suite_node_list_start; + + return stats_snapshot + index; +} + +/** + * Reset the global state between phases. This function can be thought of similarly to making a + * change affecting the state of the application being tested. + * + * @param phase The new phase of the application. + */ +static void reset_state(enum phase phase) +{ + execution_results.test_phase = phase; + execution_results.test_run_count = 0; + global_state.phase = phase; + + for (int i = 0; i < num_registered_suites; ++i) { + stats_snapshot[i] = _ztest_suite_node_list_start[i].stats; + } +} + +/** + * Create a snapshot of the tests' stats. This function should be called after each run in order + * to assert on only the changes in the stats. + */ +static void take_stats_snapshot(void) +{ + for (int i = 0; i < num_registered_suites; ++i) { + struct ztest_suite_stats *snapshot = stats_snapshot + i; + struct ztest_suite_stats *current = &_ztest_suite_node_list_start[i].stats; + + snapshot->run_count = current->run_count - snapshot->run_count; + snapshot->skip_count = current->skip_count - snapshot->skip_count; + snapshot->fail_count = current->fail_count - snapshot->fail_count; + } +} + +static void test_verify_execution(void) +{ + const struct ztest_suite_stats *stats; + + zassert_true(ARRAY_SIZE(stats_snapshot) >= num_registered_suites, + "Not enough stats snapshots, please allocate more."); + switch (execution_results.test_phase) { + case PHASE_NULL_PREDICATE_0: + /* Verify that only remove_first_node suite was run and removed. */ + stats = find_snapshot("run_null_predicate_once"); + zassert_equal(1, execution_results.test_run_count, NULL); + zassert_equal(1, stats->run_count, NULL); + zassert_equal(0, stats->skip_count, NULL); + zassert_equal(0, stats->fail_count, NULL); + break; + case PHASE_NULL_PREDICATE_1: + /* Verify that only remove_first_two_nodes_* were run. */ + zassert_equal(0, execution_results.test_run_count, NULL); + stats = find_snapshot("run_null_predicate_once"); + zassert_equal(0, stats->run_count, NULL); + zassert_equal(1, stats->skip_count, NULL); + zassert_equal(0, stats->fail_count, NULL); + break; + case PHASE_STEPS_0: + /* Verify that steps_0 and steps_all suites were run. */ + zassert_equal(2, execution_results.test_run_count, NULL); + stats = find_snapshot("test_step_0"); + zassert_equal(1, stats->run_count, NULL); + zassert_equal(0, stats->skip_count, NULL); + zassert_equal(0, stats->fail_count, NULL); + stats = find_snapshot("test_step_1"); + zassert_equal(0, stats->run_count, NULL); + zassert_equal(1, stats->skip_count, NULL); + zassert_equal(0, stats->fail_count, NULL); + stats = find_snapshot("test_step_all"); + zassert_equal(1, stats->run_count, NULL); + zassert_equal(0, stats->skip_count, NULL); + zassert_equal(0, stats->fail_count, NULL); + break; + case PHASE_STEPS_1: + /* Verify that steps_1 and steps_all suites were run. */ + zassert_equal(2, execution_results.test_run_count, NULL); + stats = find_snapshot("test_step_0"); + zassert_equal(0, stats->run_count, NULL); + zassert_equal(1, stats->skip_count, NULL); + zassert_equal(0, stats->fail_count, NULL); + stats = find_snapshot("test_step_1"); + zassert_equal(1, stats->run_count, NULL); + zassert_equal(0, stats->skip_count, NULL); + zassert_equal(0, stats->fail_count, NULL); + stats = find_snapshot("test_step_all"); + zassert_equal(1, stats->run_count, NULL); + zassert_equal(0, stats->skip_count, NULL); + zassert_equal(0, stats->fail_count, NULL); + break; + default: + ztest_test_fail(); + } +} + +static bool verify_predicate(const void *state) +{ + const struct global_test_state *s = state; + + return s->phase == PHASE_VERIFY; +} + +ztest_register_test_suite(verify, verify_predicate, + ztest_unit_test(test_verify_execution)); + +void test_main(void) +{ + /* Make sure that when predicate is set to NULL, the test is run. */ + reset_state(PHASE_NULL_PREDICATE_0); + execution_results.test_run_count = ztest_run_registered_test_suites(&global_state); + take_stats_snapshot(); + global_state.phase = PHASE_VERIFY; + ztest_run_registered_test_suites(&global_state); + + /* Try running the tests again, nothing should run. */ + reset_state(PHASE_NULL_PREDICATE_1); + execution_results.test_run_count = ztest_run_registered_test_suites(&global_state); + take_stats_snapshot(); + global_state.phase = PHASE_VERIFY; + ztest_run_registered_test_suites(&global_state); + + /* Run filter tests for step 0. */ + reset_state(PHASE_STEPS_0); + execution_results.test_run_count = ztest_run_registered_test_suites(&global_state); + global_state.phase = PHASE_VERIFY; + take_stats_snapshot(); + ztest_run_registered_test_suites(&global_state); + + /* Run filter tests for step 1. */ + reset_state(PHASE_STEPS_1); + execution_results.test_run_count = ztest_run_registered_test_suites(&global_state); + global_state.phase = PHASE_VERIFY; + take_stats_snapshot(); + ztest_run_registered_test_suites(&global_state); +} diff --git a/tests/ztest/register/src/test_null_predicate.c b/tests/ztest/register/src/test_null_predicate.c new file mode 100644 index 00000000000..aadb0147cab --- /dev/null +++ b/tests/ztest/register/src/test_null_predicate.c @@ -0,0 +1,14 @@ +/* + * Copyright 2021 Google LLC + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include "common.h" + +static void test_stub(void) +{ +} + +ztest_register_test_suite(run_null_predicate_once, NULL, ztest_unit_test(test_stub)); diff --git a/tests/ztest/register/src/test_step_0.c b/tests/ztest/register/src/test_step_0.c new file mode 100644 index 00000000000..1ab31bc5612 --- /dev/null +++ b/tests/ztest/register/src/test_step_0.c @@ -0,0 +1,21 @@ +/* + * Copyright 2021 Google LLC + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include "common.h" + +static void test(void) +{ +} + +static bool predicate(const void *state) +{ + const struct global_test_state *s = state; + + return s->phase == PHASE_STEPS_0; +} + +ztest_register_test_suite(test_step_0, predicate, ztest_unit_test(test)); diff --git a/tests/ztest/register/src/test_step_1.c b/tests/ztest/register/src/test_step_1.c new file mode 100644 index 00000000000..4a3538f5e81 --- /dev/null +++ b/tests/ztest/register/src/test_step_1.c @@ -0,0 +1,21 @@ +/* + * Copyright 2021 Google LLC + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include "common.h" + +static void test(void) +{ +} + +static bool predicate(const void *state) +{ + const struct global_test_state *s = state; + + return s->phase == PHASE_STEPS_1; +} + +ztest_register_test_suite(test_step_1, predicate, ztest_unit_test(test)); diff --git a/tests/ztest/register/src/test_step_all.c b/tests/ztest/register/src/test_step_all.c new file mode 100644 index 00000000000..0851ad35985 --- /dev/null +++ b/tests/ztest/register/src/test_step_all.c @@ -0,0 +1,21 @@ +/* + * Copyright 2021 Google LLC + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include "common.h" + +static void test(void) +{ +} + +static bool predicate(const void *state) +{ + const struct global_test_state *s = state; + + return s->phase == PHASE_STEPS_0 || s->phase == PHASE_STEPS_1; +} + +ztest_register_test_suite(test_step_all, predicate, ztest_unit_test(test)); diff --git a/tests/ztest/register/testcase.yaml b/tests/ztest/register/testcase.yaml new file mode 100644 index 00000000000..e9a0744f027 --- /dev/null +++ b/tests/ztest/register/testcase.yaml @@ -0,0 +1,5 @@ +tests: + testing.ztest.register: + tags: test_framework + integration_platforms: + - native_posix