michaelh/zephyr
1
0
Fork 0
Code Releases Activity

ztest: add a weak implementation of test_main()

Browse source 
Introduce a weak implementation of test_main() which calls:
* ztest_run_registered_test_suites(NULL);
* ztest_verify_all_registered_test_suites_ran();

This will attempt to run all registered test suites and verify that
they each ran.

Signed-off-by: Yuval Peress <peress@chromium.org>
This commit is contained in:
Yuval Peress 2021-08-01 22:12:44 -06:00 • committed by Anas Nashif
commit 27f6a5e07d
8 changed files with 106 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

33
scripts/pylib/twister/twisterlib.py
View file

@ -1589,16 +1589,20 @@ class ScanPathResult:
has_run_registered_test_suites Whether or not the path contained at
least one call to
ztest_run_registered_test_suites.
has_test_main Whether or not the path contains a
definition of test_main(void)
"""
def __init__(self,
matches: List[str] = None,
warnings: str = None,
has_registered_test_suites: bool = False,
has_run_registered_test_suites: bool = False):
has_run_registered_test_suites: bool = False,
has_test_main: bool = False):
self.matches = matches
self.warnings = warnings
self.has_registered_test_suites = has_registered_test_suites
self.has_run_registered_test_suites = has_run_registered_test_suites
self.has_test_main = has_test_main
def __eq__(self, other):
if not isinstance(other, ScanPathResult):
@ -1608,7 +1612,8 @@ class ScanPathResult:
(self.has_registered_test_suites ==
other.has_registered_test_suites) and
(self.has_run_registered_test_suites ==
other.has_run_registered_test_suites))
other.has_run_registered_test_suites) and
self.has_test_main == other.has_test_main)
class TestCase(DisablePyTestCollectionMixin):
@ -1701,6 +1706,15 @@ Tests should reference the category and subsystem with a dot as a separator.
br"^\s*ztest_register_test_suite"
br"\(\s*(?P<suite_name>[a-zA-Z0-9_]+)\s*,",
re.MULTILINE)
# Checks if the file contains a definition of "void test_main(void)"
# Since ztest provides a plain test_main implementation it is OK to:
# 1. register test suites and not call the run function iff the test
# doesn't have a custom test_main.
# 2. register test suites and a custom test_main definition iff the test
# also calls ztest_run_registered_test_suites.
test_main_regex = re.compile(
br"^\s*void\s+test_main\(void\)",
re.MULTILINE)
stc_regex = re.compile(
br"""^\s* # empy space at the beginning is ok
# catch the case where it is declared in the same sentence, e.g:
@ -1733,6 +1747,7 @@ Tests should reference the category and subsystem with a dot as a separator.
warnings = None
has_registered_test_suites = False
has_run_registered_test_suites = False
has_test_main = False
with open(inf_name) as inf:
if os.name == 'nt':
@ -1750,6 +1765,8 @@ Tests should reference the category and subsystem with a dot as a separator.
has_registered_test_suites = True
if registered_suite_run_regex.search(main_c):
has_run_registered_test_suites = True
if test_main_regex.search(main_c):
has_test_main = True
if not suite_regex_match and not has_registered_test_suites:
# can't find ztest_test_suite, maybe a client, because
@ -1758,7 +1775,8 @@ Tests should reference the category and subsystem with a dot as a separator.
matches=None,
warnings=None,
has_registered_test_suites=has_registered_test_suites,
has_run_registered_test_suites=has_run_registered_test_suites)
has_run_registered_test_suites=has_run_registered_test_suites,
has_test_main=has_test_main)
suite_run_match = suite_run_regex.search(main_c)
if suite_regex_match and not suite_run_match:
@ -1792,12 +1810,14 @@ Tests should reference the category and subsystem with a dot as a separator.
matches=matches,
warnings=warnings,
has_registered_test_suites=has_registered_test_suites,
has_run_registered_test_suites=has_run_registered_test_suites)
has_run_registered_test_suites=has_run_registered_test_suites,
has_test_main=has_test_main)
def scan_path(self, path):
subcases = []
has_registered_test_suites = False
has_run_registered_test_suites = False
has_test_main = False
for filename in glob.glob(os.path.join(path, "src", "*.c*")):
try:
result: ScanPathResult = self.scan_file(filename)
@ -1811,6 +1831,8 @@ Tests should reference the category and subsystem with a dot as a separator.
has_registered_test_suites = True
if result.has_run_registered_test_suites:
has_run_registered_test_suites = True
if result.has_test_main:
has_test_main = True
except ValueError as e:
logger.error("%s: can't find: %s" % (filename, e))
@ -1824,7 +1846,8 @@ Tests should reference the category and subsystem with a dot as a separator.
except ValueError as e:
logger.error("%s: can't find: %s" % (filename, e))
if has_registered_test_suites and not has_run_registered_test_suites:
if (has_registered_test_suites and has_test_main and
not has_run_registered_test_suites):
warning = \
"Found call to 'ztest_register_test_suite()' but no "\
"call to 'ztest_run_registered_test_suites()'"

12
scripts/tests/twister/test_testinstance.py
View file

@ -123,24 +123,28 @@ TESTDATA_5 = [
'test_test_aa',
'user', 'last'],
has_registered_test_suites=False,
has_run_registered_test_suites=False)),
has_run_registered_test_suites=False,
has_test_main=False)),
("testcases/tests/test_a/test_ztest_error.c",
ScanPathResult(
warnings="Found a test that does not start with test_",
matches=['1a', '1c', '2a', '2b'],
has_registered_test_suites=False,
has_run_registered_test_suites=False)),
has_run_registered_test_suites=False,
has_test_main=True)),
("testcases/tests/test_a/test_ztest_error_1.c",
ScanPathResult(
warnings="found invalid #ifdef, #endif in ztest_test_suite()",
matches=['unit_1a', 'unit_1b', 'Unit_1c'],
has_registered_test_suites=False,
has_run_registered_test_suites=False)),
has_run_registered_test_suites=False,
has_test_main=False)),
("testcases/tests/test_d/test_ztest_error_register_test_suite.c",
ScanPathResult(
warnings=None, matches=['unit_1a', 'unit_1b'],
has_registered_test_suites=True,
has_run_registered_test_suites=False)),
has_run_registered_test_suites=False,
has_test_main=False)),
]
@pytest.mark.parametrize("test_file, expected", TESTDATA_5)

11
subsys/testsuite/ztest/include/ztest_test.h
View file

@ -92,6 +92,17 @@ extern struct ztest_suite_node _ztest_suite_node_list_end[];
*/
int ztest_run_registered_test_suites(const void *state);
/**
* @brief Fails the test if any of the registered tests did not run.
*
* When registering test suites, a pragma function can be provided to determine WHEN the test should
* run. It is possible that a test suite could be registered but the pragma always prevents it from
* running. In cases where a test should make sure that ALL suites ran at least once, this function
* may be called at the end of test_main(). It will cause the test to fail if any suite was
* registered but never ran.
*/
void ztest_verify_all_registered_test_suites_ran(void);
/**
* @brief Run a test suite.
*

23
subsys/testsuite/ztest/src/ztest.c
View file

@ -465,6 +465,29 @@ int ztest_run_registered_test_suites(const void *state)
return count;
}
void ztest_verify_all_registered_test_suites_ran(void)
{
bool all_tests_run = true;
struct ztest_suite_node *ptr;
for (ptr = _ztest_suite_node_list_start; ptr < _ztest_suite_node_list_end; ++ptr) {
if (ptr->stats.run_count < 1) {
PRINT("ERROR: Test '%s' did not run.\n", ptr->name);
all_tests_run = false;
}
}
if (!all_tests_run) {
test_status = 1;
}
}
void __weak test_main(void)
{
ztest_run_registered_test_suites(NULL);
ztest_verify_all_registered_test_suites_ran();
}
#ifndef KERNEL
int main(void)
{

9
tests/ztest/automain/CMakeLists.txt Normal file
View file

@ -0,0 +1,9 @@
# Copyright 2021 Google LLC
# SPDX-License-Identifier: Apache-2.0
cmake_minimum_required(VERSION 3.20.0)
find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
project(automain)
FILE(GLOB app_sources src/*.c)
target_sources(app PRIVATE ${app_sources})

1
tests/ztest/automain/prj.conf Normal file
View file

@ -0,0 +1 @@
CONFIG_ZTEST=y

21
tests/ztest/automain/src/test_feature.c Normal file
View file

@ -0,0 +1,21 @@
/*
* Copyright 2021 Google LLC
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <ztest.h>
/**
* @brief A stub unit test.
*
* This is a mock unit test that is expected to run. Note that it is not added directly to a test
* suite and run via ztest_run_test_suite(). It is instead registered below using
* ztest_register_test_suite and will be run by the automatically generated test_main() function.
*/
static void test_stub(void)
{
}
ztest_register_test_suite(test_suite, NULL,
ztest_unit_test(test_stub));

5
tests/ztest/automain/testcase.yaml Normal file
View file

@ -0,0 +1,5 @@
tests:
testing.ztest.automain:
tags: test_framework
integration_platforms:
- native_posix