subsys/testsuite: make tc_util overridable

A Kconfig boolean is added to allow users to provide their own
output strings when running tests via ztest.
This allows changing e.g. the PASS/FAIL/SKIPPED strings,
add counters, change separators, and similar.

A test using the feature and relevant documentation is added.

Signed-off-by: Torstein Grindvik <torstein.grindvik@nordicsemi.no>
This commit is contained in:
Torstein Grindvik 2019-09-12 10:57:41 +02:00 committed by Anas Nashif
commit d888e012ed
9 changed files with 157 additions and 5 deletions

View file

@ -341,3 +341,21 @@ expect the values ``a=2`` and ``b=3``, and telling ``returns_int`` to return
.. doxygengroup:: ztest_mock
:project: Zephyr
Customizing Test Output
***********************
The way output is presented when running tests can be customized.
An example can be found in :zephyr_file:`tests/ztest/custom_output`.
Customization is enabled by setting :option:`CONFIG_ZTEST_TC_UTIL_USER_OVERRIDE` to "y"
and adding a file :file:`tc_util_user_override.h` with your overrides.
Add the line ``zephyr_include_directories(my_folder)`` to
your project's :file:`CMakeLists.txt` to let Zephyr find your header file during builds.
See the file :zephyr_file:`subsys/testsuite/include/tc_util.h` to see which macros and/or defines can be overridden.
These will be surrounded by blocks such as::
#ifndef SOMETHING
#define SOMETHING <default implementation>
#endif /* SOMETHING */

View file

@ -15,7 +15,13 @@
#include <shell/shell.h>
#include <sys/printk.h>
#if defined CONFIG_ZTEST_TC_UTIL_USER_OVERRIDE
#include <tc_util_user_override.h>
#endif
#ifndef PRINT_DATA
#define PRINT_DATA(fmt, ...) printk(fmt, ##__VA_ARGS__)
#endif
#if defined CONFIG_ARCH_POSIX
#include "posix_board_if.h"
@ -44,10 +50,12 @@
#define TC_PRINT_RUNID do {} while (0)
#endif
#ifndef PRINT_LINE
#define PRINT_LINE \
PRINT_DATA( \
"============================================================" \
"=======\n")
#endif
/* stack size and priority for test suite task */
#define TASK_STACK_SIZE (1024 * 2)
@ -58,38 +66,63 @@
#define TC_FAIL 1
#define TC_SKIP 2
#ifndef TC_PASS_STR
#define TC_PASS_STR "PASS"
#endif
#ifndef TC_FAIL_STR
#define TC_FAIL_STR "FAIL"
#endif
#ifndef TC_SKIP_STR
#define TC_SKIP_STR "SKIP"
#endif
static inline const char *TC_RESULT_TO_STR(int result)
{
switch (result) {
case TC_PASS:
return "PASS";
return TC_PASS_STR;
case TC_FAIL:
return "FAIL";
return TC_FAIL_STR;
case TC_SKIP:
return "SKIP";
return TC_SKIP_STR;
default:
return "?";
}
}
#ifndef TC_ERROR
#define TC_ERROR(fmt, ...) \
do { \
PRINT_DATA(FMT_ERROR, "FAIL", __func__, __LINE__); \
PRINT_DATA(fmt, ##__VA_ARGS__); \
} while (0)
#endif
#ifndef TC_PRINT
#define TC_PRINT(fmt, ...) PRINT_DATA(fmt, ##__VA_ARGS__)
#define TC_START(name) PRINT_DATA("starting test - %s\n", name)
#define TC_END(result, fmt, ...) PRINT_DATA(fmt, ##__VA_ARGS__)
#endif
#ifndef TC_START
#define TC_START(name) PRINT_DATA("starting test - %s\n", name)
#endif
#ifndef TC_END
#define TC_END(result, fmt, ...) PRINT_DATA(fmt, ##__VA_ARGS__)
#endif
#ifndef Z_TC_END_RESULT
/* prints result and the function name */
#define Z_TC_END_RESULT(result, func) \
do { \
TC_END(result, "%s - %s\n", TC_RESULT_TO_STR(result), func); \
PRINT_LINE; \
} while (0)
#endif
#ifndef TC_END_RESULT
#define TC_END_RESULT(result) \
Z_TC_END_RESULT((result), __func__)
#endif
#if defined(CONFIG_ARCH_POSIX)
#define TC_END_POST(result) posix_exit(result)
@ -97,6 +130,7 @@ static inline const char *TC_RESULT_TO_STR(int result)
#define TC_END_POST(result)
#endif /* CONFIG_ARCH_POSIX */
#ifndef TC_END_REPORT
#define TC_END_REPORT(result) \
do { \
PRINT_LINE; \
@ -106,6 +140,7 @@ static inline const char *TC_RESULT_TO_STR(int result)
(result) == TC_PASS ? "SUCCESSFUL" : "FAILED"); \
TC_END_POST(result); \
} while (0)
#endif
#if defined(CONFIG_SHELL)
#define TC_CMD_DEFINE(name) \

View file

@ -42,6 +42,16 @@ config ZTEST_MOCKING
Enable mocking support for Ztest. This allows the test to set
return values and expected parameters to functions.
config ZTEST_TC_UTIL_USER_OVERRIDE
bool "Override tc_util.h"
depends on ZTEST
help
Enable overriding defines in tc_util.h.
If True the user should provide tc_util_user_override.h in Zephyr's include path,
e.g. by adding zephyr_include_directories(project PRIVATE my_folder) to a project's CMakeLists.txt.
The override header may now #define the various macros and strings in tc_util.h which are
surrounded by #ifndef ... #endif blocks.
config ZTEST_PARAMETER_COUNT
int "Count of parameters or return values reserved"
depends on ZTEST_MOCKING

View file

@ -0,0 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
cmake_minimum_required(VERSION 3.13.1)
include($ENV{ZEPHYR_BASE}/cmake/app/boilerplate.cmake NO_POLICY_SCOPE)
project(integration)
FILE(GLOB app_sources src/*.c)
target_sources(app PRIVATE ${app_sources})
zephyr_include_directories(include)

View file

@ -0,0 +1,37 @@
/*
* Copyright (c) 2019 Nordic Semiconductor ASA
*
* SPDX-License-Identifier: Apache-2.0
*/
#ifndef __TC_UTIL_USER_OVERRIDE_H__
#define __TC_UTIL_USER_OVERRIDE_H__
/* This header provides a sample user override of various tc_util.h defines. */
/* Example: Reduce vertical line verbosity by
* redefining the separator to nothing.
*/
#define PRINT_LINE
/* Example: Prepend test cases with a counter. */
#define TC_START(original) do { \
static int count; \
printk("%d: Test [%s]", ++count, original); \
} while (0)
/* Example: Change result string output formats. */
#define TC_PASS_STR "(PASS)"
#define TC_FAIL_STR "[FAILED]"
#define TC_SKIP_STR "{SKIPPED}"
/* Example: Also count the number of pass/fail/skips and display it. */
#define Z_TC_END_RESULT(result, s) do { \
static int result_keeper[3] = {0}; \
result_keeper[result]++; \
printk(" reported %s no. %d\n", \
TC_RESULT_TO_STR(result), \
result_keeper[result]); \
} while (0)
#endif /* __TC_UTIL_USER_OVERRIDE_H__ */

View file

@ -0,0 +1 @@
CONFIG_ZTEST=y

View file

@ -0,0 +1,2 @@
CONFIG_ZTEST=y
CONFIG_ZTEST_TC_UTIL_USER_OVERRIDE=y

View file

@ -0,0 +1,33 @@
/*
* Copyright (c) 2016 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <ztest.h>
static void test_assert_pass(void)
{
ztest_test_pass();
}
static void test_assert_skip(void)
{
ztest_test_skip();
}
/*
* Same tests called several times to show custom handling of output.
*/
void test_main(void)
{
ztest_test_suite(framework_tests,
ztest_unit_test(test_assert_pass),
ztest_unit_test(test_assert_pass),
ztest_unit_test(test_assert_pass),
ztest_unit_test(test_assert_skip),
ztest_unit_test(test_assert_skip)
);
ztest_run_test_suite(framework_tests);
}

View file

@ -0,0 +1,7 @@
tests:
testing.ztest.regular_output:
tags: test_framework
testing.ztest.customized_output:
extra_args: CONF_FILE=prj_customized_output.conf
tags: test_framework