subsys/testsuite: make tc_util overridable

A Kconfig boolean is added to allow users to provide their own
output strings when running tests via ztest.
This allows changing e.g. the PASS/FAIL/SKIPPED strings,
add counters, change separators, and similar.

A test using the feature and relevant documentation is added.

Signed-off-by: Torstein Grindvik <torstein.grindvik@nordicsemi.no>

doc/guides/test/ztest.rst

@@ -341,3 +341,21 @@ expect the values ``a=2`` and ``b=3``, and telling ``returns_int`` to return
 
 .. doxygengroup:: ztest_mock
    :project: Zephyr
+
+Customizing Test Output
+***********************
+The way output is presented when running tests can be customized.
+An example can be found in :zephyr_file:`tests/ztest/custom_output`.
+
+Customization is enabled by setting :option:`CONFIG_ZTEST_TC_UTIL_USER_OVERRIDE` to "y"
+and adding a file :file:`tc_util_user_override.h` with your overrides.
+
+Add the line ``zephyr_include_directories(my_folder)`` to
+your project's :file:`CMakeLists.txt` to let Zephyr find your header file during builds.
+
+See the file :zephyr_file:`subsys/testsuite/include/tc_util.h` to see which macros and/or defines can be overridden.
+These will be surrounded by blocks such as::
+
+        #ifndef SOMETHING
+        #define SOMETHING <default implementation>
+        #endif /* SOMETHING */

subsys/testsuite/include/tc_util.h

@@ -15,7 +15,13 @@
 #include <shell/shell.h>
 #include <sys/printk.h>
 
+#if defined CONFIG_ZTEST_TC_UTIL_USER_OVERRIDE
+#include <tc_util_user_override.h>
+#endif
+
+#ifndef PRINT_DATA
 #define PRINT_DATA(fmt, ...) printk(fmt, ##__VA_ARGS__)
+#endif
 
 #if defined CONFIG_ARCH_POSIX
 #include "posix_board_if.h"
@@ -44,10 +50,12 @@
 #define TC_PRINT_RUNID do {} while (0)
 #endif
 
+#ifndef PRINT_LINE
 #define PRINT_LINE                          \
 	PRINT_DATA(                                                        \
 		"============================================================" \
 		"=======\n")
+#endif
 
 /* stack size and priority for test suite task */
 #define TASK_STACK_SIZE (1024 * 2)
@@ -58,38 +66,63 @@
 #define TC_FAIL 1
 #define TC_SKIP 2
 
+#ifndef TC_PASS_STR
+#define TC_PASS_STR "PASS"
+#endif
+#ifndef TC_FAIL_STR
+#define TC_FAIL_STR "FAIL"
+#endif
+#ifndef TC_SKIP_STR
+#define TC_SKIP_STR "SKIP"
+#endif
+
 static inline const char *TC_RESULT_TO_STR(int result)
 {
 	switch (result) {
 	case TC_PASS:
-		return "PASS";
+		return TC_PASS_STR;
 	case TC_FAIL:
-		return "FAIL";
+		return TC_FAIL_STR;
 	case TC_SKIP:
-		return "SKIP";
+		return TC_SKIP_STR;
 	default:
 		return "?";
 	}
 }
 
+#ifndef TC_ERROR
 #define TC_ERROR(fmt, ...)                               \
 	do {                                                 \
 		PRINT_DATA(FMT_ERROR, "FAIL", __func__, __LINE__); \
 		PRINT_DATA(fmt, ##__VA_ARGS__);                  \
 	} while (0)
+#endif
 
+#ifndef TC_PRINT
 #define TC_PRINT(fmt, ...) PRINT_DATA(fmt, ##__VA_ARGS__)
-#define TC_START(name) PRINT_DATA("starting test - %s\n", name)
+#endif
-#define TC_END(result, fmt, ...) PRINT_DATA(fmt, ##__VA_ARGS__)
 
+#ifndef TC_START
+#define TC_START(name) PRINT_DATA("starting test - %s\n", name)
+#endif
+
+#ifndef TC_END
+#define TC_END(result, fmt, ...) PRINT_DATA(fmt, ##__VA_ARGS__)
+#endif
+
+#ifndef Z_TC_END_RESULT
 /* prints result and the function name */
 #define Z_TC_END_RESULT(result, func)					\
 	do {								\
 		TC_END(result, "%s - %s\n", TC_RESULT_TO_STR(result), func); \
 		PRINT_LINE;						\
 	} while (0)
+#endif
+
+#ifndef TC_END_RESULT
 #define TC_END_RESULT(result)                           \
 	Z_TC_END_RESULT((result), __func__)
+#endif
 
 #if defined(CONFIG_ARCH_POSIX)
 #define TC_END_POST(result) posix_exit(result)
@@ -97,6 +130,7 @@ static inline const char *TC_RESULT_TO_STR(int result)
 #define TC_END_POST(result)
 #endif /* CONFIG_ARCH_POSIX */
 
+#ifndef TC_END_REPORT
 #define TC_END_REPORT(result)                               \
 	do {                                                    \
 		PRINT_LINE;                                         \
@@ -106,6 +140,7 @@ static inline const char *TC_RESULT_TO_STR(int result)
 		       (result) == TC_PASS ? "SUCCESSFUL" : "FAILED");	\
 		TC_END_POST(result);                                    \
 	} while (0)
+#endif
 
 #if defined(CONFIG_SHELL)
 #define TC_CMD_DEFINE(name)						\

subsys/testsuite/ztest/Kconfig

@@ -42,6 +42,16 @@ config ZTEST_MOCKING
 	  Enable mocking support for Ztest. This allows the test to set
 	  return values and expected parameters to functions.
 
+config ZTEST_TC_UTIL_USER_OVERRIDE
+	bool "Override tc_util.h"
+	depends on ZTEST
+	help
+	  Enable overriding defines in tc_util.h.
+	  If True the user should provide tc_util_user_override.h in Zephyr's include path,
+	  e.g. by adding zephyr_include_directories(project PRIVATE my_folder) to a project's CMakeLists.txt.
+	  The override header may now #define the various macros and strings in tc_util.h which are
+	  surrounded by #ifndef ... #endif blocks.
+
 config ZTEST_PARAMETER_COUNT
 	int "Count of parameters or return values reserved"
 	depends on ZTEST_MOCKING

tests/ztest/custom_output/CMakeLists.txt

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.13.1)
+include($ENV{ZEPHYR_BASE}/cmake/app/boilerplate.cmake NO_POLICY_SCOPE)
+project(integration)
+
+FILE(GLOB app_sources src/*.c)
+target_sources(app PRIVATE ${app_sources})
+zephyr_include_directories(include)

tests/ztest/custom_output/include/tc_util_user_override.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2019 Nordic Semiconductor ASA
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef __TC_UTIL_USER_OVERRIDE_H__
+#define __TC_UTIL_USER_OVERRIDE_H__
+
+/* This header provides a sample user override of various tc_util.h defines. */
+
+/* Example: Reduce vertical line verbosity by
+ * redefining the separator to nothing.
+ */
+#define PRINT_LINE
+
+/* Example: Prepend test cases with a counter. */
+#define TC_START(original)  do { \
+	static int count; \
+	printk("%d: Test [%s]", ++count, original); \
+} while (0)
+
+/* Example: Change result string output formats. */
+#define TC_PASS_STR "(PASS)"
+#define TC_FAIL_STR "[FAILED]"
+#define TC_SKIP_STR "{SKIPPED}"
+
+/* Example: Also count the number of pass/fail/skips and display it. */
+#define Z_TC_END_RESULT(result, s) do { \
+	static int result_keeper[3] = {0}; \
+	result_keeper[result]++; \
+	printk(" reported %s no. %d\n", \
+		TC_RESULT_TO_STR(result), \
+		result_keeper[result]); \
+} while (0)
+
+#endif /* __TC_UTIL_USER_OVERRIDE_H__ */

tests/ztest/custom_output/prj.conf

@@ -0,0 +1 @@
+CONFIG_ZTEST=y

tests/ztest/custom_output/prj_customized_output.conf

@@ -0,0 +1,2 @@
+CONFIG_ZTEST=y
+CONFIG_ZTEST_TC_UTIL_USER_OVERRIDE=y

tests/ztest/custom_output/src/main.c

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <ztest.h>
+
+static void test_assert_pass(void)
+{
+	ztest_test_pass();
+}
+
+static void test_assert_skip(void)
+{
+	ztest_test_skip();
+}
+
+/*
+ * Same tests called several times to show custom handling of output.
+ */
+void test_main(void)
+{
+	ztest_test_suite(framework_tests,
+		ztest_unit_test(test_assert_pass),
+		ztest_unit_test(test_assert_pass),
+		ztest_unit_test(test_assert_pass),
+		ztest_unit_test(test_assert_skip),
+		ztest_unit_test(test_assert_skip)
+	);
+
+	ztest_run_test_suite(framework_tests);
+}

tests/ztest/custom_output/testcase.yaml

@@ -0,0 +1,7 @@
+tests:
+  testing.ztest.regular_output:
+    tags: test_framework
+
+  testing.ztest.customized_output:
+    extra_args: CONF_FILE=prj_customized_output.conf
+    tags: test_framework