ztest: Add register functionality

Add new functionality to ztest to improve test modularity. The two
primary new entry points are:
* ztest_register_test_suite
* ztest_run_registered_test_suites

When registering a new test suite, users provide the name as well as
an optional predicate used to filter the tests for each run. Using NULL
as the predicate ensures that the test is run exactly once (after which
it is automatically filtered from future runs).

Calls to ztest_run_registered_test_suites take a state pointer as an
argument. This allows the the pragma functions to decide whether the
test should be run.

The biggest benefit of this system (other than the ability to filter
tests and maintain a larger test state) is the ability to better
modularize the test source code. Instead of all the various tests
having to coordinate and the main function having to know which tests
to run, each source file manages registering its own test
suite and handling the conditions for running the suite.

Signed-off-by: Yuval Peress <peress@chromium.org>

subsys/testsuite/include/ztest.ld

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+SECTIONS
+{
+	.data.ztest_suite_node_area : ALIGN(4)
+	{
+		_ztest_suite_node_list_start = .;
+		KEEP(*(SORT_BY_NAME(._ztest_suite_node.static.*)))
+		_ztest_suite_node_list_end = .;
+	}
+}
+INSERT AFTER .data;

subsys/testsuite/unittest.cmake

@@ -62,6 +62,10 @@ target_compile_options(testbinary PRIVATE
   $<$<COMPILE_LANGUAGE:ASM>:${EXTRA_AFLAGS_AS_LIST}>
   )
 
+target_link_options(testbinary PRIVATE
+  -T "${ZEPHYR_BASE}/subsys/testsuite/include/ztest.ld"
+  )
+
 target_link_libraries(testbinary PRIVATE
   ${EXTRA_LDFLAGS_AS_LIST}
   )

subsys/testsuite/ztest/include/ztest_test.h

@@ -14,6 +14,8 @@
 #define ZEPHYR_TESTSUITE_ZTEST_TEST_H_
 
 #include <app_memory/app_memdomain.h>
+#include <init.h>
+#include <stdbool.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -27,7 +29,80 @@ struct unit_test {
 	uint32_t thread_options;
 };
 
-void z_ztest_run_test_suite(const char *name, struct unit_test *suite);
+/**
+ * Stats about a ztest suite
+ */
+struct ztest_suite_stats {
+	/** The number of times that the suite ran */
+	uint32_t run_count;
+	/** The number of times that the suite was skipped */
+	uint32_t skip_count;
+	/** The number of times that the suite failed */
+	uint32_t fail_count;
+};
+
+/**
+ * A single node of test suite. Each node should be added to a single linker section which will
+ * allow ztest_run_registered_test_suites() to iterate over the various nodes.
+ */
+struct ztest_suite_node {
+	/** The name of the test suite. */
+	const char *name;
+	/** Pointer to the test suite. */
+	struct unit_test *suite;
+	/**
+	 * An optional predicate function to determine if the test should run. If NULL, then the
+	 * test will only run once on the first attempt.
+	 *
+	 * @param state The current state of the test application.
+	 * @return True if the suite should be run; false to skip.
+	 */
+	bool (*predicate)(const void *state);
+	/** Stats */
+	struct ztest_suite_stats stats;
+};
+
+extern struct ztest_suite_node _ztest_suite_node_list_start[];
+extern struct ztest_suite_node _ztest_suite_node_list_end[];
+
+/**
+ * Create and register a ztest suite. Using this macro creates a new test suite (using
+ * ztest_test_suite). It then creates a struct ztest_suite_node in a specific linker section.
+ *
+ * Tests can then be run by calling ztest_run_registered_test_suites(const void *state) by passing
+ * in the current state. See the documentation for ztest_run_registered_test_suites for more info.
+ *
+ * @param SUITE_NAME The name of the suite (see ztest_test_suite for more info)
+ * @param PREDICATE A function to test against the state and determine if the test should run.
+ * @param args Varargs placeholder for the remaining arguments passed for the unit tests.
+ */
+#define ztest_register_test_suite(SUITE_NAME, PREDICATE, args...)                                  \
+	ztest_test_suite(SUITE_NAME, ##args);                                                      \
+	static STRUCT_SECTION_ITERABLE(ztest_suite_node, z_ztest_test_node_##SUITE_NAME) = {       \
+		.name = #SUITE_NAME,                                                               \
+		.suite = _##SUITE_NAME,                                                            \
+		.predicate = PREDICATE,                                                            \
+	};
+
+/**
+ * Run the registered unit tests which return true from their pragma function.
+ *
+ * @param state The current state of the machine as it relates to the test executable.
+ * @return The number of tests that ran.
+ */
+int ztest_run_registered_test_suites(const void *state);
+
+/**
+ * @brief Run a test suite.
+ *
+ * Internal implementation. Do not call directly. This will run the full test suite along with some
+ * checks for fast failures and initialization.
+ *
+ * @param name The name of the suite to run.
+ * @param suite Pointer to the first unit test.
+ * @return Negative value if the test suite never ran; otherwise, return the number of failures.
+ */
+int z_ztest_run_test_suite(const char *name, struct unit_test *suite);
 
 /**
  * @defgroup ztest_test Ztest testing macros

subsys/testsuite/ztest/src/ztest.c

@@ -396,12 +396,12 @@ static int run_test(struct unit_test *test)
 
 #endif /* !KERNEL */
 
-void z_ztest_run_test_suite(const char *name, struct unit_test *suite)
+int z_ztest_run_test_suite(const char *name, struct unit_test *suite)
 {
 	int fail = 0;
 
 	if (test_status < 0) {
-		return;
+		return test_status;
 	}
 
 	init_testing();
@@ -418,6 +418,8 @@ void z_ztest_run_test_suite(const char *name, struct unit_test *suite)
 	TC_SUITE_END(name, (fail > 0 ? TC_FAIL : TC_PASS));
 
 	test_status = (test_status || fail) ? 1 : 0;
+
+	return fail;
 }
 
 void end_report(void)
@@ -433,6 +435,36 @@ void end_report(void)
 K_APPMEM_PARTITION_DEFINE(ztest_mem_partition);
 #endif
 
+int ztest_run_registered_test_suites(const void *state)
+{
+	struct ztest_suite_node *ptr;
+	int count = 0;
+
+	for (ptr = _ztest_suite_node_list_start; ptr < _ztest_suite_node_list_end; ++ptr) {
+		struct ztest_suite_stats *stats = &ptr->stats;
+		bool should_run = true;
+
+		if (ptr->predicate != NULL) {
+			should_run = ptr->predicate(state);
+		} else  {
+			/* If pragma is NULL, only run this test once. */
+			should_run = stats->run_count == 0;
+		}
+
+		if (should_run) {
+			int fail = z_ztest_run_test_suite(ptr->name, ptr->suite);
+
+			count++;
+			stats->run_count++;
+			stats->fail_count += (fail != 0) ? 1 : 0;
+		} else {
+			stats->skip_count++;
+		}
+	}
+
+	return count;
+}
+
 #ifndef KERNEL
 int main(void)
 {