ztest: Update ztest with more powerful testing APIs

1. Test suites in prior ztest serve no purpose other than logical
ordering of tests into a named-group. Move the construct of setup and
teardown into the test suite and away from individual tests.
Additionally, add the constructs of before/after to the test suites.
This model more closely resembels other testing frameworks such as gTest
and Junit.
2. Test can be added to a suite by using ZTEST() or ZTEST_F() where _F
stands for fixture. In the case where _F is used, the argument `this`
will be provided with the type `struct suite_name##_fixture*`. Again,
this models other modern testing frameworks and allows the test to
directly access the already set up data related to the test suite.
3. Add the concept of test rules (from Junit). Rules are similar to the
before/after functions of the test suites but are global and run on all
suites. An example of a test rule can be to check that nothing was
logged to ERROR. The rule can cause the test to fail if anything was
logged to ERROR during an integration test. Another example would be a
rule that verifies that tests ran within some defined timeout.

Signed-off-by: Yuval Peress <peress@google.com>

tests/subsys/logging/log_api/prj.conf

@@ -1,5 +1,6 @@
 CONFIG_MAIN_THREAD_PRIORITY=5
 CONFIG_ZTEST=y
+CONFIG_ZTEST_NEW_API=y
 CONFIG_TEST_USERSPACE=y
 CONFIG_TEST_LOGGING_DEFAULTS=n
 CONFIG_LOG=y

tests/subsys/logging/log_api/src/main.c

@@ -9,12 +9,3 @@
 #include <string.h>
 
 #include "test.inc"
-
-void test_cxx(void);
-void test_cc(void);
-
-void test_main(void)
-{
-	test_cc();
-	test_cxx();
-}

tests/subsys/logging/log_api/src/test.inc

@@ -627,20 +627,15 @@ static void test_log_panic(void)
 	mock_log_backend_validate(&backend1, true);
 }
 
-/* Disable backends because same suite may be excuted again but compiled by C++ */
-static void log_api_suite_teardown(void)
+/* Disable backends because same suite may be executed again but compiled by C++ */
+static void log_api_suite_teardown(void *data)
 {
+	ARG_UNUSED(data);
 	log_backend_disable(&backend1);
 	log_backend_disable(&backend2);
 }
 
-/*test case main entry*/
-#if __cplusplus
-extern "C" void test_cxx(void);
-void test_cxx(void)
-#else
-void test_cc(void)
-#endif
+static void *log_api_suite_setup(void)
 {
 	PRINT("Configuration:\n");
 	PRINT("\t Mode: %s\n",
@@ -654,17 +649,35 @@ void test_cc(void)
 #if __cplusplus
 	PRINT("\t C++: yes\n");
 #endif
-
-	ztest_test_suite(test_log_api,
-			 ztest_unit_test(test_log_various_messages),
-			 ztest_unit_test(test_log_backend_runtime_filtering),
-			 ztest_unit_test(test_log_overflow),
-			 ztest_unit_test(test_log_arguments),
-			 ztest_unit_test(test_log_from_declared_module),
-			 ztest_unit_test(test_log_msg_dropped_notification),
-			 ztest_unit_test(test_log_panic)
-			);
-	ztest_run_test_suite(test_log_api);
-
-	log_api_suite_teardown();
+	return NULL;
 }
+
+static void log_api_suite_before(void *data)
+{
+	ARG_UNUSED(data);
+	while(LOG_PROCESS()) {
+	}
+}
+
+#define WRAP_TEST(test_name, suffix)                       \
+	ZTEST(test_log_api_##suffix, test_name##_##suffix) \
+	{                                                  \
+		test_name();                               \
+	}
+
+#if __cplusplus
+#define TEST_SUFFIX cxx
+#else
+#define TEST_SUFFIX cc
+#endif
+#define TEST_SUITE_NAME test_log_api_ ## TEST_SUFFIX
+
+ZTEST_SUITE(TEST_SUITE_NAME, NULL, log_api_suite_setup,
+	    log_api_suite_before, NULL, log_api_suite_teardown);
+WRAP_TEST(test_log_various_messages, TEST_SUFFIX)
+WRAP_TEST(test_log_backend_runtime_filtering, TEST_SUFFIX)
+WRAP_TEST(test_log_overflow, TEST_SUFFIX)
+WRAP_TEST(test_log_arguments, TEST_SUFFIX)
+WRAP_TEST(test_log_from_declared_module, TEST_SUFFIX)
+WRAP_TEST(test_log_msg_dropped_notification, TEST_SUFFIX)
+WRAP_TEST(test_log_panic, TEST_SUFFIX)

tests/ztest/base/CMakeLists.txt

@@ -2,7 +2,7 @@
 
 cmake_minimum_required(VERSION 3.20.0)
 if(BOARD STREQUAL unit_testing)
-  list(APPEND SOURCES src/main.c)
+  list(APPEND SOURCES src/main_deprecated.c)
 
   find_package(ZephyrUnittest REQUIRED HINTS $ENV{ZEPHYR_BASE})
   project(base)
@@ -10,6 +10,5 @@ else()
   find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
   project(base)
 
-  FILE(GLOB app_sources src/*.c)
-  target_sources(app PRIVATE ${app_sources})
+  target_sources(app PRIVATE src/main.c)
 endif()

tests/ztest/base/prj_verbose_0.conf

@@ -1,2 +1,3 @@
 CONFIG_ZTEST=y
+CONFIG_ZTEST_NEW_API=y
 CONFIG_ZTEST_ASSERT_VERBOSE=0

tests/ztest/base/prj_verbose_1.conf

@@ -1,2 +1,3 @@
 CONFIG_ZTEST=y
+CONFIG_ZTEST_NEW_API=y
 CONFIG_ZTEST_ASSERT_VERBOSE=1

tests/ztest/base/prj_verbose_2.conf

@@ -1,2 +1,3 @@
 CONFIG_ZTEST=y
+CONFIG_ZTEST_NEW_API=y
 CONFIG_ZTEST_ASSERT_VERBOSE=2

tests/ztest/base/src/main.c

@@ -6,11 +6,13 @@
 
 #include <ztest.h>
 
-static void test_empty_test(void)
+ZTEST_SUITE(framework_tests, NULL, NULL, NULL, NULL, NULL);
+
+ZTEST(framework_tests, test_empty_test)
 {
 }
 
-static void test_assert_tests(void)
+ZTEST(framework_tests, test_assert_tests)
 {
 	zassert_true(1, NULL);
 	zassert_false(0, NULL);
@@ -20,7 +22,7 @@ static void test_assert_tests(void)
 	zassert_equal_ptr(NULL, NULL, NULL);
 }
 
-static void test_assert_mem_equal(void)
+ZTEST(framework_tests, test_assert_mem_equal)
 {
 	static const uint32_t expected[4] = {
 		0x1234,
@@ -33,13 +35,94 @@ static void test_assert_mem_equal(void)
 	zassert_mem_equal(actual, expected, sizeof(expected), NULL);
 }
 
-void test_main(void)
-{
-	ztest_test_suite(framework_tests,
-			 ztest_unit_test(test_empty_test),
-			 ztest_unit_test(test_assert_tests),
-			 ztest_unit_test(test_assert_mem_equal)
-			 );
+/***************************************************************************************************
+ * Sample fixture tests
+ **************************************************************************************************/
 
-	ztest_run_test_suite(framework_tests);
+struct fixture_tests_fixture {
+};
+
+static struct fixture_tests_fixture test_fixture;
+
+static void *fixture_tests_setup(void)
+{
+	return &test_fixture;
+}
+
+ZTEST_SUITE(fixture_tests, NULL, fixture_tests_setup, NULL, NULL, NULL);
+
+ZTEST_F(fixture_tests, test_fixture_pointer)
+{
+	zassert_equal_ptr(&test_fixture, this, "Test fixture should be at 0x%x but was at 0x%x",
+			  &test_fixture, this);
+}
+
+/***************************************************************************************************
+ * Sample rule tests
+ **************************************************************************************************/
+
+enum rule_state {
+	RULE_STATE_SETUP = 0,
+	RULE_STATE_BEFORE_EACH,
+	RULE_STATE_TEST,
+	RULE_STATE_AFTER_EACH,
+};
+
+struct rules_tests_fixture {
+	enum rule_state state;
+};
+
+static struct rules_tests_fixture rule_tests_fixture;
+
+static void rule_before_each(const struct ztest_unit_test *test, void *data)
+{
+	if (strcmp(test->test_suite_name, "rules_tests") == 0 &&
+	    strcmp(test->name, "test_rules_before_after") == 0) {
+		struct rules_tests_fixture *fixture = data;
+
+		zassert_equal_ptr(&rule_tests_fixture, data,
+				  "Data expected to point to rule_state");
+		zassert_equal(fixture->state, RULE_STATE_SETUP, "Unexpected state");
+		fixture->state = RULE_STATE_BEFORE_EACH;
+	}
+}
+
+static void rule_after_each(const struct ztest_unit_test *test, void *data)
+{
+	if (strcmp(test->test_suite_name, "rules_tests") == 0 &&
+	    strcmp(test->name, "test_rules_before_after") == 0) {
+		struct rules_tests_fixture *fixture = data;
+
+		zassert_equal_ptr(&rule_tests_fixture, data,
+				  "Data expected to point to rule_state");
+		zassert_equal(fixture->state, RULE_STATE_TEST, "Unexpected state");
+		fixture->state = RULE_STATE_AFTER_EACH;
+	}
+}
+
+static void *rule_test_setup(void)
+{
+	rule_tests_fixture.state = RULE_STATE_SETUP;
+	return &rule_tests_fixture;
+}
+
+static void rule_test_teardown(void *data)
+{
+	struct rules_tests_fixture *fixture = data;
+
+	/*
+	 * Normally, we wouldn't assert here, but it's the only way to test that the rule's
+	 * after_each function was called.
+	 */
+	zassert_equal(fixture->state, RULE_STATE_AFTER_EACH, "Unexpected state");
+}
+
+ZTEST_RULE(verify_before_after_rule, rule_before_each, rule_after_each);
+
+ZTEST_SUITE(rules_tests, NULL, rule_test_setup, NULL, NULL, rule_test_teardown);
+
+ZTEST_F(rules_tests, test_rules_before_after)
+{
+	zassert_equal(this->state, RULE_STATE_BEFORE_EACH, "Unexpected state");
+	this->state = RULE_STATE_TEST;
 }

tests/ztest/base/src/main_deprecated.c

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <ztest.h>
+
+static void test_empty_test(void)
+{
+}
+
+static void test_assert_tests(void)
+{
+	zassert_true(1, NULL);
+	zassert_false(0, NULL);
+	zassert_is_null(NULL, NULL);
+	zassert_not_null("foo", NULL);
+	zassert_equal(1, 1, NULL);
+	zassert_equal_ptr(NULL, NULL, NULL);
+}
+
+static void test_assert_mem_equal(void)
+{
+	static const uint32_t expected[4] = {
+		0x1234,
+		0x5678,
+		0x9ABC,
+		0xDEF0
+	};
+	uint32_t actual[4] = {0};
+
+	memcpy(actual, expected, sizeof(actual));
+	zassert_mem_equal(actual, expected, sizeof(expected), NULL);
+}
+
+void test_main(void)
+{
+	ztest_test_suite(framework_tests,
+			 ztest_unit_test(test_empty_test),
+			 ztest_unit_test(test_assert_tests),
+			 ztest_unit_test(test_assert_mem_equal)
+			 );
+
+	ztest_run_test_suite(framework_tests);
+}