ztest: Add initial zexpect API for delayed failing

Add the zexpect API, inspired by GoogleTest's EXPECT API. This API reports
test failures while allowing test execution to continue. This enables test
reports to show more than a singule failing property on a failing test.

Signed-off-by: Aaron Massey <aaronmassey@google.com>

doc/develop/test/ztest.rst

@@ -458,6 +458,30 @@ Example output for a failed macro from
 
 .. doxygengroup:: ztest_assert
 
+
+Expectations
+============
+
+These macros will continue test execution if the related expectation fails and subsequently fail the
+test at the end of its execution.  When an expectation fails, it will print the current file, line,
+and function, alongside a reason for the failure and an optional message but continue executing the
+test. If the config option:`CONFIG_ZTEST_ASSERT_VERBOSE` is 0, the expectations will only print the
+file and line numbers, reducing the binary size of the test.
+
+Example output for a failed macro from::
+
+  zexpect_equal(buf->ref, 2, "Invalid refcount");
+  zexpect_equal(buf->ref, 1337, "Invalid refcount");
+
+.. code-block::none
+
+    START - test_get_single_buffer
+        Expectation failed at main.c:62: test_get_single_buffer: Invalid refcount (buf->ref not equal to 2)
+        Expectation failed at main.c:63: test_get_single_buffer: Invalid refcount (buf->ref not equal to 1337)
+     FAIL - test_get_single_buffer in 0.0 seconds
+
+.. doxygengroup:: ztest_expect
+
 Assumptions
 ===========
 

subsys/testsuite/ztest/include/zephyr/ztest_assert.h

@@ -27,6 +27,7 @@ extern "C" {
 const char *ztest_relative_filename(const char *file);
 void ztest_test_fail(void);
 void ztest_test_skip(void);
+void ztest_test_expect_fail(void);
 void ztest_skip_failed_assumption(void);
 #if CONFIG_ZTEST_ASSERT_VERBOSE == 0
 
@@ -56,6 +57,19 @@ static inline bool z_zassume_(bool cond, const char *file, int line)
 
 #define z_zassume(cond, default_msg, file, line, func, msg, ...) z_zassume_(cond, file, line)
 
+static inline bool z_zexpect_(bool cond, const char *file, int line)
+{
+	if (cond == false) {
+		PRINT("\n    Expectation failed at %s:%d\n", ztest_relative_filename(file), line);
+		ztest_test_expect_fail();
+		return false;
+	}
+
+	return true;
+}
+
+#define z_zexpect(cond, default_msg, file, line, func, msg, ...) z_zexpect_(cond, file, line)
+
 #else /* CONFIG_ZTEST_ASSERT_VERBOSE != 0 */
 
 static inline bool z_zassert(bool cond, const char *default_msg, const char *file, int line,
@@ -106,6 +120,30 @@ static inline bool z_zassume(bool cond, const char *default_msg, const char *fil
 	return true;
 }
 
+static inline bool z_zexpect(bool cond, const char *default_msg, const char *file, int line,
+			     const char *func, const char *msg, ...)
+{
+	if (cond == false) {
+		va_list vargs;
+
+		va_start(vargs, msg);
+		PRINT("\n    Expectation failed at %s:%d: %s: %s\n", ztest_relative_filename(file),
+		      line, func, default_msg);
+		vprintk(msg, vargs);
+		printk("\n");
+		va_end(vargs);
+		ztest_test_expect_fail();
+		return false;
+	}
+#if CONFIG_ZTEST_ASSERT_VERBOSE == 2
+	else {
+		PRINT("\n   Expectation succeeded at %s:%d (%s)\n", ztest_relative_filename(file),
+		      line, func);
+	}
+#endif
+	return true;
+}
+
 #endif /* CONFIG_ZTEST_ASSERT_VERBOSE */
 
 /**
@@ -187,6 +225,36 @@ static inline bool z_zassume(bool cond, const char *default_msg, const char *fil
 #define zassume(cond, default_msg, ...)                                                            \
 	_zassume_va(cond, default_msg, COND_CODE_1(__VA_OPT__(1), (__VA_ARGS__), (NULL)))
 
+/**
+ * @brief If @a cond is false, fail the test but continue its execution.
+ *
+ * You probably don't need to call this macro directly. You should
+ * instead use zexpect_{condition} macros below.
+ *
+ * @param cond Condition to check
+ * @param default_msg Message to print if @a cond is false
+ * @param msg Optional, can be NULL. Message to print if @a cond is false.
+ */
+#define _zexpect_base(cond, default_msg, msg, ...)                                                 \
+	do {                                                                                       \
+		bool _msg = (msg != NULL);                                                         \
+		bool _ret =                                                                        \
+			z_zexpect(cond, _msg ? ("(" default_msg ")") : (default_msg), __FILE__,    \
+				  __LINE__, __func__, _msg ? msg : "", ##__VA_ARGS__);             \
+		(void)_msg;                                                                        \
+		if (!_ret) {                                                                       \
+			/* If kernel but without multithreading return. */                         \
+			COND_CODE_1(KERNEL, (COND_CODE_1(CONFIG_MULTITHREADING, (), (return;))),   \
+				    ())                                                            \
+		}                                                                                  \
+	} while (0)
+
+#define _zexpect_va(cond, default_msg, msg, ...)                                                   \
+	_zexpect_base(cond, default_msg, msg, ##__VA_ARGS__)
+
+#define zexpect(cond, default_msg, ...)                                                            \
+	_zexpect_va(cond, default_msg, COND_CODE_1(__VA_OPT__(1), (__VA_ARGS__), (NULL)))
+
 /**
  * @brief Assert that this function call won't be reached
  * @param ... Optional message and variables to print if the assertion fails
@@ -476,6 +544,132 @@ static inline bool z_zassume(bool cond, const char *default_msg, const char *fil
  * @}
  */
 
+/**
+ * @defgroup ztest_expect Ztest expectation macros
+ * @ingroup ztest
+ *
+ * This module provides expectations when using Ztest.
+ *
+ * @{
+ */
+
+/**
+ * @brief Expect that @a cond is true, otherwise mark test as failed but continue its execution.
+ *
+ * @param cond Condition to check
+ * @param ... Optional message and variables to print if the expectation fails
+ */
+#define zexpect_true(cond, ...) zexpect(cond, #cond " is false", ##__VA_ARGS__)
+
+/**
+ * @brief Expect that @a cond is false, otherwise mark test as failed but continue its execution.
+ *
+ * @param cond Condition to check
+ * @param ... Optional message and variables to print if the expectation fails
+ */
+#define zexpect_false(cond, ...) zexpect(!(cond), #cond " is true", ##__VA_ARGS__)
+
+/**
+ * @brief Expect that @a cond is 0 (success), otherwise mark test as failed but continue its
+ * execution.
+ *
+ * @param cond Condition to check
+ * @param ... Optional message and variables to print if the expectation fails
+ */
+#define zexpect_ok(cond, ...) zexpect(!(cond), #cond " is non-zero", ##__VA_ARGS__)
+
+/**
+ * @brief Expect that @a ptr is NULL, otherwise mark test as failed but continue its execution.
+ *
+ * @param ptr Pointer to compare
+ * @param ... Optional message and variables to print if the expectation fails
+ */
+#define zexpect_is_null(ptr, ...) zexpect((ptr) == NULL, #ptr " is not NULL", ##__VA_ARGS__)
+
+/**
+ * @brief Expect that @a ptr is not NULL, otherwise mark test as failed but continue its execution.
+ *
+ * @param ptr Pointer to compare
+ * @param ... Optional message and variables to print if the expectation fails
+ */
+#define zexpect_not_null(ptr, ...) zexpect((ptr) != NULL, #ptr " is NULL", ##__VA_ARGS__)
+
+/**
+ * @brief Expect that @a a equals @a b, otherwise mark test as failed but continue its execution.
+ * expectation fails, the test will be marked as "skipped".
+ *
+ * @param a Value to compare
+ * @param b Value to compare
+ * @param ... Optional message and variables to print if the expectation fails
+ */
+#define zexpect_equal(a, b, ...) zexpect((a) == (b), #a " not equal to " #b, ##__VA_ARGS__)
+
+/**
+ * @brief Expect that @a a does not equal @a b, otherwise mark test as failed but continue its
+ * execution.
+ *
+ * @a a and @a b won't be converted and will be compared directly.
+ *
+ * @param a Value to compare
+ * @param b Value to compare
+ * @param ... Optional message and variables to print if the expectation fails
+ */
+#define zexpect_not_equal(a, b, ...) zexpect((a) != (b), #a " equal to " #b, ##__VA_ARGS__)
+
+/**
+ * @brief Expect that @a a equals @a b, otherwise mark test as failed but continue its execution.
+ *
+ * @a a and @a b will be converted to `void *` before comparing.
+ *
+ * @param a Value to compare
+ * @param b Value to compare
+ * @param ... Optional message and variables to print if the expectation fails
+ */
+#define zexpect_equal_ptr(a, b, ...)                                                               \
+	zexpect((void *)(a) == (void *)(b), #a " not equal to " #b, ##__VA_ARGS__)
+
+/**
+ * @brief Expect that @a a is within @a b with delta @a d, otherwise mark test as failed but
+ * continue its execution.
+ *
+ * @param a Value to compare
+ * @param b Value to compare
+ * @param delta Difference between a and b
+ * @param ... Optional message and variables to print if the expectation fails
+ */
+#define zexpect_within(a, b, delta, ...)                                                           \
+	zexpect(((a) >= ((b) - (delta))) && ((a) <= ((b) + (delta))),                              \
+		#a " not within " #b " +/- " #delta, ##__VA_ARGS__)
+
+/**
+ * @brief Expect that @a a is greater than or equal to @a l and less
+ *        than or equal to @a u, otherwise mark test as failed but continue its execution.
+ *
+ * @param a Value to compare
+ * @param lower Lower limit
+ * @param upper Upper limit
+ * @param ... Optional message and variables to print if the expectation fails
+ */
+#define zexpect_between_inclusive(a, lower, upper, ...)                                            \
+	zexpect(((a) >= (lower)) && ((a) <= (upper)),                                              \
+		#a " not between " #lower " and " #upper " inclusive", ##__VA_ARGS__)
+
+/**
+ * @brief Expect that 2 memory buffers have the same contents, otherwise mark test as failed but
+ * continue its execution.
+ *
+ * @param buf Buffer to compare
+ * @param exp Buffer with expected contents
+ * @param size Size of buffers
+ * @param ... Optional message and variables to print if the expectation fails
+ */
+#define zexpect_mem_equal(buf, exp, size, ...)                                                     \
+	zexpect(memcmp(buf, exp, size) == 0, #buf " not equal to " #exp, ##__VA_ARGS__)
+
+/**
+ * @}
+ */
+
 #ifdef __cplusplus
 }
 #endif

subsys/testsuite/ztest/src/ztest_new.c

@@ -16,6 +16,7 @@
 #ifdef KERNEL
 static struct k_thread ztest_thread;
 #endif
+static bool failed_expectation;
 
 #ifdef CONFIG_ZTEST_SHUFFLE
 #include <stdlib.h>
@@ -366,6 +367,27 @@ void ztest_test_skip(void)
 	}
 }
 
+void ztest_test_expect_fail(void)
+{
+	failed_expectation = true;
+
+	switch (phase) {
+	case TEST_PHASE_SETUP:
+		PRINT(" at %s function\n", get_friendly_phase_name(phase));
+		break;
+	case TEST_PHASE_BEFORE:
+	case TEST_PHASE_TEST:
+		PRINT(" at %s function\n", get_friendly_phase_name(phase));
+		break;
+	case TEST_PHASE_AFTER:
+	case TEST_PHASE_TEARDOWN:
+	case TEST_PHASE_FRAMEWORK:
+		PRINT(" ERROR: cannot fail in test phase '%s()', bailing\n",
+		      get_friendly_phase_name(phase));
+		longjmp(stack_fail, 1);
+	}
+}
+
 static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test, void *data)
 {
 	int ret = TC_PASS;
@@ -399,6 +421,11 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test
 	}
 	run_test_functions(suite, test, data);
 out:
+	if (failed_expectation) {
+		failed_expectation = false;
+		ret = TC_FAIL;
+	}
+
 	phase = TEST_PHASE_AFTER;
 	if (test_result != ZTEST_RESULT_SUITE_FAIL) {
 		if (suite->after != NULL) {
@@ -494,6 +521,11 @@ void ztest_test_skip(void)
 	}
 }
 
+void ztest_test_expect_fail(void)
+{
+	failed_expectation = true;
+}
+
 void ztest_simple_1cpu_before(void *data)
 {
 	ARG_UNUSED(data);
@@ -582,8 +614,10 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test
 		k_msleep(100);
 	}
 
-	if (test_result == ZTEST_RESULT_FAIL || test_result == ZTEST_RESULT_SUITE_FAIL) {
+	if (test_result == ZTEST_RESULT_FAIL || test_result == ZTEST_RESULT_SUITE_FAIL ||
+	    failed_expectation) {
 		ret = TC_FAIL;
+		failed_expectation = false;
 	} else if (test_result == ZTEST_RESULT_SKIP || test_result == ZTEST_RESULT_SUITE_SKIP) {
 		ret = TC_SKIP;
 	}

tests/ztest/zexpect/CMakeLists.txt

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20.0)
+if(BOARD STREQUAL unit_testing)
+  find_package(Zephyr COMPONENTS unittest REQUIRED HINTS $ENV{ZEPHYR_BASE})
+  project(base)
+
+  target_sources(testbinary PRIVATE src/main.c)
+else()
+  find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+  project(base)
+
+  if(CONFIG_CPLUSPLUS)
+    message(STATUS "adding main.cpp")
+    target_sources(app PRIVATE src/main.cpp)
+  else()
+    target_sources(app PRIVATE src/main.c)
+
+    target_sources_ifdef(CONFIG_USERSPACE app PRIVATE src/main_userspace.c)
+  endif()
+endif()

tests/ztest/zexpect/prj.conf

@@ -0,0 +1,2 @@
+CONFIG_ZTEST=y
+CONFIG_ZTEST_NEW_API=y

tests/ztest/zexpect/src/main.c

@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2022 Google Inc
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stddef.h>
+
+#include <zephyr/ztest.h>
+
+ZTEST_SUITE(expect, NULL, NULL, NULL, NULL, NULL);
+
+ZTEST_EXPECT_FAIL(expect, test_fail_later);
+ZTEST(expect, test_fail_later)
+{
+	void *empty_ptr = NULL;
+	uint32_t val = 5;
+
+	zexpect_equal(val, 2);
+	zexpect_not_equal(val, 5);
+
+	zexpect_not_null(empty_ptr);
+
+	zassert_true(true);
+}
+
+ZTEST(expect, test_pass_expect_true)
+{
+	zexpect_true(true);
+}
+
+ZTEST_EXPECT_FAIL(expect, test_fail_expect_true);
+ZTEST(expect, test_fail_expect_true)
+{
+	zexpect_true(false);
+}
+
+ZTEST(expect, test_expect_false)
+{
+	zexpect_false(false);
+}
+
+ZTEST_EXPECT_FAIL(expect, test_fail_expect_false);
+ZTEST(expect, test_fail_expect_false)
+{
+	zexpect_false(true);
+}
+
+ZTEST(expect, test_expect_ok)
+{
+	zexpect_ok(0);
+}
+
+ZTEST_EXPECT_FAIL(expect, test_fail_expect_ok);
+ZTEST(expect, test_fail_expect_ok)
+{
+	zexpect_ok(5);
+}
+
+ZTEST(expect, test_expect_is_null)
+{
+	void *ptr = NULL;
+
+	zexpect_is_null(ptr);
+}
+
+ZTEST_EXPECT_FAIL(expect, test_fail_expect_is_null);
+ZTEST(expect, test_fail_expect_is_null)
+{
+	void *ptr = (void *)0x32137899;
+
+	zexpect_is_null(ptr);
+}
+
+ZTEST(expect, test_expect_not_null)
+{
+	void *ptr = (void *)0x91517141;
+
+	zexpect_not_null(ptr);
+}
+
+ZTEST_EXPECT_FAIL(expect, test_fail_expect_not_null);
+ZTEST(expect, test_fail_expect_not_null)
+{
+	zexpect_not_null(NULL);
+}
+
+ZTEST(expect, test_expect_equal)
+{
+	zexpect_equal(5, 5);
+}
+
+ZTEST_EXPECT_FAIL(expect, test_fail_expect_equal);
+ZTEST(expect, test_fail_expect_equal)
+{
+	zexpect_equal(5, 1);
+}
+
+ZTEST(expect, test_expect_not_equal)
+{
+	zexpect_not_equal(5, 1);
+}
+
+ZTEST_EXPECT_FAIL(expect, test_fail_expect_not_equal);
+ZTEST(expect, test_fail_expect_not_equal)
+{
+	zexpect_not_equal(5, 5);
+}
+
+ZTEST(expect, test_expect_equal_ptr)
+{
+	int v = 9;
+	int *a = &v;
+	int *b = &v;
+
+	zexpect_equal_ptr(a, b);
+}
+
+ZTEST_EXPECT_FAIL(expect, test_fail_expect_equal_ptr);
+ZTEST(expect, test_fail_expect_equal_ptr)
+{
+	int v = 9;
+	int *a = &v;
+	int *b = NULL;
+
+	zexpect_equal_ptr(a, b);
+}
+
+ZTEST(expect, test_expect_within)
+{
+	zexpect_within(7, 5, 2);
+	zexpect_within(7, 7, 0);
+	zexpect_within(7, 7, 3);
+	zexpect_within(7, 7 + 3, 3);
+}
+
+ZTEST_EXPECT_FAIL(expect, test_fail_expect_within);
+ZTEST(expect, test_fail_expect_within)
+{
+	zexpect_within(7, 5, 1);
+}
+
+ZTEST(expect, test_expect_between_inclusive)
+{
+	zexpect_between_inclusive(-5, -10, 0);
+
+	zexpect_between_inclusive(5, 0, 10);
+	zexpect_between_inclusive(0, 0, 10);
+	zexpect_between_inclusive(10, 0, 10);
+}
+
+ZTEST_EXPECT_FAIL(expect, test_fail_expect_between_inclusive);
+ZTEST(expect, test_fail_expect_between_inclusive)
+{
+	zexpect_between_inclusive(-50, -20, 30);
+
+	zexpect_between_inclusive(5, 6, 10);
+	zexpect_between_inclusive(5, 0, 4);
+	zexpect_between_inclusive(5, 0, 4);
+	zexpect_between_inclusive(5, 6, 10);
+}

tests/ztest/zexpect/src/main.cpp

@@ -0,0 +1 @@
+main.c

tests/ztest/zexpect/testcase.yaml

@@ -0,0 +1,19 @@
+# Copyright (c) 2022 Google Inc
+# SPDX-License-Identifier: Apache-2.0
+
+common:
+  timeout: 15
+  integration_platforms:
+    - native_posix
+tests:
+  testing.ztest.expect:
+    integration_platforms:
+      - native_posix
+  testing.ztest.expect_cpp:
+    extra_configs:
+      - CONFIG_CPLUSPLUS=y
+      - CONFIG_LIB_CPLUSPLUS=y
+    integration_platforms:
+      - native_posix
+  testing.ztest.expect.unit:
+    type: unit