scripts: tests: Blackbox test expansion - error

Adds tests related to filter flags:
--overflow-as-errors

Signed-off-by: Kamil Paszkiet <kamilx.paszkiet@intel.com>

scripts/tests/twister_blackbox/test_data/tests/qemu_overflow/dummy/CMakeLists.txt

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20.0)
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(integration)
+
+FILE(GLOB app_sources src/*.c)
+target_sources(app PRIVATE ${app_sources})

scripts/tests/twister_blackbox/test_data/tests/qemu_overflow/dummy/prj.conf

@@ -0,0 +1 @@
+CONFIG_ZTEST=y

scripts/tests/twister_blackbox/test_data/tests/qemu_overflow/dummy/qemu_x86.overlay

@@ -0,0 +1,3 @@
+&dram0 {
+        reg = < 0x100000 DT_SIZE_M(1) >;
+};

scripts/tests/twister_blackbox/test_data/tests/qemu_overflow/dummy/src/main.c

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2024 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <zephyr/ztest.h>
+
+
+ZTEST_SUITE(a1_1_tests, NULL, NULL, NULL, NULL, NULL);
+
+/**
+ * @brief Test Asserts
+ *
+ * This test verifies various assert macros provided by ztest.
+ *
+ */
+ZTEST(a1_1_tests, test_assert)
+{
+	zassert_true(1, "1 was false");
+	zassert_false(0, "0 was true");
+	zassert_is_null(NULL, "NULL was not NULL");
+	zassert_not_null("foo", "\"foo\" was NULL");
+	zassert_equal(1, 1, "1 was not equal to 1");
+	zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL");
+}

scripts/tests/twister_blackbox/test_data/tests/qemu_overflow/dummy/test_data.yaml

@@ -0,0 +1,8 @@
+tests:
+  always_overflow.dummy:
+    platform_allow:
+      - native_posix
+      - qemu_x86
+      - qemu_x86_64
+    integration_platforms:
+      - native_posix

scripts/tests/twister_blackbox/test_error.py

@@ -11,6 +11,7 @@ import mock
 import os
 import pytest
 import sys
+import re
 
 from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock
 from twisterlib.testplan import TestPlan
@@ -37,6 +38,16 @@ class TestError:
             SystemExit
         )
     ]
+    TESTDATA_2 = [
+        (
+            '',
+            r'always_overflow.dummy SKIPPED \(RAM overflow\)'
+        ),
+        (
+            '--overflow-as-errors',
+            r'always_overflow.dummy  ERROR Build failure \(build\)'
+        )
+    ]
 
     @classmethod
     def setup_class(cls):
@@ -72,3 +83,40 @@ class TestError:
         if expected_exception == SystemExit:
             assert str(exc.value) == '0'
         assert True
+
+    @pytest.mark.parametrize(
+        'switch, expected',
+        TESTDATA_2,
+        ids=[
+            'overflow skip',
+            'overflow error',
+        ],
+    )
+
+    @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
+    def test_overflow_as_errors(self, capfd, out_path, switch, expected):
+        path = os.path.join(TEST_DATA, 'tests', 'qemu_overflow')
+        test_platforms = ['qemu_x86']
+        args = ['--outdir', out_path, '-T', path, '-vv'] + \
+               ['--build-only'] + \
+               [val for pair in zip(
+                   ['-p'] * len(test_platforms), test_platforms
+               ) for val in pair]
+        if switch:
+            args += [switch]
+
+        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
+            pytest.raises(SystemExit) as sys_exit:
+            self.loader.exec_module(self.twister_module)
+
+        out, err = capfd.readouterr()
+        sys.stdout.write(out)
+        sys.stderr.write(err)
+
+        print(args)
+        if switch:
+            assert str(sys_exit.value) == '1'
+        else:
+            assert str(sys_exit.value) == '0'
+
+        assert re.search(expected, err)