scripts: tests: Blackbox test expansion - error

Adds tests related to filter flags:
--overflow-as-errors

Signed-off-by: Kamil Paszkiet <kamilx.paszkiet@intel.com>
This commit is contained in:
Kamil Paszkiet 2024-03-21 14:43:21 +01:00 committed by David Leach
commit 2a4332ba5f
6 changed files with 94 additions and 0 deletions

View file

@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
cmake_minimum_required(VERSION 3.20.0)
find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
project(integration)
FILE(GLOB app_sources src/*.c)
target_sources(app PRIVATE ${app_sources})

View file

@ -0,0 +1 @@
CONFIG_ZTEST=y

View file

@ -0,0 +1,3 @@
&dram0 {
reg = < 0x100000 DT_SIZE_M(1) >;
};

View file

@ -0,0 +1,26 @@
/*
* Copyright (c) 2024 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <zephyr/ztest.h>
ZTEST_SUITE(a1_1_tests, NULL, NULL, NULL, NULL, NULL);
/**
* @brief Test Asserts
*
* This test verifies various assert macros provided by ztest.
*
*/
ZTEST(a1_1_tests, test_assert)
{
zassert_true(1, "1 was false");
zassert_false(0, "0 was true");
zassert_is_null(NULL, "NULL was not NULL");
zassert_not_null("foo", "\"foo\" was NULL");
zassert_equal(1, 1, "1 was not equal to 1");
zassert_equal_ptr(NULL, NULL, "NULL was not equal to NULL");
}

View file

@ -0,0 +1,8 @@
tests:
always_overflow.dummy:
platform_allow:
- native_posix
- qemu_x86
- qemu_x86_64
integration_platforms:
- native_posix

View file

@ -11,6 +11,7 @@ import mock
import os
import pytest
import sys
import re
from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock
from twisterlib.testplan import TestPlan
@ -37,6 +38,16 @@ class TestError:
SystemExit
)
]
TESTDATA_2 = [
(
'',
r'always_overflow.dummy SKIPPED \(RAM overflow\)'
),
(
'--overflow-as-errors',
r'always_overflow.dummy ERROR Build failure \(build\)'
)
]
@classmethod
def setup_class(cls):
@ -72,3 +83,40 @@ class TestError:
if expected_exception == SystemExit:
assert str(exc.value) == '0'
assert True
@pytest.mark.parametrize(
'switch, expected',
TESTDATA_2,
ids=[
'overflow skip',
'overflow error',
],
)
@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
def test_overflow_as_errors(self, capfd, out_path, switch, expected):
path = os.path.join(TEST_DATA, 'tests', 'qemu_overflow')
test_platforms = ['qemu_x86']
args = ['--outdir', out_path, '-T', path, '-vv'] + \
['--build-only'] + \
[val for pair in zip(
['-p'] * len(test_platforms), test_platforms
) for val in pair]
if switch:
args += [switch]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
out, err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
print(args)
if switch:
assert str(sys_exit.value) == '1'
else:
assert str(sys_exit.value) == '0'
assert re.search(expected, err)