michaelh/zephyr
1
0
Fork 0
Code Releases Activity

scripts: twister: decouple debug and verbosity

Browse source 
Currently, debug logging in the console and verbosity
are tightly coupled - verbosity of level 2 and higher
enables logging at the debug level.

This change introduces a separate Twister flag
responsible for controlling the debug logging,
while leaving the rest of verbosity unchanged.

This allows for controlling the verbosity on
both logging levels, according to one's needs.

Signed-off-by: Lukasz Mrugala <lukaszx.mrugala@intel.com>
This commit is contained in:
Lukasz Mrugala 2024-09-17 11:32:43 +00:00 • committed by Henrik Brix Andersen
commit 558c74be04
8 changed files with 58 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

12
scripts/pylib/twister/twisterlib/environment.py
View file

@ -717,8 +717,16 @@ structure in the main Zephyr tree: boards/<vendor>/<board_name>/""")
"--verbose",
action="count",
default=0,
help="Emit debugging information, call multiple times to increase "
"verbosity.")
help="Call multiple times to increase verbosity.")
parser.add_argument(
"-ll",
"--log-level",
type=str.upper,
default='INFO',
choices=['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help="Select the threshold event severity for which you'd like to receive logs in console."
" Default is INFO.")
parser.add_argument("-W", "--disable-warnings-as-errors", action="store_true",
help="Do not treat warning conditions as errors.")

15
scripts/pylib/twister/twisterlib/twister_main.py
View file

@ -26,7 +26,7 @@ logger = logging.getLogger("twister")
logger.setLevel(logging.DEBUG)
def setup_logging(outdir, log_file, verbose, timestamps):
def setup_logging(outdir, log_file, log_level, timestamps):
# create file handler which logs even debug messages
if log_file:
fh = logging.FileHandler(log_file)
@ -37,11 +37,7 @@ def setup_logging(outdir, log_file, verbose, timestamps):
# create console handler with a higher log level
ch = logging.StreamHandler()
if verbose > 1:
ch.setLevel(logging.DEBUG)
else:
ch.setLevel(logging.INFO)
ch.setLevel(getattr(logging, log_level))
# create formatter and add it to the handlers
if timestamps:
@ -107,8 +103,7 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace):
with open(previous_results_file, "w") as fp:
fp.write(previous_results)
VERBOSE = options.verbose
setup_logging(options.outdir, options.log_file, VERBOSE, options.timestamps)
setup_logging(options.outdir, options.log_file, options.log_level, options.timestamps)
env = TwisterEnv(options, default_options)
env.discover()
@ -136,7 +131,7 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace):
logger.error(f"{e}")
return 1
if VERBOSE > 1:
if options.verbose > 1:
# if we are using command line platform filter, no need to list every
# other platform as excluded, we know that already.
# Show only the discards that apply to the selected platforms on the
@ -206,7 +201,7 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace):
duration = time.time() - start_time
if VERBOSE > 1:
if options.verbose > 1:
runner.results.summary()
report.summary(runner.results, options.disable_unrecognized_section_test, duration)

3
scripts/tests/twister_blackbox/test_coverage.py
View file

@ -13,6 +13,7 @@ import pytest
import sys
import json
# pylint: disable=duplicate-code
from conftest import TEST_DATA, ZEPHYR_BASE, testsuite_filename_mock, clear_log_in_test
from twisterlib.testplan import TestPlan
@ -189,7 +190,7 @@ class TestCoverage:
)
def test_enable_coverage(self, capfd, test_path, test_platforms, out_path, expected):
args = ['-i','--outdir', out_path, '-T', test_path] + \
['--enable-coverage', '-vv'] + \
['--enable-coverage', '-vv', '-ll', 'DEBUG'] + \
[val for pair in zip(
['-p'] * len(test_platforms), test_platforms
) for val in pair]

2
scripts/tests/twister_blackbox/test_disable.py
View file

@ -76,7 +76,7 @@ class TestDisable:
def test_disable_suite_name_check(self, capfd, out_path, test_path, test_platforms, flag, expected, expected_none):
args = ['-i', '--outdir', out_path, '-T', test_path] + \
[flag] + \
['-vv'] + \
['-vv', '-ll', 'DEBUG'] + \
[val for pair in zip(
['-p'] * len(test_platforms), test_platforms
) for val in pair]

4
scripts/tests/twister_blackbox/test_filter.py
View file

@ -191,7 +191,7 @@ class TestFilter:
def test_arch(self, capfd, out_path, arch, expected):
path = os.path.join(TEST_DATA, 'tests', 'no_filter')
test_platforms = ['qemu_x86', 'hsdk', 'intel_adl_crb', 'it8xxx2_evb']
args = ['--outdir', out_path, '-T', path, '-vv'] + \
args = ['--outdir', out_path, '-T', path, '-vv', '-ll', 'DEBUG'] + \
['--arch', arch] + \
[val for pair in zip(
['-p'] * len(test_platforms), test_platforms
@ -224,7 +224,7 @@ class TestFilter:
def test_vendor(self, capfd, out_path, vendor, expected):
path = os.path.join(TEST_DATA, 'tests', 'no_filter')
test_platforms = ['qemu_x86', 'hsdk', 'intel_adl_crb', 'it8xxx2_evb']
args = ['--outdir', out_path, '-T', path, '-vv'] + \
args = ['--outdir', out_path, '-T', path, '-vv', '-ll', 'DEBUG'] + \
['--vendor', vendor] + \
[val for pair in zip(
['-p'] * len(test_platforms), test_platforms

76
scripts/tests/twister_blackbox/test_output.py
View file

@ -14,7 +14,6 @@ import pytest
import sys
import json
# pylint: disable=no-name-in-module
from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock, clear_log_in_test
from twisterlib.testplan import TestPlan
@ -22,10 +21,14 @@ from twisterlib.testplan import TestPlan
@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
class TestOutput:
TESTDATA_1 = [
(
os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic')
),
]
([]),
(['-ll', 'DEBUG']),
(['-v']),
(['-v', '-ll', 'DEBUG']),
(['-vv']),
(['-vv', '-ll', 'DEBUG']),
]
@classmethod
def setup_class(cls):
apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
@ -149,15 +152,15 @@ class TestOutput:
matches = []
return matches
@pytest.mark.parametrize(
'test_path',
'flags',
TESTDATA_1,
ids=[
'single_v',
]
ids=['not verbose', 'not verbose + debug', 'v', 'v + debug', 'vv', 'vv + debug']
)
def test_single_v(self, capfd, out_path, test_path):
args = ['--outdir', out_path, '-T', test_path, '-v']
def test_output_levels(self, capfd, out_path, flags):
test_path = os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic')
args = ['--outdir', out_path, '-T', test_path, *flags]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
@ -166,35 +169,26 @@ class TestOutput:
out, err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
regex_line = [r'INFO', r'-', r'\d+/\d+', r'\S+', r'\S+', r'[A-Z]+', r'\(\w+', r'[\d.]+s\)']
matches = self._get_matches(err, regex_line)
print(matches)
assert str(sys_exit.value) == '0'
assert len(matches) > 0
@pytest.mark.parametrize(
'test_path',
TESTDATA_1,
ids=[
'double_v',
]
)
def test_double_v(self, capfd, out_path, test_path):
args = ['--outdir', out_path, '-T', test_path, '-vv']
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
out, err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
regex_line = [r'INFO', r'-', r'\d+/\d+', r'\S+', r'\S+', r'[A-Z]+', r'\(\w+', r'[\d.]+s\)']
matches = self._get_matches(err, regex_line)
booting_zephyr_regex = re.compile(r'^DEBUG\s+-\s+([^*]+)\*\*\*\s+Booting\s+Zephyr\s+OS\s+build.*$', re.MULTILINE)
info_debug_line_regex = r'^\s*(INFO|DEBUG)'
assert str(sys_exit.value) == '0'
assert re.search(booting_zephyr_regex, err) is not None
assert re.search(info_debug_line_regex, err) is not None
assert len(matches) > 0
regex_debug_line = r'^\s*DEBUG'
debug_matches = re.search(regex_debug_line, err, re.MULTILINE)
if '-ll' in flags and 'DEBUG' in flags:
assert debug_matches is not None
else:
assert debug_matches is None
# Summary requires verbosity > 1
if '-vv' in flags:
assert 'Total test suites: ' in out
else:
assert 'Total test suites: ' not in out
# Brief summary shows up only on verbosity 0 - instance-by-instance otherwise
regex_info_line = [r'INFO', r'-', r'\d+/\d+', r'\S+', r'\S+', r'[A-Z]+', r'\(\w+', r'[\d.]+s\)']
info_matches = self._get_matches(err, regex_info_line)
if not any(f in flags for f in ['-v', '-vv']):
assert not info_matches
else:
assert info_matches

3
scripts/tests/twister_blackbox/test_quarantine.py
View file

@ -14,6 +14,7 @@ import re
import sys
import json
# pylint: disable=duplicate-code
from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock
from twisterlib.testplan import TestPlan
@ -75,7 +76,7 @@ class TestQuarantine:
def test_quarantine_list(self, capfd, out_path, test_path, test_platforms, quarantine_directory):
args = ['--outdir', out_path, '-T', test_path] +\
['--quarantine-list', quarantine_directory] + \
['-vv'] + \
['-vv', '-ll', 'DEBUG'] + \
[val for pair in zip(
['-p'] * len(test_platforms), test_platforms
) for val in pair]

2
scripts/tests/twister_blackbox/test_runner.py
View file

@ -556,7 +556,7 @@ class TestRunner:
],
)
def test_tag(self, capfd, out_path, test_path, test_platforms, tags, expected):
args = ['--outdir', out_path, '-T', test_path, '-vv'] + \
args = ['--outdir', out_path, '-T', test_path, '-vv', '-ll', 'DEBUG'] + \
[val for pair in zip(
['-p'] * len(test_platforms), test_platforms
) for val in pair] + \