michaelh/zephyr
1
0
Fork 0
Code Releases Activity

twister: Improve recording at Harness

Browse source 
The Console Harness is able to parse its log with patterns to compose
extracted fields into records in 'recording.csv' file in the test's build
directory. This feature allows to extract custom test results like
performance counters.
With this change the extracted records are also written into 'twister.json'
as a part of each test suite object. This makes easier to store
all the data collected by the test for its further processing.

Other improvements:
 - compile parsing pattern only once instead of at each input line;
 - quote fields in '.csv' to avoid unexpected field separators;
 - make 'regex' a required schema field of 'harness_config';
 - Twister documentation update.

Signed-off-by: Dmitrii Golovanov <dmitrii.golovanov@intel.com>
This commit is contained in:
Dmitrii Golovanov 2023-12-01 21:19:09 +01:00 • committed by Anas Nashif
commit 47da4e2e76
8 changed files with 69 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

27
doc/develop/test/twister.rst
View file

@ -478,15 +478,9 @@ harness_config: <harness configuration options>
type: <one_line|multi_line> (required)
Depends on the regex string to be matched
record: <recording options>
regex: <expression> (required)
Any string that the particular test case prints to record test
results.
regex: <expression> (required)
Any string that the particular test case prints to confirm test
runs as expected.
regex: <list of regular expressions> (required)
Strings with regular expressions to match with the test's output
to confirm the test runs as expected.
ordered: <True|False> (default False)
Check the regular expression strings in orderly or randomly fashion
@ -494,6 +488,21 @@ harness_config: <harness configuration options>
repeat: <integer>
Number of times to validate the repeated regex expression
record: <recording options> (optional)
regex: <regular expression> (required)
The regular experssion with named subgroups to match data fields
at the test's output lines where the test provides some custom data
for further analysis. These records will be written into the build
directory 'recording.csv' file as well as 'recording' property
of the test suite object in 'twister.json'.
For example, to extract three data fields 'metric', 'cycles', 'nanoseconds':
.. code-block:: yaml
record:
regex: "(?P<metric>.*):(?P<cycles>.*) cycles, (?P<nanoseconds>.*) ns"
fixture: <expression>
Specify a test case dependency on an external device(e.g., sensor),
and identify setups that fulfill this dependency. It depends on

15
scripts/pylib/twister/twisterlib/handlers.py
View file

@ -100,12 +100,19 @@ class Handler:
def record(self, harness):
if harness.recording:
if self.instance.recording is None:
self.instance.recording = harness.recording.copy()
else:
self.instance.recording.extend(harness.recording)
filename = os.path.join(self.build_dir, "recording.csv")
with open(filename, "at") as csvfile:
cw = csv.writer(csvfile, harness.fieldnames, lineterminator=os.linesep)
cw.writerow(harness.fieldnames)
for instance in harness.recording:
cw.writerow(instance)
cw = csv.DictWriter(csvfile,
fieldnames = harness.recording[0].keys(),
lineterminator = os.linesep,
quoting = csv.QUOTE_NONNUMERIC)
cw.writeheader()
cw.writerows(harness.recording)
def terminate(self, proc):
terminate_process(proc)

29
scripts/pylib/twister/twisterlib/harness.py
View file

@ -55,8 +55,8 @@ class Harness:
self.capture_coverage = False
self.next_pattern = 0
self.record = None
self.record_pattern = None
self.recording = []
self.fieldnames = []
self.ztest = False
self.detected_suite_names = []
self.run_id = None
@ -80,6 +80,16 @@ class Harness:
self.repeat = config.get('repeat', 1)
self.ordered = config.get('ordered', True)
self.record = config.get('record', {})
if self.record:
self.record_pattern = re.compile(self.record.get("regex", ""))
def get_testcase_name(self):
"""
Get current TestCase name.
"""
return self.id
def process_test(self, line):
@ -172,7 +182,7 @@ class Console(Harness):
'''
if self.instance and len(self.instance.testcases) == 1:
return self.instance.testcases[0].name
return self.id
return super(Console, self).get_testcase_name()
def configure(self, instance):
super(Console, self).configure(instance)
@ -240,19 +250,10 @@ class Console(Harness):
elif self.GCOV_END in line:
self.capture_coverage = False
if self.record:
pattern = re.compile(self.record.get("regex", ""))
match = pattern.search(line)
if self.record_pattern:
match = self.record_pattern.search(line)
if match:
csv = []
if not self.fieldnames:
for k,v in match.groupdict().items():
self.fieldnames.append(k)
for k,v in match.groupdict().items():
csv.append(v.strip())
self.recording.append(csv)
self.recording.append({ k:v.strip() for k,v in match.groupdict(default="").items() })
self.process_test(line)
# Reset the resulting test state to 'failed' when not all of the patterns were

4
scripts/pylib/twister/twisterlib/reports.py
View file

@ -347,6 +347,10 @@ class Reporting:
testcases.append(testcase)
suite['testcases'] = testcases
if instance.recording is not None:
suite['recording'] = instance.recording
suites.append(suite)
report["testsuites"] = suites

1
scripts/pylib/twister/twisterlib/testinstance.py
View file

@ -48,6 +48,7 @@ class TestInstance:
self.reason = "Unknown"
self.metrics = dict()
self.handler = None
self.recording = None
self.outdir = outdir
self.execution_time = 0
self.build_time = 0

4
scripts/schemas/twister/testsuite-schema.yaml
View file

@ -122,7 +122,7 @@ mapping:
mapping:
"regex":
type: str
required: false
required: true
"min_ram":
type: int
required: false
@ -326,7 +326,7 @@ mapping:
mapping:
"regex":
type: str
required: false
required: true
"min_ram":
type: int
required: false

37
scripts/tests/twister/test_handlers.py
View file

@ -209,40 +209,35 @@ def test_handler_record(mocked_instance):
instance.testcases = [mock.Mock()]
handler = Handler(instance)
handler.suite_name_check = True
harness = twisterlib.harness.Test()
harness.recording = ['dummy recording']
type(harness).fieldnames = mock.PropertyMock(return_value=[])
mock_writerow = mock.Mock()
mock_writer = mock.Mock(writerow=mock_writerow)
harness = twisterlib.harness.Harness()
harness.recording = [ {'field_1': 'recording_1_1', 'field_2': 'recording_1_2'},
{'field_1': 'recording_2_1', 'field_2': 'recording_2_2'}
]
with mock.patch(
'builtins.open',
mock.mock_open(read_data='')
) as mock_file, \
mock.patch(
'csv.writer',
mock.Mock(return_value=mock_writer)
) as mock_writer_constructor:
mock.patch(
'csv.DictWriter.writerow',
mock.Mock()
) as mock_writeheader, \
mock.patch(
'csv.DictWriter.writerows',
mock.Mock()
) as mock_writerows:
handler.record(harness)
print(mock_file.mock_calls)
mock_file.assert_called_with(
os.path.join(instance.build_dir, 'recording.csv'),
'at'
)
mock_writer_constructor.assert_called_with(
mock_file(),
harness.fieldnames,
lineterminator=os.linesep
)
mock_writerow.assert_has_calls(
[mock.call(harness.fieldnames)] + \
[mock.call(recording) for recording in harness.recording]
)
mock_writeheader.assert_has_calls([mock.call({ k:k for k in harness.recording[0].keys()})])
mock_writerows.assert_has_calls([mock.call(harness.recording)])
def test_handler_terminate(mocked_instance):

2
scripts/tests/twister/test_harness.py
View file

@ -43,6 +43,8 @@ def gtest():
mock_testsuite.detailed_test_id = True
mock_testsuite.id = "id"
mock_testsuite.testcases = []
mock_testsuite.harness_config = {}
instance = TestInstance(testsuite=mock_testsuite, platform=mock_platform, outdir="")
harness = Gtest()