twister: Improve recording at Harness
The Console Harness is able to parse its log with patterns to compose extracted fields into records in 'recording.csv' file in the test's build directory. This feature allows to extract custom test results like performance counters. With this change the extracted records are also written into 'twister.json' as a part of each test suite object. This makes easier to store all the data collected by the test for its further processing. Other improvements: - compile parsing pattern only once instead of at each input line; - quote fields in '.csv' to avoid unexpected field separators; - make 'regex' a required schema field of 'harness_config'; - Twister documentation update. Signed-off-by: Dmitrii Golovanov <dmitrii.golovanov@intel.com>
This commit is contained in:
parent
b8856aed0d
commit
47da4e2e76
8 changed files with 69 additions and 50 deletions
|
@ -478,15 +478,9 @@ harness_config: <harness configuration options>
|
|||
type: <one_line|multi_line> (required)
|
||||
Depends on the regex string to be matched
|
||||
|
||||
|
||||
record: <recording options>
|
||||
regex: <expression> (required)
|
||||
Any string that the particular test case prints to record test
|
||||
results.
|
||||
|
||||
regex: <expression> (required)
|
||||
Any string that the particular test case prints to confirm test
|
||||
runs as expected.
|
||||
regex: <list of regular expressions> (required)
|
||||
Strings with regular expressions to match with the test's output
|
||||
to confirm the test runs as expected.
|
||||
|
||||
ordered: <True|False> (default False)
|
||||
Check the regular expression strings in orderly or randomly fashion
|
||||
|
@ -494,6 +488,21 @@ harness_config: <harness configuration options>
|
|||
repeat: <integer>
|
||||
Number of times to validate the repeated regex expression
|
||||
|
||||
record: <recording options> (optional)
|
||||
regex: <regular expression> (required)
|
||||
The regular experssion with named subgroups to match data fields
|
||||
at the test's output lines where the test provides some custom data
|
||||
for further analysis. These records will be written into the build
|
||||
directory 'recording.csv' file as well as 'recording' property
|
||||
of the test suite object in 'twister.json'.
|
||||
|
||||
For example, to extract three data fields 'metric', 'cycles', 'nanoseconds':
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
record:
|
||||
regex: "(?P<metric>.*):(?P<cycles>.*) cycles, (?P<nanoseconds>.*) ns"
|
||||
|
||||
fixture: <expression>
|
||||
Specify a test case dependency on an external device(e.g., sensor),
|
||||
and identify setups that fulfill this dependency. It depends on
|
||||
|
|
|
@ -100,12 +100,19 @@ class Handler:
|
|||
|
||||
def record(self, harness):
|
||||
if harness.recording:
|
||||
if self.instance.recording is None:
|
||||
self.instance.recording = harness.recording.copy()
|
||||
else:
|
||||
self.instance.recording.extend(harness.recording)
|
||||
|
||||
filename = os.path.join(self.build_dir, "recording.csv")
|
||||
with open(filename, "at") as csvfile:
|
||||
cw = csv.writer(csvfile, harness.fieldnames, lineterminator=os.linesep)
|
||||
cw.writerow(harness.fieldnames)
|
||||
for instance in harness.recording:
|
||||
cw.writerow(instance)
|
||||
cw = csv.DictWriter(csvfile,
|
||||
fieldnames = harness.recording[0].keys(),
|
||||
lineterminator = os.linesep,
|
||||
quoting = csv.QUOTE_NONNUMERIC)
|
||||
cw.writeheader()
|
||||
cw.writerows(harness.recording)
|
||||
|
||||
def terminate(self, proc):
|
||||
terminate_process(proc)
|
||||
|
|
|
@ -55,8 +55,8 @@ class Harness:
|
|||
self.capture_coverage = False
|
||||
self.next_pattern = 0
|
||||
self.record = None
|
||||
self.record_pattern = None
|
||||
self.recording = []
|
||||
self.fieldnames = []
|
||||
self.ztest = False
|
||||
self.detected_suite_names = []
|
||||
self.run_id = None
|
||||
|
@ -80,6 +80,16 @@ class Harness:
|
|||
self.repeat = config.get('repeat', 1)
|
||||
self.ordered = config.get('ordered', True)
|
||||
self.record = config.get('record', {})
|
||||
if self.record:
|
||||
self.record_pattern = re.compile(self.record.get("regex", ""))
|
||||
|
||||
|
||||
def get_testcase_name(self):
|
||||
"""
|
||||
Get current TestCase name.
|
||||
"""
|
||||
return self.id
|
||||
|
||||
|
||||
def process_test(self, line):
|
||||
|
||||
|
@ -172,7 +182,7 @@ class Console(Harness):
|
|||
'''
|
||||
if self.instance and len(self.instance.testcases) == 1:
|
||||
return self.instance.testcases[0].name
|
||||
return self.id
|
||||
return super(Console, self).get_testcase_name()
|
||||
|
||||
def configure(self, instance):
|
||||
super(Console, self).configure(instance)
|
||||
|
@ -240,19 +250,10 @@ class Console(Harness):
|
|||
elif self.GCOV_END in line:
|
||||
self.capture_coverage = False
|
||||
|
||||
|
||||
if self.record:
|
||||
pattern = re.compile(self.record.get("regex", ""))
|
||||
match = pattern.search(line)
|
||||
if self.record_pattern:
|
||||
match = self.record_pattern.search(line)
|
||||
if match:
|
||||
csv = []
|
||||
if not self.fieldnames:
|
||||
for k,v in match.groupdict().items():
|
||||
self.fieldnames.append(k)
|
||||
|
||||
for k,v in match.groupdict().items():
|
||||
csv.append(v.strip())
|
||||
self.recording.append(csv)
|
||||
self.recording.append({ k:v.strip() for k,v in match.groupdict(default="").items() })
|
||||
|
||||
self.process_test(line)
|
||||
# Reset the resulting test state to 'failed' when not all of the patterns were
|
||||
|
|
|
@ -347,6 +347,10 @@ class Reporting:
|
|||
testcases.append(testcase)
|
||||
|
||||
suite['testcases'] = testcases
|
||||
|
||||
if instance.recording is not None:
|
||||
suite['recording'] = instance.recording
|
||||
|
||||
suites.append(suite)
|
||||
|
||||
report["testsuites"] = suites
|
||||
|
|
|
@ -48,6 +48,7 @@ class TestInstance:
|
|||
self.reason = "Unknown"
|
||||
self.metrics = dict()
|
||||
self.handler = None
|
||||
self.recording = None
|
||||
self.outdir = outdir
|
||||
self.execution_time = 0
|
||||
self.build_time = 0
|
||||
|
|
|
@ -122,7 +122,7 @@ mapping:
|
|||
mapping:
|
||||
"regex":
|
||||
type: str
|
||||
required: false
|
||||
required: true
|
||||
"min_ram":
|
||||
type: int
|
||||
required: false
|
||||
|
@ -326,7 +326,7 @@ mapping:
|
|||
mapping:
|
||||
"regex":
|
||||
type: str
|
||||
required: false
|
||||
required: true
|
||||
"min_ram":
|
||||
type: int
|
||||
required: false
|
||||
|
|
|
@ -209,40 +209,35 @@ def test_handler_record(mocked_instance):
|
|||
instance.testcases = [mock.Mock()]
|
||||
|
||||
handler = Handler(instance)
|
||||
handler.suite_name_check = True
|
||||
|
||||
harness = twisterlib.harness.Test()
|
||||
harness.recording = ['dummy recording']
|
||||
type(harness).fieldnames = mock.PropertyMock(return_value=[])
|
||||
|
||||
mock_writerow = mock.Mock()
|
||||
mock_writer = mock.Mock(writerow=mock_writerow)
|
||||
harness = twisterlib.harness.Harness()
|
||||
harness.recording = [ {'field_1': 'recording_1_1', 'field_2': 'recording_1_2'},
|
||||
{'field_1': 'recording_2_1', 'field_2': 'recording_2_2'}
|
||||
]
|
||||
|
||||
with mock.patch(
|
||||
'builtins.open',
|
||||
mock.mock_open(read_data='')
|
||||
) as mock_file, \
|
||||
mock.patch(
|
||||
'csv.writer',
|
||||
mock.Mock(return_value=mock_writer)
|
||||
) as mock_writer_constructor:
|
||||
mock.patch(
|
||||
'csv.DictWriter.writerow',
|
||||
mock.Mock()
|
||||
) as mock_writeheader, \
|
||||
mock.patch(
|
||||
'csv.DictWriter.writerows',
|
||||
mock.Mock()
|
||||
) as mock_writerows:
|
||||
handler.record(harness)
|
||||
|
||||
print(mock_file.mock_calls)
|
||||
|
||||
mock_file.assert_called_with(
|
||||
os.path.join(instance.build_dir, 'recording.csv'),
|
||||
'at'
|
||||
)
|
||||
|
||||
mock_writer_constructor.assert_called_with(
|
||||
mock_file(),
|
||||
harness.fieldnames,
|
||||
lineterminator=os.linesep
|
||||
)
|
||||
|
||||
mock_writerow.assert_has_calls(
|
||||
[mock.call(harness.fieldnames)] + \
|
||||
[mock.call(recording) for recording in harness.recording]
|
||||
)
|
||||
mock_writeheader.assert_has_calls([mock.call({ k:k for k in harness.recording[0].keys()})])
|
||||
mock_writerows.assert_has_calls([mock.call(harness.recording)])
|
||||
|
||||
|
||||
def test_handler_terminate(mocked_instance):
|
||||
|
|
|
@ -43,6 +43,8 @@ def gtest():
|
|||
mock_testsuite.detailed_test_id = True
|
||||
mock_testsuite.id = "id"
|
||||
mock_testsuite.testcases = []
|
||||
mock_testsuite.harness_config = {}
|
||||
|
||||
instance = TestInstance(testsuite=mock_testsuite, platform=mock_platform, outdir="")
|
||||
|
||||
harness = Gtest()
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue