From 47da4e2e76db4854d02a5b6977c7bd24ee4293bb Mon Sep 17 00:00:00 2001 From: Dmitrii Golovanov Date: Fri, 1 Dec 2023 21:19:09 +0100 Subject: [PATCH] twister: Improve recording at Harness The Console Harness is able to parse its log with patterns to compose extracted fields into records in 'recording.csv' file in the test's build directory. This feature allows to extract custom test results like performance counters. With this change the extracted records are also written into 'twister.json' as a part of each test suite object. This makes easier to store all the data collected by the test for its further processing. Other improvements: - compile parsing pattern only once instead of at each input line; - quote fields in '.csv' to avoid unexpected field separators; - make 'regex' a required schema field of 'harness_config'; - Twister documentation update. Signed-off-by: Dmitrii Golovanov --- doc/develop/test/twister.rst | 27 +++++++++----- scripts/pylib/twister/twisterlib/handlers.py | 15 ++++++-- scripts/pylib/twister/twisterlib/harness.py | 29 ++++++++------- scripts/pylib/twister/twisterlib/reports.py | 4 ++ .../pylib/twister/twisterlib/testinstance.py | 1 + scripts/schemas/twister/testsuite-schema.yaml | 4 +- scripts/tests/twister/test_handlers.py | 37 ++++++++----------- scripts/tests/twister/test_harness.py | 2 + 8 files changed, 69 insertions(+), 50 deletions(-) diff --git a/doc/develop/test/twister.rst b/doc/develop/test/twister.rst index 122e9c3871b..4677244725b 100644 --- a/doc/develop/test/twister.rst +++ b/doc/develop/test/twister.rst @@ -478,15 +478,9 @@ harness_config: type: (required) Depends on the regex string to be matched - - record: - regex: (required) - Any string that the particular test case prints to record test - results. - - regex: (required) - Any string that the particular test case prints to confirm test - runs as expected. + regex: (required) + Strings with regular expressions to match with the test's output + to confirm the test runs as expected. ordered: (default False) Check the regular expression strings in orderly or randomly fashion @@ -494,6 +488,21 @@ harness_config: repeat: Number of times to validate the repeated regex expression + record: (optional) + regex: (required) + The regular experssion with named subgroups to match data fields + at the test's output lines where the test provides some custom data + for further analysis. These records will be written into the build + directory 'recording.csv' file as well as 'recording' property + of the test suite object in 'twister.json'. + + For example, to extract three data fields 'metric', 'cycles', 'nanoseconds': + + .. code-block:: yaml + + record: + regex: "(?P.*):(?P.*) cycles, (?P.*) ns" + fixture: Specify a test case dependency on an external device(e.g., sensor), and identify setups that fulfill this dependency. It depends on diff --git a/scripts/pylib/twister/twisterlib/handlers.py b/scripts/pylib/twister/twisterlib/handlers.py index eed3cdae84f..0d6ec5976c7 100755 --- a/scripts/pylib/twister/twisterlib/handlers.py +++ b/scripts/pylib/twister/twisterlib/handlers.py @@ -100,12 +100,19 @@ class Handler: def record(self, harness): if harness.recording: + if self.instance.recording is None: + self.instance.recording = harness.recording.copy() + else: + self.instance.recording.extend(harness.recording) + filename = os.path.join(self.build_dir, "recording.csv") with open(filename, "at") as csvfile: - cw = csv.writer(csvfile, harness.fieldnames, lineterminator=os.linesep) - cw.writerow(harness.fieldnames) - for instance in harness.recording: - cw.writerow(instance) + cw = csv.DictWriter(csvfile, + fieldnames = harness.recording[0].keys(), + lineterminator = os.linesep, + quoting = csv.QUOTE_NONNUMERIC) + cw.writeheader() + cw.writerows(harness.recording) def terminate(self, proc): terminate_process(proc) diff --git a/scripts/pylib/twister/twisterlib/harness.py b/scripts/pylib/twister/twisterlib/harness.py index 8b8ad92fc51..c1d0b6fd23c 100644 --- a/scripts/pylib/twister/twisterlib/harness.py +++ b/scripts/pylib/twister/twisterlib/harness.py @@ -55,8 +55,8 @@ class Harness: self.capture_coverage = False self.next_pattern = 0 self.record = None + self.record_pattern = None self.recording = [] - self.fieldnames = [] self.ztest = False self.detected_suite_names = [] self.run_id = None @@ -80,6 +80,16 @@ class Harness: self.repeat = config.get('repeat', 1) self.ordered = config.get('ordered', True) self.record = config.get('record', {}) + if self.record: + self.record_pattern = re.compile(self.record.get("regex", "")) + + + def get_testcase_name(self): + """ + Get current TestCase name. + """ + return self.id + def process_test(self, line): @@ -172,7 +182,7 @@ class Console(Harness): ''' if self.instance and len(self.instance.testcases) == 1: return self.instance.testcases[0].name - return self.id + return super(Console, self).get_testcase_name() def configure(self, instance): super(Console, self).configure(instance) @@ -240,19 +250,10 @@ class Console(Harness): elif self.GCOV_END in line: self.capture_coverage = False - - if self.record: - pattern = re.compile(self.record.get("regex", "")) - match = pattern.search(line) + if self.record_pattern: + match = self.record_pattern.search(line) if match: - csv = [] - if not self.fieldnames: - for k,v in match.groupdict().items(): - self.fieldnames.append(k) - - for k,v in match.groupdict().items(): - csv.append(v.strip()) - self.recording.append(csv) + self.recording.append({ k:v.strip() for k,v in match.groupdict(default="").items() }) self.process_test(line) # Reset the resulting test state to 'failed' when not all of the patterns were diff --git a/scripts/pylib/twister/twisterlib/reports.py b/scripts/pylib/twister/twisterlib/reports.py index 7425841a723..3d4b155fa9d 100644 --- a/scripts/pylib/twister/twisterlib/reports.py +++ b/scripts/pylib/twister/twisterlib/reports.py @@ -347,6 +347,10 @@ class Reporting: testcases.append(testcase) suite['testcases'] = testcases + + if instance.recording is not None: + suite['recording'] = instance.recording + suites.append(suite) report["testsuites"] = suites diff --git a/scripts/pylib/twister/twisterlib/testinstance.py b/scripts/pylib/twister/twisterlib/testinstance.py index d3cbbaf4986..b5e537892a2 100644 --- a/scripts/pylib/twister/twisterlib/testinstance.py +++ b/scripts/pylib/twister/twisterlib/testinstance.py @@ -48,6 +48,7 @@ class TestInstance: self.reason = "Unknown" self.metrics = dict() self.handler = None + self.recording = None self.outdir = outdir self.execution_time = 0 self.build_time = 0 diff --git a/scripts/schemas/twister/testsuite-schema.yaml b/scripts/schemas/twister/testsuite-schema.yaml index 1e198173c72..387a9b884c9 100644 --- a/scripts/schemas/twister/testsuite-schema.yaml +++ b/scripts/schemas/twister/testsuite-schema.yaml @@ -122,7 +122,7 @@ mapping: mapping: "regex": type: str - required: false + required: true "min_ram": type: int required: false @@ -326,7 +326,7 @@ mapping: mapping: "regex": type: str - required: false + required: true "min_ram": type: int required: false diff --git a/scripts/tests/twister/test_handlers.py b/scripts/tests/twister/test_handlers.py index 6675c4ae451..e66b73055c8 100644 --- a/scripts/tests/twister/test_handlers.py +++ b/scripts/tests/twister/test_handlers.py @@ -209,40 +209,35 @@ def test_handler_record(mocked_instance): instance.testcases = [mock.Mock()] handler = Handler(instance) - handler.suite_name_check = True - harness = twisterlib.harness.Test() - harness.recording = ['dummy recording'] - type(harness).fieldnames = mock.PropertyMock(return_value=[]) - - mock_writerow = mock.Mock() - mock_writer = mock.Mock(writerow=mock_writerow) + harness = twisterlib.harness.Harness() + harness.recording = [ {'field_1': 'recording_1_1', 'field_2': 'recording_1_2'}, + {'field_1': 'recording_2_1', 'field_2': 'recording_2_2'} + ] with mock.patch( 'builtins.open', mock.mock_open(read_data='') ) as mock_file, \ - mock.patch( - 'csv.writer', - mock.Mock(return_value=mock_writer) - ) as mock_writer_constructor: + mock.patch( + 'csv.DictWriter.writerow', + mock.Mock() + ) as mock_writeheader, \ + mock.patch( + 'csv.DictWriter.writerows', + mock.Mock() + ) as mock_writerows: handler.record(harness) + print(mock_file.mock_calls) + mock_file.assert_called_with( os.path.join(instance.build_dir, 'recording.csv'), 'at' ) - mock_writer_constructor.assert_called_with( - mock_file(), - harness.fieldnames, - lineterminator=os.linesep - ) - - mock_writerow.assert_has_calls( - [mock.call(harness.fieldnames)] + \ - [mock.call(recording) for recording in harness.recording] - ) + mock_writeheader.assert_has_calls([mock.call({ k:k for k in harness.recording[0].keys()})]) + mock_writerows.assert_has_calls([mock.call(harness.recording)]) def test_handler_terminate(mocked_instance): diff --git a/scripts/tests/twister/test_harness.py b/scripts/tests/twister/test_harness.py index 1da2aed3f46..247a9426f7b 100644 --- a/scripts/tests/twister/test_harness.py +++ b/scripts/tests/twister/test_harness.py @@ -43,6 +43,8 @@ def gtest(): mock_testsuite.detailed_test_id = True mock_testsuite.id = "id" mock_testsuite.testcases = [] + mock_testsuite.harness_config = {} + instance = TestInstance(testsuite=mock_testsuite, platform=mock_platform, outdir="") harness = Gtest()