diff --git a/doc/develop/test/twister.rst b/doc/develop/test/twister.rst index 122e9c3871b..4677244725b 100644 --- a/doc/develop/test/twister.rst +++ b/doc/develop/test/twister.rst @@ -478,15 +478,9 @@ harness_config: type: (required) Depends on the regex string to be matched - - record: - regex: (required) - Any string that the particular test case prints to record test - results. - - regex: (required) - Any string that the particular test case prints to confirm test - runs as expected. + regex: (required) + Strings with regular expressions to match with the test's output + to confirm the test runs as expected. ordered: (default False) Check the regular expression strings in orderly or randomly fashion @@ -494,6 +488,21 @@ harness_config: repeat: Number of times to validate the repeated regex expression + record: (optional) + regex: (required) + The regular experssion with named subgroups to match data fields + at the test's output lines where the test provides some custom data + for further analysis. These records will be written into the build + directory 'recording.csv' file as well as 'recording' property + of the test suite object in 'twister.json'. + + For example, to extract three data fields 'metric', 'cycles', 'nanoseconds': + + .. code-block:: yaml + + record: + regex: "(?P.*):(?P.*) cycles, (?P.*) ns" + fixture: Specify a test case dependency on an external device(e.g., sensor), and identify setups that fulfill this dependency. It depends on diff --git a/scripts/pylib/twister/twisterlib/handlers.py b/scripts/pylib/twister/twisterlib/handlers.py index eed3cdae84f..0d6ec5976c7 100755 --- a/scripts/pylib/twister/twisterlib/handlers.py +++ b/scripts/pylib/twister/twisterlib/handlers.py @@ -100,12 +100,19 @@ class Handler: def record(self, harness): if harness.recording: + if self.instance.recording is None: + self.instance.recording = harness.recording.copy() + else: + self.instance.recording.extend(harness.recording) + filename = os.path.join(self.build_dir, "recording.csv") with open(filename, "at") as csvfile: - cw = csv.writer(csvfile, harness.fieldnames, lineterminator=os.linesep) - cw.writerow(harness.fieldnames) - for instance in harness.recording: - cw.writerow(instance) + cw = csv.DictWriter(csvfile, + fieldnames = harness.recording[0].keys(), + lineterminator = os.linesep, + quoting = csv.QUOTE_NONNUMERIC) + cw.writeheader() + cw.writerows(harness.recording) def terminate(self, proc): terminate_process(proc) diff --git a/scripts/pylib/twister/twisterlib/harness.py b/scripts/pylib/twister/twisterlib/harness.py index 8b8ad92fc51..c1d0b6fd23c 100644 --- a/scripts/pylib/twister/twisterlib/harness.py +++ b/scripts/pylib/twister/twisterlib/harness.py @@ -55,8 +55,8 @@ class Harness: self.capture_coverage = False self.next_pattern = 0 self.record = None + self.record_pattern = None self.recording = [] - self.fieldnames = [] self.ztest = False self.detected_suite_names = [] self.run_id = None @@ -80,6 +80,16 @@ class Harness: self.repeat = config.get('repeat', 1) self.ordered = config.get('ordered', True) self.record = config.get('record', {}) + if self.record: + self.record_pattern = re.compile(self.record.get("regex", "")) + + + def get_testcase_name(self): + """ + Get current TestCase name. + """ + return self.id + def process_test(self, line): @@ -172,7 +182,7 @@ class Console(Harness): ''' if self.instance and len(self.instance.testcases) == 1: return self.instance.testcases[0].name - return self.id + return super(Console, self).get_testcase_name() def configure(self, instance): super(Console, self).configure(instance) @@ -240,19 +250,10 @@ class Console(Harness): elif self.GCOV_END in line: self.capture_coverage = False - - if self.record: - pattern = re.compile(self.record.get("regex", "")) - match = pattern.search(line) + if self.record_pattern: + match = self.record_pattern.search(line) if match: - csv = [] - if not self.fieldnames: - for k,v in match.groupdict().items(): - self.fieldnames.append(k) - - for k,v in match.groupdict().items(): - csv.append(v.strip()) - self.recording.append(csv) + self.recording.append({ k:v.strip() for k,v in match.groupdict(default="").items() }) self.process_test(line) # Reset the resulting test state to 'failed' when not all of the patterns were diff --git a/scripts/pylib/twister/twisterlib/reports.py b/scripts/pylib/twister/twisterlib/reports.py index 7425841a723..3d4b155fa9d 100644 --- a/scripts/pylib/twister/twisterlib/reports.py +++ b/scripts/pylib/twister/twisterlib/reports.py @@ -347,6 +347,10 @@ class Reporting: testcases.append(testcase) suite['testcases'] = testcases + + if instance.recording is not None: + suite['recording'] = instance.recording + suites.append(suite) report["testsuites"] = suites diff --git a/scripts/pylib/twister/twisterlib/testinstance.py b/scripts/pylib/twister/twisterlib/testinstance.py index d3cbbaf4986..b5e537892a2 100644 --- a/scripts/pylib/twister/twisterlib/testinstance.py +++ b/scripts/pylib/twister/twisterlib/testinstance.py @@ -48,6 +48,7 @@ class TestInstance: self.reason = "Unknown" self.metrics = dict() self.handler = None + self.recording = None self.outdir = outdir self.execution_time = 0 self.build_time = 0 diff --git a/scripts/schemas/twister/testsuite-schema.yaml b/scripts/schemas/twister/testsuite-schema.yaml index 1e198173c72..387a9b884c9 100644 --- a/scripts/schemas/twister/testsuite-schema.yaml +++ b/scripts/schemas/twister/testsuite-schema.yaml @@ -122,7 +122,7 @@ mapping: mapping: "regex": type: str - required: false + required: true "min_ram": type: int required: false @@ -326,7 +326,7 @@ mapping: mapping: "regex": type: str - required: false + required: true "min_ram": type: int required: false diff --git a/scripts/tests/twister/test_handlers.py b/scripts/tests/twister/test_handlers.py index 6675c4ae451..e66b73055c8 100644 --- a/scripts/tests/twister/test_handlers.py +++ b/scripts/tests/twister/test_handlers.py @@ -209,40 +209,35 @@ def test_handler_record(mocked_instance): instance.testcases = [mock.Mock()] handler = Handler(instance) - handler.suite_name_check = True - harness = twisterlib.harness.Test() - harness.recording = ['dummy recording'] - type(harness).fieldnames = mock.PropertyMock(return_value=[]) - - mock_writerow = mock.Mock() - mock_writer = mock.Mock(writerow=mock_writerow) + harness = twisterlib.harness.Harness() + harness.recording = [ {'field_1': 'recording_1_1', 'field_2': 'recording_1_2'}, + {'field_1': 'recording_2_1', 'field_2': 'recording_2_2'} + ] with mock.patch( 'builtins.open', mock.mock_open(read_data='') ) as mock_file, \ - mock.patch( - 'csv.writer', - mock.Mock(return_value=mock_writer) - ) as mock_writer_constructor: + mock.patch( + 'csv.DictWriter.writerow', + mock.Mock() + ) as mock_writeheader, \ + mock.patch( + 'csv.DictWriter.writerows', + mock.Mock() + ) as mock_writerows: handler.record(harness) + print(mock_file.mock_calls) + mock_file.assert_called_with( os.path.join(instance.build_dir, 'recording.csv'), 'at' ) - mock_writer_constructor.assert_called_with( - mock_file(), - harness.fieldnames, - lineterminator=os.linesep - ) - - mock_writerow.assert_has_calls( - [mock.call(harness.fieldnames)] + \ - [mock.call(recording) for recording in harness.recording] - ) + mock_writeheader.assert_has_calls([mock.call({ k:k for k in harness.recording[0].keys()})]) + mock_writerows.assert_has_calls([mock.call(harness.recording)]) def test_handler_terminate(mocked_instance): diff --git a/scripts/tests/twister/test_harness.py b/scripts/tests/twister/test_harness.py index 1da2aed3f46..247a9426f7b 100644 --- a/scripts/tests/twister/test_harness.py +++ b/scripts/tests/twister/test_harness.py @@ -43,6 +43,8 @@ def gtest(): mock_testsuite.detailed_test_id = True mock_testsuite.id = "id" mock_testsuite.testcases = [] + mock_testsuite.harness_config = {} + instance = TestInstance(testsuite=mock_testsuite, platform=mock_platform, outdir="") harness = Gtest()