twister: Improve recording at Harness

The Console Harness is able to parse its log with patterns to compose
extracted fields into records in 'recording.csv' file in the test's build
directory. This feature allows to extract custom test results like
performance counters.
With this change the extracted records are also written into 'twister.json'
as a part of each test suite object. This makes easier to store
all the data collected by the test for its further processing.

Other improvements:
 - compile parsing pattern only once instead of at each input line;
 - quote fields in '.csv' to avoid unexpected field separators;
 - make 'regex' a required schema field of 'harness_config';
 - Twister documentation update.

Signed-off-by: Dmitrii Golovanov <dmitrii.golovanov@intel.com>

doc/develop/test/twister.rst

@@ -478,15 +478,9 @@ harness_config: <harness configuration options>
     type: <one_line|multi_line> (required)
         Depends on the regex string to be matched
 
-
+    regex: <list of regular expressions> (required)
-    record: <recording options>
+        Strings with regular expressions to match with the test's output
-      regex: <expression> (required)
+        to confirm the test runs as expected.
-        Any string that the particular test case prints to record test
-        results.
-
-    regex: <expression> (required)
-        Any string that the particular test case prints to confirm test
-        runs as expected.
 
     ordered: <True|False> (default False)
         Check the regular expression strings in orderly or randomly fashion
@@ -494,6 +488,21 @@ harness_config: <harness configuration options>
     repeat: <integer>
         Number of times to validate the repeated regex expression
 
+    record: <recording options> (optional)
+      regex: <regular expression> (required)
+        The regular experssion with named subgroups to match data fields
+        at the test's output lines where the test provides some custom data
+        for further analysis. These records will be written into the build
+        directory 'recording.csv' file as well as 'recording' property
+        of the test suite object in 'twister.json'.
+
+        For example, to extract three data fields 'metric', 'cycles', 'nanoseconds':
+
+        .. code-block:: yaml
+
+          record:
+            regex: "(?P<metric>.*):(?P<cycles>.*) cycles, (?P<nanoseconds>.*) ns"
+
     fixture: <expression>
         Specify a test case dependency on an external device(e.g., sensor),
         and identify setups that fulfill this dependency. It depends on

scripts/pylib/twister/twisterlib/handlers.py

@@ -100,12 +100,19 @@ class Handler:
 
     def record(self, harness):
         if harness.recording:
+            if self.instance.recording is None:
+                self.instance.recording = harness.recording.copy()
+            else:
+                self.instance.recording.extend(harness.recording)
+
             filename = os.path.join(self.build_dir, "recording.csv")
             with open(filename, "at") as csvfile:
-                cw = csv.writer(csvfile, harness.fieldnames, lineterminator=os.linesep)
+                cw = csv.DictWriter(csvfile,
-                cw.writerow(harness.fieldnames)
+                                    fieldnames = harness.recording[0].keys(),
-                for instance in harness.recording:
+                                    lineterminator = os.linesep,
-                    cw.writerow(instance)
+                                    quoting = csv.QUOTE_NONNUMERIC)
+                cw.writeheader()
+                cw.writerows(harness.recording)
 
     def terminate(self, proc):
         terminate_process(proc)

scripts/pylib/twister/twisterlib/harness.py

@@ -55,8 +55,8 @@ class Harness:
         self.capture_coverage = False
         self.next_pattern = 0
         self.record = None
+        self.record_pattern = None
         self.recording = []
-        self.fieldnames = []
         self.ztest = False
         self.detected_suite_names = []
         self.run_id = None
@@ -80,6 +80,16 @@ class Harness:
             self.repeat = config.get('repeat', 1)
             self.ordered = config.get('ordered', True)
             self.record = config.get('record', {})
+            if self.record:
+                self.record_pattern = re.compile(self.record.get("regex", ""))
+
+
+    def get_testcase_name(self):
+        """
+        Get current TestCase name.
+        """
+        return self.id
+
 
     def process_test(self, line):
 
@@ -172,7 +182,7 @@ class Console(Harness):
         '''
         if self.instance and len(self.instance.testcases) == 1:
             return self.instance.testcases[0].name
-        return self.id
+        return super(Console, self).get_testcase_name()
 
     def configure(self, instance):
         super(Console, self).configure(instance)
@@ -240,19 +250,10 @@ class Console(Harness):
         elif self.GCOV_END in line:
             self.capture_coverage = False
 
-
+        if self.record_pattern:
-        if self.record:
+            match = self.record_pattern.search(line)
-            pattern = re.compile(self.record.get("regex", ""))
-            match = pattern.search(line)
             if match:
-                csv = []
+                self.recording.append({ k:v.strip() for k,v in match.groupdict(default="").items() })
-                if not self.fieldnames:
-                    for k,v in match.groupdict().items():
-                        self.fieldnames.append(k)
-
-                for k,v in match.groupdict().items():
-                    csv.append(v.strip())
-                self.recording.append(csv)
 
         self.process_test(line)
         # Reset the resulting test state to 'failed' when not all of the patterns were

scripts/pylib/twister/twisterlib/reports.py

@@ -347,6 +347,10 @@ class Reporting:
                 testcases.append(testcase)
 
             suite['testcases'] = testcases
+
+            if instance.recording is not None:
+                suite['recording'] = instance.recording
+
             suites.append(suite)
 
         report["testsuites"] = suites

scripts/pylib/twister/twisterlib/testinstance.py

@@ -48,6 +48,7 @@ class TestInstance:
         self.reason = "Unknown"
         self.metrics = dict()
         self.handler = None
+        self.recording = None
         self.outdir = outdir
         self.execution_time = 0
         self.build_time = 0

scripts/schemas/twister/testsuite-schema.yaml

@@ -122,7 +122,7 @@ mapping:
             mapping:
               "regex":
                 type: str
-                required: false
+                required: true
       "min_ram":
         type: int
         required: false
@@ -326,7 +326,7 @@ mapping:
                 mapping:
                   "regex":
                     type: str
-                    required: false
+                    required: true
           "min_ram":
             type: int
             required: false

scripts/tests/twister/test_handlers.py

@@ -209,40 +209,35 @@ def test_handler_record(mocked_instance):
     instance.testcases = [mock.Mock()]
 
     handler = Handler(instance)
-    handler.suite_name_check = True
 
-    harness = twisterlib.harness.Test()
+    harness = twisterlib.harness.Harness()
-    harness.recording = ['dummy recording']
+    harness.recording = [ {'field_1':  'recording_1_1', 'field_2': 'recording_1_2'},
-    type(harness).fieldnames = mock.PropertyMock(return_value=[])
+                          {'field_1':  'recording_2_1', 'field_2': 'recording_2_2'}
-
+                        ]
-    mock_writerow = mock.Mock()
-    mock_writer = mock.Mock(writerow=mock_writerow)
 
     with mock.patch(
         'builtins.open',
         mock.mock_open(read_data='')
     ) as mock_file, \
-         mock.patch(
+        mock.patch(
-        'csv.writer',
+        'csv.DictWriter.writerow',
-        mock.Mock(return_value=mock_writer)
+        mock.Mock()
-    ) as mock_writer_constructor:
+    ) as mock_writeheader, \
+        mock.patch(
+        'csv.DictWriter.writerows',
+        mock.Mock()
+    ) as mock_writerows:
         handler.record(harness)
 
+    print(mock_file.mock_calls)
+
     mock_file.assert_called_with(
         os.path.join(instance.build_dir, 'recording.csv'),
         'at'
     )
 
-    mock_writer_constructor.assert_called_with(
+    mock_writeheader.assert_has_calls([mock.call({ k:k for k in harness.recording[0].keys()})])
-        mock_file(),
+    mock_writerows.assert_has_calls([mock.call(harness.recording)])
-        harness.fieldnames,
-        lineterminator=os.linesep
-    )
-
-    mock_writerow.assert_has_calls(
-        [mock.call(harness.fieldnames)] + \
-        [mock.call(recording) for recording in harness.recording]
-    )
 
 
 def test_handler_terminate(mocked_instance):

scripts/tests/twister/test_harness.py

@@ -43,6 +43,8 @@ def gtest():
     mock_testsuite.detailed_test_id = True
     mock_testsuite.id = "id"
     mock_testsuite.testcases = []
+    mock_testsuite.harness_config = {}
+
     instance = TestInstance(testsuite=mock_testsuite, platform=mock_platform, outdir="")
 
     harness = Gtest()