michaelh/zephyr
1
0
Fork 0
Code Releases Activity

twister: pytest: Improve reporting failed pytest scenarios

Browse source 
When pytest scenario fails, then 'handler.log' is printed.
Changed to print 'twister_harness.log' that is a log from
pytest execution. That file tells much more when test fails.

Signed-off-by: Grzegorz Chwierut <grzegorz.chwierut@nordicsemi.no>
This commit is contained in:
Grzegorz Chwierut 2023-10-26 09:08:54 +02:00 • committed by Anas Nashif
commit e466b7ac26
5 changed files with 29 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

1
scripts/pylib/pytest-twister-harness/src/twister_harness/device/hardware_adapter.py
View file

@ -116,7 +116,6 @@ class HardwareAdapter(DeviceAdapter):
stdout_decoded = stdout.decode(errors='ignore')
with open(self.device_log_path, 'a+') as log_file:
log_file.write(stdout_decoded)
logger.debug(f'Flashing output:\n{stdout_decoded}')
if self.device_config.post_flash_script:
self._run_custom_script(self.device_config.post_flash_script, self.base_timeout)
if process is not None and process.returncode == 0:

2
scripts/pylib/twister/twisterlib/harness.py
View file

@ -421,8 +421,10 @@ class Pytest(Harness):
if elem_ts := root.find('testsuite'):
if elem_ts.get('failures') != '0':
self.state = 'failed'
self.instance.reason = f"{elem_ts.get('failures')}/{elem_ts.get('tests')} pytest scenario(s) failed"
elif elem_ts.get('errors') != '0':
self.state = 'error'
self.instance.reason = 'Error during pytest execution'
elif elem_ts.get('skipped') == elem_ts.get('tests'):
self.state = 'skipped'
else:

5
scripts/pylib/twister/twisterlib/reports.py
View file

@ -246,6 +246,7 @@ class Reporting:
for instance in self.instances.values():
suite = {}
handler_log = os.path.join(instance.build_dir, "handler.log")
pytest_log = os.path.join(instance.build_dir, "twister_harness.log")
build_log = os.path.join(instance.build_dir, "build.log")
device_log = os.path.join(instance.build_dir, "device.log")
@ -284,7 +285,9 @@ class Reporting:
suite['status'] = instance.status
suite["reason"] = instance.reason
# FIXME
if os.path.exists(handler_log):
if os.path.exists(pytest_log):
suite["log"] = self.process_log(pytest_log)
elif os.path.exists(handler_log):
suite["log"] = self.process_log(handler_log)
elif os.path.exists(device_log):
suite["log"] = self.process_log(device_log)

26
scripts/pylib/twister/twisterlib/runner.py
View file

@ -40,6 +40,9 @@ if sys.platform == 'linux':
from twisterlib.log_helper import log_command
from twisterlib.testinstance import TestInstance
from twisterlib.environment import TwisterEnv
from twisterlib.testsuite import TestSuite
from twisterlib.platform import Platform
from twisterlib.testplan import change_skip_to_error_if_integration
from twisterlib.harness import HarnessImporter, Pytest
@ -220,7 +223,7 @@ class CMake:
config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
def __init__(self, testsuite, platform, source_dir, build_dir, jobserver):
def __init__(self, testsuite: TestSuite, platform: Platform, source_dir, build_dir, jobserver):
self.cwd = None
self.capture_output = True
@ -414,7 +417,7 @@ class CMake:
class FilterBuilder(CMake):
def __init__(self, testsuite, platform, source_dir, build_dir, jobserver):
def __init__(self, testsuite: TestSuite, platform: Platform, source_dir, build_dir, jobserver):
super().__init__(testsuite, platform, source_dir, build_dir, jobserver)
self.log = "config-twister.log"
@ -517,7 +520,7 @@ class FilterBuilder(CMake):
class ProjectBuilder(FilterBuilder):
def __init__(self, instance, env, jobserver, **kwargs):
def __init__(self, instance: TestInstance, env: TwisterEnv, jobserver, **kwargs):
super().__init__(instance.testsuite, instance.platform, instance.testsuite.source_dir, instance.build_dir, jobserver)
self.log = "build.log"
@ -527,8 +530,7 @@ class ProjectBuilder(FilterBuilder):
self.env = env
self.duts = None
@staticmethod
def log_info(filename, inline_logs):
def log_info(self, filename, inline_logs, log_testcases=False):
filename = os.path.abspath(os.path.realpath(filename))
if inline_logs:
logger.info("{:-^100}".format(filename))
@ -542,6 +544,17 @@ class ProjectBuilder(FilterBuilder):
logger.error(data)
logger.info("{:-^100}".format(filename))
if log_testcases:
for tc in self.instance.testcases:
if not tc.reason:
continue
logger.info(
f"\n{str(tc.name).center(100, '_')}\n"
f"{tc.reason}\n"
f"{100*'_'}\n"
f"{tc.output}"
)
else:
logger.error("see: " + Fore.YELLOW + filename + Fore.RESET)
@ -551,9 +564,12 @@ class ProjectBuilder(FilterBuilder):
b_log = "{}/build.log".format(build_dir)
v_log = "{}/valgrind.log".format(build_dir)
d_log = "{}/device.log".format(build_dir)
pytest_log = "{}/twister_harness.log".format(build_dir)
if os.path.exists(v_log) and "Valgrind" in self.instance.reason:
self.log_info("{}".format(v_log), inline_logs)
elif os.path.exists(pytest_log) and os.path.getsize(pytest_log) > 0:
self.log_info("{}".format(pytest_log), inline_logs, log_testcases=True)
elif os.path.exists(h_log) and os.path.getsize(h_log) > 0:
self.log_info("{}".format(h_log), inline_logs)
elif os.path.exists(d_log) and os.path.getsize(d_log) > 0:

2
scripts/tests/twister/pytest_integration/test_harness_pytest.py
View file

@ -188,6 +188,8 @@ def test_if_report_with_error(pytester, testinstance: TestInstance):
assert tc.status == "failed"
assert tc.output
assert tc.reason
assert testinstance.reason
assert '2/2' in testinstance.reason
def test_if_report_with_skip(pytester, testinstance: TestInstance):