twister: pytest: Improve reporting failed pytest scenarios
When pytest scenario fails, then 'handler.log' is printed. Changed to print 'twister_harness.log' that is a log from pytest execution. That file tells much more when test fails. Signed-off-by: Grzegorz Chwierut <grzegorz.chwierut@nordicsemi.no>
This commit is contained in:
parent
014666bf05
commit
e466b7ac26
5 changed files with 29 additions and 7 deletions
|
@ -116,7 +116,6 @@ class HardwareAdapter(DeviceAdapter):
|
|||
stdout_decoded = stdout.decode(errors='ignore')
|
||||
with open(self.device_log_path, 'a+') as log_file:
|
||||
log_file.write(stdout_decoded)
|
||||
logger.debug(f'Flashing output:\n{stdout_decoded}')
|
||||
if self.device_config.post_flash_script:
|
||||
self._run_custom_script(self.device_config.post_flash_script, self.base_timeout)
|
||||
if process is not None and process.returncode == 0:
|
||||
|
|
|
@ -421,8 +421,10 @@ class Pytest(Harness):
|
|||
if elem_ts := root.find('testsuite'):
|
||||
if elem_ts.get('failures') != '0':
|
||||
self.state = 'failed'
|
||||
self.instance.reason = f"{elem_ts.get('failures')}/{elem_ts.get('tests')} pytest scenario(s) failed"
|
||||
elif elem_ts.get('errors') != '0':
|
||||
self.state = 'error'
|
||||
self.instance.reason = 'Error during pytest execution'
|
||||
elif elem_ts.get('skipped') == elem_ts.get('tests'):
|
||||
self.state = 'skipped'
|
||||
else:
|
||||
|
|
|
@ -246,6 +246,7 @@ class Reporting:
|
|||
for instance in self.instances.values():
|
||||
suite = {}
|
||||
handler_log = os.path.join(instance.build_dir, "handler.log")
|
||||
pytest_log = os.path.join(instance.build_dir, "twister_harness.log")
|
||||
build_log = os.path.join(instance.build_dir, "build.log")
|
||||
device_log = os.path.join(instance.build_dir, "device.log")
|
||||
|
||||
|
@ -284,7 +285,9 @@ class Reporting:
|
|||
suite['status'] = instance.status
|
||||
suite["reason"] = instance.reason
|
||||
# FIXME
|
||||
if os.path.exists(handler_log):
|
||||
if os.path.exists(pytest_log):
|
||||
suite["log"] = self.process_log(pytest_log)
|
||||
elif os.path.exists(handler_log):
|
||||
suite["log"] = self.process_log(handler_log)
|
||||
elif os.path.exists(device_log):
|
||||
suite["log"] = self.process_log(device_log)
|
||||
|
|
|
@ -40,6 +40,9 @@ if sys.platform == 'linux':
|
|||
|
||||
from twisterlib.log_helper import log_command
|
||||
from twisterlib.testinstance import TestInstance
|
||||
from twisterlib.environment import TwisterEnv
|
||||
from twisterlib.testsuite import TestSuite
|
||||
from twisterlib.platform import Platform
|
||||
from twisterlib.testplan import change_skip_to_error_if_integration
|
||||
from twisterlib.harness import HarnessImporter, Pytest
|
||||
|
||||
|
@ -220,7 +223,7 @@ class CMake:
|
|||
config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
|
||||
dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
|
||||
|
||||
def __init__(self, testsuite, platform, source_dir, build_dir, jobserver):
|
||||
def __init__(self, testsuite: TestSuite, platform: Platform, source_dir, build_dir, jobserver):
|
||||
|
||||
self.cwd = None
|
||||
self.capture_output = True
|
||||
|
@ -414,7 +417,7 @@ class CMake:
|
|||
|
||||
class FilterBuilder(CMake):
|
||||
|
||||
def __init__(self, testsuite, platform, source_dir, build_dir, jobserver):
|
||||
def __init__(self, testsuite: TestSuite, platform: Platform, source_dir, build_dir, jobserver):
|
||||
super().__init__(testsuite, platform, source_dir, build_dir, jobserver)
|
||||
|
||||
self.log = "config-twister.log"
|
||||
|
@ -517,7 +520,7 @@ class FilterBuilder(CMake):
|
|||
|
||||
class ProjectBuilder(FilterBuilder):
|
||||
|
||||
def __init__(self, instance, env, jobserver, **kwargs):
|
||||
def __init__(self, instance: TestInstance, env: TwisterEnv, jobserver, **kwargs):
|
||||
super().__init__(instance.testsuite, instance.platform, instance.testsuite.source_dir, instance.build_dir, jobserver)
|
||||
|
||||
self.log = "build.log"
|
||||
|
@ -527,8 +530,7 @@ class ProjectBuilder(FilterBuilder):
|
|||
self.env = env
|
||||
self.duts = None
|
||||
|
||||
@staticmethod
|
||||
def log_info(filename, inline_logs):
|
||||
def log_info(self, filename, inline_logs, log_testcases=False):
|
||||
filename = os.path.abspath(os.path.realpath(filename))
|
||||
if inline_logs:
|
||||
logger.info("{:-^100}".format(filename))
|
||||
|
@ -542,6 +544,17 @@ class ProjectBuilder(FilterBuilder):
|
|||
logger.error(data)
|
||||
|
||||
logger.info("{:-^100}".format(filename))
|
||||
|
||||
if log_testcases:
|
||||
for tc in self.instance.testcases:
|
||||
if not tc.reason:
|
||||
continue
|
||||
logger.info(
|
||||
f"\n{str(tc.name).center(100, '_')}\n"
|
||||
f"{tc.reason}\n"
|
||||
f"{100*'_'}\n"
|
||||
f"{tc.output}"
|
||||
)
|
||||
else:
|
||||
logger.error("see: " + Fore.YELLOW + filename + Fore.RESET)
|
||||
|
||||
|
@ -551,9 +564,12 @@ class ProjectBuilder(FilterBuilder):
|
|||
b_log = "{}/build.log".format(build_dir)
|
||||
v_log = "{}/valgrind.log".format(build_dir)
|
||||
d_log = "{}/device.log".format(build_dir)
|
||||
pytest_log = "{}/twister_harness.log".format(build_dir)
|
||||
|
||||
if os.path.exists(v_log) and "Valgrind" in self.instance.reason:
|
||||
self.log_info("{}".format(v_log), inline_logs)
|
||||
elif os.path.exists(pytest_log) and os.path.getsize(pytest_log) > 0:
|
||||
self.log_info("{}".format(pytest_log), inline_logs, log_testcases=True)
|
||||
elif os.path.exists(h_log) and os.path.getsize(h_log) > 0:
|
||||
self.log_info("{}".format(h_log), inline_logs)
|
||||
elif os.path.exists(d_log) and os.path.getsize(d_log) > 0:
|
||||
|
|
|
@ -188,6 +188,8 @@ def test_if_report_with_error(pytester, testinstance: TestInstance):
|
|||
assert tc.status == "failed"
|
||||
assert tc.output
|
||||
assert tc.reason
|
||||
assert testinstance.reason
|
||||
assert '2/2' in testinstance.reason
|
||||
|
||||
|
||||
def test_if_report_with_skip(pytester, testinstance: TestInstance):
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue