8213a1509f
test_reporting_testsuite.py: Adding testcases for reporting functions of Class testSuite. test_testsuite_class.py: Testcase for add_instances function conftest.py: Added fixture for instances_fixture Signed-off-by: Aastha Grover <aastha.grover@intel.com>
169 lines
7.8 KiB
Python
169 lines
7.8 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright (c) 2020 Intel Corporation
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
'''
|
|
This test file contains testcases for reporting functionality of Testsuite class of sanitycheck
|
|
'''
|
|
import sys
|
|
import os
|
|
import xml.etree.ElementTree as ET
|
|
import csv
|
|
from collections import defaultdict
|
|
from mock import MagicMock
|
|
import pytest
|
|
|
|
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
|
|
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/sanity_chk"))
|
|
|
|
|
|
def test_discard_report(class_testsuite, platforms_list, all_testcases_dict, caplog, tmpdir):
|
|
""" Testing discard_report function of Testsuite class in sanitycheck
|
|
Test 1: Check if apply_filters function has been run before running
|
|
discard_report
|
|
Test 2: Test if the generated report is not empty
|
|
Test 3: Test if the gerenrated report contains the expected columns"""
|
|
class_testsuite.platforms = platforms_list
|
|
class_testsuite.testcases = all_testcases_dict
|
|
filename = tmpdir.mkdir("test_discard").join("discard_report.csv")
|
|
with pytest.raises(SystemExit):
|
|
class_testsuite.discard_report(filename)
|
|
assert "apply_filters() hasn't been run!" in caplog.text
|
|
|
|
kwargs = {"exclude_tag" : ['test_a'], "exclude_platform" : ['demo_board_1'],
|
|
"platform" : ['demo_board_2']}
|
|
class_testsuite.apply_filters(**kwargs)
|
|
class_testsuite.discard_report(filename)
|
|
assert os.stat(filename).st_size != 0
|
|
with open(filename, "r") as file:
|
|
csv_reader = csv.reader(file)
|
|
assert set(['test', 'arch', 'platform', 'reason']) == set(list(csv_reader)[0])
|
|
|
|
def test_csv_report(class_testsuite, instances_fixture, tmpdir):
|
|
""" Testing csv_report function of Testsuite class in sanitycheck
|
|
Test 1: Assert the csv_report isnt empty after execution of csv_report function
|
|
Test 2: Assert on the columns and values of the generated csv_report"""
|
|
class_testsuite.instances = instances_fixture
|
|
filename = tmpdir.mkdir("test_csv").join("sanitycheck_csv_report.csv")
|
|
class_testsuite.csv_report(filename)
|
|
assert os.path.exists(filename)
|
|
assert os.stat(filename).st_size != 0
|
|
|
|
mydict = {'test': [], 'arch' : [], 'platform' : [], 'status': [],
|
|
'extra_args': [], 'handler': [], 'handler_time': [],
|
|
'ram_size': [], 'rom_size': []}
|
|
|
|
with open(filename, "r") as file:
|
|
csv_reader = csv.reader(file)
|
|
assert set(mydict.keys()) == set(list(csv_reader)[0])
|
|
|
|
for instance in class_testsuite.instances.values():
|
|
mydict["test"].append(instance.testcase.name)
|
|
mydict["arch"].append(instance.platform.arch)
|
|
mydict["platform"].append(instance.platform.name)
|
|
instance_status = instance.status if instance.status is not None else ""
|
|
mydict["status"].append(instance_status)
|
|
args = " ".join(instance.testcase.extra_args)
|
|
mydict["extra_args"].append(args)
|
|
mydict["handler"].append(instance.platform.simulation)
|
|
mydict["handler_time"].append(instance.metrics.get("handler_time", ""))
|
|
mydict["ram_size"].append(instance.metrics.get("ram_size", '0'))
|
|
mydict["rom_size"].append(instance.metrics.get("rom_size", '0'))
|
|
|
|
dict_file = open(filename, "r")
|
|
dict_reader = csv.DictReader(dict_file)
|
|
columns = defaultdict(list)
|
|
for row in dict_reader:
|
|
for (key, value) in row.items():
|
|
columns[key].append(value)
|
|
for _, value in enumerate(mydict):
|
|
assert columns[value] == mydict[value]
|
|
dict_file.close()
|
|
|
|
def test_xunit_report(class_testsuite, test_data,
|
|
instances_fixture, platforms_list, all_testcases_dict):
|
|
""" Testing xunit_report function of Testsuite class in sanitycheck
|
|
Test 1: Assert sanitycheck.xml file exists after execution of xunit_report function
|
|
Test 2: Assert on fails, passes, skips, errors values
|
|
Test 3: Assert on the tree structure of sanitycheck.xml file"""
|
|
class_testsuite.platforms = platforms_list
|
|
class_testsuite.testcases = all_testcases_dict
|
|
kwargs = {"exclude_tag" : ['test_a'], "exclude_platform" : ['demo_board_1'],
|
|
"platform" : ['demo_board_2']}
|
|
class_testsuite.apply_filters(**kwargs)
|
|
class_testsuite.instances = instances_fixture
|
|
inst1 = class_testsuite.instances.get("demo_board_2/scripts/tests/sanitycheck/test_data/testcases/tests/test_a/test_a.check_1")
|
|
inst2 = class_testsuite.instances.get("demo_board_2/scripts/tests/sanitycheck/test_data/testcases/tests/test_a/test_a.check_2")
|
|
inst1.status = "failed"
|
|
inst2.status = "skipped"
|
|
|
|
filename = test_data + "sanitycheck.xml"
|
|
class_testsuite.xunit_report(filename)
|
|
assert os.path.exists(filename)
|
|
|
|
filesize = os.path.getsize(filename)
|
|
assert filesize != 0
|
|
|
|
fails, passes, errors, skips = 0, 0, 0, 0
|
|
for instance in class_testsuite.instances.values():
|
|
if instance.status in ["failed", "timeout"]:
|
|
if instance.reason in ['build_error', 'handler_crash']:
|
|
errors += 1
|
|
else:
|
|
fails += 1
|
|
elif instance.status == 'skipped':
|
|
skips += 1
|
|
else:
|
|
passes += 1
|
|
|
|
tree = ET.parse(filename)
|
|
assert int(tree.findall('testsuite')[0].attrib["skip"]) == int(skips)
|
|
assert int(tree.findall('testsuite')[0].attrib["failures"]) == int(fails)
|
|
assert int(tree.findall('testsuite')[0].attrib["errors"]) == int(errors)
|
|
assert int(tree.findall('testsuite')[0].attrib["tests"]) == int(passes+fails+skips+errors)
|
|
|
|
for index in range(0, len(class_testsuite.instances)):
|
|
if len(list(tree.findall('testsuite')[0][index])) != 0:
|
|
if tree.findall('testsuite')[0][index][0].attrib["type"] == "failure":
|
|
assert tree.findall('testsuite')[0][index].attrib["name"] == \
|
|
(inst1.testcase.name)
|
|
elif tree.findall('testsuite')[0][index][0].attrib["type"] == "skipped":
|
|
assert tree.findall('testsuite')[0][index].attrib["name"] == \
|
|
(inst2.testcase.name)
|
|
os.remove(filename)
|
|
|
|
def test_compare_metrics(class_testsuite, test_data, instances_fixture, caplog):
|
|
""" Testing compare_metrics function of Testsuite class in sanitycheck
|
|
Test 1: Error message is raised if file sanitycheck.csv file doesnt exist
|
|
Test 2: Assert on compare_metrics results for expected values"""
|
|
class_testsuite.instances = instances_fixture
|
|
for instance in class_testsuite.instances.values():
|
|
instance.metrics["ram_size"] = 5
|
|
instance.metrics["rom_size"] = 9
|
|
filename_not_exist = test_data + "sanitycheck_file_not_exist.csv"
|
|
class_testsuite.compare_metrics(filename_not_exist)
|
|
assert "Cannot compare metrics, " + filename_not_exist + " not found" in caplog.text
|
|
|
|
filename = test_data + "sanitycheck.csv"
|
|
results = class_testsuite.compare_metrics(filename)
|
|
for instance in class_testsuite.instances.values():
|
|
for res in results:
|
|
assert res[0].platform.name == instance.platform.name
|
|
if (res[0].platform.name == instance.platform.name) and \
|
|
(res[0].testcase.name == instance.testcase.name):
|
|
if res[1] == "ram_size":
|
|
assert res[2] == instance.metrics["ram_size"]
|
|
elif res[1] == "rom_size":
|
|
assert res[2] == instance.metrics["rom_size"]
|
|
|
|
def test_target_report(class_testsuite, instances_fixture, tmpdir_factory):
|
|
""" Testing target_report function of Testsuite class in sanitycheck
|
|
Test: Assert xunit_report function is called from target_report function"""
|
|
class_testsuite.instances = instances_fixture
|
|
outdir = tmpdir_factory.mktemp("tmp")
|
|
class_testsuite.xunit_report = MagicMock(side_effect=class_testsuite.xunit_report)
|
|
class_testsuite.target_report(outdir, "abc", append=False)
|
|
assert class_testsuite.instances
|
|
class_testsuite.xunit_report.assert_called()
|