twister: move coverage classes out
Move coverage related classes out into own file. Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif
parent
78d9dab7f3
commit
adfa6c7777
3 changed files with 211 additions and 197 deletions
209
scripts/pylib/twister/twister/coverage.py
Normal file
209
scripts/pylib/twister/twister/coverage.py
Normal file
|
|
@ -0,0 +1,209 @@
|
||||||
|
# vim: set syntax=python ts=4 :
|
||||||
|
#
|
||||||
|
# Copyright (c) 2018-2022 Intel Corporation
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
|
import subprocess
|
||||||
|
import glob
|
||||||
|
import re
|
||||||
|
|
||||||
|
logger = logging.getLogger('twister')
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
class CoverageTool:
|
||||||
|
""" Base class for every supported coverage tool
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.gcov_tool = None
|
||||||
|
self.base_dir = None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def factory(tool):
|
||||||
|
if tool == 'lcov':
|
||||||
|
t = Lcov()
|
||||||
|
elif tool == 'gcovr':
|
||||||
|
t = Gcovr()
|
||||||
|
else:
|
||||||
|
logger.error("Unsupported coverage tool specified: {}".format(tool))
|
||||||
|
return None
|
||||||
|
|
||||||
|
logger.debug(f"Select {tool} as the coverage tool...")
|
||||||
|
return t
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def retrieve_gcov_data(input_file):
|
||||||
|
logger.debug("Working on %s" % input_file)
|
||||||
|
extracted_coverage_info = {}
|
||||||
|
capture_data = False
|
||||||
|
capture_complete = False
|
||||||
|
with open(input_file, 'r') as fp:
|
||||||
|
for line in fp.readlines():
|
||||||
|
if re.search("GCOV_COVERAGE_DUMP_START", line):
|
||||||
|
capture_data = True
|
||||||
|
continue
|
||||||
|
if re.search("GCOV_COVERAGE_DUMP_END", line):
|
||||||
|
capture_complete = True
|
||||||
|
break
|
||||||
|
# Loop until the coverage data is found.
|
||||||
|
if not capture_data:
|
||||||
|
continue
|
||||||
|
if line.startswith("*"):
|
||||||
|
sp = line.split("<")
|
||||||
|
if len(sp) > 1:
|
||||||
|
# Remove the leading delimiter "*"
|
||||||
|
file_name = sp[0][1:]
|
||||||
|
# Remove the trailing new line char
|
||||||
|
hex_dump = sp[1][:-1]
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
extracted_coverage_info.update({file_name: hex_dump})
|
||||||
|
if not capture_data:
|
||||||
|
capture_complete = True
|
||||||
|
return {'complete': capture_complete, 'data': extracted_coverage_info}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def create_gcda_files(extracted_coverage_info):
|
||||||
|
logger.debug("Generating gcda files")
|
||||||
|
for filename, hexdump_val in extracted_coverage_info.items():
|
||||||
|
# if kobject_hash is given for coverage gcovr fails
|
||||||
|
# hence skipping it problem only in gcovr v4.1
|
||||||
|
if "kobject_hash" in filename:
|
||||||
|
filename = (filename[:-4]) + "gcno"
|
||||||
|
try:
|
||||||
|
os.remove(filename)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
continue
|
||||||
|
|
||||||
|
with open(filename, 'wb') as fp:
|
||||||
|
fp.write(bytes.fromhex(hexdump_val))
|
||||||
|
|
||||||
|
def generate(self, outdir):
|
||||||
|
for filename in glob.glob("%s/**/handler.log" % outdir, recursive=True):
|
||||||
|
gcov_data = self.__class__.retrieve_gcov_data(filename)
|
||||||
|
capture_complete = gcov_data['complete']
|
||||||
|
extracted_coverage_info = gcov_data['data']
|
||||||
|
if capture_complete:
|
||||||
|
self.__class__.create_gcda_files(extracted_coverage_info)
|
||||||
|
logger.debug("Gcov data captured: {}".format(filename))
|
||||||
|
else:
|
||||||
|
logger.error("Gcov data capture incomplete: {}".format(filename))
|
||||||
|
|
||||||
|
with open(os.path.join(outdir, "coverage.log"), "a") as coveragelog:
|
||||||
|
ret = self._generate(outdir, coveragelog)
|
||||||
|
if ret == 0:
|
||||||
|
logger.info("HTML report generated: {}".format(
|
||||||
|
os.path.join(outdir, "coverage", "index.html")))
|
||||||
|
|
||||||
|
|
||||||
|
class Lcov(CoverageTool):
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.ignores = []
|
||||||
|
|
||||||
|
def add_ignore_file(self, pattern):
|
||||||
|
self.ignores.append('*' + pattern + '*')
|
||||||
|
|
||||||
|
def add_ignore_directory(self, pattern):
|
||||||
|
self.ignores.append('*/' + pattern + '/*')
|
||||||
|
|
||||||
|
def _generate(self, outdir, coveragelog):
|
||||||
|
coveragefile = os.path.join(outdir, "coverage.info")
|
||||||
|
ztestfile = os.path.join(outdir, "ztest.info")
|
||||||
|
cmd = ["lcov", "--gcov-tool", self.gcov_tool,
|
||||||
|
"--capture", "--directory", outdir,
|
||||||
|
"--rc", "lcov_branch_coverage=1",
|
||||||
|
"--output-file", coveragefile]
|
||||||
|
cmd_str = " ".join(cmd)
|
||||||
|
logger.debug(f"Running {cmd_str}...")
|
||||||
|
subprocess.call(cmd, stdout=coveragelog)
|
||||||
|
# We want to remove tests/* and tests/ztest/test/* but save tests/ztest
|
||||||
|
subprocess.call(["lcov", "--gcov-tool", self.gcov_tool, "--extract",
|
||||||
|
coveragefile,
|
||||||
|
os.path.join(self.base_dir, "tests", "ztest", "*"),
|
||||||
|
"--output-file", ztestfile,
|
||||||
|
"--rc", "lcov_branch_coverage=1"], stdout=coveragelog)
|
||||||
|
|
||||||
|
if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0:
|
||||||
|
subprocess.call(["lcov", "--gcov-tool", self.gcov_tool, "--remove",
|
||||||
|
ztestfile,
|
||||||
|
os.path.join(self.base_dir, "tests/ztest/test/*"),
|
||||||
|
"--output-file", ztestfile,
|
||||||
|
"--rc", "lcov_branch_coverage=1"],
|
||||||
|
stdout=coveragelog)
|
||||||
|
files = [coveragefile, ztestfile]
|
||||||
|
else:
|
||||||
|
files = [coveragefile]
|
||||||
|
|
||||||
|
for i in self.ignores:
|
||||||
|
subprocess.call(
|
||||||
|
["lcov", "--gcov-tool", self.gcov_tool, "--remove",
|
||||||
|
coveragefile, i, "--output-file",
|
||||||
|
coveragefile, "--rc", "lcov_branch_coverage=1"],
|
||||||
|
stdout=coveragelog)
|
||||||
|
|
||||||
|
# The --ignore-errors source option is added to avoid it exiting due to
|
||||||
|
# samples/application_development/external_lib/
|
||||||
|
return subprocess.call(["genhtml", "--legend", "--branch-coverage",
|
||||||
|
"--ignore-errors", "source",
|
||||||
|
"-output-directory",
|
||||||
|
os.path.join(outdir, "coverage")] + files,
|
||||||
|
stdout=coveragelog)
|
||||||
|
|
||||||
|
|
||||||
|
class Gcovr(CoverageTool):
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.ignores = []
|
||||||
|
|
||||||
|
def add_ignore_file(self, pattern):
|
||||||
|
self.ignores.append('.*' + pattern + '.*')
|
||||||
|
|
||||||
|
def add_ignore_directory(self, pattern):
|
||||||
|
self.ignores.append(".*/" + pattern + '/.*')
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _interleave_list(prefix, list):
|
||||||
|
tuple_list = [(prefix, item) for item in list]
|
||||||
|
return [item for sublist in tuple_list for item in sublist]
|
||||||
|
|
||||||
|
def _generate(self, outdir, coveragelog):
|
||||||
|
coveragefile = os.path.join(outdir, "coverage.json")
|
||||||
|
ztestfile = os.path.join(outdir, "ztest.json")
|
||||||
|
|
||||||
|
excludes = Gcovr._interleave_list("-e", self.ignores)
|
||||||
|
|
||||||
|
# We want to remove tests/* and tests/ztest/test/* but save tests/ztest
|
||||||
|
cmd = ["gcovr", "-r", self.base_dir, "--gcov-executable",
|
||||||
|
self.gcov_tool, "-e", "tests/*"] + excludes + ["--json", "-o",
|
||||||
|
coveragefile, outdir]
|
||||||
|
cmd_str = " ".join(cmd)
|
||||||
|
logger.debug(f"Running {cmd_str}...")
|
||||||
|
subprocess.call(cmd, stdout=coveragelog)
|
||||||
|
|
||||||
|
subprocess.call(["gcovr", "-r", self.base_dir, "--gcov-executable",
|
||||||
|
self.gcov_tool, "-f", "tests/ztest", "-e",
|
||||||
|
"tests/ztest/test/*", "--json", "-o", ztestfile,
|
||||||
|
outdir], stdout=coveragelog)
|
||||||
|
|
||||||
|
if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0:
|
||||||
|
files = [coveragefile, ztestfile]
|
||||||
|
else:
|
||||||
|
files = [coveragefile]
|
||||||
|
|
||||||
|
subdir = os.path.join(outdir, "coverage")
|
||||||
|
os.makedirs(subdir, exist_ok=True)
|
||||||
|
|
||||||
|
tracefiles = self._interleave_list("--add-tracefile", files)
|
||||||
|
|
||||||
|
return subprocess.call(["gcovr", "-r", self.base_dir, "--html",
|
||||||
|
"--html-details"] + tracefiles +
|
||||||
|
["-o", os.path.join(subdir, "index.html")],
|
||||||
|
stdout=coveragelog)
|
||||||
|
|
@ -2119,202 +2119,6 @@ class TestPlan:
|
||||||
self.link_dir_counter += 1
|
self.link_dir_counter += 1
|
||||||
|
|
||||||
|
|
||||||
class CoverageTool:
|
|
||||||
""" Base class for every supported coverage tool
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.gcov_tool = None
|
|
||||||
self.base_dir = None
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def factory(tool):
|
|
||||||
if tool == 'lcov':
|
|
||||||
t = Lcov()
|
|
||||||
elif tool == 'gcovr':
|
|
||||||
t = Gcovr()
|
|
||||||
else:
|
|
||||||
logger.error("Unsupported coverage tool specified: {}".format(tool))
|
|
||||||
return None
|
|
||||||
|
|
||||||
logger.debug(f"Select {tool} as the coverage tool...")
|
|
||||||
return t
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def retrieve_gcov_data(input_file):
|
|
||||||
logger.debug("Working on %s" % input_file)
|
|
||||||
extracted_coverage_info = {}
|
|
||||||
capture_data = False
|
|
||||||
capture_complete = False
|
|
||||||
with open(input_file, 'r') as fp:
|
|
||||||
for line in fp.readlines():
|
|
||||||
if re.search("GCOV_COVERAGE_DUMP_START", line):
|
|
||||||
capture_data = True
|
|
||||||
continue
|
|
||||||
if re.search("GCOV_COVERAGE_DUMP_END", line):
|
|
||||||
capture_complete = True
|
|
||||||
break
|
|
||||||
# Loop until the coverage data is found.
|
|
||||||
if not capture_data:
|
|
||||||
continue
|
|
||||||
if line.startswith("*"):
|
|
||||||
sp = line.split("<")
|
|
||||||
if len(sp) > 1:
|
|
||||||
# Remove the leading delimiter "*"
|
|
||||||
file_name = sp[0][1:]
|
|
||||||
# Remove the trailing new line char
|
|
||||||
hex_dump = sp[1][:-1]
|
|
||||||
else:
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
continue
|
|
||||||
extracted_coverage_info.update({file_name: hex_dump})
|
|
||||||
if not capture_data:
|
|
||||||
capture_complete = True
|
|
||||||
return {'complete': capture_complete, 'data': extracted_coverage_info}
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def create_gcda_files(extracted_coverage_info):
|
|
||||||
logger.debug("Generating gcda files")
|
|
||||||
for filename, hexdump_val in extracted_coverage_info.items():
|
|
||||||
# if kobject_hash is given for coverage gcovr fails
|
|
||||||
# hence skipping it problem only in gcovr v4.1
|
|
||||||
if "kobject_hash" in filename:
|
|
||||||
filename = (filename[:-4]) + "gcno"
|
|
||||||
try:
|
|
||||||
os.remove(filename)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
continue
|
|
||||||
|
|
||||||
with open(filename, 'wb') as fp:
|
|
||||||
fp.write(bytes.fromhex(hexdump_val))
|
|
||||||
|
|
||||||
def generate(self, outdir):
|
|
||||||
for filename in glob.glob("%s/**/handler.log" % outdir, recursive=True):
|
|
||||||
gcov_data = self.__class__.retrieve_gcov_data(filename)
|
|
||||||
capture_complete = gcov_data['complete']
|
|
||||||
extracted_coverage_info = gcov_data['data']
|
|
||||||
if capture_complete:
|
|
||||||
self.__class__.create_gcda_files(extracted_coverage_info)
|
|
||||||
logger.debug("Gcov data captured: {}".format(filename))
|
|
||||||
else:
|
|
||||||
logger.error("Gcov data capture incomplete: {}".format(filename))
|
|
||||||
|
|
||||||
with open(os.path.join(outdir, "coverage.log"), "a") as coveragelog:
|
|
||||||
ret = self._generate(outdir, coveragelog)
|
|
||||||
if ret == 0:
|
|
||||||
logger.info("HTML report generated: {}".format(
|
|
||||||
os.path.join(outdir, "coverage", "index.html")))
|
|
||||||
|
|
||||||
|
|
||||||
class Lcov(CoverageTool):
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
super().__init__()
|
|
||||||
self.ignores = []
|
|
||||||
|
|
||||||
def add_ignore_file(self, pattern):
|
|
||||||
self.ignores.append('*' + pattern + '*')
|
|
||||||
|
|
||||||
def add_ignore_directory(self, pattern):
|
|
||||||
self.ignores.append('*/' + pattern + '/*')
|
|
||||||
|
|
||||||
def _generate(self, outdir, coveragelog):
|
|
||||||
coveragefile = os.path.join(outdir, "coverage.info")
|
|
||||||
ztestfile = os.path.join(outdir, "ztest.info")
|
|
||||||
cmd = ["lcov", "--gcov-tool", self.gcov_tool,
|
|
||||||
"--capture", "--directory", outdir,
|
|
||||||
"--rc", "lcov_branch_coverage=1",
|
|
||||||
"--output-file", coveragefile]
|
|
||||||
cmd_str = " ".join(cmd)
|
|
||||||
logger.debug(f"Running {cmd_str}...")
|
|
||||||
subprocess.call(cmd, stdout=coveragelog)
|
|
||||||
# We want to remove tests/* and tests/ztest/test/* but save tests/ztest
|
|
||||||
subprocess.call(["lcov", "--gcov-tool", self.gcov_tool, "--extract",
|
|
||||||
coveragefile,
|
|
||||||
os.path.join(self.base_dir, "tests", "ztest", "*"),
|
|
||||||
"--output-file", ztestfile,
|
|
||||||
"--rc", "lcov_branch_coverage=1"], stdout=coveragelog)
|
|
||||||
|
|
||||||
if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0:
|
|
||||||
subprocess.call(["lcov", "--gcov-tool", self.gcov_tool, "--remove",
|
|
||||||
ztestfile,
|
|
||||||
os.path.join(self.base_dir, "tests/ztest/test/*"),
|
|
||||||
"--output-file", ztestfile,
|
|
||||||
"--rc", "lcov_branch_coverage=1"],
|
|
||||||
stdout=coveragelog)
|
|
||||||
files = [coveragefile, ztestfile]
|
|
||||||
else:
|
|
||||||
files = [coveragefile]
|
|
||||||
|
|
||||||
for i in self.ignores:
|
|
||||||
subprocess.call(
|
|
||||||
["lcov", "--gcov-tool", self.gcov_tool, "--remove",
|
|
||||||
coveragefile, i, "--output-file",
|
|
||||||
coveragefile, "--rc", "lcov_branch_coverage=1"],
|
|
||||||
stdout=coveragelog)
|
|
||||||
|
|
||||||
# The --ignore-errors source option is added to avoid it exiting due to
|
|
||||||
# samples/application_development/external_lib/
|
|
||||||
return subprocess.call(["genhtml", "--legend", "--branch-coverage",
|
|
||||||
"--ignore-errors", "source",
|
|
||||||
"-output-directory",
|
|
||||||
os.path.join(outdir, "coverage")] + files,
|
|
||||||
stdout=coveragelog)
|
|
||||||
|
|
||||||
|
|
||||||
class Gcovr(CoverageTool):
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
super().__init__()
|
|
||||||
self.ignores = []
|
|
||||||
|
|
||||||
def add_ignore_file(self, pattern):
|
|
||||||
self.ignores.append('.*' + pattern + '.*')
|
|
||||||
|
|
||||||
def add_ignore_directory(self, pattern):
|
|
||||||
self.ignores.append(".*/" + pattern + '/.*')
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _interleave_list(prefix, list):
|
|
||||||
tuple_list = [(prefix, item) for item in list]
|
|
||||||
return [item for sublist in tuple_list for item in sublist]
|
|
||||||
|
|
||||||
def _generate(self, outdir, coveragelog):
|
|
||||||
coveragefile = os.path.join(outdir, "coverage.json")
|
|
||||||
ztestfile = os.path.join(outdir, "ztest.json")
|
|
||||||
|
|
||||||
excludes = Gcovr._interleave_list("-e", self.ignores)
|
|
||||||
|
|
||||||
# We want to remove tests/* and tests/ztest/test/* but save tests/ztest
|
|
||||||
cmd = ["gcovr", "-r", self.base_dir, "--gcov-executable",
|
|
||||||
self.gcov_tool, "-e", "tests/*"] + excludes + ["--json", "-o",
|
|
||||||
coveragefile, outdir]
|
|
||||||
cmd_str = " ".join(cmd)
|
|
||||||
logger.debug(f"Running {cmd_str}...")
|
|
||||||
subprocess.call(cmd, stdout=coveragelog)
|
|
||||||
|
|
||||||
subprocess.call(["gcovr", "-r", self.base_dir, "--gcov-executable",
|
|
||||||
self.gcov_tool, "-f", "tests/ztest", "-e",
|
|
||||||
"tests/ztest/test/*", "--json", "-o", ztestfile,
|
|
||||||
outdir], stdout=coveragelog)
|
|
||||||
|
|
||||||
if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0:
|
|
||||||
files = [coveragefile, ztestfile]
|
|
||||||
else:
|
|
||||||
files = [coveragefile]
|
|
||||||
|
|
||||||
subdir = os.path.join(outdir, "coverage")
|
|
||||||
os.makedirs(subdir, exist_ok=True)
|
|
||||||
|
|
||||||
tracefiles = self._interleave_list("--add-tracefile", files)
|
|
||||||
|
|
||||||
return subprocess.call(["gcovr", "-r", self.base_dir, "--html",
|
|
||||||
"--html-details"] + tracefiles +
|
|
||||||
["-o", os.path.join(subdir, "index.html")],
|
|
||||||
stdout=coveragelog)
|
|
||||||
|
|
||||||
|
|
||||||
def init(colorama_strip):
|
def init(colorama_strip):
|
||||||
colorama.init(strip=colorama_strip)
|
colorama.init(strip=colorama_strip)
|
||||||
|
|
|
||||||
|
|
@ -206,10 +206,11 @@ except ImportError:
|
||||||
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
|
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
|
||||||
|
|
||||||
import twisterlib
|
import twisterlib
|
||||||
from twisterlib import TestPlan, CoverageTool, ExecutionCounter
|
from twisterlib import TestPlan, ExecutionCounter
|
||||||
from twister.enviornment import TwisterEnv, canonical_zephyr_base
|
from twister.enviornment import TwisterEnv, canonical_zephyr_base
|
||||||
from twister.reports import Reporting
|
from twister.reports import Reporting
|
||||||
from twister.hardwaremap import HardwareMap
|
from twister.hardwaremap import HardwareMap
|
||||||
|
from twister.coverage import CoverageTool
|
||||||
|
|
||||||
logger = logging.getLogger('twister')
|
logger = logging.getLogger('twister')
|
||||||
logger.setLevel(logging.DEBUG)
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue