sanitycheck: add harness classes
Add 2 classes, one to handle the current TestCase scenario, and one more for handling generic Console with regex matching. Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
parent
e722db14ad
commit
576be98525
2 changed files with 192 additions and 90 deletions
62
scripts/sanity_chk/harness.py
Normal file
62
scripts/sanity_chk/harness.py
Normal file
|
@ -0,0 +1,62 @@
|
|||
import re
|
||||
from collections import OrderedDict
|
||||
|
||||
class Harness:
|
||||
def __init__(self):
|
||||
self.state = None
|
||||
self.type = None
|
||||
self.regex = []
|
||||
self.matches = OrderedDict()
|
||||
self.ordered = True
|
||||
self.repeat = 1
|
||||
|
||||
def configure(self, instance):
|
||||
config = instance.test.harness_config
|
||||
if config:
|
||||
self.type = config.get('type', None)
|
||||
self.regex = config.get('regex', [] )
|
||||
self.repeat = config.get('repeat', 1)
|
||||
self.ordered = config.get('ordered', True)
|
||||
|
||||
class Console(Harness):
|
||||
|
||||
def handle(self, line):
|
||||
if self.type == "one_line":
|
||||
pattern = re.compile(self.regex[0])
|
||||
if pattern.match(line):
|
||||
self.state = "passed"
|
||||
elif self.type == "multi_line":
|
||||
for r in self.regex:
|
||||
pattern = re.compile(r)
|
||||
if pattern.match(line) and not r in self.matches:
|
||||
self.matches[r] = line
|
||||
|
||||
if len(self.matches) == len(self.regex):
|
||||
# check ordering
|
||||
if not self.ordered:
|
||||
self.state = "passed"
|
||||
return
|
||||
ordered = True
|
||||
pos = 0
|
||||
for k,v in self.matches.items():
|
||||
if k != self.regex[pos]:
|
||||
ordered = False
|
||||
pos += 1
|
||||
|
||||
if ordered:
|
||||
self.state = "passed"
|
||||
else:
|
||||
self.state = "failed"
|
||||
|
||||
|
||||
|
||||
class Test(Harness):
|
||||
RUN_PASSED = "PROJECT EXECUTION SUCCESSFUL"
|
||||
RUN_FAILED = "PROJECT EXECUTION FAILED"
|
||||
|
||||
def handle(self, line):
|
||||
if self.RUN_PASSED in line:
|
||||
self.state = "passed"
|
||||
|
||||
if self.RUN_FAILED in line:
|
||||
self.state = "failed"
|
|
@ -217,7 +217,6 @@ else:
|
|||
COLOR_GREEN = ""
|
||||
COLOR_YELLOW = ""
|
||||
|
||||
|
||||
class SanityCheckException(Exception):
|
||||
pass
|
||||
|
||||
|
@ -275,12 +274,20 @@ def verbose(what):
|
|||
if VERBOSE >= 2:
|
||||
info(what)
|
||||
|
||||
class HarnessImporter:
|
||||
|
||||
def __init__(self, name):
|
||||
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/sanity_chk"))
|
||||
module = __import__("harness")
|
||||
if name:
|
||||
my_class = getattr(module, name)
|
||||
else:
|
||||
my_class = getattr(module, "Test")
|
||||
|
||||
self.instance = my_class()
|
||||
|
||||
class Handler:
|
||||
RUN_PASSED = "PROJECT EXECUTION SUCCESSFUL"
|
||||
RUN_FAILED = "PROJECT EXECUTION FAILED"
|
||||
|
||||
def __init__(self, name, outdir, log_fn, timeout, unit=False):
|
||||
def __init__(self, instance):
|
||||
"""Constructor
|
||||
|
||||
@param name Arbitrary name of the created thread
|
||||
|
@ -296,7 +303,6 @@ class Handler:
|
|||
self.metrics["handler_time"] = 0
|
||||
self.metrics["ram_size"] = 0
|
||||
self.metrics["rom_size"] = 0
|
||||
self.unit = unit
|
||||
|
||||
def set_state(self, state, metrics):
|
||||
self.lock.acquire()
|
||||
|
@ -311,7 +317,7 @@ class Handler:
|
|||
return ret
|
||||
|
||||
class NativeHandler(Handler):
|
||||
def __init__(self, name, sourcedir, outdir, run_log, valgrind_log, timeout):
|
||||
def __init__(self, instance):
|
||||
"""Constructor
|
||||
|
||||
@param name Arbitrary name of the created thread
|
||||
|
@ -321,58 +327,68 @@ class NativeHandler(Handler):
|
|||
@param timeout Kill the QEMU process if it doesn't finish up within
|
||||
the given number of seconds
|
||||
"""
|
||||
super().__init__(name, outdir, run_log, timeout, True)
|
||||
super().__init__(instance)
|
||||
|
||||
self.timeout = timeout
|
||||
self.sourcedir = sourcedir
|
||||
self.outdir = outdir
|
||||
self.run_log = run_log
|
||||
self.instance = instance
|
||||
self.timeout = instance.test.timeout
|
||||
self.sourcedir = instance.test.code_location
|
||||
self.outdir = instance.outdir
|
||||
self.run_log = os.path.join(self.outdir, "run.log")
|
||||
self.valgrind_log = os.path.join(self.outdir, "valgrind.log")
|
||||
self.valgrind = False
|
||||
self.valgrind_log = valgrind_log
|
||||
self.returncode = 0
|
||||
self.set_state("running", {})
|
||||
|
||||
def output_reader(self, proc, harness):
|
||||
for line in iter(proc.stdout.readline, b''):
|
||||
verbose("NATIVE: {0}".format(line.decode('utf-8').rstrip()))
|
||||
harness.handle(line.decode('utf-8').rstrip())
|
||||
if harness.state:
|
||||
proc.terminate()
|
||||
break
|
||||
|
||||
def handle(self):
|
||||
out_state = "failed"
|
||||
|
||||
with open(self.run_log, "wt") as rl, open(self.valgrind_log, "wt") as vl:
|
||||
try:
|
||||
binary = os.path.join(self.outdir, "zephyr", "zephyr.exe")
|
||||
command = [binary]
|
||||
if shutil.which("valgrind") and self.valgrind:
|
||||
command = ["valgrind", "--error-exitcode=2",
|
||||
"--leak-check=full"] + command
|
||||
returncode = subprocess.call(command, timeout=self.timeout,
|
||||
stdout=rl, stderr=vl)
|
||||
self.returncode = returncode
|
||||
if returncode != 0:
|
||||
if self.returncode == 1:
|
||||
out_state = "failed"
|
||||
else:
|
||||
out_state = "failed valgrind"
|
||||
harness_name = self.instance.test.harness.capitalize()
|
||||
harness_import = HarnessImporter(harness_name)
|
||||
harness = harness_import.instance
|
||||
harness.configure(self.instance)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
binary = os.path.join(self.outdir, "zephyr", "zephyr.exe")
|
||||
command = [binary]
|
||||
if shutil.which("valgrind") and self.valgrind:
|
||||
command = ["valgrind", "--error-exitcode=2",
|
||||
"--leak-check=full"] + command
|
||||
|
||||
with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc:
|
||||
t = threading.Thread(target=self.output_reader, args=(proc, harness, ))
|
||||
t.start()
|
||||
t.join(self.timeout)
|
||||
if t.is_alive():
|
||||
proc.terminate()
|
||||
out_state = "timeout"
|
||||
self.returncode = 1
|
||||
t.join()
|
||||
|
||||
with open(self.run_log, "r") as rl:
|
||||
for line in rl.readlines():
|
||||
line = line.strip()
|
||||
if self.RUN_PASSED in line:
|
||||
out_state = "passed"
|
||||
break
|
||||
|
||||
if self.RUN_FAILED in line:
|
||||
proc.wait()
|
||||
self.returncode = proc.returncode
|
||||
if proc.returncode != 0:
|
||||
if self.returncode == 1:
|
||||
out_state = "failed"
|
||||
break
|
||||
else:
|
||||
out_state = "failed valgrind"
|
||||
|
||||
returncode = subprocess.call(["GCOV_PREFIX=" + self.outdir, "gcov", self.sourcedir, "-s", self.outdir], shell=True)
|
||||
#print(" ".join(["GCOV_PREFIX=" + self.outdir, "gcov", self.sourcedir, "-b", "-s", self.outdir]))
|
||||
returncode = subprocess.call(["GCOV_PREFIX=" + self.outdir, "gcov", self.sourcedir, "-b", "-s", self.outdir], shell=True)
|
||||
|
||||
if harness.state:
|
||||
self.set_state(harness.state, {})
|
||||
else:
|
||||
self.set_state(out_state, {})
|
||||
|
||||
self.set_state(out_state, {})
|
||||
|
||||
class UnitHandler(Handler):
|
||||
def __init__(self, name, sourcedir, outdir,
|
||||
run_log, valgrind_log, timeout):
|
||||
def __init__(self, instance):
|
||||
"""Constructor
|
||||
|
||||
@param name Arbitrary name of the created thread
|
||||
|
@ -382,13 +398,13 @@ class UnitHandler(Handler):
|
|||
@param timeout Kill the QEMU process if it doesn't finish up within
|
||||
the given number of seconds
|
||||
"""
|
||||
super().__init__(name, outdir, run_log, timeout, True)
|
||||
super().__init__(instance)
|
||||
|
||||
self.timeout = timeout
|
||||
self.sourcedir = sourcedir
|
||||
self.outdir = outdir
|
||||
self.run_log = run_log
|
||||
self.valgrind_log = valgrind_log
|
||||
self.timeout = instance.test.timeout
|
||||
self.sourcedir = instance.test.code_location
|
||||
self.outdir = instance.outdir
|
||||
self.run_log = os.path.join(self.outdir, "run.log")
|
||||
self.valgrind_log = os.path.join(self.outdir, "valgrind.log")
|
||||
self.returncode = 0
|
||||
self.set_state("running", {})
|
||||
|
||||
|
@ -432,7 +448,7 @@ class QEMUHandler(Handler):
|
|||
"""
|
||||
|
||||
@staticmethod
|
||||
def _thread(handler, timeout, outdir, logfile, fifo_fn, pid_fn, results):
|
||||
def _thread(handler, timeout, outdir, logfile, fifo_fn, pid_fn, results, harness):
|
||||
fifo_in = fifo_fn + ".in"
|
||||
fifo_out = fifo_fn + ".out"
|
||||
|
||||
|
@ -486,12 +502,9 @@ class QEMUHandler(Handler):
|
|||
line = line.strip()
|
||||
verbose("QEMU: %s" % line)
|
||||
|
||||
if line == handler.RUN_PASSED:
|
||||
out_state = "passed"
|
||||
break
|
||||
|
||||
if line == handler.RUN_FAILED:
|
||||
out_state = "failed"
|
||||
harness.handle(line)
|
||||
if harness.state:
|
||||
out_state = harness.state
|
||||
break
|
||||
|
||||
# TODO: Add support for getting numerical performance data
|
||||
|
@ -519,7 +532,7 @@ class QEMUHandler(Handler):
|
|||
os.unlink(fifo_in)
|
||||
os.unlink(fifo_out)
|
||||
|
||||
def __init__(self, name, outdir, log_fn, timeout):
|
||||
def __init__(self, instance):
|
||||
"""Constructor
|
||||
|
||||
@param name Arbitrary name of the created thread
|
||||
|
@ -529,22 +542,34 @@ class QEMUHandler(Handler):
|
|||
@param timeout Kill the QEMU process if it doesn't finish up within
|
||||
the given number of seconds
|
||||
"""
|
||||
super().__init__(name, outdir, log_fn, timeout)
|
||||
|
||||
|
||||
super().__init__(instance)
|
||||
outdir = instance.outdir
|
||||
timeout = instance.test.timeout
|
||||
name = instance.name
|
||||
run_log = os.path.join(outdir, "run.log")
|
||||
qemu_log = os.path.join(outdir, "qemu.log")
|
||||
|
||||
self.results = {}
|
||||
|
||||
# We pass this to QEMU which looks for fifos with .in and .out
|
||||
# suffixes.
|
||||
self.fifo_fn = os.path.join(outdir, "qemu-fifo")
|
||||
self.fifo_fn = os.path.join(instance.outdir, "qemu-fifo")
|
||||
|
||||
self.pid_fn = os.path.join(outdir, "qemu.pid")
|
||||
self.pid_fn = os.path.join(instance.outdir, "qemu.pid")
|
||||
if os.path.exists(self.pid_fn):
|
||||
os.unlink(self.pid_fn)
|
||||
|
||||
self.log_fn = log_fn
|
||||
self.log_fn = qemu_log
|
||||
|
||||
harness_import = HarnessImporter(instance.test.harness.capitalize())
|
||||
harness = harness_import.instance
|
||||
harness.configure(instance)
|
||||
self.thread = threading.Thread(name=name, target=QEMUHandler._thread,
|
||||
args=(self, timeout, outdir,
|
||||
self.log_fn, self.fifo_fn,
|
||||
self.pid_fn, self.results))
|
||||
self.pid_fn, self.results, harness))
|
||||
self.thread.daemon = True
|
||||
verbose("Spawning QEMU process for %s" % name)
|
||||
self.thread.start()
|
||||
|
@ -843,6 +868,11 @@ class MakeGenerator:
|
|||
if not os.path.exists(outdir):
|
||||
os.makedirs(outdir)
|
||||
|
||||
def add_instance_build_goal(self, instance, args, buildlog, make_args=""):
|
||||
|
||||
self.add_build_goal(instance.name, instance.test.code_location,
|
||||
instance.outdir, args, buildlog, make_args)
|
||||
|
||||
def add_build_goal(self, name, directory, outdir,
|
||||
args, buildlog, make_args=""):
|
||||
"""Add a goal to invoke a Kbuild session
|
||||
|
@ -878,7 +908,7 @@ class MakeGenerator:
|
|||
None,
|
||||
None)
|
||||
|
||||
def add_qemu_goal(self, name, directory, outdir, args, timeout=30):
|
||||
def add_qemu_goal(self, instance, args):
|
||||
"""Add a goal to build a Zephyr project and then run it under QEMU
|
||||
|
||||
The generated make goal invokes Make twice, the first time it will
|
||||
|
@ -898,12 +928,16 @@ class MakeGenerator:
|
|||
to run before automatically killing it. Default is 30 seconds.
|
||||
"""
|
||||
|
||||
self._add_goal(outdir)
|
||||
name = instance.name
|
||||
directory = instance.test.code_location
|
||||
outdir = instance.outdir
|
||||
|
||||
build_logfile = os.path.join(outdir, "build.log")
|
||||
run_logfile = os.path.join(outdir, "run.log")
|
||||
qemu_logfile = os.path.join(outdir, "qemu.log")
|
||||
self._add_goal(outdir)
|
||||
|
||||
qemu_handler = QEMUHandler(name, outdir, qemu_logfile, timeout)
|
||||
qemu_handler = QEMUHandler(instance)
|
||||
args.append("QEMU_PIPE=%s" % qemu_handler.get_fifo())
|
||||
text = (self._get_rule_header(name) +
|
||||
self._get_sub_make(name, "building", directory,
|
||||
|
@ -915,8 +949,12 @@ class MakeGenerator:
|
|||
self.goals[name] = MakeGoal(name, text, qemu_handler, self.logfile, build_logfile,
|
||||
run_logfile, qemu_logfile)
|
||||
|
||||
def add_unit_goal(self, name, directory, outdir,
|
||||
args, timeout=30, coverage=False):
|
||||
def add_unit_goal(self, instance, args, timeout=30, coverage=False):
|
||||
outdir = instance.outdir
|
||||
timeout = instance.test.timeout
|
||||
name = instance.name
|
||||
directory = instance.test.code_location
|
||||
|
||||
self._add_goal(outdir)
|
||||
build_logfile = os.path.join(outdir, "build.log")
|
||||
run_logfile = os.path.join(outdir, "run.log")
|
||||
|
@ -929,11 +967,17 @@ class MakeGenerator:
|
|||
self._get_sub_make(name, "building", directory,
|
||||
outdir, build_logfile, args) +
|
||||
self._get_rule_footer(name))
|
||||
unit_handler = UnitHandler(name, directory, outdir, run_logfile, valgrind_logfile, timeout)
|
||||
unit_handler = UnitHandler(instance)
|
||||
self.goals[name] = MakeGoal(name, text, unit_handler, self.logfile, build_logfile,
|
||||
run_logfile, valgrind_logfile)
|
||||
|
||||
def add_native_goal(self, name, directory, outdir, args, timeout=30, coverage=False):
|
||||
def add_native_goal(self, instance, args, coverage=False):
|
||||
|
||||
outdir = instance.outdir
|
||||
timeout = instance.test.timeout
|
||||
name = instance.name
|
||||
directory = instance.test.code_location
|
||||
|
||||
self._add_goal(outdir)
|
||||
build_logfile = os.path.join(outdir, "build.log")
|
||||
run_logfile = os.path.join(outdir, "run.log")
|
||||
|
@ -944,7 +988,7 @@ class MakeGenerator:
|
|||
self._get_sub_make(name, "building", directory,
|
||||
outdir, build_logfile, args) +
|
||||
self._get_rule_footer(name))
|
||||
native_handler = NativeHandler(name, directory, outdir, run_logfile, valgrind_logfile, timeout)
|
||||
native_handler = NativeHandler(instance)
|
||||
self.goals[name] = MakeGoal(name, text, native_handler, self.logfile, build_logfile,
|
||||
run_logfile, valgrind_logfile)
|
||||
|
||||
|
@ -964,17 +1008,13 @@ class MakeGenerator:
|
|||
args.extend(extra_args)
|
||||
if (ti.platform.qemu_support and (not ti.build_only) and
|
||||
(not build_only) and (enable_slow or not ti.test.slow)):
|
||||
self.add_qemu_goal(ti.name, ti.test.code_location, ti.outdir,
|
||||
args, ti.test.timeout)
|
||||
self.add_qemu_goal(ti, args)
|
||||
elif ti.test.type == "unit":
|
||||
self.add_unit_goal(ti.name, ti.test.code_location, ti.outdir,
|
||||
args, ti.test.timeout, coverage)
|
||||
self.add_unit_goal(ti, args, coverage)
|
||||
elif ti.platform.type == "native" and (not ti.build_only) and (not build_only):
|
||||
self.add_native_goal(ti.name, ti.test.code_location, ti.outdir,
|
||||
args, ti.test.timeout, coverage)
|
||||
self.add_native_goal(ti, args, coverage)
|
||||
else:
|
||||
self.add_build_goal(ti.name, ti.test.code_location, ti.outdir,
|
||||
args, "build.log")
|
||||
self.add_instance_build_goal(ti, args, "build.log")
|
||||
|
||||
def execute(self, callback_fn=None, context=None):
|
||||
"""Execute all the registered build goals
|
||||
|
@ -1029,8 +1069,7 @@ class MakeGenerator:
|
|||
|
||||
if state == "finished":
|
||||
if goal.handler:
|
||||
if goal.handler.unit:
|
||||
# We can't run unit tests with Make
|
||||
if hasattr(goal.handler, "handle"):
|
||||
goal.handler.handle()
|
||||
if goal.handler.returncode == 2:
|
||||
goal.handler_log = goal.handler.valgrind_log
|
||||
|
@ -1087,7 +1126,8 @@ testcase_valid_keys = {"tags": {"type": "set", "required": False},
|
|||
"toolchain_exclude": {"type": "set"},
|
||||
"toolchain_whitelist": {"type": "set"},
|
||||
"filter": {"type": "str"},
|
||||
"harness": {"type": "str"}
|
||||
"harness": {"type": "str"},
|
||||
"harness_config": {"type": "map"}
|
||||
}
|
||||
|
||||
|
||||
|
@ -1140,6 +1180,8 @@ class SanityConfigParser:
|
|||
else:
|
||||
return set(vs)
|
||||
|
||||
elif typestr.startswith("map"):
|
||||
return value
|
||||
else:
|
||||
raise ConfigurationError(
|
||||
self.filename, "unknown type '%s'" % value)
|
||||
|
@ -1319,6 +1361,7 @@ class TestCase:
|
|||
self.tc_filter = tc_dict["filter"]
|
||||
self.timeout = tc_dict["timeout"]
|
||||
self.harness = tc_dict["harness"]
|
||||
self.harness_config = tc_dict["harness_config"]
|
||||
self.build_only = tc_dict["build_only"]
|
||||
self.build_on_all = tc_dict["build_on_all"]
|
||||
self.slow = tc_dict["slow"]
|
||||
|
@ -1352,9 +1395,7 @@ class TestInstance:
|
|||
self.platform = platform
|
||||
self.name = os.path.join(platform.name, test.name)
|
||||
self.outdir = os.path.join(base_outdir, platform.name, test.path)
|
||||
# TODO: Support harness in sanitycheck, now we do not run on anything
|
||||
# that requires a harness
|
||||
self.build_only = build_only or test.build_only or (test.harness != '')
|
||||
self.build_only = build_only or test.build_only or (test.harness and test.harness != 'console')
|
||||
|
||||
def create_overlay(self):
|
||||
if len(self.test.extra_configs) > 0:
|
||||
|
@ -1629,11 +1670,10 @@ class TestSuite:
|
|||
"/")[-1]] = os.path.join(o, "zephyr", ".config")
|
||||
goal = "_".join([plat.name, "_".join(
|
||||
tc.name.split("/")), "config-sanitycheck"])
|
||||
mg.add_build_goal(
|
||||
goal, os.path.join(
|
||||
ZEPHYR_BASE, tc.code_location),
|
||||
o, args, "config-sanitycheck.log",
|
||||
make_args="config-sanitycheck")
|
||||
mg.add_build_goal(goal,
|
||||
os.path.join(ZEPHYR_BASE, tc.code_location),
|
||||
o, args,
|
||||
"config-sanitycheck.log", make_args="config-sanitycheck")
|
||||
|
||||
info("Building testcase defconfigs...")
|
||||
results = mg.execute(defconfig_cb)
|
||||
|
@ -2289,7 +2329,7 @@ def generate_coverage(outdir, ignores):
|
|||
["lcov", "--remove", coveragefile, i, "--output-file",
|
||||
coveragefile],
|
||||
stdout=coveragelog)
|
||||
subprocess.call(["genhtml", "-output-directory",
|
||||
subprocess.call(["genhtml", "--legend", "-output-directory",
|
||||
os.path.join(outdir, "coverage"),
|
||||
coveragefile, ztestfile], stdout=coveragelog)
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue