michaelh/zephyr
1
0
Fork 0
Code Releases Activity

sanitycheck: use Python 3

Browse source 
Python 2 isn't supported anymore and doesn't get fixes/updates.
For the long-term maintenance health of sanitycheck, convert to
use Python 3 interpreter, which mostly involved changes to how
strings are handled.

Change-Id: Ic42f5b2328001f04d876fd650986c4433968a76b
Signed-off-by: Andrew Boie <andrew.p.boie@intel.com>
This commit is contained in:
Andrew Boie 2016-02-22 13:28:10 -08:00 • committed by Daniel Kalowsky
commit 08ce5a5da7
1 changed files with 47 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

87
scripts/sanitycheck
View file

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3
"""Zephyr Sanity Tests """Zephyr Sanity Tests
This script scans for the set of unit test applications in the git This script scans for the set of unit test applications in the git
@ -100,7 +100,7 @@ Most everyday users will run with no arguments.
import argparse import argparse
import os import os
import sys import sys
import ConfigParser import configparser
import re import re
import tempfile import tempfile
import subprocess import subprocess
@ -162,20 +162,19 @@ class ExecutionError(MakeError):
pass pass
# Debug Functions # Debug Functions
def info(what):
def debug(what): sys.stdout.write(what + "\n")
if VERBOSE >= 1:
print what
def error(what): def error(what):
sys.stderr.write(COLOR_RED + what + COLOR_NORMAL + "\n") sys.stderr.write(COLOR_RED + what + COLOR_NORMAL + "\n")
def debug(what):
if VERBOSE >= 1:
info(what)
def verbose(what): def verbose(what):
if VERBOSE >= 2: if VERBOSE >= 2:
print what info(what)
def info(what):
sys.stdout.write(what + "\n")
# Utility functions # Utility functions
class QEMUHandler: class QEMUHandler:
@ -208,7 +207,7 @@ class QEMUHandler:
# Disable internal buffering, we don't # Disable internal buffering, we don't
# want read() or poll() to ever block if there is data in there # want read() or poll() to ever block if there is data in there
in_fp = open(fifo_out, "rb", buffering=0) in_fp = open(fifo_out, "rb", buffering=0)
log_out_fp = open(logfile, "w") log_out_fp = open(logfile, "wt")
start_time = time.time() start_time = time.time()
timeout_time = start_time + timeout timeout_time = start_time + timeout
@ -223,7 +222,7 @@ class QEMUHandler:
out_state = "timeout" out_state = "timeout"
break break
c = in_fp.read(1) c = in_fp.read(1).decode("utf-8")
if c == "": if c == "":
# EOF, this shouldn't happen unless QEMU crashes # EOF, this shouldn't happen unless QEMU crashes
out_state = "unexpected eof" out_state = "unexpected eof"
@ -262,7 +261,12 @@ class QEMUHandler:
pid = int(open(pid_fn).read()) pid = int(open(pid_fn).read())
os.unlink(pid_fn) os.unlink(pid_fn)
try:
os.kill(pid, signal.SIGTERM) os.kill(pid, signal.SIGTERM)
except ProcessLookupError:
# Oh well, as long as it's dead! User probably sent Ctrl-C
pass
os.unlink(fifo_in) os.unlink(fifo_in)
os.unlink(fifo_out) os.unlink(fifo_out)
@ -333,13 +337,13 @@ class SizeCalculator:
with open(filename, "rb") as f: with open(filename, "rb") as f:
magic = f.read(4) magic = f.read(4)
if (magic != "\x7fELF"): if (magic != b'\x7fELF'):
raise SanityRuntimeError("%s is not an ELF binary" % filename) raise SanityRuntimeError("%s is not an ELF binary" % filename)
# Search for CONFIG_XIP in the ELF's list of symbols using NM and AWK. # Search for CONFIG_XIP in the ELF's list of symbols using NM and AWK.
# GREP can not be used as it returns an error if the symbol is not found. # GREP can not be used as it returns an error if the symbol is not found.
is_xip_command = "nm " + filename + " | awk '/CONFIG_XIP/ { print $3 }'" is_xip_command = "nm " + filename + " | awk '/CONFIG_XIP/ { print $3 }'"
is_xip_output = subprocess.check_output(is_xip_command, shell=True) is_xip_output = subprocess.check_output(is_xip_command, shell=True).decode("utf-8")
self.is_xip = (len(is_xip_output) != 0) self.is_xip = (len(is_xip_output) != 0)
self.filename = filename self.filename = filename
@ -390,7 +394,7 @@ class SizeCalculator:
""" Calculate RAM and ROM usage by section """ """ Calculate RAM and ROM usage by section """
objdump_command = "objdump -h " + self.filename objdump_command = "objdump -h " + self.filename
objdump_output = subprocess.check_output(objdump_command, objdump_output = subprocess.check_output(objdump_command,
shell=True).splitlines() shell=True).decode("utf-8").splitlines()
for line in objdump_output: for line in objdump_output:
words = line.split() words = line.split()
@ -652,13 +656,13 @@ class MakeGenerator:
@return A dictionary mapping goal names to final status. @return A dictionary mapping goal names to final status.
""" """
with open(self.makefile, "w") as tf, \ with open(self.makefile, "wt") as tf, \
open(os.devnull, "wb") as devnull, \ open(os.devnull, "wb") as devnull, \
open(self.logfile, "w") as make_log: open(self.logfile, "wt") as make_log:
# Create our dynamic Makefile and execute it. # Create our dynamic Makefile and execute it.
# Watch stderr output which is where we will keep # Watch stderr output which is where we will keep
# track of build state # track of build state
for name, goal in self.goals.iteritems(): for name, goal in self.goals.items():
tf.write(goal.text) tf.write(goal.text)
tf.write("all: %s\n" % (" ".join(self.goals.keys()))) tf.write("all: %s\n" % (" ".join(self.goals.keys())))
tf.flush() tf.flush()
@ -670,6 +674,7 @@ class MakeGenerator:
stdout=devnull) stdout=devnull)
for line in iter(p.stderr.readline, b''): for line in iter(p.stderr.readline, b''):
line = line.decode("utf-8")
make_log.write(line) make_log.write(line)
verbose("MAKE: " + repr(line.strip())) verbose("MAKE: " + repr(line.strip()))
m = MakeGenerator.re_make.match(line) m = MakeGenerator.re_make.match(line)
@ -745,7 +750,7 @@ class SanityConfigParser:
@param filename Source .ini file to read @param filename Source .ini file to read
""" """
cp = ConfigParser.SafeConfigParser() cp = configparser.SafeConfigParser()
cp.readfp(open(filename)) cp.readfp(open(filename))
self.filename = filename self.filename = filename
self.cp = cp self.cp = cp
@ -832,7 +837,7 @@ class SanityConfigParser:
% (k, section)) % (k, section))
d[k] = v d[k] = v
for k, kinfo in valid_keys.iteritems(): for k, kinfo in valid_keys.items():
if k not in d: if k not in d:
if "required" in kinfo: if "required" in kinfo:
required = kinfo["required"] required = kinfo["required"]
@ -852,7 +857,7 @@ class SanityConfigParser:
else: else:
try: try:
d[k] = self._cast_value(d[k], kinfo["type"]) d[k] = self._cast_value(d[k], kinfo["type"])
except ValueError, ve: except ValueError as ve:
raise ConfigurationError(self.filename, raise ConfigurationError(self.filename,
"bad %s value '%s' for key '%s' in section '%s'" "bad %s value '%s' for key '%s' in section '%s'"
% (kinfo["type"], d[k], k, section)) % (kinfo["type"], d[k], k, section))
@ -1047,7 +1052,7 @@ def defconfig_cb(context, goals, goal):
with open(goal.get_error_log()) as fp: with open(goal.get_error_log()) as fp:
sys.stdout.write(fp.read()) sys.stdout.write(fp.read())
else: else:
print "\tsee: " + COLOR_YELLOW + goal.get_error_log() + COLOR_NORMAL info("\tsee: " + COLOR_YELLOW + goal.get_error_log() + COLOR_NORMAL)
class TestSuite: class TestSuite:
@ -1133,8 +1138,8 @@ class TestSuite:
mg = MakeGenerator(self.outdir) mg = MakeGenerator(self.outdir)
dlist = {} dlist = {}
for tc_name, tc in self.testcases.iteritems(): for tc_name, tc in self.testcases.items():
for arch_name, arch in self.arches.iteritems(): for arch_name, arch in self.arches.items():
instance_list = [] instance_list = []
for plat in arch.platforms: for plat in arch.platforms:
instance = TestInstance(tc, plat, self.outdir) instance = TestInstance(tc, plat, self.outdir)
@ -1189,11 +1194,11 @@ class TestSuite:
info("Building testcase defconfigs...") info("Building testcase defconfigs...")
results = mg.execute(defconfig_cb) results = mg.execute(defconfig_cb)
for name, goal in results.iteritems(): for name, goal in results.items():
if goal.failed: if goal.failed:
raise SanityRuntimeError("Couldn't build some defconfigs") raise SanityRuntimeError("Couldn't build some defconfigs")
for k, out_config in dlist.iteritems(): for k, out_config in dlist.items():
test, plat, ktype, name = k test, plat, ktype, name = k
defconfig = {} defconfig = {}
with open(out_config, "r") as fp: with open(out_config, "r") as fp:
@ -1204,8 +1209,8 @@ class TestSuite:
defconfig[m.group(1)] = m.group(2).strip() defconfig[m.group(1)] = m.group(2).strip()
test.defconfig[plat,ktype] = defconfig test.defconfig[plat,ktype] = defconfig
for tc_name, tc in self.testcases.iteritems(): for tc_name, tc in self.testcases.items():
for arch_name, arch in self.arches.iteritems(): for arch_name, arch in self.arches.items():
instance_list = [] instance_list = []
for plat in arch.platforms: for plat in arch.platforms:
instance = TestInstance(tc, plat, self.outdir) instance = TestInstance(tc, plat, self.outdir)
@ -1255,7 +1260,7 @@ class TestSuite:
continue continue
defconfig = {} defconfig = {}
for tcase, tdefconfig in tc.defconfig.iteritems(): for tcase, tdefconfig in tc.defconfig.items():
p, k = tcase p, k = tcase
if k == tc.ktype and p == plat: if k == tc.ktype and p == plat:
defconfig = tdefconfig defconfig = tdefconfig
@ -1317,7 +1322,7 @@ class TestSuite:
for i in self.instances.values(): for i in self.instances.values():
mg.add_test_instance(i, build_only, enable_slow) mg.add_test_instance(i, build_only, enable_slow)
self.goals = mg.execute(cb, cb_context) self.goals = mg.execute(cb, cb_context)
for name, goal in self.goals.iteritems(): for name, goal in self.goals.items():
i = self.instances[name] i = self.instances[name]
if goal.failed: if goal.failed:
continue continue
@ -1336,7 +1341,7 @@ class TestSuite:
fieldnames = ["test", "arch", "platform", "reason"] fieldnames = ["test", "arch", "platform", "reason"]
cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep) cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep)
cw.writeheader() cw.writeheader()
for instance, reason in self.discards.iteritems(): for instance, reason in self.discards.items():
rowdict = {"test" : i.test.name, rowdict = {"test" : i.test.name,
"arch" : i.platform.arch.name, "arch" : i.platform.arch.name,
"platform" : i.platform.name, "platform" : i.platform.name,
@ -1365,7 +1370,7 @@ class TestSuite:
d[m] = row[m] d[m] = row[m]
saved_metrics[(row["test"], row["platform"])] = d saved_metrics[(row["test"], row["platform"])] = d
for name, goal in self.goals.iteritems(): for name, goal in self.goals.items():
i = self.instances[name] i = self.instances[name]
mkey = (i.test.name, i.platform.name) mkey = (i.test.name, i.platform.name)
if mkey not in saved_metrics: if mkey not in saved_metrics:
@ -1387,13 +1392,13 @@ class TestSuite:
if self.goals == None: if self.goals == None:
raise SanityRuntimeException("execute() hasn't been run!") raise SanityRuntimeException("execute() hasn't been run!")
with open(filename, "wb") as csvfile: with open(filename, "wt") as csvfile:
fieldnames = ["test", "arch", "platform", "passed", "status", fieldnames = ["test", "arch", "platform", "passed", "status",
"extra_args", "qemu", "qemu_time", "ram_size", "extra_args", "qemu", "qemu_time", "ram_size",
"rom_size"] "rom_size"]
cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep) cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep)
cw.writeheader() cw.writeheader()
for name, goal in self.goals.iteritems(): for name, goal in self.goals.items():
i = self.instances[name] i = self.instances[name]
rowdict = {"test" : i.test.name, rowdict = {"test" : i.test.name,
"arch" : i.platform.arch.name, "arch" : i.platform.arch.name,
@ -1519,19 +1524,19 @@ def parse_arguments():
def log_info(filename): def log_info(filename):
filename = os.path.relpath(filename) filename = os.path.relpath(filename)
if INLINE_LOGS: if INLINE_LOGS:
print "{:-^100}".format(filename) info("{:-^100}".format(filename))
with open(filename) as fp: with open(filename) as fp:
sys.stdout.write(fp.read()) sys.stdout.write(fp.read())
print "{:-^100}".format(filename) info("{:-^100}".format(filename))
else: else:
print "\tsee: " + COLOR_YELLOW + filename + COLOR_NORMAL info("\tsee: " + COLOR_YELLOW + filename + COLOR_NORMAL)
def terse_test_cb(instances, goals, goal): def terse_test_cb(instances, goals, goal):
total_tests = len(goals) total_tests = len(goals)
total_done = 0 total_done = 0
total_failed = 0 total_failed = 0
for k, g in goals.iteritems(): for k, g in goals.items():
if g.finished: if g.finished:
total_done += 1 total_done += 1
if g.failed: if g.failed:
@ -1616,7 +1621,7 @@ def main():
ts.discard_report(args.discard_report) ts.discard_report(args.discard_report)
if VERBOSE: if VERBOSE:
for i, reason in discards.iteritems(): for i, reason in discards.items():
debug("{:<25} {:<50} {}SKIPPED{}: {}".format(i.platform.name, debug("{:<25} {:<50} {}SKIPPED{}: {}".format(i.platform.name,
i.test.name, COLOR_YELLOW, COLOR_NORMAL, reason)) i.test.name, COLOR_YELLOW, COLOR_NORMAL, reason))
@ -1632,7 +1637,7 @@ def main():
else: else:
goals = ts.execute(terse_test_cb, ts.instances, args.build_only, goals = ts.execute(terse_test_cb, ts.instances, args.build_only,
args.enable_slow) args.enable_slow)
print info("")
deltas = ts.compare_metrics(LAST_SANITY if args.last_metrics deltas = ts.compare_metrics(LAST_SANITY if args.last_metrics
else RELEASE_DATA) else RELEASE_DATA)
@ -1659,7 +1664,7 @@ def main():
("release" if not args.last_metrics else "run")) ("release" if not args.last_metrics else "run"))
failed = 0 failed = 0
for name, goal in goals.iteritems(): for name, goal in goals.items():
if goal.failed: if goal.failed:
failed += 1 failed += 1
elif goal.metrics["unrecognized"]: elif goal.metrics["unrecognized"]: