michaelh/zephyr
1
0
Fork 0
Code Releases Activity

sanitycheck: add subsets support

Browse source 
This new options makes it possible to only run a subset of the tests
which will allow us to run sanitycheck on multiple hosts and merge the
results into one report. This way we do not need to worry about
selecting specific architectures to be run on a certain host.

The option accepts a string value: x/y where x is the subset and y is
the total, so if we specify --subset 1/5, we will only run the first
fifth of the total tests, --subset 5/5 would only select the last fifth.

To get consistent results, the testcase instance list is now ordered,
to avoid duplications and have full coverage.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif 2017-05-13 21:31:53 -04:00 • committed by Anas Nashif
commit 035799fe45
1 changed files with 30 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

30
scripts/sanitycheck
View file

@ -173,6 +173,8 @@ import glob
import concurrent
import concurrent.futures
import xml.etree.ElementTree as ET
from collections import OrderedDict
from itertools import islice
if "ZEPHYR_BASE" not in os.environ:
sys.stderr.write("$ZEPHYR_BASE environment variable undefined.\n")
@ -1707,6 +1709,11 @@ def parse_arguments():
parser.add_argument("--ccache", action="store_const", const=1, default=0,
help="Enable the use of ccache when building")
parser.add_argument("-B", "--subset",
help="Only run a subset of the tests, 1/4 for running the first 25%%, "
"3/5 means run the 3rd fifth of the total. "
"This option is useful when running a large number of tests on "
"different hosts to speed up execution time.")
parser.add_argument("-y", "--dry-run", action="store_true",
help="Create the filtered list of test cases, but don't actually "
"run them. Useful if you're just interested in "
@ -1897,6 +1904,14 @@ def main():
if args.jobs:
CPU_COUNTS = args.jobs
if args.subset:
subset, sets = args.subset.split("/")
if int(subset) > 0 and int(sets) >= int(subset):
info("Running only a subset: %s/%s" %(subset,sets))
else:
error("You have provided a wrong subset value: %s." %args.subset)
return
if os.path.exists(args.outdir) and not args.no_clean:
info("Cleaning output directory " + args.outdir)
shutil.rmtree(args.outdir)
@ -1918,6 +1933,21 @@ def main():
debug("{:<25} {:<50} {}SKIPPED{}: {}".format(i.platform.name,
i.test.name, COLOR_YELLOW, COLOR_NORMAL, reason))
ts.instancets = OrderedDict(sorted(ts.instances.items(), key=lambda t: t[0]))
if args.subset:
subset, sets = args.subset.split("/")
total = len(ts.instancets)
per_set = round(total / int(sets))
start = (int(subset) - 1 ) * per_set
if subset == sets:
end = total
else:
end = start + per_set
sliced_instances = islice(ts.instancets.items(),start, end)
ts.instances = OrderedDict(sliced_instances)
info("%d tests selected, %d tests discarded due to filters" %
(len(ts.instances), len(discards)))