ci: limit CI to only changed architecture
If files are changed for architecture code, then only run CI on those architectures. Combine all scripts into one and make it simpler. Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif
Kumar Gala
parent
28bc79b49f
commit
82daea910b
5 changed files with 137 additions and 170 deletions
|
|
@ -1,85 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# A script to generate a list of boards that have changed or added and create an
|
||||
# arguments file for twister to allow running more tests for those boards.
|
||||
|
||||
import re, os
|
||||
import sh
|
||||
import logging
|
||||
import argparse
|
||||
import glob
|
||||
|
||||
if "ZEPHYR_BASE" not in os.environ:
|
||||
logging.error("$ZEPHYR_BASE environment variable undefined.\n")
|
||||
exit(1)
|
||||
|
||||
logger = None
|
||||
|
||||
repository_path = os.environ['ZEPHYR_BASE']
|
||||
sh_special_args = {
|
||||
'_tty_out': False,
|
||||
'_cwd': repository_path
|
||||
}
|
||||
|
||||
def init_logs():
|
||||
log_lev = os.environ.get('LOG_LEVEL', None)
|
||||
level = logging.INFO
|
||||
global logger
|
||||
|
||||
if log_lev == "DEBUG":
|
||||
level = logging.DEBUG
|
||||
elif log_lev == "ERROR":
|
||||
level = logging.ERROR
|
||||
|
||||
console = logging.StreamHandler()
|
||||
format = logging.Formatter('%(levelname)-8s: %(message)s')
|
||||
console.setFormatter(format)
|
||||
logger = logging.getLogger('')
|
||||
logger.addHandler(console)
|
||||
logger.setLevel(level)
|
||||
|
||||
logging.debug("Log init completed")
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate a twister argument for for boards "
|
||||
" that have changed")
|
||||
parser.add_argument('-c', '--commits', default=None,
|
||||
help="Commit range in the form: a..b")
|
||||
return parser.parse_args()
|
||||
|
||||
def main():
|
||||
boards = set()
|
||||
all_boards = set()
|
||||
|
||||
args = parse_args()
|
||||
if not args.commits:
|
||||
exit(1)
|
||||
|
||||
# pylint does not like the 'sh' library
|
||||
# pylint: disable=too-many-function-args,unexpected-keyword-arg
|
||||
commit = sh.git("diff", "--name-only", args.commits, **sh_special_args)
|
||||
files = commit.split("\n")
|
||||
|
||||
for f in files:
|
||||
if f.endswith(".rst") or f.endswith(".png") or f.endswith(".jpg"):
|
||||
continue
|
||||
p = re.match(r"^boards\/[^/]+\/([^/]+)\/", f)
|
||||
if p and p.groups():
|
||||
boards.add(p.group(1))
|
||||
|
||||
for b in boards:
|
||||
suboards = glob.glob("boards/*/%s/*.yaml" %(b))
|
||||
for subboard in suboards:
|
||||
name = os.path.splitext(os.path.basename(subboard))[0]
|
||||
if name:
|
||||
all_boards.add(name)
|
||||
|
||||
if all_boards:
|
||||
print("-p\n%s" %("\n-p\n".join(all_boards)))
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# A script to generate a list of tests that have changed or added and create an
|
||||
# arguments file for twister to allow running those tests with --all
|
||||
|
||||
import os
|
||||
import sh
|
||||
import logging
|
||||
import argparse
|
||||
|
||||
if "ZEPHYR_BASE" not in os.environ:
|
||||
logging.error("$ZEPHYR_BASE environment variable undefined.\n")
|
||||
exit(1)
|
||||
|
||||
logger = None
|
||||
|
||||
repository_path = os.environ['ZEPHYR_BASE']
|
||||
sh_special_args = {
|
||||
'_tty_out': False,
|
||||
'_cwd': repository_path
|
||||
}
|
||||
|
||||
def init_logs():
|
||||
global logger
|
||||
log_lev = os.environ.get('LOG_LEVEL', None)
|
||||
level = logging.INFO
|
||||
if log_lev == "DEBUG":
|
||||
level = logging.DEBUG
|
||||
elif log_lev == "ERROR":
|
||||
level = logging.ERROR
|
||||
|
||||
console = logging.StreamHandler()
|
||||
format = logging.Formatter('%(levelname)-8s: %(message)s')
|
||||
console.setFormatter(format)
|
||||
logger = logging.getLogger('')
|
||||
logger.addHandler(console)
|
||||
logger.setLevel(level)
|
||||
|
||||
logging.debug("Log init completed")
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate a twister argument for for tests "
|
||||
" that have changed")
|
||||
parser.add_argument('-c', '--commits', default=None,
|
||||
help="Commit range in the form: a..b")
|
||||
return parser.parse_args()
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
if not args.commits:
|
||||
exit(1)
|
||||
|
||||
# pylint does not like the 'sh' library
|
||||
# pylint: disable=too-many-function-args,unexpected-keyword-arg
|
||||
commit = sh.git("diff", "--name-only", args.commits, **sh_special_args)
|
||||
files = commit.split("\n")
|
||||
tests = set()
|
||||
for f in files:
|
||||
if f.endswith(".rst"):
|
||||
continue
|
||||
d = os.path.dirname(f)
|
||||
while d:
|
||||
if os.path.exists(os.path.join(d, "testcase.yaml")) or os.path.exists(os.path.join(d, "sample.yaml")):
|
||||
tests.add(d)
|
||||
break
|
||||
else:
|
||||
d = os.path.dirname(d)
|
||||
|
||||
if tests:
|
||||
print("-T\n%s\n--all" %("\n-T\n".join(tests)))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
103
scripts/ci/get_twister_opt.py
Executable file
103
scripts/ci/get_twister_opt.py
Executable file
|
|
@ -0,0 +1,103 @@
|
|||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright (c) 2021 Intel Corporation
|
||||
|
||||
# A script to generate twister options based on modified files.
|
||||
|
||||
import re, os
|
||||
import sh
|
||||
import argparse
|
||||
import glob
|
||||
|
||||
if "ZEPHYR_BASE" not in os.environ:
|
||||
exit("$ZEPHYR_BASE environment variable undefined.")
|
||||
|
||||
repository_path = os.environ['ZEPHYR_BASE']
|
||||
sh_special_args = {
|
||||
'_tty_out': False,
|
||||
'_cwd': repository_path
|
||||
}
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate twister argument files based on modified file")
|
||||
parser.add_argument('-c', '--commits', default=None,
|
||||
help="Commit range in the form: a..b")
|
||||
return parser.parse_args()
|
||||
|
||||
def find_archs(files):
|
||||
# we match both arch/<arch>/* and include/arch/<arch> and skip common.
|
||||
# Some architectures like riscv require special handling, i.e. riscv
|
||||
# directory covers 2 architectures known to twister: riscv32 and riscv64.
|
||||
archs = set()
|
||||
|
||||
for f in files:
|
||||
p = re.match(r"^arch\/([^/]+)\/", f)
|
||||
if not p:
|
||||
p = re.match(r"^include\/arch\/([^/]+)\/", f)
|
||||
if p:
|
||||
if p.group(1) != 'common':
|
||||
if p.group(1) == 'riscv':
|
||||
archs.add('riscv32')
|
||||
archs.add('riscv64')
|
||||
else:
|
||||
archs.add(p.group(1))
|
||||
|
||||
if archs:
|
||||
with open("modified_archs.args", "w") as fp:
|
||||
fp.write("-a\n%s" %("\n-a\n".join(archs)))
|
||||
|
||||
def find_boards(files):
|
||||
boards = set()
|
||||
all_boards = set()
|
||||
|
||||
for f in files:
|
||||
if f.endswith(".rst") or f.endswith(".png") or f.endswith(".jpg"):
|
||||
continue
|
||||
p = re.match(r"^boards\/[^/]+\/([^/]+)\/", f)
|
||||
if p and p.groups():
|
||||
boards.add(p.group(1))
|
||||
|
||||
for b in boards:
|
||||
suboards = glob.glob("boards/*/%s/*.yaml" %(b))
|
||||
for subboard in suboards:
|
||||
name = os.path.splitext(os.path.basename(subboard))[0]
|
||||
if name:
|
||||
all_boards.add(name)
|
||||
|
||||
if all_boards:
|
||||
with open("modified_boards.args", "w") as fp:
|
||||
fp.write("-p\n%s" %("\n-p\n".join(all_boards)))
|
||||
|
||||
def find_tests(files):
|
||||
tests = set()
|
||||
for f in files:
|
||||
if f.endswith(".rst"):
|
||||
continue
|
||||
d = os.path.dirname(f)
|
||||
while d:
|
||||
if os.path.exists(os.path.join(d, "testcase.yaml")) or \
|
||||
os.path.exists(os.path.join(d, "sample.yaml")):
|
||||
tests.add(d)
|
||||
break
|
||||
else:
|
||||
d = os.path.dirname(d)
|
||||
|
||||
if tests:
|
||||
with open("modified_tests.args", "w") as fp:
|
||||
fp.write("-T\n%s\n--all" %("\n-T\n".join(tests)))
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
args = parse_args()
|
||||
if not args.commits:
|
||||
exit(1)
|
||||
|
||||
# pylint does not like the 'sh' library
|
||||
# pylint: disable=too-many-function-args,unexpected-keyword-arg
|
||||
commit = sh.git("diff", "--name-only", args.commits, **sh_special_args)
|
||||
files = commit.split("\n")
|
||||
|
||||
find_boards(files)
|
||||
find_archs(files)
|
||||
find_tests(files)
|
||||
|
|
@ -22,7 +22,7 @@ set -xe
|
|||
|
||||
twister_options=" --inline-logs -N -v --integration"
|
||||
export BSIM_OUT_PATH="${BSIM_OUT_PATH:-/opt/bsim/}"
|
||||
if [ ! -d "${BSIM_OUT_PATH}" ]; then
|
||||
if [ ! -d "${BSIM_OUT_PATH}x" ]; then
|
||||
unset BSIM_OUT_PATH
|
||||
fi
|
||||
export BSIM_COMPONENTS_PATH="${BSIM_OUT_PATH}/components/"
|
||||
|
|
@ -104,16 +104,21 @@ function run_bsim_bt_tests() {
|
|||
|
||||
function get_tests_to_run() {
|
||||
./scripts/zephyr_module.py --twister-out module_tests.args
|
||||
./scripts/ci/get_modified_tests.py --commits ${commit_range} > modified_tests.args
|
||||
./scripts/ci/get_modified_boards.py --commits ${commit_range} > modified_boards.args
|
||||
./scripts/ci/get_twister_opt.py --commits ${commit_range}
|
||||
|
||||
if [ -s modified_boards.args ]; then
|
||||
${twister} ${twister_options} +modified_boards.args --save-tests test_file_1.txt || exit 1
|
||||
${twister} ${twister_options} +modified_boards.args \
|
||||
--save-tests test_file_1.txt || exit 1
|
||||
fi
|
||||
if [ -s modified_tests.args ]; then
|
||||
${twister} ${twister_options} +modified_tests.args --save-tests test_file_2.txt || exit 1
|
||||
${twister} ${twister_options} +modified_tests.args \
|
||||
--save-tests test_file_2.txt || exit 1
|
||||
fi
|
||||
rm -f modified_tests.args modified_boards.args
|
||||
if [ -s modified_archs.args ]; then
|
||||
${twister} ${twister_options} +modified_archs.args \
|
||||
--save-tests test_file_3.txt || exit 1
|
||||
fi
|
||||
rm -f modified_tests.args modified_boards.args modified_archs.args
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -209,6 +214,7 @@ if [ -n "$main_ci" ]; then
|
|||
# https://stackoverflow.com/questions/3398258/edit-shell-script-while-its-running
|
||||
git rebase $remote/${branch}
|
||||
else
|
||||
echo "Full Run"
|
||||
SC="full"
|
||||
fi
|
||||
$short_git_log
|
||||
|
|
@ -226,7 +232,7 @@ if [ -n "$main_ci" ]; then
|
|||
|
||||
# cleanup
|
||||
rm -f test_file.txt
|
||||
touch test_file_1.txt test_file_2.txt
|
||||
touch test_file_1.txt test_file_2.txt test_file_3.txt
|
||||
|
||||
# In a pull-request see if we have changed any tests or board definitions
|
||||
if [ -n "${pull_request_nr}" -o -n "${local_run}" ]; then
|
||||
|
|
@ -235,16 +241,18 @@ if [ -n "$main_ci" ]; then
|
|||
|
||||
if [ "$SC" == "full" ]; then
|
||||
# Save list of tests to be run
|
||||
${twister} ${twister_options} --save-tests test_file_3.txt || exit 1
|
||||
${twister} ${twister_options} --save-tests test_file_4.txt || exit 1
|
||||
else
|
||||
echo "test,arch,platform,status,extra_args,handler,handler_time,ram_size,rom_size" > test_file_3.txt
|
||||
echo "test,arch,platform,status,extra_args,handler,handler_time,ram_size,rom_size" \
|
||||
> test_file_4.txt
|
||||
fi
|
||||
|
||||
# Remove headers from all files but the first one to generate one
|
||||
# single file with only one header row
|
||||
tail -n +2 test_file_3.txt > test_file_3_in.txt
|
||||
tail -n +2 test_file_2.txt > test_file_2_in.txt
|
||||
tail -n +2 test_file_1.txt > test_file_1_in.txt
|
||||
cat test_file_3.txt test_file_2_in.txt test_file_1_in.txt > test_file.txt
|
||||
cat test_file_4.txt test_file_3_in.txt test_file_2_in.txt test_file_1_in.txt > test_file.txt
|
||||
|
||||
echo "+++ run twister"
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,22 @@ Makefile
|
|||
tests/*
|
||||
samples/*
|
||||
boards/*/*/*
|
||||
arch/xtensa/*
|
||||
arch/x86/*
|
||||
arch/posix/*
|
||||
arch/arc/*
|
||||
arch/sparc/*
|
||||
arch/arm/*
|
||||
arch/nios2/*
|
||||
arch/riscv/*
|
||||
include/arch/xtensa/*
|
||||
include/arch/x86/*
|
||||
include/arch/posix/*
|
||||
include/arch/arc/*
|
||||
include/arch/sparc/*
|
||||
include/arch/arm/*
|
||||
include/arch/nios2/*
|
||||
include/arch/riscv/*
|
||||
doc/*
|
||||
# GH action have no impact on code
|
||||
.github/*
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue