ci: move ci scripting from .shippable to a script

move CI scripting to a dedicated script. For example, to simulate what
is run in CI when a pull request is submitted:

./scripts/ci/run_ci.sh -b master -r upstream  -p

Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif 2018-11-10 09:53:13 -05:00
commit 0a63947abc
2 changed files with 141 additions and 46 deletions

View file

@ -34,52 +34,10 @@ build:
- export CCACHE_DIR=${SHIPPABLE_BUILD_DIR}/ccache/.ccache
- >
if [ "$IS_PULL_REQUEST" = "true" ]; then
git rebase origin/${PULL_REQUEST_BASE_BRANCH};
fi
- export NRF_HW_MODELS_VERSION=`cat boards/posix/nrf52_bsim/hw_models_version`
- >
pushd . ;
cd ${BSIM_COMPONENTS_PATH} ;
git clone -b ${NRF_HW_MODELS_VERSION} https://github.com/BabbleSim/ext_NRF52_hw_models.git;
cd /opt/bsim/ ;
make everything -j 8;
popd ;
- source zephyr-env.sh
- ccache -c -s --max-size=5000M
- >
if [ "$MATRIX_BUILD" = "5" -a "$IS_PULL_REQUEST" = "true" ]; then
export COMMIT_RANGE=origin/${PULL_REQUEST_BASE_BRANCH}..HEAD
echo "Building a Pull Request";
echo "- Building Documentation";
echo "Commit range:" ${COMMIT_RANGE}
make htmldocs
if [ "$?" != "0" ]; then
echo "Documentation build failed";
exit 1;
fi
if [ -s doc/_build/doc.warnings ]; then
echo " => New documentation warnings/errors";
cp doc/_build/doc.warnings doc.warnings
./scripts/ci/run_ci.sh -b master -r origin -m ${MATRIX_BUILD} -M ${MATRIX_BUILDS} -p;
else
./scripts/ci/run_ci.sh -b master -r origin -m ${MATRIX_BUILD} -M ${MATRIX_BUILDS};
fi;
echo "- Verify commit message, coding style, doc build";
./scripts/ci/check-compliance.py --commits ${COMMIT_RANGE} || true;
fi;
- >
if [ "$IS_PULL_REQUEST" = "true" ]; then
./scripts/ci/get_modified_tests.py --commits origin/${PULL_REQUEST_BASE_BRANCH}..HEAD > modified_tests.args;
./scripts/ci/get_modified_boards.py --commits origin/${PULL_REQUEST_BASE_BRANCH}..HEAD > modified_boards.args;
if [ -s modified_boards.args ]; then
./scripts/sanitycheck ${SANITYCHECK_OPTIONS} +modified_boards.args --save-tests test_file.txt || exit 1;
fi;
if [ -s modified_tests.args ]; then
./scripts/sanitycheck ${SANITYCHECK_OPTIONS} +modified_tests.args --save-tests test_file.txt || exit 1;
fi;
rm -f modified_tests.args modified_boards.args;
fi;
- ./scripts/sanitycheck ${SANITYCHECK_OPTIONS} --save-tests test_file.txt || exit 1
- ./scripts/sanitycheck ${SANITYCHECK_OPTIONS} --load-tests test_file.txt --subset ${MATRIX_BUILD}/${MATRIX_BUILDS} || ./scripts/sanitycheck ${SANITYCHECK_OPTIONS_RETRY} || ./scripts/sanitycheck ${SANITYCHECK_OPTIONS_RETRY}
- rm test_file.txt
- ccache -s
on_failure:
- >

137
scripts/ci/run_ci.sh Executable file
View file

@ -0,0 +1,137 @@
#!/bin/bash
# Copyright (c) 2017 Linaro Limited
#
# SPDX-License-Identifier: Apache-2.0
#
# Author: Anas Nashif
#
# This script is run in CI and support both pull request and commit modes. So
# it can be used also on commits to the tree or on tags.
# The following options are supports:
# -p start the script for pull requests
# -m matrix node number, for example 3 on a 5 node matrix
# -M total number of nodes in the matrix
# -b base branch
# -r the remote to rebase on
#
# The script can be run locally using for exmaple:
# ./scripts/ci/run_ci.sh -b master -r upstream -p
source zephyr-env.sh
SANITYCHECK_OPTIONS=" --inline-logs --enable-coverage -N"
SANITYCHECK_OPTIONS_RETRY="${SANITYCHECK_OPTIONS} --only-failed --outdir=out-2nd-pass"
SANITYCHECK_OPTIONS_RETRY_2="${SANITYCHECK_OPTIONS} --only-failed --outdir=out-3nd-pass"
SANITYCHECK="${ZEPHYR_BASE}/scripts/sanitycheck"
MATRIX_BUILDS=1
MATRIX=1
DOC_MATRIX=${MATRIX_BUILDS}
while getopts ":pm:b:r:M:" opt; do
case $opt in
p)
echo "Testing a Pull Request." >&2
PULL_REQUEST=1
;;
m)
echo "Running on Matrix $OPTARG" >&2
MATRIX=$OPTARG
;;
M)
echo "Running a matrix of $OPTARG slaves" >&2
MATRIX_BUILDS=$OPTARG
;;
b)
echo "Base Branch: $OPTARG" >&2
BRANCH=$OPTARG
;;
r)
echo "Remote: $OPTARG" >&2
REMOTE=$OPTARG
;;
\?)
echo "Invalid option: -$OPTARG" >&2
;;
esac
done
if [ -z "$BRANCH" ]; then
echo "No base branch given"
exit
fi
if [ -n "$PULL_REQUEST" ]; then
git rebase $REMOTE/${BRANCH};
fi
if [ -n "$BRANCH" ]; then
COMMIT_RANGE=$REMOTE/${BRANCH}..HEAD
echo "Commit range:" ${COMMIT_RANGE}
else
exit
fi
function build_btsim() {
NRF_HW_MODELS_VERSION=`cat boards/posix/nrf52_bsim/hw_models_version`
pushd . ;
cd ${BSIM_COMPONENTS_PATH} ;
git clone -b ${NRF_HW_MODELS_VERSION} https://github.com/BabbleSim/ext_NRF52_hw_models.git;
cd /opt/bsim/ ;
make everything -j 8;
popd ;
}
function build_docs() {
echo "- Building Documentation";
make htmldocs
if [ "$?" != "0" ]; then
echo "Documentation build failed";
exit 1;
fi
if [ -s doc/_build/doc.warnings ]; then
echo " => New documentation warnings/errors";
cp doc/_build/doc.warnings doc.warnings
fi
echo "- Verify commit message, coding style, doc build";
}
function get_tests_to_run() {
./scripts/ci/get_modified_tests.py --commits ${COMMIT_RANGE} > modified_tests.args;
./scripts/ci/get_modified_boards.py --commits ${COMMIT_RANGE} > modified_boards.args;
if [ -s modified_boards.args ]; then
${SANITYCHECK} ${SANITYCHECK_OPTIONS} +modified_boards.args --save-tests test_file.txt || exit 1;
fi
if [ -s modified_tests.args ]; then
${SANITYCHECK} ${SANITYCHECK_OPTIONS} +modified_tests.args --save-tests test_file.txt || exit 1;
fi
rm -f modified_tests.args modified_boards.args;
}
# Build BT Simulator
test -z ${BSIM_OUT_PATH} || build_btsim
# Build Docs on matrix 5 in a pull request
if [ "${MATRIX}" = "${DOC_MATRIX}" -a -n "${PULL_REQUEST}" ]; then
build_docs
./scripts/ci/check-compliance.py --commits ${COMMIT_RANGE} || true;
fi
# In a pull-request see if we have changed any tests or board definitions
if [ -n "${PULL_REQUEST}" ]; then
get_tests_to_run
fi
# Save list of tests to be run
${SANITYCHECK} ${SANITYCHECK_OPTIONS} --save-tests test_file.txt || exit 1
# Run a subset of tests based on matrix size
${SANITYCHECK} ${SANITYCHECK_OPTIONS} --load-tests test_file.txt --subset ${MATRIX}/${MATRIX_BUILDS}
if [ "$?" != 0 ]; then
# let the host settle down
sleep 10
${SANITYCHECK} ${SANITYCHECK_OPTIONS_RETRY} || \
( sleep 10; ${SANITYCHECK} ${SANITYCHECK_OPTIONS_RETRY}; )
fi
# cleanup
rm test_file.txt