From bea131835d61bb7c035d074adca066493b63e54e Mon Sep 17 00:00:00 2001 From: Kumar Gala Date: Wed, 3 Jun 2020 07:02:35 -0500 Subject: [PATCH] ci: Add initial buildkite ci setup Add setup to utilize buildkite for CI purposes: 1. .buildkite/hooks/pre-command: * Handles getting git checkout setup against upstream repo * Setup some west module cache (dirs, clean out files & locks) * init dir for ccache 2. .buildkite/hooks/post-command: * Report disk usage (meant for possible debugging) 3. .buildkite/pipeline.yml [uses to determine what to do]: * setup zephyr env vars * set which docker container to use (export some local disk caches for git, west modules, and ccache) * uses plug to general build annotation on failure (junit-annotate) 4. .buildkite/run.sh [ buildkite wrapper to invoke scripts/ci/run.sh ] Signed-off-by: Kumar Gala --- .buildkite/hooks/post-command | 8 ++++++ .buildkite/hooks/pre-command | 38 +++++++++++++++++++++++++++++ .buildkite/pipeline.yml | 28 +++++++++++++++++++++ .buildkite/run.sh | 46 +++++++++++++++++++++++++++++++++++ CODEOWNERS | 1 + 5 files changed, 121 insertions(+) create mode 100755 .buildkite/hooks/post-command create mode 100755 .buildkite/hooks/pre-command create mode 100644 .buildkite/pipeline.yml create mode 100755 .buildkite/run.sh diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command new file mode 100755 index 00000000000..839d86ed2a5 --- /dev/null +++ b/.buildkite/hooks/post-command @@ -0,0 +1,8 @@ +#!/bin/bash +# Copyright (c) 2020 Linaro Limited +# +# SPDX-License-Identifier: Apache-2.0 + +# report disk usage: +echo "--- $0 disk usage" +df -h diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100755 index 00000000000..04ea9a06b01 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright (c) 2020 Linaro Limited +# +# SPDX-License-Identifier: Apache-2.0 + +# Save off where we started so we can go back there +WORKDIR=${PWD} + +if [ -n "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" ]; then + git fetch -v origin ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + git checkout FETCH_HEAD + git config --local user.email "builds@zephyrproject.org" + git config --local user.name "Zephyr CI" + git merge --no-edit "${BUILDKITE_COMMIT}" || { + local merge_result=$? + echo "Merge failed: ${merge_result}" + git merge --abort + exit $merge_result + } +fi + +mkdir -p /var/lib/buildkite-agent/zephyr-ccache/ + +# create cache dirs, no-op if they already exist +mkdir -p /var/lib/buildkite-agent/zephyr-module-cache/modules +mkdir -p /var/lib/buildkite-agent/zephyr-module-cache/tools +mkdir -p /var/lib/buildkite-agent/zephyr-module-cache/bootloader + +# Clean cache - if it already exists +cd /var/lib/buildkite-agent/zephyr-module-cache +find -type f -not -path "*/.git/*" -not -name ".git" -delete + +# Remove any stale locks +find -name index.lock -delete + +# return from where we started so we can find pipeline files from +# git repo +cd ${WORKDIR} diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 00000000000..ffe699e514a --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,28 @@ +steps: + - command: + - .buildkite/run.sh + env: + ZEPHYR_TOOLCHAIN_VARIANT: "zephyr" + ZEPHYR_SDK_INSTALL_DIR: "/opt/sdk/zephyr-sdk-0.11.3" + parallelism: 20 + timeout_in_minutes: 120 + retry: + manual: true + plugins: + - docker#v3.5.0: + image: "zephyrprojectrtos/ci:v0.11.8" + propagate-environment: true + volumes: + - "/var/lib/buildkite-agent/git-mirrors:/var/lib/buildkite-agent/git-mirrors" + - "/var/lib/buildkite-agent/zephyr-module-cache:/var/lib/buildkite-agent/zephyr-module-cache" + - "/var/lib/buildkite-agent/zephyr-ccache:/root/.ccache" + workdir: "/workdir/zephyr" + agents: + - "queue=default" + + - wait: ~ + continue_on_failure: true + + - plugins: + - junit-annotate#v1.7.0: + artifacts: sanitycheck-*.xml diff --git a/.buildkite/run.sh b/.buildkite/run.sh new file mode 100755 index 00000000000..f47671e6ea5 --- /dev/null +++ b/.buildkite/run.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) 2020 Linaro Limited +# +# SPDX-License-Identifier: Apache-2.0 + +echo "--- run $0" + +git log -n 5 --oneline --decorate --abbrev=12 + +# Setup module cache +cd /workdir +ln -s /var/lib/buildkite-agent/zephyr-module-cache/modules +ln -s /var/lib/buildkite-agent/zephyr-module-cache/tools +ln -s /var/lib/buildkite-agent/zephyr-module-cache/bootloader +cd /workdir/zephyr + +export JOB_NUM=$((${BUILDKITE_PARALLEL_JOB}+1)) + +# ccache stats +echo "" +echo "--- ccache stats at start" +ccache -s + +if [ -n "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" ]; then + ./scripts/ci/run_ci.sh -c -b ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} -r origin \ + -m ${JOB_NUM} -M ${BUILDKITE_PARALLEL_JOB_COUNT} -p ${BUILDKITE_PULL_REQUEST} +else + ./scripts/ci/run_ci.sh -c -b ${BUILDKITE_BRANCH} -r origin \ + -m ${JOB_NUM} -M ${BUILDKITE_PARALLEL_JOB_COUNT}; +fi; + +SANITY_EXIT_STATUS=$? + +# Rename sanitycheck junit xml for use with junit-annotate-buildkite-plugin +mv sanity-out/sanitycheck.xml sanitycheck-${BUILDKITE_JOB_ID}.xml +buildkite-agent artifact upload sanitycheck-${BUILDKITE_JOB_ID}.xml + +# ccache stats +echo "--- ccache stats at finish" +ccache -s + +# disk usage +echo "--- disk usage at finish" +df -h + +exit ${SANITY_EXIT_STATUS} diff --git a/CODEOWNERS b/CODEOWNERS index 499e7f23601..76bbf80bdfa 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -16,6 +16,7 @@ /.known-issues/ @nashif /.github/ @nashif /.github/workflows/ @galak @nashif +/.buildkite/ @galak /arch/arc/ @vonhust @ruuddw /arch/arm/ @MaureenHelm @galak @ioannisg /arch/arm/core/aarch32/cortex_m/cmse/ @ioannisg