ci: twister: use workflow_call for prep job
Use workflow_call to allow for different type of nodes depending on event. On push, we do not need to use zephyr runners, GH runners are enough and are much faster to deploy and start. This resolves an issue where push jobs will have to be queued for a longer time waiting for the prep step, once the prep step is done, we will have to wait one more time in the queue for requested nodes. This should speed up execution of push events in CI. Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif
parent
0a8b16693f
commit
ac08acafdd
2 changed files with 151 additions and 102 deletions
150
.github/workflows/twister-prep.yaml
vendored
Normal file
150
.github/workflows/twister-prep.yaml
vendored
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
name: Prep
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
outputs:
|
||||
subset:
|
||||
description: subset
|
||||
value: ${{ jobs.prep_push.outputs.subset }} || ${{ jobs.prep_pr.outputs.subset }}
|
||||
size:
|
||||
description: size
|
||||
value: ${{ jobs.prep_push.outputs.size }} || ${{ jobs.prep_pr.outputs.size }}
|
||||
fullrun:
|
||||
description: fullrun
|
||||
value: ${{ jobs.prep_push.outputs.fullrun }} || ${{ jobs.prep_pr.outputs.size }}
|
||||
|
||||
jobs:
|
||||
prep_pr:
|
||||
if: github.repository_owner == 'zephyrproject-rtos' && github.event_name == 'pull_request_target'
|
||||
runs-on:
|
||||
group: zephyr-runner-v2-linux-x64-4xlarge
|
||||
container:
|
||||
image: ghcr.io/zephyrproject-rtos/ci-repo-cache:v0.27.4.20241026
|
||||
options: '--entrypoint /bin/bash'
|
||||
outputs:
|
||||
subset: ${{ steps.output-services.outputs.subset }}
|
||||
size: ${{ steps.output-services.outputs.size }}
|
||||
fullrun: ${{ steps.output-services.outputs.fullrun }}
|
||||
env:
|
||||
MATRIX_SIZE: 10
|
||||
PUSH_MATRIX_SIZE: 20
|
||||
DAILY_MATRIX_SIZE: 80
|
||||
BSIM_OUT_PATH: /opt/bsim/
|
||||
BSIM_COMPONENTS_PATH: /opt/bsim/components
|
||||
TESTS_PER_BUILDER: 700
|
||||
COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
|
||||
BASE_REF: ${{ github.base_ref }}
|
||||
steps:
|
||||
- name: Apply container owner mismatch workaround
|
||||
run: |
|
||||
# FIXME: The owner UID of the GITHUB_WORKSPACE directory may not
|
||||
# match the container user UID because of the way GitHub
|
||||
# Actions runner is implemented. Remove this workaround when
|
||||
# GitHub comes up with a fundamental fix for this problem.
|
||||
git config --global --add safe.directory ${GITHUB_WORKSPACE}
|
||||
|
||||
- name: Print cloud service information
|
||||
run: |
|
||||
echo "ZEPHYR_RUNNER_CLOUD_PROVIDER = ${ZEPHYR_RUNNER_CLOUD_PROVIDER}"
|
||||
echo "ZEPHYR_RUNNER_CLOUD_NODE = ${ZEPHYR_RUNNER_CLOUD_NODE}"
|
||||
echo "ZEPHYR_RUNNER_CLOUD_POD = ${ZEPHYR_RUNNER_CLOUD_POD}"
|
||||
|
||||
- name: Clone cached Zephyr repository
|
||||
if: github.event_name == 'pull_request_target'
|
||||
continue-on-error: true
|
||||
run: |
|
||||
git clone --shared /repo-cache/zephyrproject/zephyr .
|
||||
git remote set-url origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}
|
||||
|
||||
- name: Checkout
|
||||
if: github.event_name == 'pull_request_target'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Environment Setup
|
||||
if: github.event_name == 'pull_request_target'
|
||||
run: |
|
||||
git config --global user.email "bot@zephyrproject.org"
|
||||
git config --global user.name "Zephyr Bot"
|
||||
rm -fr ".git/rebase-apply"
|
||||
git rebase origin/${BASE_REF}
|
||||
git clean -f -d
|
||||
git log --pretty=oneline | head -n 10
|
||||
west init -l . || true
|
||||
west config manifest.group-filter -- +ci,+optional
|
||||
west config --global update.narrow true
|
||||
west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || ( rm -rf ../modules ../bootloader ../tools && west update --path-cache /repo-cache/zephyrproject)
|
||||
west forall -c 'git reset --hard HEAD'
|
||||
|
||||
echo "ZEPHYR_SDK_INSTALL_DIR=/opt/toolchains/zephyr-sdk-$( cat SDK_VERSION )" >> $GITHUB_ENV
|
||||
|
||||
- name: Generate Test Plan with Twister
|
||||
if: github.event_name == 'pull_request_target'
|
||||
id: test-plan
|
||||
run: |
|
||||
export ZEPHYR_BASE=${PWD}
|
||||
export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
|
||||
python3 ./scripts/ci/test_plan.py -c origin/${BASE_REF}.. --pull-request -t $TESTS_PER_BUILDER
|
||||
if [ -s .testplan ]; then
|
||||
cat .testplan >> $GITHUB_ENV
|
||||
else
|
||||
echo "TWISTER_NODES=${MATRIX_SIZE}" >> $GITHUB_ENV
|
||||
fi
|
||||
rm -f testplan.json .testplan
|
||||
|
||||
- name: Determine matrix size
|
||||
id: output-services
|
||||
run: |
|
||||
if [ -n "${TWISTER_NODES}" ]; then
|
||||
subset="[$(seq -s',' 1 ${TWISTER_NODES})]"
|
||||
else
|
||||
subset="[$(seq -s',' 1 ${MATRIX_SIZE})]"
|
||||
fi
|
||||
size=${TWISTER_NODES}
|
||||
|
||||
echo "subset=${subset}" >> $GITHUB_OUTPUT
|
||||
echo "size=${size}" >> $GITHUB_OUTPUT
|
||||
echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT
|
||||
|
||||
prep_push:
|
||||
if: github.repository_owner == 'zephyrproject-rtos' && (github.event_name == 'push' || github.event_name == 'schedule')
|
||||
runs-on: ubuntu-22.04
|
||||
outputs:
|
||||
subset: ${{ steps.output-services.outputs.subset }}
|
||||
size: ${{ steps.output-services.outputs.size }}
|
||||
fullrun: ${{ steps.output-services.outputs.fullrun }}
|
||||
env:
|
||||
MATRIX_SIZE: 10
|
||||
PUSH_MATRIX_SIZE: 20
|
||||
DAILY_MATRIX_SIZE: 80
|
||||
BSIM_OUT_PATH: /opt/bsim/
|
||||
BSIM_COMPONENTS_PATH: /opt/bsim/components
|
||||
TESTS_PER_BUILDER: 700
|
||||
COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
|
||||
BASE_REF: ${{ github.base_ref }}
|
||||
steps:
|
||||
- name: Print cloud service information
|
||||
run: |
|
||||
echo "ZEPHYR_RUNNER_CLOUD_PROVIDER = ${ZEPHYR_RUNNER_CLOUD_PROVIDER}"
|
||||
echo "ZEPHYR_RUNNER_CLOUD_NODE = ${ZEPHYR_RUNNER_CLOUD_NODE}"
|
||||
echo "ZEPHYR_RUNNER_CLOUD_POD = ${ZEPHYR_RUNNER_CLOUD_POD}"
|
||||
|
||||
- name: Determine matrix size
|
||||
id: output-services
|
||||
run: |
|
||||
if [ "${{github.event_name}}" = "push" ]; then
|
||||
subset="[$(seq -s',' 1 ${PUSH_MATRIX_SIZE})]"
|
||||
size=${MATRIX_SIZE}
|
||||
elif [ "${{github.event_name}}" = "schedule" -a "${{github.repository}}" = "zephyrproject-rtos/zephyr" ]; then
|
||||
subset="[$(seq -s',' 1 ${DAILY_MATRIX_SIZE})]"
|
||||
size=${DAILY_MATRIX_SIZE}
|
||||
else
|
||||
size=0
|
||||
fi
|
||||
|
||||
echo "subset=${subset}" >> $GITHUB_OUTPUT
|
||||
echo "size=${size}" >> $GITHUB_OUTPUT
|
||||
echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT
|
||||
103
.github/workflows/twister.yaml
vendored
103
.github/workflows/twister.yaml
vendored
|
|
@ -21,108 +21,7 @@ concurrency:
|
|||
|
||||
jobs:
|
||||
twister-build-prep:
|
||||
if: github.repository_owner == 'zephyrproject-rtos'
|
||||
runs-on:
|
||||
group: zephyr-runner-v2-linux-x64-4xlarge
|
||||
container:
|
||||
image: ghcr.io/zephyrproject-rtos/ci-repo-cache:v0.27.4.20241026
|
||||
options: '--entrypoint /bin/bash'
|
||||
outputs:
|
||||
subset: ${{ steps.output-services.outputs.subset }}
|
||||
size: ${{ steps.output-services.outputs.size }}
|
||||
fullrun: ${{ steps.output-services.outputs.fullrun }}
|
||||
env:
|
||||
MATRIX_SIZE: 10
|
||||
PUSH_MATRIX_SIZE: 20
|
||||
DAILY_MATRIX_SIZE: 80
|
||||
BSIM_OUT_PATH: /opt/bsim/
|
||||
BSIM_COMPONENTS_PATH: /opt/bsim/components
|
||||
TESTS_PER_BUILDER: 700
|
||||
COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
|
||||
BASE_REF: ${{ github.base_ref }}
|
||||
steps:
|
||||
- name: Apply container owner mismatch workaround
|
||||
run: |
|
||||
# FIXME: The owner UID of the GITHUB_WORKSPACE directory may not
|
||||
# match the container user UID because of the way GitHub
|
||||
# Actions runner is implemented. Remove this workaround when
|
||||
# GitHub comes up with a fundamental fix for this problem.
|
||||
git config --global --add safe.directory ${GITHUB_WORKSPACE}
|
||||
|
||||
- name: Print cloud service information
|
||||
run: |
|
||||
echo "ZEPHYR_RUNNER_CLOUD_PROVIDER = ${ZEPHYR_RUNNER_CLOUD_PROVIDER}"
|
||||
echo "ZEPHYR_RUNNER_CLOUD_NODE = ${ZEPHYR_RUNNER_CLOUD_NODE}"
|
||||
echo "ZEPHYR_RUNNER_CLOUD_POD = ${ZEPHYR_RUNNER_CLOUD_POD}"
|
||||
|
||||
- name: Clone cached Zephyr repository
|
||||
if: github.event_name == 'pull_request_target'
|
||||
continue-on-error: true
|
||||
run: |
|
||||
git clone --shared /repo-cache/zephyrproject/zephyr .
|
||||
git remote set-url origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}
|
||||
|
||||
- name: Checkout
|
||||
if: github.event_name == 'pull_request_target'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Environment Setup
|
||||
if: github.event_name == 'pull_request_target'
|
||||
run: |
|
||||
git config --global user.email "bot@zephyrproject.org"
|
||||
git config --global user.name "Zephyr Bot"
|
||||
rm -fr ".git/rebase-apply"
|
||||
git rebase origin/${BASE_REF}
|
||||
git clean -f -d
|
||||
git log --pretty=oneline | head -n 10
|
||||
west init -l . || true
|
||||
west config manifest.group-filter -- +ci,+optional
|
||||
west config --global update.narrow true
|
||||
west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || ( rm -rf ../modules ../bootloader ../tools && west update --path-cache /repo-cache/zephyrproject)
|
||||
west forall -c 'git reset --hard HEAD'
|
||||
|
||||
echo "ZEPHYR_SDK_INSTALL_DIR=/opt/toolchains/zephyr-sdk-$( cat SDK_VERSION )" >> $GITHUB_ENV
|
||||
|
||||
- name: Generate Test Plan with Twister
|
||||
if: github.event_name == 'pull_request_target'
|
||||
id: test-plan
|
||||
run: |
|
||||
export ZEPHYR_BASE=${PWD}
|
||||
export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
|
||||
python3 ./scripts/ci/test_plan.py -c origin/${BASE_REF}.. --pull-request -t $TESTS_PER_BUILDER
|
||||
if [ -s .testplan ]; then
|
||||
cat .testplan >> $GITHUB_ENV
|
||||
else
|
||||
echo "TWISTER_NODES=${MATRIX_SIZE}" >> $GITHUB_ENV
|
||||
fi
|
||||
rm -f testplan.json .testplan
|
||||
|
||||
- name: Determine matrix size
|
||||
id: output-services
|
||||
run: |
|
||||
if [ "${{github.event_name}}" = "pull_request_target" ]; then
|
||||
if [ -n "${TWISTER_NODES}" ]; then
|
||||
subset="[$(seq -s',' 1 ${TWISTER_NODES})]"
|
||||
else
|
||||
subset="[$(seq -s',' 1 ${MATRIX_SIZE})]"
|
||||
fi
|
||||
size=${TWISTER_NODES}
|
||||
elif [ "${{github.event_name}}" = "push" ]; then
|
||||
subset="[$(seq -s',' 1 ${PUSH_MATRIX_SIZE})]"
|
||||
size=${MATRIX_SIZE}
|
||||
elif [ "${{github.event_name}}" = "schedule" -a "${{github.repository}}" = "zephyrproject-rtos/zephyr" ]; then
|
||||
subset="[$(seq -s',' 1 ${DAILY_MATRIX_SIZE})]"
|
||||
size=${DAILY_MATRIX_SIZE}
|
||||
else
|
||||
size=0
|
||||
fi
|
||||
echo "subset=${subset}" >> $GITHUB_OUTPUT
|
||||
echo "size=${size}" >> $GITHUB_OUTPUT
|
||||
echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT
|
||||
uses: ./.github/workflows/twister-prep.yaml
|
||||
|
||||
twister-build:
|
||||
runs-on:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue