ci: workflows: pin python dependencies

Pin python dependencies to hashes and cleanup/unify python setup steps in
various workflows.

We now have one dependency file containing all requirements for github
actions that is managed centrally with hashes. No direct pip installs
are needed in workflow files and everything shall go via the
requirements file.

Pinning to specific version and hashes helps with preventing supply
chain attacks.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>

.github/workflows/assigner.yml

@@ -28,13 +28,20 @@ jobs:
       issues: write        # to add assignees to issues
 
     steps:
-    - name: Install Python dependencies
-      run: |
-        pip install -U PyGithub>=1.55 west
-
     - name: Check out source code
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
+    - name: Set up Python
+      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+      with:
+        python-version: 3.12
+        cache: pip
+        cache-dependency-path: scripts/requirements-actions.txt
+
+    - name: Install Python packages
+      run: |
+        pip install -r scripts/requirements-actions.txt --require-hashes
+
     - name: Run assignment script
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/backport_issue_check.yml

@@ -28,16 +28,23 @@ jobs:
       - name: Check out source code
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - name: Install Python dependencies
+      - name: Set up Python
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: 3.12
+          cache: pip
+          cache-dependency-path: scripts/requirements-actions.txt
+
+      - name: Install Python packages
         run: |
-          pip install -U pygithub
+          pip install -r scripts/requirements-actions.txt --require-hashes
 
       - name: Run backport issue checker
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           ./scripts/release/list_backports.py \
-            -o ${{ github.event.repository.owner.login }} \
-            -r ${{ github.event.repository.name }} \
-            -b ${{ github.event.pull_request.base.ref }} \
-            -p ${{ github.event.pull_request.number }}
+          -o ${{ github.event.repository.owner.login }} \
+          -r ${{ github.event.repository.name }} \
+          -b ${{ github.event.pull_request.base.ref }} \
+          -p ${{ github.event.pull_request.number }}

.github/workflows/bsim-tests.yaml

@@ -178,7 +178,6 @@ jobs:
 
       - name: Merge Test Results
         run: |
-          pip install junitparser junit2html
           junitparser merge --glob "./bsim_*/*bsim_results.*.xml" "./twister-out/twister.xml" junit.xml
           junit2html junit.xml junit.html
 

.github/workflows/bug_snapshot.yaml

@@ -26,9 +26,17 @@ jobs:
     - name: Checkout
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-    - name: Install Python dependencies
+    - name: Set up Python
+      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+      with:
+        python-version: 3.12
+        cache: pip
+        cache-dependency-path: scripts/requirements-actions.txt
+
+    - name: Install Python packages
       run: |
-        pip install -U pygithub
+        pip install -r scripts/requirements-actions.txt --require-hashes
+
 
     - name: Snapshot bugs
       env:

.github/workflows/clang.yaml

@@ -135,13 +135,30 @@ jobs:
       checks: write # to create GitHub annotations
     if: (success() || failure())
     steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
       - name: Download Artifacts
         uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
         with:
           path: artifacts
+
+      - name: Set up Python
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: 3.12
+          cache: pip
+          cache-dependency-path: scripts/requirements-actions.txt
+
+      - name: Install Python packages
+        run: |
+          pip install -r scripts/requirements-actions.txt --require-hashes
+
       - name: Merge Test Results
         run: |
-          pip install junitparser junit2html
           junitparser merge artifacts/*/twister.xml junit.xml
           junit2html junit.xml junit-clang.html
 

.github/workflows/codecov.yaml

@@ -104,7 +104,6 @@ jobs:
           export ZEPHYR_BASE=${PWD}
           export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
           mkdir -p coverage/reports
-          pip install gcovr==6.0
           ./scripts/twister -E ${{matrix.normalized}}-testplan.json
           ls -la
           ./scripts/twister \
@@ -144,6 +143,17 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Set up Python
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: 3.12
+          cache: pip
+          cache-dependency-path: scripts/requirements-actions.txt
+
+      - name: Install Python packages
+        run: |
+          pip install -r scripts/requirements-actions.txt --require-hashes
+
       - name: Download Artifacts
         uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
         with:
@@ -185,7 +195,6 @@ jobs:
       - name: Merge coverage files
         run: |
           pushd ./coverage/reports
-          pip install gcovr==6.0
           gcovr ${{ steps.get-coverage-files.outputs.mergefiles }}  --merge-mode-functions=separate --json merged.json
           gcovr ${{ steps.get-coverage-files.outputs.mergefiles }} --merge-mode-functions=separate --cobertura merged.xml
           popd
@@ -201,7 +210,6 @@ jobs:
       - name: Generate Coverage Report
         if: always()
         run: |
-          pip install xlsxwriter ijson
           python3 ./scripts/ci/coverage/coverage_analysis.py \
             -t native_sim-testplan.json \
             -m MAINTAINERS.yml \

.github/workflows/coding_guidelines.yml

@@ -16,16 +16,16 @@ jobs:
         ref: ${{ github.event.pull_request.head.sha }}
         fetch-depth: 0
 
-    - name: cache-pip
-      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
+    - name: Set up Python
+      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('.github/workflows/coding_guidelines.yml') }}
+        python-version: 3.12
+        cache: pip
+        cache-dependency-path: scripts/requirements-actions.txt
 
-    - name: Install python dependencies
+    - name: Install Python packages
       run: |
-        pip install unidiff
-        pip install sh
+        pip install -r scripts/requirements-actions.txt --require-hashes
 
     - name: Install Packages
       run: |

.github/workflows/compliance.yml

@@ -46,18 +46,13 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
-        python-version: 3.11
+        python-version: 3.12
+        cache: pip
+        cache-dependency-path: scripts/requirements-actions.txt
 
-    - name: cache-pip
-      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('.github/workflows/compliance.yml') }}
-
-    - name: Install python dependencies
+    - name: Install Python packages
       run: |
-        pip install -r scripts/requirements-compliance.txt
-        pip install west
+        pip install -r scripts/requirements-actions.txt --require-hashes
 
     - name: west setup
       run: |

.github/workflows/daily_test_version.yml

@@ -26,15 +26,22 @@ jobs:
         aws-secret-access-key: ${{ secrets.AWS_TESTING_SECRET_ACCESS_KEY }}
         aws-region: us-east-1
 
-    - name: install-pip
-      run: |
-        pip install gitpython
-
     - name: checkout
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
         fetch-depth: 0
 
+    - name: Set up Python
+      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+      with:
+        python-version: 3.12
+        cache: pip
+        cache-dependency-path: scripts/requirements-actions.txt
+
+    - name: Install Python packages
+      run: |
+        pip install -r scripts/requirements-actions.txt --require-hashes
+
     - name: Upload to AWS S3
       run: |
         python3 scripts/ci/version_mgr.py --update .

.github/workflows/devicetree_checks.yml

@@ -34,38 +34,18 @@ jobs:
     steps:
     - name: checkout
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
         python-version: ${{ matrix.python-version }}
-    - name: cache-pip-linux
-      if: startsWith(runner.os, 'Linux')
-      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ matrix.python-version }}
-        restore-keys: |
-          ${{ runner.os }}-pip-${{ matrix.python-version }}
-    - name: cache-pip-mac
-      if: startsWith(runner.os, 'macOS')
-      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
-      with:
-        path: ~/Library/Caches/pip
-        # Trailing '-' was just to get a different cache name
-        key: ${{ runner.os }}-pip-${{ matrix.python-version }}-
-        restore-keys: |
-          ${{ runner.os }}-pip-${{ matrix.python-version }}-
-    - name: cache-pip-win
-      if: startsWith(runner.os, 'Windows')
-      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
-      with:
-        path: ~\AppData\Local\pip\Cache
-        key: ${{ runner.os }}-pip-${{ matrix.python-version }}
-        restore-keys: |
-          ${{ runner.os }}-pip-${{ matrix.python-version }}
-    - name: install python dependencies
+        cache: pip
+        cache-dependency-path: scripts/requirements-actions.txt
+
+    - name: Install Python packages
       run: |
-        pip install pytest pyyaml tox
+        pip install -r scripts/requirements-actions.txt --require-hashes
+
     - name: run tox
       working-directory: scripts/dts/python-devicetree
       run: |

.github/workflows/footprint-tracking.yml

@@ -61,7 +61,6 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y python3-venv
-          pip install -U gitpython
 
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -69,6 +68,17 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
 
+      - name: Set up Python
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: 3.12
+          cache: pip
+          cache-dependency-path: scripts/requirements-actions.txt
+
+      - name: Install Python packages
+        run: |
+            pip install -r scripts/requirements-actions.txt --require-hashes
+
       - name: Environment Setup
         run: |
           echo "ZEPHYR_SDK_INSTALL_DIR=/opt/toolchains/zephyr-sdk-$( cat SDK_VERSION )" >> $GITHUB_ENV
@@ -97,7 +107,6 @@ jobs:
         run: |
           python3 -m venv .venv
           . .venv/bin/activate
-          pip install awscli
           aws s3 sync  --quiet footprint_data/ s3://testing.zephyrproject.org/footprint_data/
 
       - name: Transform Footprint data to Twister JSON reports
@@ -116,7 +125,6 @@ jobs:
           ELASTICSEARCH_INDEX: ${{ vars.FOOTPRINT_TRACKING_INDEX }}
         run: |
           shopt -s globstar
-          pip install -U elasticsearch
           run_date=`date --iso-8601=minutes`
           python3 ./scripts/ci/upload_test_results_es.py -r ${run_date} \
             --flatten footprint \

.github/workflows/manifest.yml

@@ -20,12 +20,23 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
 
+      - name: Set up Python
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: 3.12
+          cache: pip
+          cache-dependency-path: scripts/requirements-actions.txt
+
+      - name: Install Python packages
+        run: |
+          cd zephyrproject/zephyr
+          pip install -r scripts/requirements-actions.txt --require-hashes
+
       - name: west setup
         env:
           BASE_REF: ${{ github.base_ref }}
         working-directory: zephyrproject/zephyr
         run: |
-          pip install west
           git config --global user.email "you@example.com"
           git config --global user.name "Your Name"
           west init -l . || true

.github/workflows/pylib_tests.yml

@@ -33,21 +33,17 @@ jobs:
     steps:
     - name: checkout
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
         python-version: ${{ matrix.python-version }}
-    - name: cache-pip-linux
-      if: startsWith(runner.os, 'Linux')
-      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ matrix.python-version }}
-        restore-keys: |
-          ${{ runner.os }}-pip-${{ matrix.python-version }}
-    - name: install-packages
+        cache: pip
+        cache-dependency-path: scripts/requirements-actions.txt
+
+    - name: Install Python packages
       run: |
-        pip install -r scripts/requirements-base.txt -r scripts/requirements-build-test.txt
+        pip install -r scripts/requirements-actions.txt --require-hashes
     - name: Run pytest for build_helpers
       env:
         ZEPHYR_BASE: ./

.github/workflows/scripts_tests.yml

@@ -55,19 +55,12 @@ jobs:
         uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: scripts/requirements-actions.txt
 
-      - name: cache-pip-linux
-        if: startsWith(runner.os, 'Linux')
-        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ matrix.python-version }}
-          restore-keys: |
-            ${{ runner.os }}-pip-${{ matrix.python-version }}
-
-      - name: install-packages
+      - name: Install Python packages
         run: |
-          pip install -r scripts/requirements-base.txt -r scripts/requirements-build-test.txt
+          pip install -r scripts/requirements-actions.txt --require-hashes
 
       - name: Run pytest
         env:

.github/workflows/stats_merged_prs.yml

@@ -17,6 +17,17 @@ jobs:
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Set up Python
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: 3.12
+          cache: pip
+          cache-dependency-path: scripts/requirements-actions.txt
+      - name: Install Python packages
+        run: |
+          pip install -r scripts/requirements-actions.txt --require-hashes
+
       - name: PR event
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -24,5 +35,4 @@ jobs:
           ELASTICSEARCH_SERVER: "https://elasticsearch.zephyrproject.io:443"
           PR_STAT_ES_INDEX: ${{ vars.PR_STAT_ES_INDEX }}
         run: |
-          pip install pygithub elasticsearch
           python3 ./scripts/ci/stats/merged_prs.py --pull-request ${{ github.event.pull_request.number }} --repo ${{ github.repository }}

.github/workflows/twister-publish.yaml

@@ -27,6 +27,15 @@ jobs:
         with:
           fetch-depth: 0
           persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: 3.12
+          cache: pip
+          cache-dependency-path: scripts/requirements-actions.txt
+      - name: Install Python packages
+        run: |
+          pip install -r scripts/requirements-actions.txt --require-hashes
 
       - name: Download Artifacts
         id: download-artifacts
@@ -40,7 +49,6 @@ jobs:
       - name: Upload to elasticsearch
         if: steps.download-artifacts.outputs.found_artifact == 'true'
         run: |
-          pip install elasticsearch
           # set run date on upload to get consistent and unified data across the matrix.
           run_date=`date --iso-8601=minutes`
           if [ "${{github.event.workflow_run.event}}" = "push" ]; then

.github/workflows/twister.yaml

@@ -223,13 +223,23 @@ jobs:
 
     steps:
       - name: Check out source code
-        if: needs.twister-build.result == 'failure'
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
           persist-credentials: false
 
+      - name: Set up Python
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: 3.12
+          cache: pip
+          cache-dependency-path: scripts/requirements-actions.txt
+
+      - name: Install Python packages
+        run: |
+          pip install -r scripts/requirements-actions.txt --require-hashes
+
       - name: Download Artifacts
         uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
         with:
@@ -237,7 +247,6 @@ jobs:
 
       - name: Merge Test Results
         run: |
-          pip install junitparser junit2html
           junitparser merge artifacts/*/*/twister.xml junit.xml
           junit2html junit.xml junit.html
 

.github/workflows/twister_tests.yml

@@ -40,21 +40,18 @@ jobs:
     steps:
     - name: checkout
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
         python-version: ${{ matrix.python-version }}
-    - name: cache-pip-linux
-      if: startsWith(runner.os, 'Linux')
-      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ matrix.python-version }}
-        restore-keys: |
-          ${{ runner.os }}-pip-${{ matrix.python-version }}
-    - name: install-packages
+        cache: pip
+        cache-dependency-path: scripts/requirements-actions.txt
+
+    - name: Install Python packages
       run: |
-        pip install -r scripts/requirements-base.txt -r scripts/requirements-build-test.txt -r scripts/requirements-run-test.txt
+        pip install -r scripts/requirements-actions.txt --require-hashes
+
     - name: Run pytest for twisterlib
       env:
         ZEPHYR_BASE: ./

.github/workflows/west_cmds.yml

@@ -37,42 +37,23 @@ jobs:
     steps:
     - name: checkout
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
         python-version: ${{ matrix.python-version }}
-    - name: cache-pip-linux
-      if: startsWith(runner.os, 'Linux')
-      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ matrix.python-version }}
-        restore-keys: |
-          ${{ runner.os }}-pip-${{ matrix.python-version }}
-    - name: cache-pip-mac
-      if: startsWith(runner.os, 'macOS')
-      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
-      with:
-        path: ~/Library/Caches/pip
-        # Trailing '-' was just to get a different cache name
-        key: ${{ runner.os }}-pip-${{ matrix.python-version }}-
-        restore-keys: |
-          ${{ runner.os }}-pip-${{ matrix.python-version }}-
-    - name: cache-pip-win
-      if: startsWith(runner.os, 'Windows')
-      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
-      with:
-        path: ~\AppData\Local\pip\Cache
-        key: ${{ runner.os }}-pip-${{ matrix.python-version }}
-        restore-keys: |
-          ${{ runner.os }}-pip-${{ matrix.python-version }}
-    - name: install pytest
+        cache: pip
+        cache-dependency-path: scripts/requirements-actions.txt
+
+    - name: Install Python packages
       run: |
-        pip install pytest west pyelftools canopen natsort progress mypy intelhex psutil ply pyserial anytree junitparser
+        pip install -r scripts/requirements-actions.txt --require-hashes
+
     - name: run pytest-win
       if: runner.os == 'Windows'
       run: |
         python ./scripts/west_commands/run_tests.py
+
     - name: run pytest-mac-linux
       if: runner.os != 'Windows'
       run: |

scripts/requirements-actions.in

@@ -0,0 +1,40 @@
+# zephyr-keep-sorted-start
+anytree
+awscli
+canopen
+clang-format>=15.0.0
+elasticsearch
+exceptiongroup>=1.0.0rc8
+gcovr
+gitlint>=0.19.1
+ijson
+intelhex
+junit2html
+junitparser>=2
+lxml>=5.3.0
+mock
+mypy
+natsort
+ply>=3.10
+progress
+psutil>=5.6.6
+pyelftools>=0.29
+pygithub
+pykwalify
+pylint>=3
+pyserial
+pytest
+python-magic-bin; sys_platform == "win32"
+python-magic; sys_platform != "win32"
+pyyaml
+ruff==0.8.1
+setuptools>=70.2.0
+sphinx-lint
+tabulate
+tomli>=1.1.0
+tox
+unidiff
+west>=0.14.0
+xlsxwriter
+yamllint
+# zephyr-keep-sorted-stop