[ci] Checkout only relevant submodules (#17250)

* [ci] Checkout only relevant submodules 1. Associate submodules to platforms for which they're needed. 2. Add a script that is able to checkout submodules for specific platforms only. 3. Use the script in CI workflows. * Fix cirque build * Address code reviews
project-chip · Aug 10, 2023 · 2052191 · 2052191
1 parent 5af5edf
commit 2052191
Show file tree

Hide file tree

Showing 25 changed files with 207 additions and 32 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -52,10 +52,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform linux
             - name: Bootstrap
               timeout-minutes: 10
               run: scripts/build/gn_bootstrap.sh
@@ -126,7 +127,6 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
@@ -135,6 +135,8 @@ jobs:
             #   uses: github/codeql-action/init@v1
             #   with:
             #      languages: "cpp"
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform linux
             - name: Bootstrap
               timeout-minutes: 10
               run: scripts/build/gn_bootstrap.sh
@@ -255,10 +257,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform linux
             - name: Bootstrap
               timeout-minutes: 10
               run: scripts/build/gn_bootstrap.sh
@@ -286,7 +289,6 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
@@ -295,6 +297,8 @@ jobs:
             #  uses: github/codeql-action/init@v1
             #  with:
             #     languages: "cpp"
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform darwin
             - name: Setup Environment
               run: brew install openssl pkg-config
             - name: Try to ensure the directory for diagnostic log collection exists

diff --git a/.github/workflows/cirque.yaml b/.github/workflows/cirque.yaml
@@ -53,10 +53,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform linux
 
             - name: Get cirque cache key
               id: cirque-key

diff --git a/.github/workflows/darwin.yaml b/.github/workflows/darwin.yaml
@@ -37,10 +37,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform darwin
             - name: Setup Environment
               run: brew install openssl pkg-config python@3.9
             - name: Fix pkgconfig link

diff --git a/.github/workflows/examples-ameba.yaml b/.github/workflows/examples-ameba.yaml
@@ -41,9 +41,10 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform ameba
             - name: Build example Ameba All Clusters App
               run: scripts/run_in_build_env.sh "./scripts/build/build_examples.py --target-glob 'ameba-*' build"
diff --git a/.github/workflows/examples-cc13x2x7_26x2x7.yaml b/.github/workflows/examples-cc13x2x7_26x2x7.yaml
@@ -43,10 +43,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform cc13x2_26x2
             - name: Set up environment for size reports
               if: ${{ !env.ACT }}
               env:

diff --git a/.github/workflows/examples-cyw30739.yaml b/.github/workflows/examples-cyw30739.yaml
@@ -40,10 +40,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform cyw30739
 
             - name: Set up environment for size reports
               if: ${{ !env.ACT }}

diff --git a/.github/workflows/examples-efr32.yaml b/.github/workflows/examples-efr32.yaml
@@ -44,10 +44,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform efr32
 
             - name: Set up environment for size reports
               if: ${{ !env.ACT }}

diff --git a/.github/workflows/examples-esp32.yaml b/.github/workflows/examples-esp32.yaml
@@ -42,10 +42,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform esp32
 
             - name: Set up environment for size reports
               if: ${{ !env.ACT }}
@@ -127,10 +128,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform esp32
 
             - name: Bootstrap
               timeout-minutes: 10

diff --git a/.github/workflows/examples-infineon.yaml b/.github/workflows/examples-infineon.yaml
@@ -40,10 +40,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform p6
 
             - name: Set up environment for size reports
               if: ${{ !env.ACT }}

diff --git a/.github/workflows/examples-k32w.yaml b/.github/workflows/examples-k32w.yaml
@@ -43,10 +43,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform k32w0
 
             - name: Set up environment for size reports
               if: ${{ !env.ACT }}

diff --git a/.github/workflows/examples-linux-arm.yaml b/.github/workflows/examples-linux-arm.yaml
@@ -41,10 +41,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform linux
 
             - name: Set up environment for size reports
               if: ${{ !env.ACT }}

diff --git a/.github/workflows/examples-linux-imx.yaml b/.github/workflows/examples-linux-imx.yaml
@@ -39,10 +39,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform linux
 
             - name: Build App
               timeout-minutes: 30

diff --git a/.github/workflows/examples-linux-standalone.yaml b/.github/workflows/examples-linux-standalone.yaml
@@ -44,10 +44,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform linux
 
             - name: Set up environment for size reports
               if: ${{ !env.ACT }}

diff --git a/.github/workflows/examples-mbed.yaml b/.github/workflows/examples-mbed.yaml
@@ -47,10 +47,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform mbed
 
             - name: Detect changed paths
               uses: dorny/paths-filter@v2

diff --git a/.github/workflows/examples-nrfconnect.yaml b/.github/workflows/examples-nrfconnect.yaml
@@ -44,10 +44,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform nrfconnect
             - name: Detect changed paths
               uses: dorny/paths-filter@v2
               id: changed_paths

diff --git a/.github/workflows/examples-qpg.yaml b/.github/workflows/examples-qpg.yaml
@@ -43,10 +43,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform qpg
 
             - name: Set up environment for size reports
               if: ${{ !env.ACT }}

diff --git a/.github/workflows/examples-telink.yaml b/.github/workflows/examples-telink.yaml
@@ -42,10 +42,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform telink
 
             - name: Set up environment for size reports
               if: ${{ !env.ACT }}

diff --git a/.github/workflows/examples-tizen.yaml b/.github/workflows/examples-tizen.yaml
@@ -44,9 +44,10 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform tizen
             - name: Build example Tizen lighting app
               run: scripts/run_in_build_env.sh "./scripts/build/build_examples.py --target-glob 'tizen-*' build"
diff --git a/.github/workflows/qemu.yaml b/.github/workflows/qemu.yaml
@@ -44,10 +44,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform esp32
             - name: Bootstrap
               timeout-minutes: 25
               run: scripts/build/gn_bootstrap.sh

diff --git a/.github/workflows/smoketest-android.yaml b/.github/workflows/smoketest-android.yaml
@@ -44,10 +44,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform android
             - name: Bootstrap
               timeout-minutes: 10
               run: scripts/build/gn_bootstrap.sh

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -53,10 +53,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform linux
             - name: Try to ensure the directories for core dumping exist and we
                   can write them.
               run: |
@@ -136,10 +137,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform darwin
             - name: Setup Environment
               # coreutils for stdbuf
               run: brew install openssl pkg-config coreutils
@@ -240,8 +242,8 @@ jobs:
         steps:
             - name: Checkout
               uses: actions/checkout@v2
-              with:
-                  submodules: true
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform linux
             - name:
                   Try to ensure the directories for core dumping exist and we
                   can write them.
@@ -311,8 +313,8 @@ jobs:
         steps:
             - name: Checkout
               uses: actions/checkout@v2
-              with:
-                  submodules: true
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform darwin
             - name: Setup Environment
               # coreutils for stdbuf
               run: brew install openssl pkg-config coreutils

diff --git a/.github/workflows/unit_integration_test.yaml b/.github/workflows/unit_integration_test.yaml
@@ -48,10 +48,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform linux
             - name: Bootstrap
               timeout-minutes: 10
               run: |

diff --git a/.github/workflows/zap_templates.yaml b/.github/workflows/zap_templates.yaml
@@ -41,10 +41,11 @@ jobs:
               with:
                   action: actions/checkout@v3
                   with: |
-                      submodules: true
                       token: ${{ github.token }}
                   attempt_limit: 3
                   attempt_delay: 2000
+            - name: Checkout submodules
+              run: scripts/checkout_submodules.py --shallow --platform linux
             - name: Setup ZAP
               timeout-minutes: 10
               run: |