From d8ad9da9b38edde298d1515cc5246e629113efc4 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 09:09:10 -0500 Subject: [PATCH 01/62] Add GitHub Action --- .github/workflows/generate_sbom.yml | 82 +++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/generate_sbom.yml diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml new file mode 100644 index 0000000000..18cf5bcc8a --- /dev/null +++ b/.github/workflows/generate_sbom.yml @@ -0,0 +1,82 @@ +name: Generate SBOM + +on: + workflow_dispatch: + push: + branches: + - 'master' + - 'releases/**' + - 'CXX**' + +jobs: + configure-and-scan: + permissions: + id-token: write # Required to request a json web token (JWT) for keyless authentication with Endor Labs + #packages: write + contents: read + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v6 + with: + submodules: recursive + + - name: Install dev libs + run: sudo apt install -y libsasl2-dev libsnappy-dev libssl-dev libmongocrypt-dev + + - name: Configure CMake and fetch dependency source + env: + BUILD_TYPE: Release + BUILD: ${{github.workspace}}/build + CXX_STANDARD: 17 + working-directory: ${{env.BUILD}} + run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} + + # - name: Install endorctl and Scan with Endor Labs + # uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 + # with: + # additional_args: --languages=c + # log_level: info + # log_verbose: false + # namespace: mongodb.${{github.repository_owner}} + # pr: false + # scan_dependencies: true + # tags: github_action + # env: + # ENDOR_SCAN_EMBEDDINGS: true + + - name: Setup Endor Labs Endorctl + uses: endorlabs/github-action/setup@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 + with: + namespace: mongodb.${{github.repository_owner}} + enable_github_action_token: true + + - name: Run Endorctl + env: + ENDOR_GITHUB_ACTION_TOKEN_ENABLE: true + ENDOR_SCAN_DEPENDENCIES: true + ENDOR_SCAN_EMBEDDINGS: true + ENDOR_SCAN_INCLUDE_PATH: "build/_deps/**" + ENDOR_SCAN_LANGUAGES: c + #ENDOR_SCAN_SUMMARY_OUTPUT_TYPE: json + ENDOR_SCAN_TAGS: github_action + run: | + endorctl scan + + # - uses: actions/setup-python@v6 + # with: + # python-version: '3.10' + # - run: python my_script.py + + # ${{ github.sha }} + # - name: Run Endorctl + # env: + # ENDOR_GITHUB_ACTION_TOKEN_ENABLE: true + # ENDOR_SCAN_DEPENDENCIES: true + # ENDOR_SCAN_EMBEDDINGS: true + # ENDOR_SCAN_INCLUDE_PATH: + # ENDOR_SCAN_LANGUAGES: c + # ENDOR_SCAN_SUMMARY_OUTPUT_TYPE: json + # ENDOR_SCAN_TAGS: github_action + # run: | + # endorctl scan From 579c8d7705bf6e5493fe317ea4a770a054fd56ab Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 09:12:16 -0500 Subject: [PATCH 02/62] Adjust trigger --- .github/workflows/generate_sbom.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 18cf5bcc8a..987189f0b4 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -3,10 +3,10 @@ name: Generate SBOM on: workflow_dispatch: push: - branches: - - 'master' - - 'releases/**' - - 'CXX**' + # branches: + # - 'master' + # - 'releases/**' + # - 'CXX**' jobs: configure-and-scan: From e404ce539cc6f2865f96e61707f7cae58934ca53 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 09:16:31 -0500 Subject: [PATCH 03/62] Set working dir and verbose --- .github/workflows/generate_sbom.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 987189f0b4..34853a4d77 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -52,6 +52,7 @@ jobs: enable_github_action_token: true - name: Run Endorctl + working-directory: ${{github.workspace}} env: ENDOR_GITHUB_ACTION_TOKEN_ENABLE: true ENDOR_SCAN_DEPENDENCIES: true @@ -60,6 +61,7 @@ jobs: ENDOR_SCAN_LANGUAGES: c #ENDOR_SCAN_SUMMARY_OUTPUT_TYPE: json ENDOR_SCAN_TAGS: github_action + ENDOR_LOG_VERBOSE: true run: | endorctl scan From 87e563a3c92ad3668c19a40a7b05ed0e3af76e96 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 09:19:47 -0500 Subject: [PATCH 04/62] Disable exclude path --- .github/workflows/generate_sbom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 34853a4d77..2e8aa55f83 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -57,7 +57,7 @@ jobs: ENDOR_GITHUB_ACTION_TOKEN_ENABLE: true ENDOR_SCAN_DEPENDENCIES: true ENDOR_SCAN_EMBEDDINGS: true - ENDOR_SCAN_INCLUDE_PATH: "build/_deps/**" + #ENDOR_SCAN_INCLUDE_PATH: "build/_deps/**" ENDOR_SCAN_LANGUAGES: c #ENDOR_SCAN_SUMMARY_OUTPUT_TYPE: json ENDOR_SCAN_TAGS: github_action From 602e0beaf2b633b8a867957ebd6dba45baad5f66 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 09:38:12 -0500 Subject: [PATCH 05/62] Delete unneeded source --- .github/workflows/generate_sbom.yml | 61 ++++++++++++++++------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 2e8aa55f83..cdeb95bba6 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -22,7 +22,7 @@ jobs: submodules: recursive - name: Install dev libs - run: sudo apt install -y libsasl2-dev libsnappy-dev libssl-dev libmongocrypt-dev + run: sudo apt install -y libsasl2-dev libsnappy-dev libssl-dev libmongocrypt-dev=1.13.0-1 - name: Configure CMake and fetch dependency source env: @@ -32,38 +32,43 @@ jobs: working-directory: ${{env.BUILD}} run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} - # - name: Install endorctl and Scan with Endor Labs - # uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 - # with: - # additional_args: --languages=c - # log_level: info - # log_verbose: false - # namespace: mongodb.${{github.repository_owner}} - # pr: false - # scan_dependencies: true - # tags: github_action - # env: - # ENDOR_SCAN_EMBEDDINGS: true + - name: Remove unneeded source files + working-directory: ${{github.workspace}} + run: | + rm -rf benchmark cmake data docs etc examples extras generate_uninstall src - - name: Setup Endor Labs Endorctl - uses: endorlabs/github-action/setup@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 + - name: Install endorctl and Scan with Endor Labs + uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 with: + additional_args: --languages=c + log_level: info + log_verbose: false namespace: mongodb.${{github.repository_owner}} - enable_github_action_token: true - - - name: Run Endorctl - working-directory: ${{github.workspace}} + pr: false + scan_dependencies: true + tags: github_action env: - ENDOR_GITHUB_ACTION_TOKEN_ENABLE: true - ENDOR_SCAN_DEPENDENCIES: true ENDOR_SCAN_EMBEDDINGS: true - #ENDOR_SCAN_INCLUDE_PATH: "build/_deps/**" - ENDOR_SCAN_LANGUAGES: c - #ENDOR_SCAN_SUMMARY_OUTPUT_TYPE: json - ENDOR_SCAN_TAGS: github_action - ENDOR_LOG_VERBOSE: true - run: | - endorctl scan + + # - name: Setup Endor Labs Endorctl + # uses: endorlabs/github-action/setup@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 + # with: + # namespace: mongodb.${{github.repository_owner}} + # enable_github_action_token: true + + # - name: Run Endorctl + # working-directory: ${{github.workspace}} + # env: + # ENDOR_GITHUB_ACTION_TOKEN_ENABLE: true + # ENDOR_SCAN_DEPENDENCIES: true + # ENDOR_SCAN_EMBEDDINGS: true + # #ENDOR_SCAN_INCLUDE_PATH: "build/_deps/**" + # ENDOR_SCAN_LANGUAGES: c + # ENDOR_SCAN_SUMMARY_OUTPUT_TYPE: json + # ENDOR_SCAN_TAGS: github_action + # ENDOR_LOG_VERBOSE: true + # run: | + # endorctl scan # - uses: actions/setup-python@v6 # with: From 191f47c94d2918defbfd01e155737b4467292845 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 09:42:06 -0500 Subject: [PATCH 06/62] Disable Install dev libs --- .github/workflows/generate_sbom.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index cdeb95bba6..2e028bd33a 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -21,8 +21,8 @@ jobs: with: submodules: recursive - - name: Install dev libs - run: sudo apt install -y libsasl2-dev libsnappy-dev libssl-dev libmongocrypt-dev=1.13.0-1 +# - name: Install dev libs +# run: sudo apt install -y libsasl2-dev libsnappy-dev libssl-dev libmongocrypt-dev - name: Configure CMake and fetch dependency source env: From 8498f94d4160e94662c8e4e3f9f796ed7761e559 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 09:45:45 -0500 Subject: [PATCH 07/62] Add exclude path src/** w/o quotes --- .github/workflows/generate_sbom.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 2e028bd33a..31d9162f57 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -36,11 +36,12 @@ jobs: working-directory: ${{github.workspace}} run: | rm -rf benchmark cmake data docs etc examples extras generate_uninstall src + ls -al - name: Install endorctl and Scan with Endor Labs uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 with: - additional_args: --languages=c + additional_args: "--languages=c --exclude-path=src/**" log_level: info log_verbose: false namespace: mongodb.${{github.repository_owner}} From 7dce2aaf15bf6c64da6255aebf645c6015f92be9 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 09:51:21 -0500 Subject: [PATCH 08/62] Exclude pathwith quotes --- .github/workflows/generate_sbom.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 31d9162f57..b6aab61b7b 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -32,16 +32,16 @@ jobs: working-directory: ${{env.BUILD}} run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} - - name: Remove unneeded source files - working-directory: ${{github.workspace}} - run: | - rm -rf benchmark cmake data docs etc examples extras generate_uninstall src - ls -al + # - name: Remove unneeded source files + # working-directory: ${{github.workspace}} + # run: | + # git rm -r -- benchmark cmake data docs etc examples extras generate_uninstall src + # ls -al - name: Install endorctl and Scan with Endor Labs uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 with: - additional_args: "--languages=c --exclude-path=src/**" + additional_args: "--languages=c --exclude-path=\"src/**\"" log_level: info log_verbose: false namespace: mongodb.${{github.repository_owner}} From 6cbeb35a696a40fbb1a424c9de6dc19af089985e Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 10:54:04 -0500 Subject: [PATCH 09/62] Add ENABLE_TESTS=ON for Catch2 --- .github/workflows/generate_sbom.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index b6aab61b7b..55f026ae8f 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -30,7 +30,7 @@ jobs: BUILD: ${{github.workspace}}/build CXX_STANDARD: 17 working-directory: ${{env.BUILD}} - run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} + run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON # - name: Remove unneeded source files # working-directory: ${{github.workspace}} @@ -41,7 +41,7 @@ jobs: - name: Install endorctl and Scan with Endor Labs uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 with: - additional_args: "--languages=c --exclude-path=\"src/**\"" + additional_args: "--languages=c --exclude-path=\"benchmark/**\" --exclude-path=\"src/**\" --exclude-path=\"build/CMakeFiles/**\"" log_level: info log_verbose: false namespace: mongodb.${{github.repository_owner}} From ae82151c3d7eff4c16859725c5ea03ee7c9aee51 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 11:17:23 -0500 Subject: [PATCH 10/62] Use scan profile for extra args --- .github/workflows/generate_sbom.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 55f026ae8f..9a2ef0ebca 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -38,10 +38,25 @@ jobs: # git rm -r -- benchmark cmake data docs etc examples extras generate_uninstall src # ls -al + - name: Create and populate .endorctl/scanprofile.yaml file + run: | + md .endorctl + cat < .endorctl/scanprofile.yaml + kind: AutomatedScanParameters + spec: + automated_scan_parameters: + included_paths: + - build/_deps/** + languages: + - c + EOF + echo "cat .endorctl/scanprofile.yaml" + cat .endorctl/scanprofile.yaml + - name: Install endorctl and Scan with Endor Labs uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 with: - additional_args: "--languages=c --exclude-path=\"benchmark/**\" --exclude-path=\"src/**\" --exclude-path=\"build/CMakeFiles/**\"" + #additional_args: "--languages=c --exclude-path=\"benchmark/**\" --exclude-path=\"src/**\" --exclude-path=\"build/CMakeFiles/**\"" log_level: info log_verbose: false namespace: mongodb.${{github.repository_owner}} From 06876c83862a5b2050b7581bd7092c4ebe15f285 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 11:18:47 -0500 Subject: [PATCH 11/62] Use scan profile for extra args fix --- .github/workflows/generate_sbom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 9a2ef0ebca..0aab0e7e06 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -40,7 +40,7 @@ jobs: - name: Create and populate .endorctl/scanprofile.yaml file run: | - md .endorctl + mkdir .endorctl cat < .endorctl/scanprofile.yaml kind: AutomatedScanParameters spec: From 73ed54d456a4fa3be2936b6ded3c3d0df6494381 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 11:29:57 -0500 Subject: [PATCH 12/62] Scan profile with shell run --- .github/workflows/generate_sbom.yml | 60 ++++++++++++++--------------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 0aab0e7e06..939ef7a91f 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -38,6 +38,25 @@ jobs: # git rm -r -- benchmark cmake data docs etc examples extras generate_uninstall src # ls -al + # - name: Install endorctl and Scan with Endor Labs + # uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 + # with: + # #additional_args: "--languages=c --exclude-path=\"benchmark/**\" --exclude-path=\"src/**\" --exclude-path=\"build/CMakeFiles/**\"" + # log_level: info + # log_verbose: false + # namespace: mongodb.${{github.repository_owner}} + # pr: false + # scan_dependencies: true + # tags: github_action + # env: + # ENDOR_SCAN_EMBEDDINGS: true + + - name: Setup Endor Labs Endorctl + uses: endorlabs/github-action/setup@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 + with: + namespace: mongodb.${{github.repository_owner}} + enable_github_action_token: true + - name: Create and populate .endorctl/scanprofile.yaml file run: | mkdir .endorctl @@ -45,46 +64,23 @@ jobs: kind: AutomatedScanParameters spec: automated_scan_parameters: + additional_environment_variables: + - ENDOR_SCAN_EMBEDDINGS=true included_paths: - build/_deps/** + #excluded_paths: + # - benchmark/** + # - src/** languages: - c + scan_dependencies: true + tags: github_action EOF echo "cat .endorctl/scanprofile.yaml" cat .endorctl/scanprofile.yaml - - name: Install endorctl and Scan with Endor Labs - uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 - with: - #additional_args: "--languages=c --exclude-path=\"benchmark/**\" --exclude-path=\"src/**\" --exclude-path=\"build/CMakeFiles/**\"" - log_level: info - log_verbose: false - namespace: mongodb.${{github.repository_owner}} - pr: false - scan_dependencies: true - tags: github_action - env: - ENDOR_SCAN_EMBEDDINGS: true - - # - name: Setup Endor Labs Endorctl - # uses: endorlabs/github-action/setup@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 - # with: - # namespace: mongodb.${{github.repository_owner}} - # enable_github_action_token: true - - # - name: Run Endorctl - # working-directory: ${{github.workspace}} - # env: - # ENDOR_GITHUB_ACTION_TOKEN_ENABLE: true - # ENDOR_SCAN_DEPENDENCIES: true - # ENDOR_SCAN_EMBEDDINGS: true - # #ENDOR_SCAN_INCLUDE_PATH: "build/_deps/**" - # ENDOR_SCAN_LANGUAGES: c - # ENDOR_SCAN_SUMMARY_OUTPUT_TYPE: json - # ENDOR_SCAN_TAGS: github_action - # ENDOR_LOG_VERBOSE: true - # run: | - # endorctl scan + - name: Run Endorctl + run: endorctl scan # - uses: actions/setup-python@v6 # with: From 6fb1c637d82c779cee6e641deb4abf4f0b365dbc Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 11:32:31 -0500 Subject: [PATCH 13/62] With git add --- .github/workflows/generate_sbom.yml | 45 +++++++++++++++-------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 939ef7a91f..141ce48d7c 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -32,6 +32,29 @@ jobs: working-directory: ${{env.BUILD}} run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON + - name: Create and populate .endorctl/scanprofile.yaml file + run: | + mkdir .endorctl + cat < .endorctl/scanprofile.yaml + kind: AutomatedScanParameters + spec: + automated_scan_parameters: + additional_environment_variables: + - ENDOR_SCAN_EMBEDDINGS=true + included_paths: + - build/_deps/** + #excluded_paths: + # - benchmark/** + # - src/** + languages: + - c + scan_dependencies: true + tags: github_action + EOF + git add .endorctl/scanprofile.yaml + echo "cat .endorctl/scanprofile.yaml" + cat .endorctl/scanprofile.yaml + # - name: Remove unneeded source files # working-directory: ${{github.workspace}} # run: | @@ -57,28 +80,6 @@ jobs: namespace: mongodb.${{github.repository_owner}} enable_github_action_token: true - - name: Create and populate .endorctl/scanprofile.yaml file - run: | - mkdir .endorctl - cat < .endorctl/scanprofile.yaml - kind: AutomatedScanParameters - spec: - automated_scan_parameters: - additional_environment_variables: - - ENDOR_SCAN_EMBEDDINGS=true - included_paths: - - build/_deps/** - #excluded_paths: - # - benchmark/** - # - src/** - languages: - - c - scan_dependencies: true - tags: github_action - EOF - echo "cat .endorctl/scanprofile.yaml" - cat .endorctl/scanprofile.yaml - - name: Run Endorctl run: endorctl scan From 9966a2c19a5dfc435959c9d36e700fc58dffd8e4 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 11:38:48 -0500 Subject: [PATCH 14/62] ENDOR_SCAN_USE_SCAN_PROFILE --- .github/workflows/generate_sbom.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 141ce48d7c..761dc6506c 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -81,6 +81,8 @@ jobs: enable_github_action_token: true - name: Run Endorctl + env: + ENDOR_SCAN_USE_SCAN_PROFILE: true run: endorctl scan # - uses: actions/setup-python@v6 From 74b602c3abe1298ec606e00529150c2ebffe53cb Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 11:41:57 -0500 Subject: [PATCH 15/62] NDOR_SCAN_USE_SCAN_PROFILE with gh action --- .github/workflows/generate_sbom.yml | 39 +++++++++++++++-------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 761dc6506c..075c60487c 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -61,29 +61,30 @@ jobs: # git rm -r -- benchmark cmake data docs etc examples extras generate_uninstall src # ls -al - # - name: Install endorctl and Scan with Endor Labs - # uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 - # with: - # #additional_args: "--languages=c --exclude-path=\"benchmark/**\" --exclude-path=\"src/**\" --exclude-path=\"build/CMakeFiles/**\"" - # log_level: info - # log_verbose: false - # namespace: mongodb.${{github.repository_owner}} - # pr: false - # scan_dependencies: true - # tags: github_action - # env: - # ENDOR_SCAN_EMBEDDINGS: true - - - name: Setup Endor Labs Endorctl - uses: endorlabs/github-action/setup@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 + - name: Install endorctl and Scan with Endor Labs + uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 with: + #additional_args: "--languages=c --exclude-path=\"benchmark/**\" --exclude-path=\"src/**\" --exclude-path=\"build/CMakeFiles/**\"" + log_level: info + log_verbose: false namespace: mongodb.${{github.repository_owner}} - enable_github_action_token: true - - - name: Run Endorctl + pr: false + scan_dependencies: true + tags: github_action env: + ENDOR_SCAN_EMBEDDINGS: true ENDOR_SCAN_USE_SCAN_PROFILE: true - run: endorctl scan + + # - name: Setup Endor Labs Endorctl + # uses: endorlabs/github-action/setup@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 + # with: + # namespace: mongodb.${{github.repository_owner}} + # enable_github_action_token: true + + # - name: Run Endorctl + # env: + # ENDOR_SCAN_USE_SCAN_PROFILE: true + # run: endorctl scan # - uses: actions/setup-python@v6 # with: From 0f8e52eacfc318cc3396428c9702df2b4eac1fae Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 11:45:14 -0500 Subject: [PATCH 16/62] Remove unneeded source files with git rm --- .github/workflows/generate_sbom.yml | 57 ++++++++++++++--------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 075c60487c..05f73e05ab 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -32,39 +32,39 @@ jobs: working-directory: ${{env.BUILD}} run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON - - name: Create and populate .endorctl/scanprofile.yaml file - run: | - mkdir .endorctl - cat < .endorctl/scanprofile.yaml - kind: AutomatedScanParameters - spec: - automated_scan_parameters: - additional_environment_variables: - - ENDOR_SCAN_EMBEDDINGS=true - included_paths: - - build/_deps/** - #excluded_paths: - # - benchmark/** - # - src/** - languages: - - c - scan_dependencies: true - tags: github_action - EOF - git add .endorctl/scanprofile.yaml - echo "cat .endorctl/scanprofile.yaml" - cat .endorctl/scanprofile.yaml - - # - name: Remove unneeded source files - # working-directory: ${{github.workspace}} + # - name: Create and populate .endorctl/scanprofile.yaml file # run: | - # git rm -r -- benchmark cmake data docs etc examples extras generate_uninstall src - # ls -al + # mkdir .endorctl + # cat < .endorctl/scanprofile.yaml + # kind: AutomatedScanParameters + # spec: + # automated_scan_parameters: + # additional_environment_variables: + # - ENDOR_SCAN_EMBEDDINGS=true + # included_paths: + # - build/_deps/** + # #excluded_paths: + # # - benchmark/** + # # - src/** + # languages: + # - c + # scan_dependencies: true + # tags: github_action + # EOF + # git add .endorctl/scanprofile.yaml + # echo "cat .endorctl/scanprofile.yaml" + # cat .endorctl/scanprofile.yaml + + - name: Remove unneeded source files + working-directory: ${{github.workspace}} + run: | + git rm -r -- benchmark cmake data docs etc examples extras generate_uninstall src + ls -al - name: Install endorctl and Scan with Endor Labs uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 with: - #additional_args: "--languages=c --exclude-path=\"benchmark/**\" --exclude-path=\"src/**\" --exclude-path=\"build/CMakeFiles/**\"" + additional_args: "--languages=c" log_level: info log_verbose: false namespace: mongodb.${{github.repository_owner}} @@ -73,7 +73,6 @@ jobs: tags: github_action env: ENDOR_SCAN_EMBEDDINGS: true - ENDOR_SCAN_USE_SCAN_PROFILE: true # - name: Setup Endor Labs Endorctl # uses: endorlabs/github-action/setup@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 From 728ea0c47017f65ead72a98e6e6c4c03c0ec02e5 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 11:50:43 -0500 Subject: [PATCH 17/62] With got add . --- .github/workflows/generate_sbom.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 05f73e05ab..10be8953c6 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -58,7 +58,8 @@ jobs: - name: Remove unneeded source files working-directory: ${{github.workspace}} run: | - git rm -r -- benchmark cmake data docs etc examples extras generate_uninstall src + git add . + git rm -r --quiet benchmark cmake data docs etc examples extras generate_uninstall src ls -al - name: Install endorctl and Scan with Endor Labs From b645cd7647eda2aec8b547a15a3750a1a6fdb2af Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 11:53:11 -0500 Subject: [PATCH 18/62] add path: build/_deps --- .github/workflows/generate_sbom.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 10be8953c6..639576cd50 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -69,6 +69,7 @@ jobs: log_level: info log_verbose: false namespace: mongodb.${{github.repository_owner}} + path: build/_deps pr: false scan_dependencies: true tags: github_action From c7b7834e20ff13560bdd2f5c95efcf5752051629 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 11:56:55 -0500 Subject: [PATCH 19/62] scan_path --- .github/workflows/generate_sbom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 639576cd50..996f61a2ae 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -69,7 +69,7 @@ jobs: log_level: info log_verbose: false namespace: mongodb.${{github.repository_owner}} - path: build/_deps + scan_path: build/_deps pr: false scan_dependencies: true tags: github_action From ef75eba0e183d405264fc6b68d1565b3acf09c38 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 11:59:51 -0500 Subject: [PATCH 20/62] include-path --- .github/workflows/generate_sbom.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 996f61a2ae..b2f6453e0d 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -65,11 +65,10 @@ jobs: - name: Install endorctl and Scan with Endor Labs uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 with: - additional_args: "--languages=c" + additional_args: "--languages=c --include-path=\"build/_deps/**\"" log_level: info log_verbose: false namespace: mongodb.${{github.repository_owner}} - scan_path: build/_deps pr: false scan_dependencies: true tags: github_action From 294b846d75061a9269a0e2adf98cb7dd21570040 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 12:04:37 -0500 Subject: [PATCH 21/62] include build/** --- .github/workflows/generate_sbom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index b2f6453e0d..c61860284a 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -65,7 +65,7 @@ jobs: - name: Install endorctl and Scan with Endor Labs uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 with: - additional_args: "--languages=c --include-path=\"build/_deps/**\"" + additional_args: "--languages=c --include-path=\"build/**\"" log_level: info log_verbose: false namespace: mongodb.${{github.repository_owner}} From d287e8fa800273331947c2c68ec6de4ef758cba0 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 12:07:47 -0500 Subject: [PATCH 22/62] Rename build folder --- .github/workflows/generate_sbom.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index c61860284a..6579d8b213 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -55,17 +55,13 @@ jobs: # echo "cat .endorctl/scanprofile.yaml" # cat .endorctl/scanprofile.yaml - - name: Remove unneeded source files - working-directory: ${{github.workspace}} - run: | - git add . - git rm -r --quiet benchmark cmake data docs etc examples extras generate_uninstall src - ls -al + - name: Rename build folder # Endor Labs will automatically try to exclude "build" + run: mv build third_party - name: Install endorctl and Scan with Endor Labs uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 with: - additional_args: "--languages=c --include-path=\"build/**\"" + additional_args: "--languages=c --include-path=third_party/**" log_level: info log_verbose: false namespace: mongodb.${{github.repository_owner}} From 71156e7dc3d78418c13761831b0b5926162903ed Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 12:10:09 -0500 Subject: [PATCH 23/62] w/git add --- .github/workflows/generate_sbom.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 6579d8b213..a881b65707 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -56,7 +56,9 @@ jobs: # cat .endorctl/scanprofile.yaml - name: Rename build folder # Endor Labs will automatically try to exclude "build" - run: mv build third_party + run: | + mv build third_party + git add third_party - name: Install endorctl and Scan with Endor Labs uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 From 99fad4f4a6f0fc68e68028389b1c92b3bb041a1e Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sat, 22 Nov 2025 12:14:32 -0500 Subject: [PATCH 24/62] revert to full scan --- .github/workflows/generate_sbom.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index a881b65707..2d0eb3c20a 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -55,15 +55,15 @@ jobs: # echo "cat .endorctl/scanprofile.yaml" # cat .endorctl/scanprofile.yaml - - name: Rename build folder # Endor Labs will automatically try to exclude "build" - run: | - mv build third_party - git add third_party + # - name: Rename build folder # Endor Labs will automatically try to exclude "build" + # run: | + # mv build third_party + # git add third_party - name: Install endorctl and Scan with Endor Labs uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 with: - additional_args: "--languages=c --include-path=third_party/**" + additional_args: "--languages=c" log_level: info log_verbose: false namespace: mongodb.${{github.repository_owner}} From 696d21b7c671cf32a943ea15ccafe32056f02726 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Sun, 23 Nov 2025 10:27:37 -0500 Subject: [PATCH 25/62] Add an ignore, add python, add old sbom --- .github/workflows/generate_sbom.yml | 78 +-- etc/sbom/config.py | 204 ++++++++ etc/sbom/endorctl_utils.py | 486 ++++++++++++++++++ etc/sbom/generate_sbom.py | 758 ++++++++++++++++++++++++++++ etc/sbom/metadata.cdx.json | 304 +++++++++++ sbom.json | 85 ++++ 6 files changed, 1852 insertions(+), 63 deletions(-) create mode 100644 etc/sbom/config.py create mode 100644 etc/sbom/endorctl_utils.py create mode 100755 etc/sbom/generate_sbom.py create mode 100644 etc/sbom/metadata.cdx.json create mode 100644 sbom.json diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 2d0eb3c20a..bae69d286b 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -12,7 +12,6 @@ jobs: configure-and-scan: permissions: id-token: write # Required to request a json web token (JWT) for keyless authentication with Endor Labs - #packages: write contents: read runs-on: ubuntu-latest steps: @@ -21,10 +20,7 @@ jobs: with: submodules: recursive -# - name: Install dev libs -# run: sudo apt install -y libsasl2-dev libsnappy-dev libssl-dev libmongocrypt-dev - - - name: Configure CMake and fetch dependency source + - name: Configure CMake and fetch dependency sources env: BUILD_TYPE: Release BUILD: ${{github.workspace}}/build @@ -32,38 +28,10 @@ jobs: working-directory: ${{env.BUILD}} run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON - # - name: Create and populate .endorctl/scanprofile.yaml file - # run: | - # mkdir .endorctl - # cat < .endorctl/scanprofile.yaml - # kind: AutomatedScanParameters - # spec: - # automated_scan_parameters: - # additional_environment_variables: - # - ENDOR_SCAN_EMBEDDINGS=true - # included_paths: - # - build/_deps/** - # #excluded_paths: - # # - benchmark/** - # # - src/** - # languages: - # - c - # scan_dependencies: true - # tags: github_action - # EOF - # git add .endorctl/scanprofile.yaml - # echo "cat .endorctl/scanprofile.yaml" - # cat .endorctl/scanprofile.yaml - - # - name: Rename build folder # Endor Labs will automatically try to exclude "build" - # run: | - # mv build third_party - # git add third_party - - name: Install endorctl and Scan with Endor Labs - uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 + uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 with: - additional_args: "--languages=c" + additional_args: "--languages=c --exclude-path=\"build/CMakeFiles/**\"" log_level: info log_verbose: false namespace: mongodb.${{github.repository_owner}} @@ -73,31 +41,15 @@ jobs: env: ENDOR_SCAN_EMBEDDINGS: true - # - name: Setup Endor Labs Endorctl - # uses: endorlabs/github-action/setup@519df81de5f68536c84ae05ebb2986d0bb1d19fc # Release v1.1.8 - # with: - # namespace: mongodb.${{github.repository_owner}} - # enable_github_action_token: true - - # - name: Run Endorctl - # env: - # ENDOR_SCAN_USE_SCAN_PROFILE: true - # run: endorctl scan - - # - uses: actions/setup-python@v6 - # with: - # python-version: '3.10' - # - run: python my_script.py - - # ${{ github.sha }} - # - name: Run Endorctl - # env: - # ENDOR_GITHUB_ACTION_TOKEN_ENABLE: true - # ENDOR_SCAN_DEPENDENCIES: true - # ENDOR_SCAN_EMBEDDINGS: true - # ENDOR_SCAN_INCLUDE_PATH: - # ENDOR_SCAN_LANGUAGES: c - # ENDOR_SCAN_SUMMARY_OUTPUT_TYPE: json - # ENDOR_SCAN_TAGS: github_action - # run: | - # endorctl scan + - name: Set up Python 3.10 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: '3.10' + - uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4 + with: + python-version: "3.10" + activate-environment: true + - name: Install dependencies + run: uv sync --group make_release + - name: generate_sbom.py + run: uv run etc/sbom/generate_sbom.py --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json \ No newline at end of file diff --git a/etc/sbom/config.py b/etc/sbom/config.py new file mode 100644 index 0000000000..5c92c24fcb --- /dev/null +++ b/etc/sbom/config.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""generate_sbom.py config. Operational configuration values stored separately from the core code.""" + +import json +import logging +import re + +logger = logging.getLogger("generate_sbom") +logger.setLevel(logging.NOTSET) + + +# ################ Component Filters ################ + +# List of Endor Labs SBOM components that must be removed before processing +endor_components_remove = [ + # An incorrect match from parts of pkg:github/madler/zlib + "zlib-ng/zlib-ng", +] + +# bom-ref prefixes (Endor Labs has been changing them, so add all that we have seen) +prefixes = [ + "pkg:c/github.com/", + "pkg:generic/github.com/", + "pkg:github/", +] + +for component in endor_components_remove: + for prefix in prefixes: + endor_components_remove.append(prefix + component) + +# ################ Component Renaming ################ +# Endor does not have syntactically valid PURLs for C/C++ packages. +# e.g., +# Invalid: pkg:c/github.com/abseil/abseil-cpp@20250512.1 +# Valid: pkg:github/abseil/abseil-cpp@20250512.1 +# Run string replacements to correct for this: +endor_components_rename = [ + ["pkg:generic/zlib.net/zlib", "pkg:github/madler/zlib"], + ["pkg:github/philsquared/clara", "pkg:github/catchorg/clara"], + ["pkg:generic/github.com/", "pkg:github/"], + ["pkg:c/github.com/", "pkg:github/"], +] + +# ################ PURL Validation ################ +REGEX_STR_PURL_OPTIONAL = ( # Optional Version (any chars except ? @ #) + r"(?:@[^?@#]*)?" + # Optional Qualifiers (any chars except @ #) + r"(?:\?[^@#]*)?" + # Optional Subpath (any chars) + r"(?:#.*)?$" +) + +REGEX_PURL = { + # deb PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/deb-definition.md + "deb": re.compile( + r"^pkg:deb/" # Scheme and type + # Namespace (organization/user), letters must be lowercase + r"(debian|ubuntu)+" + r"/" + r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name + ), + # Generic PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/generic-definition.md + "generic": re.compile( + r"^pkg:generic/" # Scheme and type + r"([a-zA-Z0-9._-]+/)?" # Optional namespace segment + r"[a-zA-Z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name (required) + ), + # GitHub PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/github-definition.md + "github": re.compile( + r"^pkg:github/" # Scheme and type + # Namespace (organization/user), letters must be lowercase + r"[a-z0-9-]+" + r"/" + r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name (repository) + ), + # PyPI PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md + "pypi": re.compile( + r"^pkg:pypi/" # Scheme and type + r"[a-z0-9_-]+" # Name, letters must be lowercase, dashes, underscore + + REGEX_STR_PURL_OPTIONAL + ), +} + + +def is_valid_purl(purl: str) -> bool: + """Validate a GitHub or Generic PURL""" + for purl_type, regex in REGEX_PURL.items(): + if regex.match(purl): + logger.debug(f"PURL: {purl} matched PURL type '{purl_type}' regex '{regex.pattern}'") + return True + return False + + +# ################ Version Transformation ################ + +# In some cases we need to transform the version string to strip out tag-related text +# It is unknown what patterns may appear in the future, so we have targeted (not broad) regex +# This a list of 'pattern' and 'repl' inputs to re.sub() +RE_VER_NUM = r"(0|[1-9]\d*)" +RE_VER_LBL = r"(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?" +RE_SEMVER = rf"{RE_VER_NUM}\.{RE_VER_NUM}\.{RE_VER_NUM}{RE_VER_LBL}" +regex_semver = re.compile(RE_SEMVER) + +VERSION_PATTERN_REPL = [ + # 'debian/1.28.1-1' pkg:github/mongodb/mongo-c-driver (temporary workaround) + [re.compile(rf"^debian/({RE_SEMVER})-\d$"), r"\1"], + # 'gperftools-2.9.1' pkg:github/gperftools/gperftools + # 'mongo/v1.5.2' pkg:github/google/benchmark + # 'mongodb-8.2.0-alpha2' pkg:github/wiredtiger/wiredtiger + # 'release-1.12.0' pkg:github/apache/avro + # 'yaml-cpp-0.6.3' pkg:github/jbeder/yaml-cpp + [re.compile(rf"^[-a-z]+[-/][vr]?({RE_SEMVER})$"), r"\1"], + # 'asio-1-34-2' pkg:github/chriskohlhoff/asio + # 'cares-1_27_0' pkg:github/c-ares/c-ares + [ + re.compile(rf"^[a-z]+-{RE_VER_NUM}[_-]{RE_VER_NUM}[_-]{RE_VER_NUM}{RE_VER_LBL}$"), + r"\1.\2.\3", + ], + # 'pcre2-10.40' pkg:github/pcre2project/pcre2 + [re.compile(rf"^[a-z0-9]+-({RE_VER_NUM}\.{RE_VER_NUM})$"), r"\1"], + # 'icu-release-57-1' pkg:github/unicode-org/icu + [re.compile(rf"^[a-z]+-?[a-z]+-{RE_VER_NUM}-{RE_VER_NUM}$"), r"\1.\2"], + # 'v2.6.0' pkg:github/confluentinc/librdkafka + # 'r2.5.1' + [re.compile(rf"^[rv]({RE_SEMVER})$"), r"\1"], + # 'v2025.04.21.00' pkg:github/facebook/folly + [re.compile(r"^v(\d+\.\d+\.\d+\.\d+)$"), r"\1"], +] + + +def get_semver_from_release_version(release_ver: str) -> str: + """Extract the version number from string with tags or other annotations""" + if release_ver: + for re_obj, repl in VERSION_PATTERN_REPL: + if re_obj.match(release_ver): + return re_obj.sub(repl, release_ver) + return release_ver + + +# region special component use-case functions + + +def get_version_from_wiredtiger_import_data(file_path: str) -> str: + """Get the info in the 'import.data' file saved in the wiredtiger folder""" + try: + with open(file_path, "r") as input_json: + import_data = input_json.read() + result = json.loads(import_data) + except Exception as e: + logger.error(f"Error loading JSON file from {file_path}") + logger.error(e) + return None + return result.get("commit") + + +def get_version_sasl_from_workspace(file_path: str) -> str: + """Determine the version that is pulled for Windows Cyrus SASL by searching WORKSPACE.bazel""" + # e.g., + # "https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-2.1.28.zip", + try: + with open(file_path, "r") as file: + for line in file: + if line.strip().startswith( + '"https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-' + ): + return line.strip().split("windows_cyrus_sasl-")[1].split(".zip")[0] + except Exception as e: + logger.warning(f"Unable to load {file_path}") + logger.warning(e) + else: + return None + + +def process_component_special_cases( + component_key: str, component: dict, versions: dict, repo_root: str +) -> None: + ## Special case for Cyrus SASL ## + if component_key == "pkg:github/cyrusimap/cyrus-sasl": + # Cycrus SASL is optionally loaded as a Windows library, when needed. There is no source code for Endor Labs to scan. + # The version of Cyrus SASL that is used is defined in the WORKSPACE.bazel file: + # "https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-2.1.28.zip", + # Rather than add the complexity of Bazel queries to this script, we just search the text. + + versions["import_script"] = get_version_sasl_from_workspace(repo_root + "/WORKSPACE.bazel") + logger.info( + f"VERSION SPECIAL CASE: {component_key}: Found version '{versions['import_script']}' in 'WORKSPACE.bazel' file" + ) + + ## Special case for wiredtiger ## + elif component_key == "pkg:github/wiredtiger/wiredtiger": + # MongoDB release branches import wiredtiger commits via a bot. These commits will likely not line up with a release or tag. + # Endor labs will try to pull the nearest release/tag, but we want the more precise commit hash, which is stored in: + # src/third_party/wiredtiget/import.data + occurrences = component.get("evidence", {}).get("occurrences", []) + if occurrences: + location = occurrences[0].get("location") + versions["import_script"] = get_version_from_wiredtiger_import_data( + f"{repo_root}/{location}/import.data" + ) + logger.info( + f"VERSION SPECIAL CASE: {component_key}: Found version '{versions['import_script']}' in 'import.data' file" + ) + +# endregion special component use-case functions diff --git a/etc/sbom/endorctl_utils.py b/etc/sbom/endorctl_utils.py new file mode 100644 index 0000000000..ae2738aeb8 --- /dev/null +++ b/etc/sbom/endorctl_utils.py @@ -0,0 +1,486 @@ +#!/usr/bin/env python3 +""" +Utility functions for the Endor Labs API via endorctl + +""" + +import json +import logging +import subprocess +import time +from datetime import datetime +from enum import Enum + +logger = logging.getLogger("generate_sbom") +logger.setLevel(logging.NOTSET) + +default_field_masks = { + "PackageVersion": [ + "context", + "meta", + "processing_status", + "spec.package_name", + "spec.resolved_dependencies.dependencies", + "spec.source_code_reference", + ], + "ScanResult": [ + "context", + "meta", + "spec.end_time", + "spec.logs", + "spec.refs", + "spec.start_time", + "spec.status", + "spec.versions", + ], +} + + +def _get_default_field_mask(kind): + default_field_mask = default_field_masks.get(kind, []) + return ",".join(default_field_mask) + + +class EndorResourceKind(Enum): + """Enumeration for Endor Labs API resource kinds""" + + PROJECT = "Project" + REPOSITORY_VERSION = "RepositoryVersion" + SCAN_RESULT = "ScanResult" + PACKAGE_VERSION = "PackageVersion" + + +class EndorContextType(Enum): + """Most objects include a common nested object called Context. Contexts keep objects from different scans separated. + https://docs.endorlabs.com/rest-api/using-the-rest-api/data-model/common-fields/#context""" + + # Objects from a scan of the default branch. All objects in the OSS namespace are in the main context. The context ID is always default. + MAIN = "CONTEXT_TYPE_MAIN" + # Objects from a scan of a specific branch. The context ID is the branch reference name. + REF = "CONTEXT_TYPE_REF" + # Objects from a PR scan. The context ID is the PR UUID. Objects in this context are deleted after 30 days. + CI_RUN = "CONTEXT_TYPE_CI_RUN" + + +class EndorFilter: + """Provide standard filters for Endor Labs API resource kinds""" + + def __init__(self, context_id=None, context_type=None): + self.context_id = context_id + self.context_type = context_type + + def _base_filters(self): + base_filters = [] + if self.context_id: + base_filters.append(f"context.id=={self.context_id}") + if self.context_type: + base_filters.append(f"context.type=={self.context_type}") + + return base_filters + + def repository_version(self, project_uuid=None, sha=None, ref=None): + filters = self._base_filters() + if project_uuid: + filters.append(f"meta.parent_uuid=={project_uuid}") + if sha: + filters.append(f"spec.version.sha=={sha}") + if ref: + filters.append(f"spec.version.ref=={ref}") + + return " and ".join(filters) + + def package_version( + self, + context_type: EndorContextType = None, + context_id=None, + project_uuid=None, + name=None, + package_name=None, + ): + filters = self._base_filters() + if context_type: + filters.append(f"context.type=={context_type.value}") + if context_type: + filters.append(f"context.id=={context_id}") + if project_uuid: + filters.append(f"spec.project_uuid=={project_uuid}") + if name: + filters.append(f"spec.package_name=={name}") + if package_name: + filters.append(f"meta.name=={package_name}") + + return " and ".join(filters) + + def scan_result( + self, + context_type: EndorContextType = None, + project_uuid=None, + ref=None, + sha=None, + status=None, + ): + filters = self._base_filters() + if context_type: + filters.append(f"context.type=={context_type.value}") + if project_uuid: + filters.append(f"meta.parent_uuid=={project_uuid}") + if ref: + filters.append(f"spec.versions.ref contains '{ref}'") + if sha: + filters.append(f"spec.versions.sha contains '{sha}'") + if status: + filters.append(f"spec.status=={status}") + + return " and ".join(filters) + + +class EndorCtl: + """Interact with endorctl (Endor Labs CLI)""" + + # region internal functions + def __init__( + self, + namespace, + retry_limit=5, + sleep_duration=30, + endorctl_path="endorctl", + config_path=None, + ): + self.namespace = namespace + self.retry_limit = retry_limit + self.sleep_duration = sleep_duration + self.endorctl_path = endorctl_path + self.config_path = config_path + + def _call_endorctl(self, command, subcommand, **kwargs): + """https://docs.endorlabs.com/endorctl/""" + + try: + command = [self.endorctl_path, command, subcommand, f"--namespace={self.namespace}"] + if self.config_path: + command.append(f"--config-path={self.config_path}") + + # parse args into flags + for key, value in kwargs.items(): + # Handle endorctl flags with hyphens that are defined in the script with underscores + flag = key.replace("_", "-") + if value: + command.append(f"--{flag}={value}") + logger.info("Running: %s", " ".join(command)) + + result = subprocess.run(command, capture_output=True, text=True, check=True) + + resource = json.loads(result.stdout) + + except subprocess.CalledProcessError as e: + logger.error(f"Error executing command: {e}") + logger.error(e.stderr) + except json.JSONDecodeError as e: + logger.error(f"Error decoding JSON: {e}") + logger.error(f"Stdout: {result.stdout}") + except FileNotFoundError as e: + logger.error(f"FileNotFoundError: {e}") + logger.error( + f"'endorctl' not found in path '{self.endorctl_path}'. Supply the correct path, run 'buildscripts/install_endorctl.sh' or visit https://docs.endorlabs.com/endorctl/install-and-configure/" + ) + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + else: + return resource + + def _api_get(self, resource, **kwargs): + """https://docs.endorlabs.com/endorctl/commands/api/""" + return self._call_endorctl("api", "get", resource=resource, **kwargs) + + def _api_list(self, resource, filter=None, retry=True, **kwargs): + """https://docs.endorlabs.com/endorctl/commands/api/""" + # If this script is run immediately after making a commit, Endor Labs will likely not yet have created the assocaited ScanResult object. The wait/retry logic below handles this scenario. + tries = 0 + while True: + tries += 1 + result = self._call_endorctl("api", "list", resource=resource, filter=filter, **kwargs) + + # The expected output of 'endorctl api list' is: { "list": { "objects": [...] } } + # We want to just return the objects. In case we get an empty list, return a list + # with a single None to avoid having to handle index errors downstream. + if result and result["list"].get("objects") and len(result["list"]["objects"]) > 0: + return result["list"]["objects"] + elif retry: + logger.info( + f"API LIST: Resource not found: {resource} with filter '{filter}' in namespace '{self.namespace}'" + ) + if tries <= self.retry_limit: + logger.info( + f"API LIST: Waiting for {self.sleep_duration} seconds before retry attempt {tries} of {self.retry_limit}" + ) + time.sleep(self.sleep_duration) + else: + logger.warning( + f"API LIST: Maximum number of allowed retries {self.retry_limit} attempted with no {resource} found using filter '{filter}'" + ) + return [None] + else: + return [None] + + def _check_resource(self, resource, resource_description) -> None: + if not resource: + raise LookupError(f"Resource not found: {resource_description}") + logger.info(f"Retrieved: {resource_description}") + + # endregion internal functions + + # region resource functions + def get_resource(self, resource, uuid=None, name=None, field_mask=None, **kwargs): + """https://docs.endorlabs.com/rest-api/using-the-rest-api/data-model/resource-kinds/""" + if not field_mask: + field_mask = _get_default_field_mask(resource) + return self._api_get( + resource=resource, uuid=uuid, name=name, field_mask=field_mask, **kwargs + ) + + def get_resources( + self, + resource, + filter=None, + field_mask=None, + sort_path="meta.create_time", + sort_order="descending", + retry=True, + **kwargs, + ): + """https://docs.endorlabs.com/rest-api/using-the-rest-api/data-model/resource-kinds/""" + if not field_mask: + field_mask = _get_default_field_mask(resource) + return self._api_list( + resource=resource, + filter=filter, + field_mask=field_mask, + sort_path=sort_path, + sort_order=sort_order, + retry=retry, + **kwargs, + ) + + def get_project(self, git_url): + resource_kind = EndorResourceKind.PROJECT.value + resource_description = ( + f"{resource_kind} with name '{git_url}' in namespace '{self.namespace}'" + ) + project = self.get_resource(resource_kind, name=git_url) + self._check_resource(project, resource_description) + return project + + def get_repository_version(self, filter=None, retry=True): + resource_kind = EndorResourceKind.REPOSITORY_VERSION.value + resource_description = ( + f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'" + ) + repository_version = self.get_resources( + resource_kind, filter=filter, retry=retry, page_size=1 + )[0] + self._check_resource(repository_version, resource_description) + return repository_version + + def get_scan_result(self, filter=None, retry=True): + resource_kind = EndorResourceKind.SCAN_RESULT.value + resource_description = ( + f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'" + ) + scan_result = self.get_resources(resource_kind, filter=filter, retry=retry, page_size=1)[0] + self._check_resource(scan_result, resource_description) + uuid = scan_result.get("uuid") + start_time = scan_result["spec"].get("start_time") + refs = scan_result["spec"].get("refs") + polling_start_time = datetime.now() + while True: + status = scan_result["spec"].get("status") + end_time = scan_result["spec"].get("end_time") + if status == "STATUS_SUCCESS": + logger.info( + f" Scan completed successfully. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}." + ) + return scan_result + elif status == "STATUS_RUNNING": + logger.info( + f" Scan is running. ScanResult uuid {uuid} for refs {refs} started at {start_time}." + ) + logger.info( + f" Waiting {self.sleep_duration} seconds before checking status. Total wait time: {(datetime.now() - polling_start_time).total_seconds()/60:.2f} minutes" + ) + time.sleep(self.sleep_duration) + scan_result = self.get_resources( + resource_kind, filter=filter, retry=retry, page_size=1 + )[0] + elif status == "STATUS_PARTIAL_SUCCESS": + scan_logs = scan_result["spec"].get("logs") + raise RuntimeError( + f" Scan completed, but with critical warnings or errors. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}. Scan logs: {scan_logs}" + ) + elif status == "STATUS_FAILURE": + scan_logs = scan_result["spec"].get("logs") + raise RuntimeError( + f" Scan failed. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}. Scan logs: {scan_logs}" + ) + + def get_package_versions(self, filter): + resource_kind = EndorResourceKind.PACKAGE_VERSION.value + resource_description = ( + f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'" + ) + package_versions = self.get_resources(resource_kind, filter=filter) + self._check_resource(package_versions, resource_description) + return package_versions + + def export_sbom( + self, + package_version_uuid=None, + package_version_uuids=None, + package_version_name=None, + app_name=None, + project_name=None, + project_uuid=None, + ): + """Export an SBOM from Endor Labs + + Valid parameter sets (other combinations result in an error from 'endorctl'): + Single-Package SBOM: + package_version_uuid + package_version_name + Multi-Package SBOM: + package_version_uuids,app_name + project_uuid,app_name,app_name + project_name,app_name,app_name + + https://docs.endorlabs.com/endorctl/commands/sbom/export/ + """ + if package_version_uuids: + package_version_uuids = ",".join(package_version_uuids) + return self._call_endorctl( + "sbom", + "export", + package_version_uuid=package_version_uuid, + package_version_uuids=package_version_uuids, + package_version_name=package_version_name, + app_name=app_name, + project_name=project_name, + project_uuid=project_uuid, + ) + + # endregion resource functions + + # region workflow functions + def get_sbom_for_commit(self, git_url: str, commit_sha: str) -> dict: + """Export SBOM for the PR commit (sha)""" + + endor_filter = EndorFilter() + + try: + # Project: get uuid + project = self.get_project(git_url) + project_uuid = project["uuid"] + app_name = project["spec"]["git"]["full_name"] + + # RepositoryVersion: get the context for the PR scan + endor_filter.context_type = EndorContextType.CI_RUN.value + filter_str = endor_filter.repository_version(project_uuid, commit_sha) + repository_version = self.get_repository_version(filter_str) + context_id = repository_version["context"]["id"] + + # ScanResult: wait for a completed scan + endor_filter.context_id = context_id + filter_str = endor_filter.scan_result(project_uuid) + self.get_scan_result(filter_str) + + # PackageVersions: get package versions for SBOM + filter_str = endor_filter.package_version(project_uuid) + package_versions = self.get_package_versions(filter_str) + package_version_uuids = [ + package_version["uuid"] for package_version in package_versions + ] + package_version_names = [ + package_version["meta"]["name"] for package_version in package_versions + ] + + # Export SBOM + sbom = self.export_sbom(package_version_uuids=package_version_uuids, app_name=app_name) + print( + f"Retrieved: CycloneDX SBOM for PackageVersion(s), name: {package_version_names}, uuid: {package_version_uuids}" + ) + return sbom + + except Exception as e: + print(f"Exception: {e}") + return + + def get_sbom_for_branch(self, git_url: str, branch: str) -> dict: + """Export lastest SBOM for a monitored branch/ref""" + + endor_filter = EndorFilter() + + try: + # Project: get uuid + project = self.get_project(git_url) + project_uuid = project["uuid"] + app_name = project["spec"]["git"]["full_name"] + + # RepositoryVersion: get the context for the latest branch scan + filter_str = endor_filter.repository_version(project_uuid, ref=branch) + repository_version = self.get_repository_version(filter_str) + repository_version_uuid = repository_version["uuid"] + repository_version_ref = repository_version["spec"]["version"]["ref"] + repository_version_sha = repository_version["spec"]["version"]["sha"] + repository_version_scan_object_status = repository_version["scan_object"]["status"] + if repository_version_scan_object_status != "STATUS_SCANNED": + logger.warning( + f"RepositoryVersion (uuid: {repository_version_uuid}, ref: {repository_version_ref}, sha: {repository_version_sha}) scan status is '{repository_version_scan_object_status}' (expected 'STATUS_SCANNED')" + ) + + # ScanResult: search for a completed scan + filter_str = endor_filter.scan_result( + None, project_uuid, repository_version_ref, repository_version_sha + ) + scan_result = self.get_scan_result(filter_str, retry=False) + project_uuid = scan_result["meta"]["parent_uuid"] + + # PackageVersions: get package versions for SBOM + if branch == "master": + context_type = EndorContextType.MAIN + context_id = "default" + else: + context_type = EndorContextType.REF + context_id = branch + filter_str = endor_filter.package_version(context_type, context_id, project_uuid) + package_version = self.get_package_versions(filter_str)[0] + package_version_name = package_version["meta"]["name"] + package_version_uuid = package_version["uuid"] + + # Export SBOM + sbom = self.export_sbom(package_version_uuid=package_version_uuid, app_name=app_name) + logger.info( + f"SBOM: Retrieved CycloneDX SBOM for PackageVersion, name: {package_version_name}, uuid {package_version_uuid}" + ) + return sbom + + except Exception as e: + print(f"Exception: {e}") + return + + def get_sbom_for_project(self, git_url: str) -> dict: + """Export latest SBOM for EndorCtl project default branch""" + + try: + # Project: get uuid + project = self.get_project(git_url) + project_uuid = project["uuid"] + app_name = project["spec"]["git"]["full_name"] + + # Export SBOM + sbom = self.export_sbom(project_uuid=project_uuid, app_name=app_name) + logger.info(f"Retrieved: CycloneDX SBOM for Project {app_name}") + return sbom + + except Exception as e: + print(f"Exception: {e}") + return + + # endregion workflow functions diff --git a/etc/sbom/generate_sbom.py b/etc/sbom/generate_sbom.py new file mode 100755 index 0000000000..6335d6591d --- /dev/null +++ b/etc/sbom/generate_sbom.py @@ -0,0 +1,758 @@ +#!/usr/bin/env python3 +""" +Generate a CycloneDX SBOM using scan results from Endor Labs. +Schema validation of output is not performed. +Use 'buildscripts/sbom_linter.py' for validation. + +Invoke with ---help or -h for help message. +""" + +import argparse +import json +import logging +import os +import re +import subprocess +import sys +import urllib.parse +import uuid +from datetime import datetime, timezone +from pathlib import Path + +from config import ( + endor_components_remove, + endor_components_rename, + get_semver_from_release_version, + is_valid_purl, + process_component_special_cases, +) +from endorctl_utils import EndorCtl +from git import Commit, Repo + +# region init + + +class WarningListHandler(logging.Handler): + """Collect warnings""" + + def __init__(self): + super().__init__() + self.warnings = [] + + def emit(self, record): + if record.levelno >= logging.WARNING: + self.warnings.append(record) + + +logging.basicConfig(stream=sys.stdout) +logger = logging.getLogger("generate_sbom") +logger.setLevel(logging.INFO) + +# Create an instance of the custom handler +warning_handler = WarningListHandler() + +# Add the handler to the logger +logger.addHandler(warning_handler) + + +# Get the absolute path of the script file and directory +script_path = Path(__file__).resolve() +script_directory = script_path.parent + +# Regex for validation +REGEX_COMMIT_SHA = r"^[0-9a-fA-F]{40}$" +REGEX_GIT_BRANCH = r"^[a-zA-Z0-9_.\-/]+$" +REGEX_GITHUB_URL = r"^(https://github.com/)([a-zA-Z0-9-]{1,39}/[a-zA-Z0-9-_.]{1,100})(\.git)$" +REGEX_RELEASE_BRANCH = r"^v\d\.\d$" +REGEX_RELEASE_TAG = r"^r\d\.\d.\d(-\w*)?$" + +# endregion init + + +# region functions and classes + + +class GitInfo: + """Get, set, format git info""" + + def __init__(self): + print_banner("Gathering git info") + try: + self.repo_root = Path( + subprocess.run( + "git rev-parse --show-toplevel", shell=True, text=True, capture_output=True + ).stdout.strip() + ) + self._repo = Repo(self.repo_root) + except Exception as e: + logger.warning( + "Unable to read git repo information. All necessary script arguments must be provided." + ) + logger.warning(e) + self._repo = None + else: + try: + self.project = self._repo.remotes.origin.config_reader.get("url") + if not self.project.endswith(".git"): + self.project += ".git" + org_repo = extract_repo_from_git_url(self.project) + self.org = org_repo["org"] + self.repo = org_repo["repo"] + self.commit = self._repo.head.commit.hexsha + self.branch = self._repo.active_branch.name + + # filter tags for latest release e.g., r8.2.1 + release_tags = [] + filtered_tags = [ + tag for tag in self._repo.tags if re.fullmatch(REGEX_RELEASE_TAG, tag.name) + ] + logging.info(f"GIT: Parsing {len(filtered_tags)} release tags for match to commit") + for tag in filtered_tags: + if tag.commit == self.commit: + release_tags.append(tag.name) + if len(release_tags) > 0: + self.release_tag = release_tags[-1] + else: + self.release_tag = None + logging.debug(f"GitInfo->release_tag(): {self.release_tag}") + + logging.debug(f"GitInfo->__init__: {self}") + except Exception as e: + logger.warning("Unable to fully parse git info.") + logger.warning(e) + + def close(self): + """Closes the underlying Git repo object to release resources.""" + if self._repo: + logger.debug("Closing Git repo object.") + self._repo.close() + self._repo = None + + def added_new_3p_folder(self, commit: Commit) -> bool: + """ + Checks if a given commit added a new third-party subfolder. + + Args: + commit: The GitPython Commit object to analyze. + + Returns: + True if the commit added a new subfolder, False otherwise. + """ + if not commit.parents: + # If it's the initial commit, all folders are "new" + # You might want to refine this logic based on your definition of "new" + # Check if there are any subfolders in the initial commit + return bool(commit.tree.trees) + + parent_commit = commit.parents[0] + diff_index = commit.diff(parent_commit) + + for diff in diff_index: + # Check for added items that are directories + if diff.change_type == "A" and diff.b_is_dir: + return True + return False + + +def print_banner(text: str) -> None: + """print() a padded status message to stdout""" + print() + print(text.center(len(text) + 2, " ").center(120, "=")) + + +def extract_repo_from_git_url(git_url: str) -> dict: + """Determine org/repo for a given git url""" + git_org = git_url.split("/")[-2].replace(".git", "") + git_repo = git_url.split("/")[-1].replace(".git", "") + return { + "org": git_org, + "repo": git_repo, + } + + +def sbom_components_to_dict(sbom: dict, with_version: bool = False) -> dict: + """Create a dict of SBOM components with a version-less PURL as the key""" + components = sbom["components"] + if with_version: + components_dict = { + urllib.parse.unquote(component["bom-ref"]): component for component in components + } + else: + components_dict = { + urllib.parse.unquote(component["bom-ref"]).split("@")[0]: component + for component in components + } + return components_dict + + +def read_sbom_json_file(file_path: str) -> dict: + """Load a JSON SBOM file (schema is not validated)""" + try: + with open(file_path, "r", encoding="utf-8") as input_json: + sbom_json = input_json.read() + result = json.loads(sbom_json) + except Exception as e: + logger.error(f"Error loading SBOM file from {file_path}") + logger.error(e) + else: + logger.info(f"SBOM loaded from {file_path} with {len(result['components'])} components") + return result + + +def write_sbom_json_file(sbom_dict: dict, file_path: str) -> None: + """Save a JSON SBOM file (schema is not validated)""" + try: + file_path = os.path.abspath(file_path) + with open(file_path, "w", encoding="utf-8") as output_json: + json.dump(sbom_dict, output_json, indent=2) + output_json.write("\n") + except Exception as e: + logger.error(f"Error writing SBOM file to {file_path}") + logger.error(e) + else: + logger.info(f"SBOM file saved to {file_path}") + + +def write_list_to_text_file(str_list: list, file_path: str) -> None: + """Save a list of strings to a text file""" + try: + file_path = os.path.abspath(file_path) + with open(file_path, "w", encoding="utf-8") as output_txt: + for item in str_list: + output_txt.write(f"{item}\n") + except Exception as e: + logger.error(f"Error writing text file to {file_path}") + logger.error(e) + else: + logger.info(f"Text file saved to {file_path}") + + +def set_component_version( + component: dict, version: str, purl_version: str = None, cpe_version: str = None +) -> None: + """Update the appropriate version fields in a component from the metadata SBOM""" + if not purl_version: + purl_version = version + + if not cpe_version: + cpe_version = version + + component["bom-ref"] = component["bom-ref"].replace("{{VERSION}}", purl_version) + component["version"] = component["version"].replace("{{VERSION}}", version) + if component.get("purl"): + component["purl"] = component["purl"].replace("{{VERSION}}", purl_version) + if not is_valid_purl(component["purl"]): + logger.warning(f"PURL: Invalid PURL ({component['purl']})") + if component.get("cpe"): + component["cpe"] = component["cpe"].replace("{{VERSION}}", cpe_version) + + +def set_dependency_version(dependencies: list, meta_bom_ref: str, purl_version: str) -> None: + """Update the appropriate dependency version fields in the metadata SBOM""" + r = 0 + d = 0 + for dependency in dependencies: + if "{{VERSION}}" in dependency["ref"] and dependency["ref"] == meta_bom_ref: + dependency["ref"] = dependency["ref"].replace("{{VERSION}}", purl_version) + r += 1 + for i in range(len(dependency["dependsOn"])): + if dependency["dependsOn"][i] == meta_bom_ref: + dependency["dependsOn"][i] = dependency["dependsOn"][i].replace( + "{{VERSION}}", purl_version + ) + d += 1 + + logger.debug(f"set_dependency_version: '{meta_bom_ref}' updated {r} refs and {d} dependsOn") + + +def get_subfolders_dict(folder_path: str = ".") -> dict: + """Get list of all directories in the specified path""" + subfolders = [] + try: + # Get all entries (files and directories) in the specified path + entries = os.listdir(folder_path) + + # Filter for directories + for entry in entries: + full_path = os.path.join(folder_path, entry) + if os.path.isdir(full_path): + subfolders.append(entry) + except FileNotFoundError: + logger.error(f"Error: Directory '{folder_path}' not found.") + except Exception as e: + logger.error(f"An error occurred: {e}") + + subfolders.sort() + return {key: 0 for key in subfolders} + + +# endregion functions and classes + + +def main() -> None: + # region define args + + parser = argparse.ArgumentParser( + description="""Generate a CycloneDX v1.5 JSON SBOM file using a combination of scan results from Endor Labs, pre-defined SBOM metadata, and the existing SBOM. + Requires endorctl to be installed and configured, which can be done using 'buildscripts/sbom/install_endorctl.sh'. + For use in CI, script may be run with no arguments.""", + epilog="Note: The git-related default values are dynamically generated.", + formatter_class=argparse.MetavarTypeHelpFormatter, + ) + + endor = parser.add_argument_group("Endor Labs API (via 'endorctl')") + endor.add_argument( + "--endorctl-path", + help="Path to endorctl, the Endor Labs CLI (Default: 'endorctl')", + default="endorctl", + type=str, + ) + endor.add_argument( + "--config-path", + help="Path to endor config directory containing config.yaml (Default: '$HOME/.endorctl')", + default=None, + type=str, + ) + endor.add_argument( + "--namespace", help="Endor Labs namespace (Default: mongodb.{git org})", type=str + ) + endor.add_argument( + "--target", + help="Target for generated SBOM. Commit: results from running/completed PR scan, Branch: results from latest monitoring scan, Project: results from latest monitoring scan of the 'default' branch (default: commit)", + choices=["commit", "branch", "project"], + default="commit", + type=str, + ) + endor.add_argument( + "--project", + help="Full GitHub git URL [e.g., https://github.com/10gen/mongo.git] (Default: current git URL)", + type=str, + ) + + target = parser.add_argument_group("Target values. Apply only if --target is not 'project'") + exclusive_target = target.add_mutually_exclusive_group() + exclusive_target.add_argument( + "--commit", + help="PR commit SHA [40-character hex string] (Default: current git commit)", + type=str, + ) + exclusive_target.add_argument( + "--branch", + help="Git repo branch monitored by Endor Labs [e.g., v8.0] (Default: current git org/repo)", + type=str, + ) + + files = parser.add_argument_group("SBOM files") + files.add_argument( + "--sbom-metadata", + help="Input path for template SBOM file with metadata (Default: './buildscripts/sbom/metadata.cdx.json')", + default="./buildscripts/sbom/metadata.cdx.json", + type=str, + ) + files.add_argument( + "--sbom-in", + help="Input path for previous SBOM file (Default: './sbom.json')", + default="./sbom.json", + type=str, + ) + files.add_argument( + "--sbom-out", + help="Output path for SBOM file (Default: './sbom.json')", + default="./sbom.json", + type=str, + ) + parser.add_argument( + "--retry-limit", + help="Maximum number of times to retry when a target PR scan has not started (Default: 5)", + default=5, + type=int, + ) + parser.add_argument( + "--sleep-duration", + help="Number of seconds to wait between retries (Default: 30)", + default=30, + type=int, + ) + parser.add_argument( + "--save-warnings", + help="Save warning messages to a specified file (Default: None)", + default=None, + type=str, + ) + parser.add_argument("--debug", help="Set logging level to DEBUG", action="store_true") + + # endregion define args + + # region parse args + + args = parser.parse_args() + + git_info = GitInfo() + + # endor + endorctl_path = args.endorctl_path + config_path = args.config_path + namespace = args.namespace if args.namespace else f"mongodb.{git_info.org}" + target = args.target + + # project + if args.project and args.project != git_info.project: + if not re.fullmatch(REGEX_GITHUB_URL, args.project): + parser.error(f"Invalid Git URL: {args.project}.") + git_info.project = args.project + git_info.org, git_info.repo = map( + extract_repo_from_git_url(git_info.project).get, ("org", "repo") + ) + git_info.release_tag = None + + # targets + # commit + if args.commit and args.commit != git_info.commit: + if not re.fullmatch(REGEX_COMMIT_SHA, args.commit): + parser.error( + f"Invalid Git commit SHA: {args.commit}. Must be a 40-character hexadecimal string (SHA-1)." + ) + git_info.commit = args.commit + + # branch + if args.branch and args.branch != git_info.branch: + if len(args.branch.encode("utf-8")) > 244 or not re.fullmatch( + REGEX_GIT_BRANCH, args.branch + ): + parser.error( + f"Invalid Git branch name: {args.branch}. Limit is 244 bytes with allowed characters: [a-zA-Z0-9_.-/]" + ) + git_info.branch = args.branch + + # files + sbom_out_path = args.sbom_out + sbom_in_path = args.sbom_in + sbom_metadata_path = args.sbom_metadata + save_warnings = args.save_warnings + + # environment + retry_limit = args.retry_limit + sleep_duration = args.sleep_duration + + if args.debug: + logger.setLevel(logging.DEBUG) + + # endregion parse args + + # region export Endor Labs SBOM + + print_banner(f"Exporting Endor Labs SBOM for {target} {getattr(git_info, target)}") + endorctl = EndorCtl(namespace, retry_limit, sleep_duration, endorctl_path, config_path) + if target == "commit": + endor_bom = endorctl.get_sbom_for_commit(git_info.project, git_info.commit) + elif target == "branch": + endor_bom = endorctl.get_sbom_for_branch(git_info.project, git_info.branch) + elif target == "project": + endor_bom = endorctl.get_sbom_for_project(git_info.project) + + if not endor_bom: + logger.error("Empty result for Endor SBOM!") + if target == "commit": + logger.error("Check Endor Labs for any unanticipated issues with the target PR scan.") + else: + logger.error("Check Endor Labs for status of the target monitoring scan.") + sys.exit(1) + + # endregion export Endor Labs SBOM + + # region Pre-process Endor Labs SBOM + + print_banner("Pre-Processing Endor Labs SBOM") + + ## remove uneeded components ## + # [list]endor_components_remove is defined in config.py + # Endor Labs includes the main component in 'components'. This is not standard, so we remove it. + endor_components_remove.append(f"pkg:github/{git_info.org}/{git_info.repo}") + + # Reverse iterate the SBOM components list to safely modify in situ + for i in range(len(endor_bom["components"]) - 1, -1, -1): + component = endor_bom["components"][i] + removed = False + for remove in endor_components_remove: + if component["bom-ref"].startswith(remove): + logger.info("ENDOR SBOM PRE-PROCESS: removing " + component["bom-ref"]) + del endor_bom["components"][i] + removed = True + break + if not removed: + for rename in endor_components_rename: + old = rename[0] + new = rename[1] + component["bom-ref"] = component["bom-ref"].replace(old, new) + component["purl"] = component["purl"].replace(old, new) + + logger.info(f"Endor Labs SBOM pre-processed with {len(endor_bom['components'])} components") + + # endregion Pre-process Endor Labs SBOM + + # region load metadata and previous SBOMs + + print_banner("Loading metadata SBOM and previous SBOM") + + meta_bom = read_sbom_json_file(sbom_metadata_path) + if not meta_bom: + logger.error("No SBOM metadata. This is fatal.") + sys.exit(1) + + prev_bom = read_sbom_json_file(sbom_in_path) + if not prev_bom: + logger.warning( + "Unable to load previous SBOM data. The new SBOM will be generated without any previous context. This is unexpected, but not fatal." + ) + # Create empty prev_bom to avoid downstream processing errors + prev_bom = { + "bom-ref": None, + "metadata": { + "timestamp": endor_bom["metadata"]["timestamp"], + "component": { + "version": None, + }, + }, + "components": [], + } + + # endregion load metadata and previous SBOMs + + # region Build composite SBOM + # Note: No exception handling here. The most likely reason for an exception is missing data elements + # in SBOM files, which is fatal if it happens. Code is in place to handle the situation + # where there is no previous SBOM to include, but we want to fail if required data is absent. + print_banner("Building composite SBOM (metadata + endor + previous)") + + # Sort components by bom-ref + endor_bom["components"].sort(key=lambda c: c["bom-ref"]) + meta_bom["components"].sort(key=lambda c: c["bom-ref"]) + prev_bom["components"].sort(key=lambda c: c["bom-ref"]) + + # Create SBOM component lookup dicts + endor_components = sbom_components_to_dict(endor_bom) + prev_components = sbom_components_to_dict(prev_bom) + + # region MongoDB primary component + + # Attempt to determine the MongoDB Version being scanned + logger.debug( + f"Available MongoDB version options, tag: {git_info.release_tag}, branch: {git_info.branch}, previous SBOM: {prev_bom['metadata'].get('component',{}).get('version')}" + ) + meta_bom_ref = meta_bom["metadata"]["component"]["bom-ref"] + + # Project scan always set to 'master' or if using 'master' branch + if target == "project" or git_info.branch == "master": + version = "master" + purl_version = "master" + cpe_version = "master" + logger.info("Using branch 'master' as MongoDB version") + + # tagged release. e.g., r8.1.0, r8.2.1-rc0 + elif git_info.release_tag: + version = git_info.release_tag[1:] # remove leading 'r' + purl_version = git_info.release_tag + cpe_version = version # without leading 'r' + logger.info(f"Using release_tag '{git_info.release_tag}' as MongoDB version") + + # Release branch e.g., v7.0 or v8.2 + elif target == "branch" and re.fullmatch(REGEX_RELEASE_BRANCH, git_info.branch): + version = git_info.branch + purl_version = git_info.branch + # remove leading 'v', add wildcard. e.g. 8.2.* + cpe_version = version[1:] + ".*" + logger.info(f"Using release branch '{git_info.branch}' as MongoDB version") + + # Previous SBOM app version, if all needed specifiers exist + elif ( + prev_bom.get("metadata", {}).get("component", {}).get("version") + and prev_bom.get("metadata", {}).get("component", {}).get("purl") + and prev_bom.get("metadata", {}).get("component", {}).get("cpe") + ): + version = prev_bom["metadata"]["component"]["version"] + purl_version = prev_bom["metadata"]["component"]["purl"].split("@")[-1] + cpe_version = prev_bom["metadata"]["component"]["cpe"].split(":")[5] + logger.info(f"Using previous SBOM version '{version}' as MongoDB version") + + else: + # Fall back to the version specified in the Endor SBOM + # This is unlikely to be accurate + version = endor_bom["metadata"]["component"]["version"] + purl_version = version + cpe_version = version + logger.warning( + f"Using SBOM version '{version}' from Endor Labs scan. This is unlikely to be accurate and may specify a PR #." + ) + + # Set main component version + set_component_version(meta_bom["metadata"]["component"], version, purl_version, cpe_version) + # Run through 'dependency' objects to set main component version + set_dependency_version(meta_bom["dependencies"], meta_bom_ref, purl_version) + + # endregion MongoDB primary component + + # region SBOM components + + # region Parse metadata SBOM components + + for component in meta_bom["components"]: + versions = { + "endor": None, + "metadata": None, + } + + component_key = component["bom-ref"].split("@")[0] + + print_banner("Component: " + component_key) + + ################ Endor Labs ################ + if component_key in endor_components: + # Pop component from dict so we are left with only unmatched components + endor_component = endor_components.pop(component_key) + versions["endor"] = endor_component.get("version") + logger.debug( + f"VERSION ENDOR: {component_key}: Found version '{versions['endor']}' in Endor Labs results" + ) + + ############## Metadata ############### + # Hard-coded metadata version, if exists + if "{{VERSION}}" not in component["version"]: + versions["metadata"] = component.get("version") + + logger.info(f"VERSIONS: {component_key}: " + str(versions)) + + ############## Component Special Cases ############### + process_component_special_cases( + component_key, component, versions, git_info.repo_root.as_posix() + ) + + # For the standard workflow, we favor the Endor Labs version followed by hard coded + version = versions["endor"] or versions["metadata"] + + ############## Assign Version ############### + if version: + meta_bom_ref = component["bom-ref"] + + ## Special case for FireFox ## + # The CPE for FireFox ESR needs the 'esr' removed from the version, as it is specified in another section + if component["bom-ref"].startswith("pkg:deb/debian/firefox-esr@"): + set_component_version(component, version, cpe_version=version.replace("esr", "")) + else: + semver = get_semver_from_release_version(version) + set_component_version(component, semver, version, semver) + + set_dependency_version(meta_bom["dependencies"], meta_bom_ref, version) + else: + logger.warning( + f"VERSION NOT FOUND: Could not find a version for {component_key}! Removing from SBOM. Component may need to be removed from the {sbom_metadata_path} file." + ) + del component + + # explicit cleanup to avoid gc race condition on script temination + git_info.close() + del git_info + + # endregion Parse metadata SBOM components + + # region Parse unmatched Endor Labs components + + print_banner("New Endor Labs components") + if endor_components: + logger.info( + f"ENDOR SBOM: There are {len(endor_components)} unmatched components in the Endor Labs SBOM. Adding as-is. The applicable metadata should be added to the metadata SBOM for the next run." + ) + for component in endor_components: + # set scope to excluded by default until the component is evaluated + endor_components[component]["scope"] = "excluded" + meta_bom["components"].append(endor_components[component]) + meta_bom["dependencies"].append( + {"ref": endor_components[component]["bom-ref"], "dependsOn": []} + ) + logger.info(f"SBOM AS-IS COMPONENT: Added {component}") + + # endregion Parse unmatched Endor Labs components + + # region Finalize SBOM + + # Have the SBOM app version changed? + sbom_app_version_changed = ( + prev_bom['metadata'].get('component',{}).get('version') != meta_bom["metadata"]["component"]["version"] + ) + logger.info(f"SUMMARY: MongoDB version changed: {sbom_app_version_changed}") + + # Have the components changed? + prev_components = sbom_components_to_dict(prev_bom, with_version=True) + meta_components = sbom_components_to_dict(meta_bom, with_version=True) + sbom_components_changed = prev_components.keys() != meta_components.keys() + logger.info( + f"SBOM_DIFF: SBOM components changed (added, removed, or version): {sbom_components_changed}. Previous SBOM has {len(prev_components)} components; New SBOM has {len(meta_components)} components" + ) + + # Components in prev SBOM but not in generated SBOM + prev_components = sbom_components_to_dict(prev_bom, with_version=False) + meta_components = sbom_components_to_dict(meta_bom, with_version=False) + prev_components_diff = list(set(prev_components.keys()) - set(meta_components.keys())) + if prev_components_diff: + logger.info( + "SBOM_DIFF: Components in previous SBOM and not in generated SBOM: " + + ",".join(prev_components_diff) + ) + + # Components in generated SBOM but not in prev SBOM + meta_components_diff = list(set(meta_components.keys()) - set(prev_components.keys())) + if meta_components_diff: + logger.info( + "SBOM_DIFF: Components in generated SBOM and not in previous SBOM: " + + ",".join(meta_components_diff) + ) + + # serialNumber https://cyclonedx.org/docs/1.5/json/#serialNumber + # version (SBOM version) https://cyclonedx.org/docs/1.5/json/#version + if sbom_app_version_changed: + # New MongoDB version requires a unique serial number and version 1 + meta_bom["serialNumber"] = uuid.uuid4().urn + meta_bom["version"] = 1 + else: + # MongoDB version is the same, so reuse the serial number and SBOM version + meta_bom["serialNumber"] = prev_bom["serialNumber"] + meta_bom["version"] = prev_bom["version"] + # If the components have changed, bump the SBOM version + if sbom_components_changed: + meta_bom["version"] += 1 + + # metadata.timestamp https://cyclonedx.org/docs/1.5/json/#metadata_timestamp + # Only update the timestamp if something has changed + if sbom_app_version_changed or sbom_components_changed: + meta_bom["metadata"]["timestamp"] = ( + datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + ) + else: + meta_bom["metadata"]["timestamp"] = prev_bom["metadata"]["timestamp"] + + # metadata.tools https://cyclonedx.org/docs/1.5/json/#metadata_tools + meta_bom["metadata"]["tools"] = endor_bom["metadata"]["tools"] + + write_sbom_json_file(meta_bom, sbom_out_path) + + # Access the collected warnings + print_banner("CONSOLIDATED WARNINGS") + warnings = [] + for record in warning_handler.warnings: + warnings.append(record.getMessage()) + + print("\n".join(warnings)) + + if save_warnings: + write_list_to_text_file(warnings, save_warnings) + + print_banner("COMPLETED") + if not os.getenv("CI"): + print("Be sure to add the SBOM to your next commit if the file content has changed.") + + # endregion Finalize SBOM + + # endregion Build composite SBOM + + +if __name__ == "__main__": + main() diff --git a/etc/sbom/metadata.cdx.json b/etc/sbom/metadata.cdx.json new file mode 100644 index 0000000000..54f92fa24e --- /dev/null +++ b/etc/sbom/metadata.cdx.json @@ -0,0 +1,304 @@ +{ + "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "serialNumber": "urn:uuid:89685c6d-f505-4943-a685-0fc532b179f8", + "version": 1, + "metadata": { + "timestamp": "2025-11-22T20:32:17Z", + "lifecycles": [ + { + "phase": "pre-build" + } + ], + "component": { + "type": "application", + "bom-ref": "pkg:github/mongodb/mongo-cxx-driver@{{VERSION}}", + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "author": "MongoDB, Inc.", + "publisher": "MongoDB, Inc.", + "group": "mongodb", + "name": "mongo-cxx-driver", + "description": "C++ Driver for MongoDB", + "version": "{{VERSION}}", + "cpe": "cpe:2.3:a:mongodb:c++:{{VERSION}}:*:*:*:*:*:*:*", + "purl": "pkg:github/mongodb/mongo-cxx-driver@{{VERSION}}", + "externalReferences": [ + { + "type": "license", + "url": "https://raw.githubusercontent.com/mongodb/mongo-cxx-driver/refs/heads/master/LICENSE", + "comment": "Apache License 2.0" + }, + { + "type": "website", + "url": "https://www.mongodb.com/docs/languages/cpp/cpp-driver/", + "comment": "Documentation site for the official MongoDB C++ Driver." + }, + { + "type": "release-notes", + "url": "https://www.mongodb.com/docs/languages/cpp/cpp-driver/current/whats-new/" + }, + { + "type": "vcs", + "url": "https://github.com/mongodb/mongo-cxx-driver" + } + ] + }, + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + } + }, + "components": [ + { + "type": "library", + "bom-ref": "pkg:github/madler/zlib@{{VERSION}}", + "supplier": { + "name": "zlib", + "url": [ + "https://zlib.net/" + ] + }, + "author": "Jean-loup Gailly, Mark Adler", + "group": "madler", + "name": "zlib", + "version": "{{VERSION}}", + "description": "zlib is a general purpose data compression library.", + "licenses": [ + { + "license": { + "id": "Zlib" + } + } + ], + "copyright": "Copyright \u00a9 1995-2024 Jean-loup Gailly and Mark Adler.", + "cpe": "cpe:2.3:a:zlib:zlib:{{VERSION}}:*:*:*:*:*:*:*", + "purl": "pkg:github/madler/zlib@{{VERSION}}", + "externalReferences": [ + { + "url": "https://zlib.net/fossils/", + "type": "distribution" + } + ], + "scope": "required" + }, + { + "bom-ref": "pkg:github/catchorg/catch2@{{VERSION}}", + "type": "library", + "author": "Martin Hořeňovský", + "group": "catchorg", + "name": "Catch2", + "version": "{{VERSION}}", + "description": "A modern, C++-native, test framework for unit-tests, TDD and BDD.", + "licenses": [ + { + "license": { + "id": "BSL-1.0" + } + } + ], + "copyright": "Copyright Catch2 Authors", + "purl": "pkg:github/catchorg/catch2@{{VERSION}}", + "externalReferences": [ + { + "url": "https://github.com/catchorg/catch2.git", + "type": "distribution" + } + ], + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/juliastrings/utf8proc@{{VERSION}}", + "type": "library", + "author": "Jan Behrens, the Public Software Group, the Julia-language developers", + "supplier": { + "name": "The Julia Programming Language", + "url": [ + "https://julialang.org/" + ] + }, + "group": "JuliaStrings", + "name": "utf8proc", + "version": "{{VERSION}}", + "description": "A clean C library for processing UTF-8 Unicode data", + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "copyright": "Copyright © 2014-2021 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.", + "purl": "pkg:github/juliastrings/utf8proc@{{VERSION}}", + "externalReferences": [ + { + "url": "https://github.com/juliastrings/utf8proc.git", + "type": "distribution" + } + ] + }, + { + "type": "library", + "bom-ref": "pkg:github/mongodb/libmongocrypt@{{VERSION}}", + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "author": "MongoDB, Inc.", + "group": "mongodb", + "name": "libmongocrypt", + "version": "{{VERSION}}", + "description": "C library for Client Side and Queryable Encryption in MongoDB", + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ], + "copyright": "Copyright 2019-present MongoDB, Inc.", + "cpe": "cpe:2.3:a:mongodb:libmongocrypt:{{VERSION}}:*:*:*:*:*:*:*", + "purl": "pkg:github/mongodb/libmongocrypt@{{VERSION}}", + "externalReferences": [ + { + "url": "https://github.com/mongodb/libmongocrypt.git", + "type": "distribution" + } + ], + "scope": "optional" + }, + { + "type": "library", + "bom-ref": "pkg:github/mongodb/mongo-c-driver@{{VERSION}}", + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "author": "MongoDB, Inc.", + "group": "mongodb", + "name": "MongoDB C Driver", + "version": "{{VERSION}}", + "description": "The Official MongoDB driver for C language", + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ], + "copyright": "2009-present, MongoDB, Inc.", + "cpe": "cpe:2.3:a:mongodb:c_driver:{{VERSION}}:*:*:*:*:*:*:*", + "purl": "pkg:github/mongodb/mongo-c-driver@{{VERSION}}", + "externalReferences": [ + { + "url": "https://github.com/mongodb/mongo-c-driver.git", + "type": "distribution" + } + ], + "scope": "required" + }, + { + "bom-ref": "pkg:github/catchorg/clara@{{VERSION}}", + "type": "library", + "author": "Phil Nash", + "group": "catchorg", + "name": "Clara", + "version": "{{VERSION}}", + "description": "A simple to use, composable, command line parser for C++ 11 and beyond", + "licenses": [ + { + "license": { + "id": "BSL-1.0" + } + } + ], + "copyright": "Copyright 2017 Two Blue Cubes Ltd. All rights reserved.", + "purl": "pkg:github/catchorg/clara@{{VERSION}}", + "externalReferences": [ + { + "url": "https://github.com/catchorg/clara.git", + "type": "distribution" + } + ], + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/troydhanson/uthash@{{VERSION}}", + "type": "library", + "author": "Troy D. Hanson", + "name": "github.com/troydhanson/uthash", + "version": "{{VERSION}}", + "description": "C macros for hash tables and more", + "licenses": [ + { + "license": { + "id": "BSD-1-Clause" + } + } + ], + "copyright": "Copyright (c) 2005-2025, Troy D. Hanson", + "purl": "pkg:github/troydhanson/uthash@{{VERSION}}", + "externalReferences": [ + { + "url": "https://github.com/troydhanson/uthash.git", + "type": "distribution" + } + ] + } + ], + "dependencies": [ + { + "ref": "pkg:github/mongodb/mongo-cxx-driver@{{VERSION}}", + "dependsOn": [ + "pkg:github/catchorg/catch2@{{VERSION}}", + "pkg:github/mongodb/mongo-c-driver@{{VERSION}}" + ] + }, + { + "ref": "pkg:github/catchorg/catch2@{{VERSION}}", + "dependsOn": [ + "pkg:github/catchorg/clara@{{VERSION}}" + ] + }, + { + "ref": "pkg:github/mongodb/mongo-c-driver@{{VERSION}}", + "dependsOn": [ + "pkg:github/madler/zlib@{{VERSION}}", + "pkg:github/juliastrings/utf8proc@{{VERSION}}", + "pkg:github/mongodb/libmongocrypt@{{VERSION}}", + "pkg:github/troydhanson/uthash@{{VERSION}}" + ] + }, + { + "ref": "pkg:github/catchorg/clara@{{VERSION}}", + "dependsOn": [] + }, + { + "ref": "pkg:github/juliastrings/utf8proc@{{VERSION}}", + "dependsOn": [] + }, + { + "ref": "pkg:github/madler/zlib@{{VERSION}}", + "dependsOn": [] + }, + { + "ref": "pkg:github/mongodb/libmongocrypt@{{VERSION}}", + "dependsOn": [] + }, + { + "ref": "pkg:github/troydhanson/uthash@{{VERSION}}", + "dependsOn": [] + } + ] +} \ No newline at end of file diff --git a/sbom.json b/sbom.json new file mode 100644 index 0000000000..2cf1320900 --- /dev/null +++ b/sbom.json @@ -0,0 +1,85 @@ +{ + "components": [ + { + "bom-ref": "pkg:github/mongodb/mongo-c-driver@2.1.2", + "copyright": "Copyright 2009-present MongoDB, Inc.", + "externalReferences": [ + { + "type": "distribution", + "url": "https://github.com/mongodb/mongo-c-driver/archive/refs/tags/2.1.2.tar.gz" + }, + { + "type": "website", + "url": "https://github.com/mongodb/mongo-c-driver/tree/2.1.2" + } + ], + "group": "mongodb", + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ], + "name": "mongo-c-driver", + "purl": "pkg:github/mongodb/mongo-c-driver@2.1.2", + "type": "library", + "version": "2.1.2" + } + ], + "dependencies": [ + { + "ref": "pkg:github/mongodb/mongo-c-driver@2.1.2" + } + ], + "metadata": { + "timestamp": "2025-10-07T17:27:33.460808+00:00", + "tools": [ + { + "externalReferences": [ + { + "type": "build-system", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/actions" + }, + { + "type": "distribution", + "url": "https://pypi.org/project/cyclonedx-python-lib/" + }, + { + "type": "documentation", + "url": "https://cyclonedx-python-library.readthedocs.io/" + }, + { + "type": "issue-tracker", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/issues" + }, + { + "type": "license", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/LICENSE" + }, + { + "type": "release-notes", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/CHANGELOG.md" + }, + { + "type": "vcs", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib" + }, + { + "type": "website", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/#readme" + } + ], + "name": "cyclonedx-python-lib", + "vendor": "CycloneDX", + "version": "6.4.4" + } + ] + }, + "serialNumber": "urn:uuid:837d822c-50e1-45de-bd89-fad61a9600c7", + "version": 1, + "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "vulnerabilities": [] +} From fc2675145e1ff0dd6e80b3955e204a62539332e2 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Tue, 25 Nov 2025 17:20:46 -0500 Subject: [PATCH 26/62] Refresh --- CMakeLists.txt | 1 - etc/sbom/config.py | 121 ++++--------------------------------- etc/sbom/endorctl_utils.py | 2 +- etc/sbom/generate_sbom.py | 107 +++++++++++++++++++++++++++++--- 4 files changed, 110 insertions(+), 121 deletions(-) mode change 100644 => 100755 etc/sbom/config.py diff --git a/CMakeLists.txt b/CMakeLists.txt index c6188d4a02..826f996ae5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,7 +56,6 @@ else() message(WARNING "Unknown compiler... recklessly proceeding without a version check") endif() -# Also update etc/purls.txt. set(BSON_REQUIRED_VERSION 2.1.2) set(MONGOC_REQUIRED_VERSION 2.1.2) set(MONGOC_DOWNLOAD_VERSION 2.1.2) diff --git a/etc/sbom/config.py b/etc/sbom/config.py old mode 100644 new mode 100755 index 5c92c24fcb..01cd735166 --- a/etc/sbom/config.py +++ b/etc/sbom/config.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """generate_sbom.py config. Operational configuration values stored separately from the core code.""" -import json import logging import re @@ -24,7 +23,16 @@ "pkg:github/", ] -for component in endor_components_remove: +components_remove = [ + # Endor Labs includes the main component in 'components'. This is not standard, so we remove it. + "10gen/mongo", + # should be pkg:github/antirez/linenoise - waiting on Endor Labs fix + "amokhuginnsson/replxx", + # a transitive dependency of s2 that is not necessary to include + "sparsehash/sparsehash", +] + +for component in components_remove: for prefix in prefixes: endor_components_remove.append(prefix + component) @@ -41,56 +49,6 @@ ["pkg:c/github.com/", "pkg:github/"], ] -# ################ PURL Validation ################ -REGEX_STR_PURL_OPTIONAL = ( # Optional Version (any chars except ? @ #) - r"(?:@[^?@#]*)?" - # Optional Qualifiers (any chars except @ #) - r"(?:\?[^@#]*)?" - # Optional Subpath (any chars) - r"(?:#.*)?$" -) - -REGEX_PURL = { - # deb PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/deb-definition.md - "deb": re.compile( - r"^pkg:deb/" # Scheme and type - # Namespace (organization/user), letters must be lowercase - r"(debian|ubuntu)+" - r"/" - r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name - ), - # Generic PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/generic-definition.md - "generic": re.compile( - r"^pkg:generic/" # Scheme and type - r"([a-zA-Z0-9._-]+/)?" # Optional namespace segment - r"[a-zA-Z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name (required) - ), - # GitHub PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/github-definition.md - "github": re.compile( - r"^pkg:github/" # Scheme and type - # Namespace (organization/user), letters must be lowercase - r"[a-z0-9-]+" - r"/" - r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name (repository) - ), - # PyPI PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md - "pypi": re.compile( - r"^pkg:pypi/" # Scheme and type - r"[a-z0-9_-]+" # Name, letters must be lowercase, dashes, underscore - + REGEX_STR_PURL_OPTIONAL - ), -} - - -def is_valid_purl(purl: str) -> bool: - """Validate a GitHub or Generic PURL""" - for purl_type, regex in REGEX_PURL.items(): - if regex.match(purl): - logger.debug(f"PURL: {purl} matched PURL type '{purl_type}' regex '{regex.pattern}'") - return True - return False - - # ################ Version Transformation ################ # In some cases we need to transform the version string to strip out tag-related text @@ -140,65 +98,8 @@ def get_semver_from_release_version(release_ver: str) -> str: # region special component use-case functions -def get_version_from_wiredtiger_import_data(file_path: str) -> str: - """Get the info in the 'import.data' file saved in the wiredtiger folder""" - try: - with open(file_path, "r") as input_json: - import_data = input_json.read() - result = json.loads(import_data) - except Exception as e: - logger.error(f"Error loading JSON file from {file_path}") - logger.error(e) - return None - return result.get("commit") - - -def get_version_sasl_from_workspace(file_path: str) -> str: - """Determine the version that is pulled for Windows Cyrus SASL by searching WORKSPACE.bazel""" - # e.g., - # "https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-2.1.28.zip", - try: - with open(file_path, "r") as file: - for line in file: - if line.strip().startswith( - '"https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-' - ): - return line.strip().split("windows_cyrus_sasl-")[1].split(".zip")[0] - except Exception as e: - logger.warning(f"Unable to load {file_path}") - logger.warning(e) - else: - return None - - def process_component_special_cases( component_key: str, component: dict, versions: dict, repo_root: str ) -> None: - ## Special case for Cyrus SASL ## - if component_key == "pkg:github/cyrusimap/cyrus-sasl": - # Cycrus SASL is optionally loaded as a Windows library, when needed. There is no source code for Endor Labs to scan. - # The version of Cyrus SASL that is used is defined in the WORKSPACE.bazel file: - # "https://s3.amazonaws.com/boxes.10gen.com/build/windows_cyrus_sasl-2.1.28.zip", - # Rather than add the complexity of Bazel queries to this script, we just search the text. - - versions["import_script"] = get_version_sasl_from_workspace(repo_root + "/WORKSPACE.bazel") - logger.info( - f"VERSION SPECIAL CASE: {component_key}: Found version '{versions['import_script']}' in 'WORKSPACE.bazel' file" - ) - - ## Special case for wiredtiger ## - elif component_key == "pkg:github/wiredtiger/wiredtiger": - # MongoDB release branches import wiredtiger commits via a bot. These commits will likely not line up with a release or tag. - # Endor labs will try to pull the nearest release/tag, but we want the more precise commit hash, which is stored in: - # src/third_party/wiredtiget/import.data - occurrences = component.get("evidence", {}).get("occurrences", []) - if occurrences: - location = occurrences[0].get("location") - versions["import_script"] = get_version_from_wiredtiger_import_data( - f"{repo_root}/{location}/import.data" - ) - logger.info( - f"VERSION SPECIAL CASE: {component_key}: Found version '{versions['import_script']}' in 'import.data' file" - ) - + pass # endregion special component use-case functions diff --git a/etc/sbom/endorctl_utils.py b/etc/sbom/endorctl_utils.py index ae2738aeb8..8de36797e6 100644 --- a/etc/sbom/endorctl_utils.py +++ b/etc/sbom/endorctl_utils.py @@ -437,7 +437,7 @@ def get_sbom_for_branch(self, git_url: str, branch: str) -> dict: # ScanResult: search for a completed scan filter_str = endor_filter.scan_result( - None, project_uuid, repository_version_ref, repository_version_sha + EndorContextType.MAIN, project_uuid, repository_version_ref, repository_version_sha ) scan_result = self.get_scan_result(filter_str, retry=False) project_uuid = scan_result["meta"]["parent_uuid"] diff --git a/etc/sbom/generate_sbom.py b/etc/sbom/generate_sbom.py index 6335d6591d..0dd2f5c1b7 100755 --- a/etc/sbom/generate_sbom.py +++ b/etc/sbom/generate_sbom.py @@ -23,7 +23,6 @@ endor_components_remove, endor_components_rename, get_semver_from_release_version, - is_valid_purl, process_component_special_cases, ) from endorctl_utils import EndorCtl @@ -54,7 +53,6 @@ def emit(self, record): # Add the handler to the logger logger.addHandler(warning_handler) - # Get the absolute path of the script file and directory script_path = Path(__file__).resolve() script_directory = script_path.parent @@ -66,6 +64,65 @@ def emit(self, record): REGEX_RELEASE_BRANCH = r"^v\d\.\d$" REGEX_RELEASE_TAG = r"^r\d\.\d.\d(-\w*)?$" +# ################ PURL Validation ################ +REGEX_STR_PURL_OPTIONAL = ( # Optional Version (any chars except ? @ #) + r"(?:@[^?@#]*)?" + # Optional Qualifiers (any chars except @ #) + r"(?:\?[^@#]*)?" + # Optional Subpath (any chars) + r"(?:#.*)?$" +) + +REGEX_PURL = { + # deb PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/deb-definition.md + "deb": re.compile( + r"^pkg:deb/" # Scheme and type + # Namespace (organization/user), letters must be lowercase + r"(debian|ubuntu)+" + r"/" + r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name + ), + # Generic PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/generic-definition.md + "generic": re.compile( + r"^pkg:generic/" # Scheme and type + r"([a-zA-Z0-9._-]+/)?" # Optional namespace segment + r"[a-zA-Z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name (required) + ), + # GitHub PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/github-definition.md + "github": re.compile( + r"^pkg:github/" # Scheme and type + # Namespace (organization/user), letters must be lowercase + r"[a-z0-9-]+" + r"/" + r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name (repository) + ), + # PyPI PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md + "pypi": re.compile( + r"^pkg:pypi/" # Scheme and type + r"[a-z0-9_-]+" # Name, letters must be lowercase, dashes, underscore + + REGEX_STR_PURL_OPTIONAL + ), +} + + +# Metadata SBOM requirements +METADATA_FIELDS_REQUIRED = [ + "type", + "bom-ref", + "group", + "name", + "version", + "description", + "licenses", + "copyright", + "externalReferences", + "scope", +] +METADATA_FIELDS_ONE_OF = [ + ["author", "supplier"], + ["purl", "cpe"], +] + # endregion init @@ -80,7 +137,11 @@ def __init__(self): try: self.repo_root = Path( subprocess.run( - "git rev-parse --show-toplevel", shell=True, text=True, capture_output=True + "git rev-parse --show-toplevel", + shell=True, + text=True, + capture_output=True, + check=True, ).stdout.strip() ) self._repo = Repo(self.repo_root) @@ -170,6 +231,15 @@ def extract_repo_from_git_url(git_url: str) -> dict: } +def is_valid_purl(purl: str) -> bool: + """Validate a GitHub or Generic PURL""" + for purl_type, regex in REGEX_PURL.items(): + if regex.match(purl): + logger.debug(f"PURL: {purl} matched PURL type '{purl_type}' regex '{regex.pattern}'") + return True + return False + + def sbom_components_to_dict(sbom: dict, with_version: bool = False) -> dict: """Create a dict of SBOM components with a version-less PURL as the key""" components = sbom["components"] @@ -185,6 +255,23 @@ def sbom_components_to_dict(sbom: dict, with_version: bool = False) -> dict: return components_dict +def check_metadata_sbom(meta_bom: dict) -> None: + for component in meta_bom["components"]: + for field in METADATA_FIELDS_REQUIRED: + if field not in component: + logger.warning( + f"METADATA: '{component['bom-ref'] or component['name']} is missing required field '{field}'." + ) + for fields in METADATA_FIELDS_ONE_OF: + found = False + for field in fields: + found = found or field in component + if not found: + logger.warning( + f"METADATA: '{component['bom-ref'] or component['name']} is missing one of fields '{fields}'." + ) + + def read_sbom_json_file(file_path: str) -> dict: """Load a JSON SBOM file (schema is not validated)""" try: @@ -204,8 +291,8 @@ def write_sbom_json_file(sbom_dict: dict, file_path: str) -> None: try: file_path = os.path.abspath(file_path) with open(file_path, "w", encoding="utf-8") as output_json: - json.dump(sbom_dict, output_json, indent=2) - output_json.write("\n") + formatted_sbom = json.dumps(sbom_dict, indent=2) + "\n" + output_json.write(formatted_sbom) except Exception as e: logger.error(f"Error writing SBOM file to {file_path}") logger.error(e) @@ -449,6 +536,8 @@ def main() -> None: endor_bom = endorctl.get_sbom_for_branch(git_info.project, git_info.branch) elif target == "project": endor_bom = endorctl.get_sbom_for_project(git_info.project) + else: + endor_bom = None if not endor_bom: logger.error("Empty result for Endor SBOM!") @@ -466,9 +555,6 @@ def main() -> None: ## remove uneeded components ## # [list]endor_components_remove is defined in config.py - # Endor Labs includes the main component in 'components'. This is not standard, so we remove it. - endor_components_remove.append(f"pkg:github/{git_info.org}/{git_info.repo}") - # Reverse iterate the SBOM components list to safely modify in situ for i in range(len(endor_bom["components"]) - 1, -1, -1): component = endor_bom["components"][i] @@ -529,6 +615,9 @@ def main() -> None: meta_bom["components"].sort(key=lambda c: c["bom-ref"]) prev_bom["components"].sort(key=lambda c: c["bom-ref"]) + # Check metadata SBOM for completeness + check_metadata_sbom(meta_bom) + # Create SBOM component lookup dicts endor_components = sbom_components_to_dict(endor_bom) prev_components = sbom_components_to_dict(prev_bom) @@ -537,7 +626,7 @@ def main() -> None: # Attempt to determine the MongoDB Version being scanned logger.debug( - f"Available MongoDB version options, tag: {git_info.release_tag}, branch: {git_info.branch}, previous SBOM: {prev_bom['metadata'].get('component',{}).get('version')}" + f"Available MongoDB version options, tag: {git_info.release_tag}, branch: {git_info.branch}, previous SBOM: {prev_bom['metadata']['component']['version']}" ) meta_bom_ref = meta_bom["metadata"]["component"]["bom-ref"] From dc1865701df2a17e7b94d5a2f92774b9446e93fc Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 26 Nov 2025 10:07:23 -0500 Subject: [PATCH 27/62] Add --enable-github-action-token --- .github/workflows/generate_sbom.yml | 2 +- etc/sbom/endorctl_utils.py | 4 ++++ etc/sbom/generate_sbom.py | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index bae69d286b..9b5b8fb6cd 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -52,4 +52,4 @@ jobs: - name: Install dependencies run: uv sync --group make_release - name: generate_sbom.py - run: uv run etc/sbom/generate_sbom.py --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json \ No newline at end of file + run: uv run etc/sbom/generate_sbom.py --enable-github-action-token --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json \ No newline at end of file diff --git a/etc/sbom/endorctl_utils.py b/etc/sbom/endorctl_utils.py index 8de36797e6..02d6b2ecb9 100644 --- a/etc/sbom/endorctl_utils.py +++ b/etc/sbom/endorctl_utils.py @@ -145,12 +145,14 @@ def __init__( sleep_duration=30, endorctl_path="endorctl", config_path=None, + enable_github_action_token=False ): self.namespace = namespace self.retry_limit = retry_limit self.sleep_duration = sleep_duration self.endorctl_path = endorctl_path self.config_path = config_path + self.enable_github_action_token = enable_github_action_token def _call_endorctl(self, command, subcommand, **kwargs): """https://docs.endorlabs.com/endorctl/""" @@ -159,6 +161,8 @@ def _call_endorctl(self, command, subcommand, **kwargs): command = [self.endorctl_path, command, subcommand, f"--namespace={self.namespace}"] if self.config_path: command.append(f"--config-path={self.config_path}") + if self.enable_github_action_token: + command.append(f"--enable-github-action-token") # parse args into flags for key, value in kwargs.items(): diff --git a/etc/sbom/generate_sbom.py b/etc/sbom/generate_sbom.py index 0dd2f5c1b7..7fe5e5ea98 100755 --- a/etc/sbom/generate_sbom.py +++ b/etc/sbom/generate_sbom.py @@ -400,6 +400,7 @@ def main() -> None: default=None, type=str, ) + endor.add_argument("--enable-github-action-token", help="Enable keyless authentication using Github action OIDC tokens", action="store_true") endor.add_argument( "--namespace", help="Endor Labs namespace (Default: mongodb.{git org})", type=str ) @@ -479,6 +480,7 @@ def main() -> None: # endor endorctl_path = args.endorctl_path config_path = args.config_path + enable_github_action_token = args.enable_github_action_token namespace = args.namespace if args.namespace else f"mongodb.{git_info.org}" target = args.target @@ -529,7 +531,7 @@ def main() -> None: # region export Endor Labs SBOM print_banner(f"Exporting Endor Labs SBOM for {target} {getattr(git_info, target)}") - endorctl = EndorCtl(namespace, retry_limit, sleep_duration, endorctl_path, config_path) + endorctl = EndorCtl(namespace, retry_limit, sleep_duration, endorctl_path, config_path, enable_github_action_token=enable_github_action_token) if target == "commit": endor_bom = endorctl.get_sbom_for_commit(git_info.project, git_info.commit) elif target == "branch": From 5f1011910586129643d8b5f79ef1ca9ec8439c59 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 26 Nov 2025 10:47:44 -0500 Subject: [PATCH 28/62] Improve context_type tracking --- etc/sbom/endorctl_utils.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/etc/sbom/endorctl_utils.py b/etc/sbom/endorctl_utils.py index 02d6b2ecb9..356464bc1d 100644 --- a/etc/sbom/endorctl_utils.py +++ b/etc/sbom/endorctl_utils.py @@ -56,10 +56,21 @@ class EndorContextType(Enum): # Objects from a scan of the default branch. All objects in the OSS namespace are in the main context. The context ID is always default. MAIN = "CONTEXT_TYPE_MAIN" + CONTEXT_TYPE_MAIN = "CONTEXT_TYPE_MAIN" # Objects from a scan of a specific branch. The context ID is the branch reference name. REF = "CONTEXT_TYPE_REF" + CONTEXT_TYPE_REF = "CONTEXT_TYPE_REF" # Objects from a PR scan. The context ID is the PR UUID. Objects in this context are deleted after 30 days. CI_RUN = "CONTEXT_TYPE_CI_RUN" + CONTEXT_TYPE_CI_RUN = "CONTEXT_TYPE_CI_RUN" + # Objects from an SBOM scan. The context ID is the SBOM serial number or some other unique identifier. + SBOM = "CONTEXT_TYPE_SBOM" + CONTEXT_TYPE_SBOM = "CONTEXT_TYPE_SBOM" + # Indicates that this object is a copy/temporary value of an object in another project. Used for same-tenant dependencies. + # In source code reference this is equivalent to “vendor” folders. Package versions in the external context are only scanned for call + # graphs. No other operations are performed on them. + EXTERNAL = "CONTEXT_TYPE_EXTERNAL" + CONTEXT_TYPE_EXTERNAL = "CONTEXT_TYPE_EXTERNAL" class EndorFilter: @@ -78,8 +89,12 @@ def _base_filters(self): return base_filters - def repository_version(self, project_uuid=None, sha=None, ref=None): + def repository_version(self, project_uuid=None, sha=None, ref=None, context_type:EndorContextType=None, context_type_exclude:EndorContextType=None): filters = self._base_filters() + if context_type: + filters.append(f"context.type=={context_type.value}") + if context_type_exclude: + filters.append(f"context.type!={context_type_exclude.value}") if project_uuid: filters.append(f"meta.parent_uuid=={project_uuid}") if sha: @@ -428,8 +443,9 @@ def get_sbom_for_branch(self, git_url: str, branch: str) -> dict: app_name = project["spec"]["git"]["full_name"] # RepositoryVersion: get the context for the latest branch scan - filter_str = endor_filter.repository_version(project_uuid, ref=branch) + filter_str = endor_filter.repository_version(project_uuid, ref=branch, context_type_exclude=EndorContextType.CI_RUN) repository_version = self.get_repository_version(filter_str) + repository_version_context_type = EndorContextType[repository_version["context"]["type"]] repository_version_uuid = repository_version["uuid"] repository_version_ref = repository_version["spec"]["version"]["ref"] repository_version_sha = repository_version["spec"]["version"]["sha"] @@ -441,13 +457,13 @@ def get_sbom_for_branch(self, git_url: str, branch: str) -> dict: # ScanResult: search for a completed scan filter_str = endor_filter.scan_result( - EndorContextType.MAIN, project_uuid, repository_version_ref, repository_version_sha + repository_version_context_type, project_uuid, repository_version_ref, repository_version_sha ) scan_result = self.get_scan_result(filter_str, retry=False) project_uuid = scan_result["meta"]["parent_uuid"] # PackageVersions: get package versions for SBOM - if branch == "master": + if branch in ["master","main"]: context_type = EndorContextType.MAIN context_id = "default" else: From 23f2f1557ae80772eac16b7edf893c390f9c0281 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 26 Nov 2025 12:50:40 -0500 Subject: [PATCH 29/62] Improvements plus add PR --- .github/workflows/generate_sbom.yml | 12 +- etc/sbom/config.py | 88 +++-- etc/sbom/endorctl_utils.py | 281 +++++++-------- etc/sbom/generate_sbom.py | 526 ++++++++++++++-------------- etc/sbom/metadata.cdx.json | 7 +- pyproject.toml | 7 + 6 files changed, 475 insertions(+), 446 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 9b5b8fb6cd..4f2ba568e1 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -52,4 +52,14 @@ jobs: - name: Install dependencies run: uv sync --group make_release - name: generate_sbom.py - run: uv run etc/sbom/generate_sbom.py --enable-github-action-token --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json \ No newline at end of file + run: uv run etc/sbom/generate_sbom.py --enable-github-action-token --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json --save-warnings=${{runner.temp}}/warnings.txt + + - name: Open Pull Request + uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9 + with: + add-paths: sbom.json + body-path: ${{runner.temp}}/warnings.txt + branch: cxx-sbom-update + commit-message: Update SBOM file(s) + delete-branch: true + title: CXX Update SBOM action \ No newline at end of file diff --git a/etc/sbom/config.py b/etc/sbom/config.py index 01cd735166..82320bdec7 100755 --- a/etc/sbom/config.py +++ b/etc/sbom/config.py @@ -3,36 +3,30 @@ import logging import re +import semver +import subprocess -logger = logging.getLogger("generate_sbom") +logger = logging.getLogger('generate_sbom') logger.setLevel(logging.NOTSET) - # ################ Component Filters ################ # List of Endor Labs SBOM components that must be removed before processing endor_components_remove = [ + # A dependency erroneously matched in build/CMakeFiles + 'mozilla/cubeb', # An incorrect match from parts of pkg:github/madler/zlib - "zlib-ng/zlib-ng", + 'zlib-ng/zlib-ng', ] # bom-ref prefixes (Endor Labs has been changing them, so add all that we have seen) prefixes = [ - "pkg:c/github.com/", - "pkg:generic/github.com/", - "pkg:github/", -] - -components_remove = [ - # Endor Labs includes the main component in 'components'. This is not standard, so we remove it. - "10gen/mongo", - # should be pkg:github/antirez/linenoise - waiting on Endor Labs fix - "amokhuginnsson/replxx", - # a transitive dependency of s2 that is not necessary to include - "sparsehash/sparsehash", + 'pkg:c/github.com/', + 'pkg:generic/github.com/', + 'pkg:github/', ] -for component in components_remove: +for component in endor_components_remove: for prefix in prefixes: endor_components_remove.append(prefix + component) @@ -43,50 +37,74 @@ # Valid: pkg:github/abseil/abseil-cpp@20250512.1 # Run string replacements to correct for this: endor_components_rename = [ - ["pkg:generic/zlib.net/zlib", "pkg:github/madler/zlib"], - ["pkg:github/philsquared/clara", "pkg:github/catchorg/clara"], - ["pkg:generic/github.com/", "pkg:github/"], - ["pkg:c/github.com/", "pkg:github/"], + ['pkg:generic/zlib.net/zlib', 'pkg:github/madler/zlib'], + ['pkg:github/philsquared/clara', 'pkg:github/catchorg/clara'], + # in case of regression + ['pkg:generic/github.com/', 'pkg:github/'], + ['pkg:c/github.com/', 'pkg:github/'], ] + +# ################ Primary Component Version ################ +def get_primary_component_version() -> str: + """Attempt to determine primary component version using repo script.""" + + # mongo-cxx-driver: etc/calc_release_version.py + try: + result = subprocess.run(["python", "etc/calc_release_version.py"], capture_output=True, text=True) + version = semver.VersionInfo.parse(result.stdout) + if version.match("0.0.0"): + return None + else: + return version + except Exception as e: + logger.warning("PRIMARY COMPONENT VERSION: Unable to parse output from etc/calc_release_version.py: %s", result.stdout) + logger.warning(e) + return None + + # ################ Version Transformation ################ # In some cases we need to transform the version string to strip out tag-related text # It is unknown what patterns may appear in the future, so we have targeted (not broad) regex # This a list of 'pattern' and 'repl' inputs to re.sub() -RE_VER_NUM = r"(0|[1-9]\d*)" -RE_VER_LBL = r"(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?" -RE_SEMVER = rf"{RE_VER_NUM}\.{RE_VER_NUM}\.{RE_VER_NUM}{RE_VER_LBL}" +RE_VER_NUM = r'(0|[1-9]\d*)' +RE_VER_LBL = r'(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?' +RE_SEMVER = rf'{RE_VER_NUM}\.{RE_VER_NUM}\.{RE_VER_NUM}{RE_VER_LBL}' regex_semver = re.compile(RE_SEMVER) +# Release Naming Conventions +REGEX_RELEASE_BRANCH = rf'^releases/v{RE_SEMVER}$' # e.g., releases/v4.1 +REGEX_RELEASE_TAG = rf'^(r{RE_SEMVER})|(debian/{RE_SEMVER}-1)$' # e.g., r3.7.0-beta1, debian/4.1.4-1 + VERSION_PATTERN_REPL = [ # 'debian/1.28.1-1' pkg:github/mongodb/mongo-c-driver (temporary workaround) - [re.compile(rf"^debian/({RE_SEMVER})-\d$"), r"\1"], + [re.compile(rf'^debian/({RE_SEMVER})-\d$'), r'\1'], # 'gperftools-2.9.1' pkg:github/gperftools/gperftools # 'mongo/v1.5.2' pkg:github/google/benchmark # 'mongodb-8.2.0-alpha2' pkg:github/wiredtiger/wiredtiger # 'release-1.12.0' pkg:github/apache/avro # 'yaml-cpp-0.6.3' pkg:github/jbeder/yaml-cpp - [re.compile(rf"^[-a-z]+[-/][vr]?({RE_SEMVER})$"), r"\1"], + [re.compile(rf'^[-a-z]+[-/][vr]?({RE_SEMVER})$'), r'\1'], # 'asio-1-34-2' pkg:github/chriskohlhoff/asio # 'cares-1_27_0' pkg:github/c-ares/c-ares [ - re.compile(rf"^[a-z]+-{RE_VER_NUM}[_-]{RE_VER_NUM}[_-]{RE_VER_NUM}{RE_VER_LBL}$"), - r"\1.\2.\3", + re.compile(rf'^[a-z]+-{RE_VER_NUM}[_-]{RE_VER_NUM}[_-]{RE_VER_NUM}{RE_VER_LBL}$'), + r'\1.\2.\3', ], # 'pcre2-10.40' pkg:github/pcre2project/pcre2 - [re.compile(rf"^[a-z0-9]+-({RE_VER_NUM}\.{RE_VER_NUM})$"), r"\1"], + [re.compile(rf'^[a-z0-9]+-({RE_VER_NUM}\.{RE_VER_NUM})$'), r'\1'], # 'icu-release-57-1' pkg:github/unicode-org/icu - [re.compile(rf"^[a-z]+-?[a-z]+-{RE_VER_NUM}-{RE_VER_NUM}$"), r"\1.\2"], + [re.compile(rf'^[a-z]+-?[a-z]+-{RE_VER_NUM}-{RE_VER_NUM}$'), r'\1.\2'], # 'v2.6.0' pkg:github/confluentinc/librdkafka # 'r2.5.1' - [re.compile(rf"^[rv]({RE_SEMVER})$"), r"\1"], + [re.compile(rf'^[rv]({RE_SEMVER})$'), r'\1'], # 'v2025.04.21.00' pkg:github/facebook/folly - [re.compile(r"^v(\d+\.\d+\.\d+\.\d+)$"), r"\1"], + [re.compile(r'^v(\d+\.\d+\.\d+\.\d+)$'), r'\1'], ] -def get_semver_from_release_version(release_ver: str) -> str: +def get_semver_from_release_version(release_ver: str) -> semver: """Extract the version number from string with tags or other annotations""" if release_ver: for re_obj, repl in VERSION_PATTERN_REPL: @@ -98,8 +116,8 @@ def get_semver_from_release_version(release_ver: str) -> str: # region special component use-case functions -def process_component_special_cases( - component_key: str, component: dict, versions: dict, repo_root: str -) -> None: +def process_component_special_cases(component_key: str, component: dict, versions: dict, repo_root: str) -> None: pass + + # endregion special component use-case functions diff --git a/etc/sbom/endorctl_utils.py b/etc/sbom/endorctl_utils.py index 356464bc1d..0761edc538 100644 --- a/etc/sbom/endorctl_utils.py +++ b/etc/sbom/endorctl_utils.py @@ -11,43 +11,43 @@ from datetime import datetime from enum import Enum -logger = logging.getLogger("generate_sbom") +logger = logging.getLogger('generate_sbom') logger.setLevel(logging.NOTSET) default_field_masks = { - "PackageVersion": [ - "context", - "meta", - "processing_status", - "spec.package_name", - "spec.resolved_dependencies.dependencies", - "spec.source_code_reference", + 'PackageVersion': [ + 'context', + 'meta', + 'processing_status', + 'spec.package_name', + 'spec.resolved_dependencies.dependencies', + 'spec.source_code_reference', ], - "ScanResult": [ - "context", - "meta", - "spec.end_time", - "spec.logs", - "spec.refs", - "spec.start_time", - "spec.status", - "spec.versions", + 'ScanResult': [ + 'context', + 'meta', + 'spec.end_time', + 'spec.logs', + 'spec.refs', + 'spec.start_time', + 'spec.status', + 'spec.versions', ], } def _get_default_field_mask(kind): default_field_mask = default_field_masks.get(kind, []) - return ",".join(default_field_mask) + return ','.join(default_field_mask) class EndorResourceKind(Enum): """Enumeration for Endor Labs API resource kinds""" - PROJECT = "Project" - REPOSITORY_VERSION = "RepositoryVersion" - SCAN_RESULT = "ScanResult" - PACKAGE_VERSION = "PackageVersion" + PROJECT = 'Project' + REPOSITORY_VERSION = 'RepositoryVersion' + SCAN_RESULT = 'ScanResult' + PACKAGE_VERSION = 'PackageVersion' class EndorContextType(Enum): @@ -55,22 +55,22 @@ class EndorContextType(Enum): https://docs.endorlabs.com/rest-api/using-the-rest-api/data-model/common-fields/#context""" # Objects from a scan of the default branch. All objects in the OSS namespace are in the main context. The context ID is always default. - MAIN = "CONTEXT_TYPE_MAIN" - CONTEXT_TYPE_MAIN = "CONTEXT_TYPE_MAIN" + MAIN = 'CONTEXT_TYPE_MAIN' + CONTEXT_TYPE_MAIN = 'CONTEXT_TYPE_MAIN' # Objects from a scan of a specific branch. The context ID is the branch reference name. - REF = "CONTEXT_TYPE_REF" - CONTEXT_TYPE_REF = "CONTEXT_TYPE_REF" + REF = 'CONTEXT_TYPE_REF' + CONTEXT_TYPE_REF = 'CONTEXT_TYPE_REF' # Objects from a PR scan. The context ID is the PR UUID. Objects in this context are deleted after 30 days. - CI_RUN = "CONTEXT_TYPE_CI_RUN" - CONTEXT_TYPE_CI_RUN = "CONTEXT_TYPE_CI_RUN" + CI_RUN = 'CONTEXT_TYPE_CI_RUN' + CONTEXT_TYPE_CI_RUN = 'CONTEXT_TYPE_CI_RUN' # Objects from an SBOM scan. The context ID is the SBOM serial number or some other unique identifier. - SBOM = "CONTEXT_TYPE_SBOM" - CONTEXT_TYPE_SBOM = "CONTEXT_TYPE_SBOM" + SBOM = 'CONTEXT_TYPE_SBOM' + CONTEXT_TYPE_SBOM = 'CONTEXT_TYPE_SBOM' # Indicates that this object is a copy/temporary value of an object in another project. Used for same-tenant dependencies. # In source code reference this is equivalent to “vendor” folders. Package versions in the external context are only scanned for call # graphs. No other operations are performed on them. - EXTERNAL = "CONTEXT_TYPE_EXTERNAL" - CONTEXT_TYPE_EXTERNAL = "CONTEXT_TYPE_EXTERNAL" + EXTERNAL = 'CONTEXT_TYPE_EXTERNAL' + CONTEXT_TYPE_EXTERNAL = 'CONTEXT_TYPE_EXTERNAL' class EndorFilter: @@ -83,26 +83,33 @@ def __init__(self, context_id=None, context_type=None): def _base_filters(self): base_filters = [] if self.context_id: - base_filters.append(f"context.id=={self.context_id}") + base_filters.append(f'context.id=={self.context_id}') if self.context_type: - base_filters.append(f"context.type=={self.context_type}") + base_filters.append(f'context.type=={self.context_type}') return base_filters - def repository_version(self, project_uuid=None, sha=None, ref=None, context_type:EndorContextType=None, context_type_exclude:EndorContextType=None): + def repository_version( + self, + project_uuid=None, + sha=None, + ref=None, + context_type: EndorContextType = None, + context_type_exclude: EndorContextType = None, + ): filters = self._base_filters() if context_type: - filters.append(f"context.type=={context_type.value}") + filters.append(f'context.type=={context_type.value}') if context_type_exclude: - filters.append(f"context.type!={context_type_exclude.value}") + filters.append(f'context.type!={context_type_exclude.value}') if project_uuid: - filters.append(f"meta.parent_uuid=={project_uuid}") + filters.append(f'meta.parent_uuid=={project_uuid}') if sha: - filters.append(f"spec.version.sha=={sha}") + filters.append(f'spec.version.sha=={sha}') if ref: - filters.append(f"spec.version.ref=={ref}") + filters.append(f'spec.version.ref=={ref}') - return " and ".join(filters) + return ' and '.join(filters) def package_version( self, @@ -114,17 +121,17 @@ def package_version( ): filters = self._base_filters() if context_type: - filters.append(f"context.type=={context_type.value}") + filters.append(f'context.type=={context_type.value}') if context_type: - filters.append(f"context.id=={context_id}") + filters.append(f'context.id=={context_id}') if project_uuid: - filters.append(f"spec.project_uuid=={project_uuid}") + filters.append(f'spec.project_uuid=={project_uuid}') if name: - filters.append(f"spec.package_name=={name}") + filters.append(f'spec.package_name=={name}') if package_name: - filters.append(f"meta.name=={package_name}") + filters.append(f'meta.name=={package_name}') - return " and ".join(filters) + return ' and '.join(filters) def scan_result( self, @@ -136,17 +143,17 @@ def scan_result( ): filters = self._base_filters() if context_type: - filters.append(f"context.type=={context_type.value}") + filters.append(f'context.type=={context_type.value}') if project_uuid: - filters.append(f"meta.parent_uuid=={project_uuid}") + filters.append(f'meta.parent_uuid=={project_uuid}') if ref: filters.append(f"spec.versions.ref contains '{ref}'") if sha: filters.append(f"spec.versions.sha contains '{sha}'") if status: - filters.append(f"spec.status=={status}") + filters.append(f'spec.status=={status}') - return " and ".join(filters) + return ' and '.join(filters) class EndorCtl: @@ -158,9 +165,9 @@ def __init__( namespace, retry_limit=5, sleep_duration=30, - endorctl_path="endorctl", + endorctl_path='endorctl', config_path=None, - enable_github_action_token=False + enable_github_action_token=False, ): self.namespace = namespace self.retry_limit = retry_limit @@ -173,43 +180,43 @@ def _call_endorctl(self, command, subcommand, **kwargs): """https://docs.endorlabs.com/endorctl/""" try: - command = [self.endorctl_path, command, subcommand, f"--namespace={self.namespace}"] + command = [self.endorctl_path, command, subcommand, f'--namespace={self.namespace}'] if self.config_path: - command.append(f"--config-path={self.config_path}") + command.append(f'--config-path={self.config_path}') if self.enable_github_action_token: - command.append(f"--enable-github-action-token") + command.append(f'--enable-github-action-token') # parse args into flags for key, value in kwargs.items(): # Handle endorctl flags with hyphens that are defined in the script with underscores - flag = key.replace("_", "-") + flag = key.replace('_', '-') if value: - command.append(f"--{flag}={value}") - logger.info("Running: %s", " ".join(command)) + command.append(f'--{flag}={value}') + logger.info('Running: %s', ' '.join(command)) result = subprocess.run(command, capture_output=True, text=True, check=True) resource = json.loads(result.stdout) except subprocess.CalledProcessError as e: - logger.error(f"Error executing command: {e}") + logger.error(f'Error executing command: {e}') logger.error(e.stderr) except json.JSONDecodeError as e: - logger.error(f"Error decoding JSON: {e}") - logger.error(f"Stdout: {result.stdout}") + logger.error(f'Error decoding JSON: {e}') + logger.error(f'Stdout: {result.stdout}') except FileNotFoundError as e: - logger.error(f"FileNotFoundError: {e}") + logger.error(f'FileNotFoundError: {e}') logger.error( f"'endorctl' not found in path '{self.endorctl_path}'. Supply the correct path, run 'buildscripts/install_endorctl.sh' or visit https://docs.endorlabs.com/endorctl/install-and-configure/" ) except Exception as e: - logger.error(f"An unexpected error occurred: {e}") + logger.error(f'An unexpected error occurred: {e}') else: return resource def _api_get(self, resource, **kwargs): """https://docs.endorlabs.com/endorctl/commands/api/""" - return self._call_endorctl("api", "get", resource=resource, **kwargs) + return self._call_endorctl('api', 'get', resource=resource, **kwargs) def _api_list(self, resource, filter=None, retry=True, **kwargs): """https://docs.endorlabs.com/endorctl/commands/api/""" @@ -217,20 +224,20 @@ def _api_list(self, resource, filter=None, retry=True, **kwargs): tries = 0 while True: tries += 1 - result = self._call_endorctl("api", "list", resource=resource, filter=filter, **kwargs) + result = self._call_endorctl('api', 'list', resource=resource, filter=filter, **kwargs) # The expected output of 'endorctl api list' is: { "list": { "objects": [...] } } # We want to just return the objects. In case we get an empty list, return a list # with a single None to avoid having to handle index errors downstream. - if result and result["list"].get("objects") and len(result["list"]["objects"]) > 0: - return result["list"]["objects"] + if result and result['list'].get('objects') and len(result['list']['objects']) > 0: + return result['list']['objects'] elif retry: logger.info( f"API LIST: Resource not found: {resource} with filter '{filter}' in namespace '{self.namespace}'" ) if tries <= self.retry_limit: logger.info( - f"API LIST: Waiting for {self.sleep_duration} seconds before retry attempt {tries} of {self.retry_limit}" + f'API LIST: Waiting for {self.sleep_duration} seconds before retry attempt {tries} of {self.retry_limit}' ) time.sleep(self.sleep_duration) else: @@ -243,8 +250,8 @@ def _api_list(self, resource, filter=None, retry=True, **kwargs): def _check_resource(self, resource, resource_description) -> None: if not resource: - raise LookupError(f"Resource not found: {resource_description}") - logger.info(f"Retrieved: {resource_description}") + raise LookupError(f'Resource not found: {resource_description}') + logger.info(f'Retrieved: {resource_description}') # endregion internal functions @@ -253,17 +260,15 @@ def get_resource(self, resource, uuid=None, name=None, field_mask=None, **kwargs """https://docs.endorlabs.com/rest-api/using-the-rest-api/data-model/resource-kinds/""" if not field_mask: field_mask = _get_default_field_mask(resource) - return self._api_get( - resource=resource, uuid=uuid, name=name, field_mask=field_mask, **kwargs - ) + return self._api_get(resource=resource, uuid=uuid, name=name, field_mask=field_mask, **kwargs) def get_resources( self, resource, filter=None, field_mask=None, - sort_path="meta.create_time", - sort_order="descending", + sort_path='meta.create_time', + sort_order='descending', retry=True, **kwargs, ): @@ -282,70 +287,56 @@ def get_resources( def get_project(self, git_url): resource_kind = EndorResourceKind.PROJECT.value - resource_description = ( - f"{resource_kind} with name '{git_url}' in namespace '{self.namespace}'" - ) + resource_description = f"{resource_kind} with name '{git_url}' in namespace '{self.namespace}'" project = self.get_resource(resource_kind, name=git_url) self._check_resource(project, resource_description) return project def get_repository_version(self, filter=None, retry=True): resource_kind = EndorResourceKind.REPOSITORY_VERSION.value - resource_description = ( - f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'" - ) - repository_version = self.get_resources( - resource_kind, filter=filter, retry=retry, page_size=1 - )[0] + resource_description = f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'" + repository_version = self.get_resources(resource_kind, filter=filter, retry=retry, page_size=1)[0] self._check_resource(repository_version, resource_description) return repository_version def get_scan_result(self, filter=None, retry=True): resource_kind = EndorResourceKind.SCAN_RESULT.value - resource_description = ( - f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'" - ) + resource_description = f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'" scan_result = self.get_resources(resource_kind, filter=filter, retry=retry, page_size=1)[0] self._check_resource(scan_result, resource_description) - uuid = scan_result.get("uuid") - start_time = scan_result["spec"].get("start_time") - refs = scan_result["spec"].get("refs") + uuid = scan_result.get('uuid') + start_time = scan_result['spec'].get('start_time') + refs = scan_result['spec'].get('refs') polling_start_time = datetime.now() while True: - status = scan_result["spec"].get("status") - end_time = scan_result["spec"].get("end_time") - if status == "STATUS_SUCCESS": + status = scan_result['spec'].get('status') + end_time = scan_result['spec'].get('end_time') + if status == 'STATUS_SUCCESS': logger.info( - f" Scan completed successfully. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}." + f' Scan completed successfully. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}.' ) return scan_result - elif status == "STATUS_RUNNING": + elif status == 'STATUS_RUNNING': + logger.info(f' Scan is running. ScanResult uuid {uuid} for refs {refs} started at {start_time}.') logger.info( - f" Scan is running. ScanResult uuid {uuid} for refs {refs} started at {start_time}." - ) - logger.info( - f" Waiting {self.sleep_duration} seconds before checking status. Total wait time: {(datetime.now() - polling_start_time).total_seconds()/60:.2f} minutes" + f' Waiting {self.sleep_duration} seconds before checking status. Total wait time: {(datetime.now() - polling_start_time).total_seconds() / 60:.2f} minutes' ) time.sleep(self.sleep_duration) - scan_result = self.get_resources( - resource_kind, filter=filter, retry=retry, page_size=1 - )[0] - elif status == "STATUS_PARTIAL_SUCCESS": - scan_logs = scan_result["spec"].get("logs") + scan_result = self.get_resources(resource_kind, filter=filter, retry=retry, page_size=1)[0] + elif status == 'STATUS_PARTIAL_SUCCESS': + scan_logs = scan_result['spec'].get('logs') raise RuntimeError( - f" Scan completed, but with critical warnings or errors. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}. Scan logs: {scan_logs}" + f' Scan completed, but with critical warnings or errors. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}. Scan logs: {scan_logs}' ) - elif status == "STATUS_FAILURE": - scan_logs = scan_result["spec"].get("logs") + elif status == 'STATUS_FAILURE': + scan_logs = scan_result['spec'].get('logs') raise RuntimeError( - f" Scan failed. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}. Scan logs: {scan_logs}" + f' Scan failed. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}. Scan logs: {scan_logs}' ) def get_package_versions(self, filter): resource_kind = EndorResourceKind.PACKAGE_VERSION.value - resource_description = ( - f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'" - ) + resource_description = f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'" package_versions = self.get_resources(resource_kind, filter=filter) self._check_resource(package_versions, resource_description) return package_versions @@ -373,10 +364,10 @@ def export_sbom( https://docs.endorlabs.com/endorctl/commands/sbom/export/ """ if package_version_uuids: - package_version_uuids = ",".join(package_version_uuids) + package_version_uuids = ','.join(package_version_uuids) return self._call_endorctl( - "sbom", - "export", + 'sbom', + 'export', package_version_uuid=package_version_uuid, package_version_uuids=package_version_uuids, package_version_name=package_version_name, @@ -396,14 +387,14 @@ def get_sbom_for_commit(self, git_url: str, commit_sha: str) -> dict: try: # Project: get uuid project = self.get_project(git_url) - project_uuid = project["uuid"] - app_name = project["spec"]["git"]["full_name"] + project_uuid = project['uuid'] + app_name = project['spec']['git']['full_name'] # RepositoryVersion: get the context for the PR scan endor_filter.context_type = EndorContextType.CI_RUN.value filter_str = endor_filter.repository_version(project_uuid, commit_sha) repository_version = self.get_repository_version(filter_str) - context_id = repository_version["context"]["id"] + context_id = repository_version['context']['id'] # ScanResult: wait for a completed scan endor_filter.context_id = context_id @@ -413,22 +404,18 @@ def get_sbom_for_commit(self, git_url: str, commit_sha: str) -> dict: # PackageVersions: get package versions for SBOM filter_str = endor_filter.package_version(project_uuid) package_versions = self.get_package_versions(filter_str) - package_version_uuids = [ - package_version["uuid"] for package_version in package_versions - ] - package_version_names = [ - package_version["meta"]["name"] for package_version in package_versions - ] + package_version_uuids = [package_version['uuid'] for package_version in package_versions] + package_version_names = [package_version['meta']['name'] for package_version in package_versions] # Export SBOM sbom = self.export_sbom(package_version_uuids=package_version_uuids, app_name=app_name) print( - f"Retrieved: CycloneDX SBOM for PackageVersion(s), name: {package_version_names}, uuid: {package_version_uuids}" + f'Retrieved: CycloneDX SBOM for PackageVersion(s), name: {package_version_names}, uuid: {package_version_uuids}' ) return sbom except Exception as e: - print(f"Exception: {e}") + print(f'Exception: {e}') return def get_sbom_for_branch(self, git_url: str, branch: str) -> dict: @@ -439,18 +426,20 @@ def get_sbom_for_branch(self, git_url: str, branch: str) -> dict: try: # Project: get uuid project = self.get_project(git_url) - project_uuid = project["uuid"] - app_name = project["spec"]["git"]["full_name"] + project_uuid = project['uuid'] + app_name = project['spec']['git']['full_name'] # RepositoryVersion: get the context for the latest branch scan - filter_str = endor_filter.repository_version(project_uuid, ref=branch, context_type_exclude=EndorContextType.CI_RUN) + filter_str = endor_filter.repository_version( + project_uuid, ref=branch, context_type_exclude=EndorContextType.CI_RUN + ) repository_version = self.get_repository_version(filter_str) - repository_version_context_type = EndorContextType[repository_version["context"]["type"]] - repository_version_uuid = repository_version["uuid"] - repository_version_ref = repository_version["spec"]["version"]["ref"] - repository_version_sha = repository_version["spec"]["version"]["sha"] - repository_version_scan_object_status = repository_version["scan_object"]["status"] - if repository_version_scan_object_status != "STATUS_SCANNED": + repository_version_context_type = EndorContextType[repository_version['context']['type']] + repository_version_uuid = repository_version['uuid'] + repository_version_ref = repository_version['spec']['version']['ref'] + repository_version_sha = repository_version['spec']['version']['sha'] + repository_version_scan_object_status = repository_version['scan_object']['status'] + if repository_version_scan_object_status != 'STATUS_SCANNED': logger.warning( f"RepositoryVersion (uuid: {repository_version_uuid}, ref: {repository_version_ref}, sha: {repository_version_sha}) scan status is '{repository_version_scan_object_status}' (expected 'STATUS_SCANNED')" ) @@ -460,29 +449,29 @@ def get_sbom_for_branch(self, git_url: str, branch: str) -> dict: repository_version_context_type, project_uuid, repository_version_ref, repository_version_sha ) scan_result = self.get_scan_result(filter_str, retry=False) - project_uuid = scan_result["meta"]["parent_uuid"] + project_uuid = scan_result['meta']['parent_uuid'] # PackageVersions: get package versions for SBOM - if branch in ["master","main"]: + if branch in ['master', 'main']: context_type = EndorContextType.MAIN - context_id = "default" + context_id = 'default' else: context_type = EndorContextType.REF context_id = branch filter_str = endor_filter.package_version(context_type, context_id, project_uuid) package_version = self.get_package_versions(filter_str)[0] - package_version_name = package_version["meta"]["name"] - package_version_uuid = package_version["uuid"] + package_version_name = package_version['meta']['name'] + package_version_uuid = package_version['uuid'] # Export SBOM sbom = self.export_sbom(package_version_uuid=package_version_uuid, app_name=app_name) logger.info( - f"SBOM: Retrieved CycloneDX SBOM for PackageVersion, name: {package_version_name}, uuid {package_version_uuid}" + f'SBOM: Retrieved CycloneDX SBOM for PackageVersion, name: {package_version_name}, uuid {package_version_uuid}' ) return sbom except Exception as e: - print(f"Exception: {e}") + print(f'Exception: {e}') return def get_sbom_for_project(self, git_url: str) -> dict: @@ -491,16 +480,16 @@ def get_sbom_for_project(self, git_url: str) -> dict: try: # Project: get uuid project = self.get_project(git_url) - project_uuid = project["uuid"] - app_name = project["spec"]["git"]["full_name"] + project_uuid = project['uuid'] + app_name = project['spec']['git']['full_name'] # Export SBOM sbom = self.export_sbom(project_uuid=project_uuid, app_name=app_name) - logger.info(f"Retrieved: CycloneDX SBOM for Project {app_name}") + logger.info(f'Retrieved: CycloneDX SBOM for Project {app_name}') return sbom except Exception as e: - print(f"Exception: {e}") + print(f'Exception: {e}') return # endregion workflow functions diff --git a/etc/sbom/generate_sbom.py b/etc/sbom/generate_sbom.py index 7fe5e5ea98..18650c9c5e 100755 --- a/etc/sbom/generate_sbom.py +++ b/etc/sbom/generate_sbom.py @@ -19,12 +19,7 @@ from datetime import datetime, timezone from pathlib import Path -from config import ( - endor_components_remove, - endor_components_rename, - get_semver_from_release_version, - process_component_special_cases, -) +import config from endorctl_utils import EndorCtl from git import Commit, Repo @@ -44,7 +39,7 @@ def emit(self, record): logging.basicConfig(stream=sys.stdout) -logger = logging.getLogger("generate_sbom") +logger = logging.getLogger('generate_sbom') logger.setLevel(logging.INFO) # Create an instance of the custom handler @@ -58,69 +53,66 @@ def emit(self, record): script_directory = script_path.parent # Regex for validation -REGEX_COMMIT_SHA = r"^[0-9a-fA-F]{40}$" -REGEX_GIT_BRANCH = r"^[a-zA-Z0-9_.\-/]+$" -REGEX_GITHUB_URL = r"^(https://github.com/)([a-zA-Z0-9-]{1,39}/[a-zA-Z0-9-_.]{1,100})(\.git)$" -REGEX_RELEASE_BRANCH = r"^v\d\.\d$" -REGEX_RELEASE_TAG = r"^r\d\.\d.\d(-\w*)?$" +REGEX_COMMIT_SHA = r'^[0-9a-fA-F]{40}$' +REGEX_GIT_BRANCH = r'^[a-zA-Z0-9_.\-/]+$' +REGEX_GITHUB_URL = r'^(https://github.com/)([a-zA-Z0-9-]{1,39}/[a-zA-Z0-9-_.]{1,100})(\.git)$' # ################ PURL Validation ################ REGEX_STR_PURL_OPTIONAL = ( # Optional Version (any chars except ? @ #) - r"(?:@[^?@#]*)?" + r'(?:@[^?@#]*)?' # Optional Qualifiers (any chars except @ #) - r"(?:\?[^@#]*)?" + r'(?:\?[^@#]*)?' # Optional Subpath (any chars) - r"(?:#.*)?$" + r'(?:#.*)?$' ) REGEX_PURL = { # deb PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/deb-definition.md - "deb": re.compile( - r"^pkg:deb/" # Scheme and type + 'deb': re.compile( + r'^pkg:deb/' # Scheme and type # Namespace (organization/user), letters must be lowercase - r"(debian|ubuntu)+" - r"/" - r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name + r'(debian|ubuntu)+' + r'/' + r'[a-z0-9._-]+' + REGEX_STR_PURL_OPTIONAL # Name ), # Generic PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/generic-definition.md - "generic": re.compile( - r"^pkg:generic/" # Scheme and type - r"([a-zA-Z0-9._-]+/)?" # Optional namespace segment - r"[a-zA-Z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name (required) + 'generic': re.compile( + r'^pkg:generic/' # Scheme and type + r'([a-zA-Z0-9._-]+/)?' # Optional namespace segment + r'[a-zA-Z0-9._-]+' + REGEX_STR_PURL_OPTIONAL # Name (required) ), # GitHub PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/github-definition.md - "github": re.compile( - r"^pkg:github/" # Scheme and type + 'github': re.compile( + r'^pkg:github/' # Scheme and type # Namespace (organization/user), letters must be lowercase - r"[a-z0-9-]+" - r"/" - r"[a-z0-9._-]+" + REGEX_STR_PURL_OPTIONAL # Name (repository) + r'[a-z0-9-]+' + r'/' + r'[a-z0-9._-]+' + REGEX_STR_PURL_OPTIONAL # Name (repository) ), # PyPI PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md - "pypi": re.compile( - r"^pkg:pypi/" # Scheme and type - r"[a-z0-9_-]+" # Name, letters must be lowercase, dashes, underscore - + REGEX_STR_PURL_OPTIONAL + 'pypi': re.compile( + r'^pkg:pypi/' # Scheme and type + r'[a-z0-9_-]+' + REGEX_STR_PURL_OPTIONAL # Name, letters must be lowercase, dashes, underscore ), } # Metadata SBOM requirements METADATA_FIELDS_REQUIRED = [ - "type", - "bom-ref", - "group", - "name", - "version", - "description", - "licenses", - "copyright", - "externalReferences", - "scope", + 'type', + 'bom-ref', + 'group', + 'name', + 'version', + 'description', + 'licenses', + 'copyright', + 'externalReferences', + 'scope', ] METADATA_FIELDS_ONE_OF = [ - ["author", "supplier"], - ["purl", "cpe"], + ['author', 'supplier'], + ['purl', 'cpe'], ] # endregion init @@ -133,11 +125,11 @@ class GitInfo: """Get, set, format git info""" def __init__(self): - print_banner("Gathering git info") + print_banner('Gathering git info') try: self.repo_root = Path( subprocess.run( - "git rev-parse --show-toplevel", + 'git rev-parse --show-toplevel', shell=True, text=True, capture_output=True, @@ -146,28 +138,24 @@ def __init__(self): ) self._repo = Repo(self.repo_root) except Exception as e: - logger.warning( - "Unable to read git repo information. All necessary script arguments must be provided." - ) + logger.warning('Unable to read git repo information. All necessary script arguments must be provided.') logger.warning(e) self._repo = None else: try: - self.project = self._repo.remotes.origin.config_reader.get("url") - if not self.project.endswith(".git"): - self.project += ".git" + self.project = self._repo.remotes.origin.config_reader.get('url') + if not self.project.endswith('.git'): + self.project += '.git' org_repo = extract_repo_from_git_url(self.project) - self.org = org_repo["org"] - self.repo = org_repo["repo"] + self.org = org_repo['org'] + self.repo = org_repo['repo'] self.commit = self._repo.head.commit.hexsha self.branch = self._repo.active_branch.name # filter tags for latest release e.g., r8.2.1 release_tags = [] - filtered_tags = [ - tag for tag in self._repo.tags if re.fullmatch(REGEX_RELEASE_TAG, tag.name) - ] - logging.info(f"GIT: Parsing {len(filtered_tags)} release tags for match to commit") + filtered_tags = [tag for tag in self._repo.tags if re.fullmatch(config.REGEX_RELEASE_TAG, tag.name)] + logging.info(f'GIT: Parsing {len(filtered_tags)} release tags for match to commit') for tag in filtered_tags: if tag.commit == self.commit: release_tags.append(tag.name) @@ -175,17 +163,17 @@ def __init__(self): self.release_tag = release_tags[-1] else: self.release_tag = None - logging.debug(f"GitInfo->release_tag(): {self.release_tag}") + logging.debug(f'GitInfo->release_tag(): {self.release_tag}') - logging.debug(f"GitInfo->__init__: {self}") + logging.debug(f'GitInfo->__init__: {self}') except Exception as e: - logger.warning("Unable to fully parse git info.") + logger.warning('Unable to fully parse git info.') logger.warning(e) def close(self): """Closes the underlying Git repo object to release resources.""" if self._repo: - logger.debug("Closing Git repo object.") + logger.debug('Closing Git repo object.') self._repo.close() self._repo = None @@ -210,7 +198,7 @@ def added_new_3p_folder(self, commit: Commit) -> bool: for diff in diff_index: # Check for added items that are directories - if diff.change_type == "A" and diff.b_is_dir: + if diff.change_type == 'A' and diff.b_is_dir: return True return False @@ -218,16 +206,16 @@ def added_new_3p_folder(self, commit: Commit) -> bool: def print_banner(text: str) -> None: """print() a padded status message to stdout""" print() - print(text.center(len(text) + 2, " ").center(120, "=")) + print(text.center(len(text) + 2, ' ').center(120, '=')) def extract_repo_from_git_url(git_url: str) -> dict: """Determine org/repo for a given git url""" - git_org = git_url.split("/")[-2].replace(".git", "") - git_repo = git_url.split("/")[-1].replace(".git", "") + git_org = git_url.split('/')[-2].replace('.git', '') + git_repo = git_url.split('/')[-1].replace('.git', '') return { - "org": git_org, - "repo": git_repo, + 'org': git_org, + 'repo': git_repo, } @@ -242,21 +230,18 @@ def is_valid_purl(purl: str) -> bool: def sbom_components_to_dict(sbom: dict, with_version: bool = False) -> dict: """Create a dict of SBOM components with a version-less PURL as the key""" - components = sbom["components"] + components = sbom['components'] if with_version: - components_dict = { - urllib.parse.unquote(component["bom-ref"]): component for component in components - } + components_dict = {urllib.parse.unquote(component['bom-ref']): component for component in components} else: components_dict = { - urllib.parse.unquote(component["bom-ref"]).split("@")[0]: component - for component in components + urllib.parse.unquote(component['bom-ref']).split('@')[0]: component for component in components } return components_dict def check_metadata_sbom(meta_bom: dict) -> None: - for component in meta_bom["components"]: + for component in meta_bom['components']: for field in METADATA_FIELDS_REQUIRED: if field not in component: logger.warning( @@ -275,14 +260,14 @@ def check_metadata_sbom(meta_bom: dict) -> None: def read_sbom_json_file(file_path: str) -> dict: """Load a JSON SBOM file (schema is not validated)""" try: - with open(file_path, "r", encoding="utf-8") as input_json: + with open(file_path, 'r', encoding='utf-8') as input_json: sbom_json = input_json.read() result = json.loads(sbom_json) except Exception as e: - logger.error(f"Error loading SBOM file from {file_path}") + logger.error(f'Error loading SBOM file from {file_path}') logger.error(e) else: - logger.info(f"SBOM loaded from {file_path} with {len(result['components'])} components") + logger.info(f'SBOM loaded from {file_path} with {len(result["components"])} components') return result @@ -290,33 +275,31 @@ def write_sbom_json_file(sbom_dict: dict, file_path: str) -> None: """Save a JSON SBOM file (schema is not validated)""" try: file_path = os.path.abspath(file_path) - with open(file_path, "w", encoding="utf-8") as output_json: - formatted_sbom = json.dumps(sbom_dict, indent=2) + "\n" + with open(file_path, 'w', encoding='utf-8') as output_json: + formatted_sbom = json.dumps(sbom_dict, indent=2) + '\n' output_json.write(formatted_sbom) except Exception as e: - logger.error(f"Error writing SBOM file to {file_path}") + logger.error(f'Error writing SBOM file to {file_path}') logger.error(e) else: - logger.info(f"SBOM file saved to {file_path}") + logger.info(f'SBOM file saved to {file_path}') def write_list_to_text_file(str_list: list, file_path: str) -> None: """Save a list of strings to a text file""" try: file_path = os.path.abspath(file_path) - with open(file_path, "w", encoding="utf-8") as output_txt: + with open(file_path, 'w', encoding='utf-8') as output_txt: for item in str_list: - output_txt.write(f"{item}\n") + output_txt.write(f'{item}\n') except Exception as e: - logger.error(f"Error writing text file to {file_path}") + logger.error(f'Error writing text file to {file_path}') logger.error(e) else: - logger.info(f"Text file saved to {file_path}") + logger.info(f'Text file saved to {file_path}') -def set_component_version( - component: dict, version: str, purl_version: str = None, cpe_version: str = None -) -> None: +def set_component_version(component: dict, version: str, purl_version: str = None, cpe_version: str = None) -> None: """Update the appropriate version fields in a component from the metadata SBOM""" if not purl_version: purl_version = version @@ -324,14 +307,14 @@ def set_component_version( if not cpe_version: cpe_version = version - component["bom-ref"] = component["bom-ref"].replace("{{VERSION}}", purl_version) - component["version"] = component["version"].replace("{{VERSION}}", version) - if component.get("purl"): - component["purl"] = component["purl"].replace("{{VERSION}}", purl_version) - if not is_valid_purl(component["purl"]): - logger.warning(f"PURL: Invalid PURL ({component['purl']})") - if component.get("cpe"): - component["cpe"] = component["cpe"].replace("{{VERSION}}", cpe_version) + component['bom-ref'] = component['bom-ref'].replace('{{VERSION}}', purl_version) + component['version'] = component['version'].replace('{{VERSION}}', version) + if component.get('purl'): + component['purl'] = component['purl'].replace('{{VERSION}}', purl_version) + if not is_valid_purl(component['purl']): + logger.warning(f'PURL: Invalid PURL ({component["purl"]})') + if component.get('cpe'): + component['cpe'] = component['cpe'].replace('{{VERSION}}', cpe_version) def set_dependency_version(dependencies: list, meta_bom_ref: str, purl_version: str) -> None: @@ -339,20 +322,18 @@ def set_dependency_version(dependencies: list, meta_bom_ref: str, purl_version: r = 0 d = 0 for dependency in dependencies: - if "{{VERSION}}" in dependency["ref"] and dependency["ref"] == meta_bom_ref: - dependency["ref"] = dependency["ref"].replace("{{VERSION}}", purl_version) + if '{{VERSION}}' in dependency['ref'] and dependency['ref'] == meta_bom_ref: + dependency['ref'] = dependency['ref'].replace('{{VERSION}}', purl_version) r += 1 - for i in range(len(dependency["dependsOn"])): - if dependency["dependsOn"][i] == meta_bom_ref: - dependency["dependsOn"][i] = dependency["dependsOn"][i].replace( - "{{VERSION}}", purl_version - ) + for i in range(len(dependency['dependsOn'])): + if dependency['dependsOn'][i] == meta_bom_ref: + dependency['dependsOn'][i] = dependency['dependsOn'][i].replace('{{VERSION}}', purl_version) d += 1 logger.debug(f"set_dependency_version: '{meta_bom_ref}' updated {r} refs and {d} dependsOn") -def get_subfolders_dict(folder_path: str = ".") -> dict: +def get_subfolders_dict(folder_path: str = '.') -> dict: """Get list of all directories in the specified path""" subfolders = [] try: @@ -367,7 +348,7 @@ def get_subfolders_dict(folder_path: str = ".") -> dict: except FileNotFoundError: logger.error(f"Error: Directory '{folder_path}' not found.") except Exception as e: - logger.error(f"An error occurred: {e}") + logger.error(f'An error occurred: {e}') subfolders.sort() return {key: 0 for key in subfolders} @@ -383,91 +364,93 @@ def main() -> None: description="""Generate a CycloneDX v1.5 JSON SBOM file using a combination of scan results from Endor Labs, pre-defined SBOM metadata, and the existing SBOM. Requires endorctl to be installed and configured, which can be done using 'buildscripts/sbom/install_endorctl.sh'. For use in CI, script may be run with no arguments.""", - epilog="Note: The git-related default values are dynamically generated.", + epilog='Note: The git-related default values are dynamically generated.', formatter_class=argparse.MetavarTypeHelpFormatter, ) endor = parser.add_argument_group("Endor Labs API (via 'endorctl')") endor.add_argument( - "--endorctl-path", + '--endorctl-path', help="Path to endorctl, the Endor Labs CLI (Default: 'endorctl')", - default="endorctl", + default='endorctl', type=str, ) endor.add_argument( - "--config-path", + '--config-path', help="Path to endor config directory containing config.yaml (Default: '$HOME/.endorctl')", default=None, type=str, ) - endor.add_argument("--enable-github-action-token", help="Enable keyless authentication using Github action OIDC tokens", action="store_true") endor.add_argument( - "--namespace", help="Endor Labs namespace (Default: mongodb.{git org})", type=str + '--enable-github-action-token', + help='Enable keyless authentication using Github action OIDC tokens', + action='store_true', ) + endor.add_argument('--namespace', help='Endor Labs namespace (Default: mongodb.{git org})', type=str) endor.add_argument( - "--target", + '--target', help="Target for generated SBOM. Commit: results from running/completed PR scan, Branch: results from latest monitoring scan, Project: results from latest monitoring scan of the 'default' branch (default: commit)", - choices=["commit", "branch", "project"], - default="commit", + choices=['commit', 'branch', 'project'], + default='commit', type=str, ) endor.add_argument( - "--project", - help="Full GitHub git URL [e.g., https://github.com/10gen/mongo.git] (Default: current git URL)", + '--project', + help='Full GitHub git URL [e.g., https://github.com/10gen/mongo.git] (Default: current git URL)', type=str, ) target = parser.add_argument_group("Target values. Apply only if --target is not 'project'") exclusive_target = target.add_mutually_exclusive_group() exclusive_target.add_argument( - "--commit", - help="PR commit SHA [40-character hex string] (Default: current git commit)", + '--commit', + help='PR commit SHA [40-character hex string] (Default: current git commit)', type=str, ) exclusive_target.add_argument( - "--branch", - help="Git repo branch monitored by Endor Labs [e.g., v8.0] (Default: current git org/repo)", + '--branch', + help='Git repo branch monitored by Endor Labs [e.g., v8.0] (Default: current git org/repo)', type=str, ) - files = parser.add_argument_group("SBOM files") + files = parser.add_argument_group('SBOM files') files.add_argument( - "--sbom-metadata", + '--sbom-metadata', help="Input path for template SBOM file with metadata (Default: './buildscripts/sbom/metadata.cdx.json')", - default="./buildscripts/sbom/metadata.cdx.json", + default='./buildscripts/sbom/metadata.cdx.json', type=str, ) files.add_argument( - "--sbom-in", + '--sbom-in', help="Input path for previous SBOM file (Default: './sbom.json')", - default="./sbom.json", + default='./sbom.json', type=str, ) files.add_argument( - "--sbom-out", + '--sbom-out', help="Output path for SBOM file (Default: './sbom.json')", - default="./sbom.json", + default='./sbom.json', type=str, ) parser.add_argument( - "--retry-limit", - help="Maximum number of times to retry when a target PR scan has not started (Default: 5)", + '--retry-limit', + help='Maximum number of times to retry when a target PR scan has not started (Default: 5)', default=5, type=int, ) parser.add_argument( - "--sleep-duration", - help="Number of seconds to wait between retries (Default: 30)", + '--sleep-duration', + help='Number of seconds to wait between retries (Default: 30)', default=30, type=int, ) parser.add_argument( - "--save-warnings", - help="Save warning messages to a specified file (Default: None)", + '--save-warnings', + help='Save warning messages to a specified file (Default: None)', default=None, type=str, ) - parser.add_argument("--debug", help="Set logging level to DEBUG", action="store_true") + parser.add_argument('--debug', help='Set logging level to DEBUG', action='store_true') # endregion define args @@ -481,35 +464,29 @@ def main() -> None: endorctl_path = args.endorctl_path config_path = args.config_path enable_github_action_token = args.enable_github_action_token - namespace = args.namespace if args.namespace else f"mongodb.{git_info.org}" + namespace = args.namespace if args.namespace else f'mongodb.{git_info.org}' target = args.target # project if args.project and args.project != git_info.project: if not re.fullmatch(REGEX_GITHUB_URL, args.project): - parser.error(f"Invalid Git URL: {args.project}.") + parser.error(f'Invalid Git URL: {args.project}.') git_info.project = args.project - git_info.org, git_info.repo = map( - extract_repo_from_git_url(git_info.project).get, ("org", "repo") - ) + git_info.org, git_info.repo = map(extract_repo_from_git_url(git_info.project).get, ('org', 'repo')) git_info.release_tag = None # targets # commit if args.commit and args.commit != git_info.commit: if not re.fullmatch(REGEX_COMMIT_SHA, args.commit): - parser.error( - f"Invalid Git commit SHA: {args.commit}. Must be a 40-character hexadecimal string (SHA-1)." - ) + parser.error(f'Invalid Git commit SHA: {args.commit}. Must be a 40-character hexadecimal string (SHA-1).') git_info.commit = args.commit # branch if args.branch and args.branch != git_info.branch: - if len(args.branch.encode("utf-8")) > 244 or not re.fullmatch( - REGEX_GIT_BRANCH, args.branch - ): + if len(args.branch.encode('utf-8')) > 244 or not re.fullmatch(REGEX_GIT_BRANCH, args.branch): parser.error( - f"Invalid Git branch name: {args.branch}. Limit is 244 bytes with allowed characters: [a-zA-Z0-9_.-/]" + f'Invalid Git branch name: {args.branch}. Limit is 244 bytes with allowed characters: [a-zA-Z0-9_.-/]' ) git_info.branch = args.branch @@ -530,79 +507,104 @@ def main() -> None: # region export Endor Labs SBOM - print_banner(f"Exporting Endor Labs SBOM for {target} {getattr(git_info, target)}") - endorctl = EndorCtl(namespace, retry_limit, sleep_duration, endorctl_path, config_path, enable_github_action_token=enable_github_action_token) - if target == "commit": + print_banner(f'Exporting Endor Labs SBOM for {target} {getattr(git_info, target)}') + endorctl = EndorCtl( + namespace, + retry_limit, + sleep_duration, + endorctl_path, + config_path, + enable_github_action_token=enable_github_action_token, + ) + if target == 'commit': endor_bom = endorctl.get_sbom_for_commit(git_info.project, git_info.commit) - elif target == "branch": + elif target == 'branch': endor_bom = endorctl.get_sbom_for_branch(git_info.project, git_info.branch) - elif target == "project": + elif target == 'project': endor_bom = endorctl.get_sbom_for_project(git_info.project) else: endor_bom = None if not endor_bom: - logger.error("Empty result for Endor SBOM!") - if target == "commit": - logger.error("Check Endor Labs for any unanticipated issues with the target PR scan.") + logger.error('Empty result for Endor SBOM!') + if target == 'commit': + logger.error('Check Endor Labs for any unanticipated issues with the target PR scan.') else: - logger.error("Check Endor Labs for status of the target monitoring scan.") + logger.error('Check Endor Labs for status of the target monitoring scan.') sys.exit(1) # endregion export Endor Labs SBOM # region Pre-process Endor Labs SBOM - print_banner("Pre-Processing Endor Labs SBOM") + print_banner('Pre-Processing Endor Labs SBOM') ## remove uneeded components ## # [list]endor_components_remove is defined in config.py + # Endor Labs includes the main component in 'components'. This is not standard, so we remove it. + config.endor_components_remove.append(f"{git_info.org}/{git_info.repo}") + # Reverse iterate the SBOM components list to safely modify in situ - for i in range(len(endor_bom["components"]) - 1, -1, -1): - component = endor_bom["components"][i] + for i in range(len(endor_bom['components']) - 1, -1, -1): + component = endor_bom['components'][i] removed = False - for remove in endor_components_remove: - if component["bom-ref"].startswith(remove): - logger.info("ENDOR SBOM PRE-PROCESS: removing " + component["bom-ref"]) - del endor_bom["components"][i] + for remove in config.endor_components_remove: + if component['bom-ref'].startswith(remove): + logger.info('ENDOR SBOM PRE-PROCESS: removing ' + component['bom-ref']) + del endor_bom['components'][i] removed = True break if not removed: - for rename in endor_components_rename: + for rename in config.endor_components_rename: old = rename[0] new = rename[1] - component["bom-ref"] = component["bom-ref"].replace(old, new) - component["purl"] = component["purl"].replace(old, new) + component['bom-ref'] = component['bom-ref'].replace(old, new) + component['purl'] = component['purl'].replace(old, new) - logger.info(f"Endor Labs SBOM pre-processed with {len(endor_bom['components'])} components") + logger.info(f'Endor Labs SBOM pre-processed with {len(endor_bom["components"])} components') # endregion Pre-process Endor Labs SBOM # region load metadata and previous SBOMs - print_banner("Loading metadata SBOM and previous SBOM") + print_banner('Loading metadata SBOM and previous SBOM') meta_bom = read_sbom_json_file(sbom_metadata_path) if not meta_bom: - logger.error("No SBOM metadata. This is fatal.") + logger.error('No SBOM metadata. This is fatal.') sys.exit(1) prev_bom = read_sbom_json_file(sbom_in_path) if not prev_bom: logger.warning( - "Unable to load previous SBOM data. The new SBOM will be generated without any previous context. This is unexpected, but not fatal." + 'Unable to load previous SBOM data. The new SBOM will be generated without any previous context. This is unexpected, but not fatal.' ) # Create empty prev_bom to avoid downstream processing errors prev_bom = { - "bom-ref": None, - "metadata": { - "timestamp": endor_bom["metadata"]["timestamp"], - "component": { - "version": None, + 'bom-ref': None, + 'metadata': { + 'timestamp': endor_bom['metadata']['timestamp'], + 'component': { + 'version': None, }, }, - "components": [], + 'components': [], } + else: + if 'metadata' not in prev_bom: + prev_bom['metadata'] = { + 'timestamp': endor_bom['metadata']['timestamp'], + 'component': { + 'version': None, + }, + } + else: + if 'timestamp' not in prev_bom['metadata']: + prev_bom['metadata']['timestamp'] = endor_bom['metadata']['timestamp'] + if 'component' not in prev_bom['metadata']: + prev_bom['metadata']['component'] = { + 'version': None, + } # endregion load metadata and previous SBOMs @@ -610,12 +612,12 @@ def main() -> None: # Note: No exception handling here. The most likely reason for an exception is missing data elements # in SBOM files, which is fatal if it happens. Code is in place to handle the situation # where there is no previous SBOM to include, but we want to fail if required data is absent. - print_banner("Building composite SBOM (metadata + endor + previous)") + print_banner('Building composite SBOM (metadata + endor + previous)') # Sort components by bom-ref - endor_bom["components"].sort(key=lambda c: c["bom-ref"]) - meta_bom["components"].sort(key=lambda c: c["bom-ref"]) - prev_bom["components"].sort(key=lambda c: c["bom-ref"]) + endor_bom['components'].sort(key=lambda c: c['bom-ref']) + meta_bom['components'].sort(key=lambda c: c['bom-ref']) + prev_bom['components'].sort(key=lambda c: c['bom-ref']) # Check metadata SBOM for completeness check_metadata_sbom(meta_bom) @@ -624,118 +626,122 @@ def main() -> None: endor_components = sbom_components_to_dict(endor_bom) prev_components = sbom_components_to_dict(prev_bom) - # region MongoDB primary component + # region primary component - # Attempt to determine the MongoDB Version being scanned + # Attempt to determine the primary component version being scanned + primary_component_version = config.get_primary_component_version() + logger.debug( - f"Available MongoDB version options, tag: {git_info.release_tag}, branch: {git_info.branch}, previous SBOM: {prev_bom['metadata']['component']['version']}" + f'Available main component version options, repo script: {primary_component_version}, tag: {git_info.release_tag}, branch: {git_info.branch}, previous SBOM: {prev_bom["metadata"]["component"]["version"]}' ) - meta_bom_ref = meta_bom["metadata"]["component"]["bom-ref"] - + meta_bom_ref = meta_bom['metadata']['component']['bom-ref'] + + if primary_component_version: + version = primary_component_version + purl_version = "r" + primary_component_version + cpe_version = primary_component_version + logger.info(f"PRIMARY COMPONENT VERSION: Using repo script output '{primary_component_version}' as primary component version") + # Project scan always set to 'master' or if using 'master' branch - if target == "project" or git_info.branch == "master": - version = "master" - purl_version = "master" - cpe_version = "master" - logger.info("Using branch 'master' as MongoDB version") + if target == 'project' or git_info.branch == 'master': + version = 'master' + purl_version = 'master' + cpe_version = 'master' + logger.info("PRIMARY COMPONENT VERSION: Using branch 'master' as primary component version") # tagged release. e.g., r8.1.0, r8.2.1-rc0 elif git_info.release_tag: version = git_info.release_tag[1:] # remove leading 'r' purl_version = git_info.release_tag cpe_version = version # without leading 'r' - logger.info(f"Using release_tag '{git_info.release_tag}' as MongoDB version") + logger.info(f"PRIMARY COMPONENT VERSION: Using release_tag '{git_info.release_tag}' as primary component version") # Release branch e.g., v7.0 or v8.2 - elif target == "branch" and re.fullmatch(REGEX_RELEASE_BRANCH, git_info.branch): + elif target == 'branch' and re.fullmatch(config.REGEX_RELEASE_BRANCH, git_info.branch): version = git_info.branch purl_version = git_info.branch # remove leading 'v', add wildcard. e.g. 8.2.* - cpe_version = version[1:] + ".*" - logger.info(f"Using release branch '{git_info.branch}' as MongoDB version") + cpe_version = version.replace("releases/","")[1:] + '.*' + logger.info(f"PRIMARY COMPONENT VERSION: Using release branch '{git_info.branch}' as primary component version") # Previous SBOM app version, if all needed specifiers exist elif ( - prev_bom.get("metadata", {}).get("component", {}).get("version") - and prev_bom.get("metadata", {}).get("component", {}).get("purl") - and prev_bom.get("metadata", {}).get("component", {}).get("cpe") + prev_bom.get('metadata', {}).get('component', {}).get('version') + and prev_bom.get('metadata', {}).get('component', {}).get('purl') + and prev_bom.get('metadata', {}).get('component', {}).get('cpe') ): - version = prev_bom["metadata"]["component"]["version"] - purl_version = prev_bom["metadata"]["component"]["purl"].split("@")[-1] - cpe_version = prev_bom["metadata"]["component"]["cpe"].split(":")[5] - logger.info(f"Using previous SBOM version '{version}' as MongoDB version") + version = prev_bom['metadata']['component']['version'] + purl_version = prev_bom['metadata']['component']['purl'].split('@')[-1] + cpe_version = prev_bom['metadata']['component']['cpe'].split(':')[5] + logger.info(f"PRIMARY COMPONENT VERSION: Using previous SBOM version '{version}' as primary component version") else: # Fall back to the version specified in the Endor SBOM # This is unlikely to be accurate - version = endor_bom["metadata"]["component"]["version"] + version = endor_bom['metadata']['component']['version'] purl_version = version cpe_version = version logger.warning( - f"Using SBOM version '{version}' from Endor Labs scan. This is unlikely to be accurate and may specify a PR #." + f"PRIMARY COMPONENT VERSION: Using SBOM version '{version}' from Endor Labs scan. This is unlikely to be accurate and may specify a PR #." ) - # Set main component version - set_component_version(meta_bom["metadata"]["component"], version, purl_version, cpe_version) + # Set primary component version + set_component_version(meta_bom['metadata']['component'], version, purl_version, cpe_version) # Run through 'dependency' objects to set main component version - set_dependency_version(meta_bom["dependencies"], meta_bom_ref, purl_version) + set_dependency_version(meta_bom['dependencies'], meta_bom_ref, purl_version) - # endregion MongoDB primary component + # endregion primary component # region SBOM components # region Parse metadata SBOM components - for component in meta_bom["components"]: + for component in meta_bom['components']: versions = { - "endor": None, - "metadata": None, + 'endor': None, + 'metadata': None, } - component_key = component["bom-ref"].split("@")[0] + component_key = component['bom-ref'].split('@')[0] - print_banner("Component: " + component_key) + print_banner('Component: ' + component_key) ################ Endor Labs ################ if component_key in endor_components: # Pop component from dict so we are left with only unmatched components endor_component = endor_components.pop(component_key) - versions["endor"] = endor_component.get("version") - logger.debug( - f"VERSION ENDOR: {component_key}: Found version '{versions['endor']}' in Endor Labs results" - ) + versions['endor'] = endor_component.get('version') + logger.debug(f"VERSION ENDOR: {component_key}: Found version '{versions['endor']}' in Endor Labs results") ############## Metadata ############### # Hard-coded metadata version, if exists - if "{{VERSION}}" not in component["version"]: - versions["metadata"] = component.get("version") + if '{{VERSION}}' not in component['version']: + versions['metadata'] = component.get('version') - logger.info(f"VERSIONS: {component_key}: " + str(versions)) + logger.info(f'VERSIONS: {component_key}: ' + str(versions)) ############## Component Special Cases ############### - process_component_special_cases( - component_key, component, versions, git_info.repo_root.as_posix() - ) + config.process_component_special_cases(component_key, component, versions, git_info.repo_root.as_posix()) # For the standard workflow, we favor the Endor Labs version followed by hard coded - version = versions["endor"] or versions["metadata"] + version = versions['endor'] or versions['metadata'] ############## Assign Version ############### if version: - meta_bom_ref = component["bom-ref"] + meta_bom_ref = component['bom-ref'] ## Special case for FireFox ## # The CPE for FireFox ESR needs the 'esr' removed from the version, as it is specified in another section - if component["bom-ref"].startswith("pkg:deb/debian/firefox-esr@"): - set_component_version(component, version, cpe_version=version.replace("esr", "")) + if component['bom-ref'].startswith('pkg:deb/debian/firefox-esr@'): + set_component_version(component, version, cpe_version=version.replace('esr', '')) else: - semver = get_semver_from_release_version(version) + semver = config.get_semver_from_release_version(version) set_component_version(component, semver, version, semver) - set_dependency_version(meta_bom["dependencies"], meta_bom_ref, version) + set_dependency_version(meta_bom['dependencies'], meta_bom_ref, version) else: logger.warning( - f"VERSION NOT FOUND: Could not find a version for {component_key}! Removing from SBOM. Component may need to be removed from the {sbom_metadata_path} file." + f'VERSION NOT FOUND: Could not find a version for {component_key}! Removing from SBOM. Component may need to be removed from the {sbom_metadata_path} file.' ) del component @@ -747,19 +753,17 @@ def main() -> None: # region Parse unmatched Endor Labs components - print_banner("New Endor Labs components") + print_banner('New Endor Labs components') if endor_components: logger.info( - f"ENDOR SBOM: There are {len(endor_components)} unmatched components in the Endor Labs SBOM. Adding as-is. The applicable metadata should be added to the metadata SBOM for the next run." + f'ENDOR SBOM: There are {len(endor_components)} unmatched components in the Endor Labs SBOM. Adding as-is. The applicable metadata should be added to the metadata SBOM for the next run.' ) for component in endor_components: # set scope to excluded by default until the component is evaluated - endor_components[component]["scope"] = "excluded" - meta_bom["components"].append(endor_components[component]) - meta_bom["dependencies"].append( - {"ref": endor_components[component]["bom-ref"], "dependsOn": []} - ) - logger.info(f"SBOM AS-IS COMPONENT: Added {component}") + endor_components[component]['scope'] = 'excluded' + meta_bom['components'].append(endor_components[component]) + meta_bom['dependencies'].append({'ref': endor_components[component]['bom-ref'], 'dependsOn': []}) + logger.info(f'SBOM AS-IS COMPONENT: Added {component}') # endregion Parse unmatched Endor Labs components @@ -767,16 +771,16 @@ def main() -> None: # Have the SBOM app version changed? sbom_app_version_changed = ( - prev_bom['metadata'].get('component',{}).get('version') != meta_bom["metadata"]["component"]["version"] + prev_bom['metadata'].get('component', {}).get('version') != meta_bom['metadata']['component']['version'] ) - logger.info(f"SUMMARY: MongoDB version changed: {sbom_app_version_changed}") + logger.info(f'SUMMARY: Primary component version changed: {sbom_app_version_changed}') # Have the components changed? prev_components = sbom_components_to_dict(prev_bom, with_version=True) meta_components = sbom_components_to_dict(meta_bom, with_version=True) sbom_components_changed = prev_components.keys() != meta_components.keys() logger.info( - f"SBOM_DIFF: SBOM components changed (added, removed, or version): {sbom_components_changed}. Previous SBOM has {len(prev_components)} components; New SBOM has {len(meta_components)} components" + f'SBOM_DIFF: SBOM components changed (added, removed, or version): {sbom_components_changed}. Previous SBOM has {len(prev_components)} components; New SBOM has {len(meta_components)} components' ) # Components in prev SBOM but not in generated SBOM @@ -785,65 +789,63 @@ def main() -> None: prev_components_diff = list(set(prev_components.keys()) - set(meta_components.keys())) if prev_components_diff: logger.info( - "SBOM_DIFF: Components in previous SBOM and not in generated SBOM: " - + ",".join(prev_components_diff) + 'SBOM_DIFF: Components in previous SBOM and not in generated SBOM: ' + ','.join(prev_components_diff) ) # Components in generated SBOM but not in prev SBOM meta_components_diff = list(set(meta_components.keys()) - set(prev_components.keys())) if meta_components_diff: logger.info( - "SBOM_DIFF: Components in generated SBOM and not in previous SBOM: " - + ",".join(meta_components_diff) + 'SBOM_DIFF: Components in generated SBOM and not in previous SBOM: ' + ','.join(meta_components_diff) ) # serialNumber https://cyclonedx.org/docs/1.5/json/#serialNumber # version (SBOM version) https://cyclonedx.org/docs/1.5/json/#version if sbom_app_version_changed: - # New MongoDB version requires a unique serial number and version 1 - meta_bom["serialNumber"] = uuid.uuid4().urn - meta_bom["version"] = 1 + # New primary component version requires a unique serial number and version 1 + meta_bom['serialNumber'] = uuid.uuid4().urn + meta_bom['version'] = 1 else: - # MongoDB version is the same, so reuse the serial number and SBOM version - meta_bom["serialNumber"] = prev_bom["serialNumber"] - meta_bom["version"] = prev_bom["version"] + # Primary component version is the same, so reuse the serial number and SBOM version + meta_bom['serialNumber'] = prev_bom['serialNumber'] + meta_bom['version'] = prev_bom['version'] # If the components have changed, bump the SBOM version if sbom_components_changed: - meta_bom["version"] += 1 + meta_bom['version'] += 1 # metadata.timestamp https://cyclonedx.org/docs/1.5/json/#metadata_timestamp # Only update the timestamp if something has changed if sbom_app_version_changed or sbom_components_changed: - meta_bom["metadata"]["timestamp"] = ( - datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + meta_bom['metadata']['timestamp'] = ( + datetime.now(timezone.utc).isoformat(timespec='seconds').replace('+00:00', 'Z') ) else: - meta_bom["metadata"]["timestamp"] = prev_bom["metadata"]["timestamp"] + meta_bom['metadata']['timestamp'] = prev_bom['metadata']['timestamp'] # metadata.tools https://cyclonedx.org/docs/1.5/json/#metadata_tools - meta_bom["metadata"]["tools"] = endor_bom["metadata"]["tools"] + meta_bom['metadata']['tools'] = endor_bom['metadata']['tools'] write_sbom_json_file(meta_bom, sbom_out_path) # Access the collected warnings - print_banner("CONSOLIDATED WARNINGS") + print_banner('CONSOLIDATED WARNINGS') warnings = [] for record in warning_handler.warnings: warnings.append(record.getMessage()) - print("\n".join(warnings)) + print('\n'.join(warnings)) if save_warnings: write_list_to_text_file(warnings, save_warnings) - print_banner("COMPLETED") - if not os.getenv("CI"): - print("Be sure to add the SBOM to your next commit if the file content has changed.") + print_banner('COMPLETED') + if not os.getenv('CI'): + print('Be sure to add the SBOM to your next commit if the file content has changed.') # endregion Finalize SBOM # endregion Build composite SBOM -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/etc/sbom/metadata.cdx.json b/etc/sbom/metadata.cdx.json index 54f92fa24e..c821b2816c 100644 --- a/etc/sbom/metadata.cdx.json +++ b/etc/sbom/metadata.cdx.json @@ -142,7 +142,8 @@ "url": "https://github.com/juliastrings/utf8proc.git", "type": "distribution" } - ] + ], + "scope": "required" }, { "type": "library", @@ -237,6 +238,7 @@ "bom-ref": "pkg:github/troydhanson/uthash@{{VERSION}}", "type": "library", "author": "Troy D. Hanson", + "group": "troydhanson", "name": "github.com/troydhanson/uthash", "version": "{{VERSION}}", "description": "C macros for hash tables and more", @@ -254,7 +256,8 @@ "url": "https://github.com/troydhanson/uthash.git", "type": "distribution" } - ] + ], + "scope": "required" } ], "dependencies": [ diff --git a/pyproject.toml b/pyproject.toml index c239b1b24b..6f9cf0c359 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,13 @@ make_release = [ "pygithub>=2.1", ] +sbom = [ + # etc/sbom/generate_sbom.py + "gitpython>=3.1", + "pygithub>=2.1", + "semver>=3.0.0", +] + [tool.ruff] line-length = 120 src = [".evergreen", "etc"] From 250625c74fb7e572da48b4f8b99612a309ec3ecf Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 26 Nov 2025 12:58:30 -0500 Subject: [PATCH 30/62] use uv group sbom, bypass scan for testing --- .github/workflows/generate_sbom.yml | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 4f2ba568e1..d0218a8b35 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -28,18 +28,24 @@ jobs: working-directory: ${{env.BUILD}} run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON - - name: Install endorctl and Scan with Endor Labs - uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 + # - name: Install endorctl and Scan with Endor Labs + # uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 + # with: + # additional_args: "--languages=c --exclude-path=\"build/CMakeFiles/**\"" + # log_level: info + # log_verbose: false + # namespace: mongodb.${{github.repository_owner}} + # pr: false + # scan_dependencies: true + # tags: github_action + # env: + # ENDOR_SCAN_EMBEDDINGS: true + + - name: Install endorctl + uses: endorlabs/github-action/setup@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 with: - additional_args: "--languages=c --exclude-path=\"build/CMakeFiles/**\"" - log_level: info - log_verbose: false namespace: mongodb.${{github.repository_owner}} - pr: false - scan_dependencies: true - tags: github_action - env: - ENDOR_SCAN_EMBEDDINGS: true + enable_github_action_token: true - name: Set up Python 3.10 uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 @@ -50,7 +56,7 @@ jobs: python-version: "3.10" activate-environment: true - name: Install dependencies - run: uv sync --group make_release + run: uv sync --group sbom - name: generate_sbom.py run: uv run etc/sbom/generate_sbom.py --enable-github-action-token --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json --save-warnings=${{runner.temp}}/warnings.txt From ca06a647964a2b420bd77eedd7d2704d00bff461 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 26 Nov 2025 13:10:11 -0500 Subject: [PATCH 31/62] Skip cmake, inline uv deps --- .github/workflows/generate_sbom.yml | 23 ++++++++++++++--------- etc/sbom/generate_sbom.py | 7 +++++++ pyproject.toml | 7 ------- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index d0218a8b35..07c61dcdd8 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -20,13 +20,13 @@ jobs: with: submodules: recursive - - name: Configure CMake and fetch dependency sources - env: - BUILD_TYPE: Release - BUILD: ${{github.workspace}}/build - CXX_STANDARD: 17 - working-directory: ${{env.BUILD}} - run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON + # - name: Configure CMake and fetch dependency sources + # env: + # BUILD_TYPE: Release + # BUILD: ${{github.workspace}}/build + # CXX_STANDARD: 17 + # working-directory: ${{env.BUILD}} + # run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON # - name: Install endorctl and Scan with Endor Labs # uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 @@ -50,13 +50,18 @@ jobs: - name: Set up Python 3.10 uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: - python-version: '3.10' - - uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4 + python-version: '3.10' + + - name: Install uv + uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4 with: python-version: "3.10" activate-environment: true + enable-cache: true + - name: Install dependencies run: uv sync --group sbom + - name: generate_sbom.py run: uv run etc/sbom/generate_sbom.py --enable-github-action-token --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json --save-warnings=${{runner.temp}}/warnings.txt diff --git a/etc/sbom/generate_sbom.py b/etc/sbom/generate_sbom.py index 18650c9c5e..389704180b 100755 --- a/etc/sbom/generate_sbom.py +++ b/etc/sbom/generate_sbom.py @@ -1,3 +1,10 @@ +# /// script +# dependencies = [ +# "gitpython", +# "pygithub", +# "semver", +# ] +# /// #!/usr/bin/env python3 """ Generate a CycloneDX SBOM using scan results from Endor Labs. diff --git a/pyproject.toml b/pyproject.toml index 6f9cf0c359..c239b1b24b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,13 +53,6 @@ make_release = [ "pygithub>=2.1", ] -sbom = [ - # etc/sbom/generate_sbom.py - "gitpython>=3.1", - "pygithub>=2.1", - "semver>=3.0.0", -] - [tool.ruff] line-length = 120 src = [".evergreen", "etc"] From dea5b19c162e286c19e49ab944ea59f07e2e1069 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 26 Nov 2025 13:10:59 -0500 Subject: [PATCH 32/62] Remove install dependencies --- .github/workflows/generate_sbom.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 07c61dcdd8..84ceece6e9 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -59,8 +59,8 @@ jobs: activate-environment: true enable-cache: true - - name: Install dependencies - run: uv sync --group sbom + # - name: Install dependencies + # run: uv sync --group sbom - name: generate_sbom.py run: uv run etc/sbom/generate_sbom.py --enable-github-action-token --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json --save-warnings=${{runner.temp}}/warnings.txt From 99ee44b61c42427dab6598665d5e4a7feeda58fa Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 26 Nov 2025 13:17:22 -0500 Subject: [PATCH 33/62] fail-fast: false --- .github/workflows/generate_sbom.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 84ceece6e9..5d4713351c 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -10,6 +10,8 @@ on: jobs: configure-and-scan: + strategy: + fail-fast: false permissions: id-token: write # Required to request a json web token (JWT) for keyless authentication with Endor Labs contents: read From 17bba3d65fb70eb00cb11ae481f7d46de87c1133 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 26 Nov 2025 14:35:01 -0500 Subject: [PATCH 34/62] Fixed recursion bug --- .github/workflows/generate_sbom.yml | 5 +++-- etc/sbom/config.py | 2 +- etc/sbom/generate_sbom.py | 7 ------- pyproject.toml | 7 +++++++ 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 5d4713351c..8fac0aaf4a 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -61,8 +61,9 @@ jobs: activate-environment: true enable-cache: true - # - name: Install dependencies - # run: uv sync --group sbom + - name: Install dependencies + run: | + uv sync --group generate_sbom - name: generate_sbom.py run: uv run etc/sbom/generate_sbom.py --enable-github-action-token --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json --save-warnings=${{runner.temp}}/warnings.txt diff --git a/etc/sbom/config.py b/etc/sbom/config.py index 82320bdec7..bc693ac47f 100755 --- a/etc/sbom/config.py +++ b/etc/sbom/config.py @@ -28,7 +28,7 @@ for component in endor_components_remove: for prefix in prefixes: - endor_components_remove.append(prefix + component) + component = prefix + component # ################ Component Renaming ################ # Endor does not have syntactically valid PURLs for C/C++ packages. diff --git a/etc/sbom/generate_sbom.py b/etc/sbom/generate_sbom.py index 389704180b..18650c9c5e 100755 --- a/etc/sbom/generate_sbom.py +++ b/etc/sbom/generate_sbom.py @@ -1,10 +1,3 @@ -# /// script -# dependencies = [ -# "gitpython", -# "pygithub", -# "semver", -# ] -# /// #!/usr/bin/env python3 """ Generate a CycloneDX SBOM using scan results from Endor Labs. diff --git a/pyproject.toml b/pyproject.toml index c239b1b24b..1e00caf00b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,13 @@ make_release = [ "pygithub>=2.1", ] +generate_sbom = [ + # etc/sbom/*.py + "gitpython>=3.1", + "pygithub>=2.1", + "semver>=3.0.0", +] + [tool.ruff] line-length = 120 src = [".evergreen", "etc"] From f68bf6b5a54f353118f9560b15f73a94221e4151 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 26 Nov 2025 15:21:28 -0500 Subject: [PATCH 35/62] contents: write --- .github/workflows/generate_sbom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 8fac0aaf4a..284e6de638 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -14,7 +14,7 @@ jobs: fail-fast: false permissions: id-token: write # Required to request a json web token (JWT) for keyless authentication with Endor Labs - contents: read + contents: write # Required for PR runs-on: ubuntu-latest steps: - name: Checkout Repository From 289007abc64e9ba415668abb6e55ab4fd24b31ba Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 26 Nov 2025 15:23:37 -0500 Subject: [PATCH 36/62] pull-requests: write --- .github/workflows/generate_sbom.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 284e6de638..6f99bf53e3 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -14,7 +14,8 @@ jobs: fail-fast: false permissions: id-token: write # Required to request a json web token (JWT) for keyless authentication with Endor Labs - contents: write # Required for PR + contents: write # Required for commit + pull-requests: write # Required for PR runs-on: ubuntu-latest steps: - name: Checkout Repository From b7ec1ac64c8ca950e01c2dd58a4e9b68828d3464 Mon Sep 17 00:00:00 2001 From: jasonhills-mongodb <191278712+jasonhills-mongodb@users.noreply.github.com> Date: Wed, 26 Nov 2025 20:24:04 +0000 Subject: [PATCH 37/62] Update SBOM file(s) --- sbom.json | 439 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 378 insertions(+), 61 deletions(-) diff --git a/sbom.json b/sbom.json index 2cf1320900..f6ebb1d8d7 100644 --- a/sbom.json +++ b/sbom.json @@ -1,19 +1,197 @@ { - "components": [ - { - "bom-ref": "pkg:github/mongodb/mongo-c-driver@2.1.2", - "copyright": "Copyright 2009-present MongoDB, Inc.", + "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "serialNumber": "urn:uuid:14f3bbeb-851e-43f7-8e5c-26e087cc0970", + "version": 1, + "metadata": { + "timestamp": "2025-11-26T20:24:04Z", + "lifecycles": [ + { + "phase": "pre-build" + } + ], + "component": { + "type": "application", + "bom-ref": "pkg:github/mongodb/mongo-cxx-driver@sbom_github_action", + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "author": "MongoDB, Inc.", + "publisher": "MongoDB, Inc.", + "group": "mongodb", + "name": "mongo-cxx-driver", + "description": "C++ Driver for MongoDB", + "version": "sbom_github_action", + "cpe": "cpe:2.3:a:mongodb:c++:sbom_github_action:*:*:*:*:*:*:*", + "purl": "pkg:github/mongodb/mongo-cxx-driver@sbom_github_action", "externalReferences": [ { - "type": "distribution", - "url": "https://github.com/mongodb/mongo-c-driver/archive/refs/tags/2.1.2.tar.gz" + "type": "license", + "url": "https://raw.githubusercontent.com/mongodb/mongo-cxx-driver/refs/heads/master/LICENSE", + "comment": "Apache License 2.0" }, { "type": "website", - "url": "https://github.com/mongodb/mongo-c-driver/tree/2.1.2" + "url": "https://www.mongodb.com/docs/languages/cpp/cpp-driver/", + "comment": "Documentation site for the official MongoDB C++ Driver." + }, + { + "type": "release-notes", + "url": "https://www.mongodb.com/docs/languages/cpp/cpp-driver/current/whats-new/" + }, + { + "type": "vcs", + "url": "https://github.com/mongodb/mongo-cxx-driver" + } + ] + }, + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "tools": { + "services": [ + { + "name": "Endor Labs Inc", + "version": "v1.7.671" + } + ] + } + }, + "components": [ + { + "bom-ref": "pkg:github/catchorg/catch2@v3.8.1", + "type": "library", + "author": "Martin Ho\u0159e\u0148ovsk\u00fd", + "group": "catchorg", + "name": "Catch2", + "version": "3.8.1", + "description": "A modern, C++-native, test framework for unit-tests, TDD and BDD.", + "licenses": [ + { + "license": { + "id": "BSL-1.0" + } + } + ], + "copyright": "Copyright Catch2 Authors", + "purl": "pkg:github/catchorg/catch2@v3.8.1", + "externalReferences": [ + { + "url": "https://github.com/catchorg/catch2.git", + "type": "distribution" + } + ], + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/catchorg/clara@v1.1.5", + "type": "library", + "author": "Phil Nash", + "group": "catchorg", + "name": "Clara", + "version": "1.1.5", + "description": "A simple to use, composable, command line parser for C++ 11 and beyond", + "licenses": [ + { + "license": { + "id": "BSL-1.0" + } + } + ], + "copyright": "Copyright 2017 Two Blue Cubes Ltd. All rights reserved.", + "purl": "pkg:github/catchorg/clara@v1.1.5", + "externalReferences": [ + { + "url": "https://github.com/catchorg/clara.git", + "type": "distribution" + } + ], + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/juliastrings/utf8proc@v2.8.0", + "type": "library", + "author": "Jan Behrens, the Public Software Group, the Julia-language developers", + "supplier": { + "name": "The Julia Programming Language", + "url": [ + "https://julialang.org/" + ] + }, + "group": "JuliaStrings", + "name": "utf8proc", + "version": "2.8.0", + "description": "A clean C library for processing UTF-8 Unicode data", + "licenses": [ + { + "license": { + "id": "MIT" + } } ], + "copyright": "Copyright \u00a9 2014-2021 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.", + "purl": "pkg:github/juliastrings/utf8proc@v2.8.0", + "externalReferences": [ + { + "url": "https://github.com/juliastrings/utf8proc.git", + "type": "distribution" + } + ], + "scope": "required" + }, + { + "type": "library", + "bom-ref": "pkg:github/madler/zlib@1.3.1", + "supplier": { + "name": "zlib", + "url": [ + "https://zlib.net/" + ] + }, + "author": "Jean-loup Gailly, Mark Adler", + "group": "madler", + "name": "zlib", + "version": "1.3.1", + "description": "zlib is a general purpose data compression library.", + "licenses": [ + { + "license": { + "id": "Zlib" + } + } + ], + "copyright": "Copyright \u00a9 1995-2024 Jean-loup Gailly and Mark Adler.", + "cpe": "cpe:2.3:a:zlib:zlib:1.3.1:*:*:*:*:*:*:*", + "purl": "pkg:github/madler/zlib@1.3.1", + "externalReferences": [ + { + "url": "https://zlib.net/fossils/", + "type": "distribution" + } + ], + "scope": "required" + }, + { + "type": "library", + "bom-ref": "pkg:github/mongodb/libmongocrypt@1.16.0", + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "author": "MongoDB, Inc.", "group": "mongodb", + "name": "libmongocrypt", + "version": "1.16.0", + "description": "C library for Client Side and Queryable Encryption in MongoDB", "licenses": [ { "license": { @@ -21,65 +199,204 @@ } } ], - "name": "mongo-c-driver", + "copyright": "Copyright 2019-present MongoDB, Inc.", + "cpe": "cpe:2.3:a:mongodb:libmongocrypt:1.16.0:*:*:*:*:*:*:*", + "purl": "pkg:github/mongodb/libmongocrypt@1.16.0", + "externalReferences": [ + { + "url": "https://github.com/mongodb/libmongocrypt.git", + "type": "distribution" + } + ], + "scope": "optional" + }, + { + "type": "library", + "bom-ref": "pkg:github/mongodb/mongo-c-driver@2.1.2", + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "author": "MongoDB, Inc.", + "group": "mongodb", + "name": "MongoDB C Driver", + "version": "2.1.2", + "description": "The Official MongoDB driver for C language", + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ], + "copyright": "2009-present, MongoDB, Inc.", + "cpe": "cpe:2.3:a:mongodb:c_driver:2.1.2:*:*:*:*:*:*:*", "purl": "pkg:github/mongodb/mongo-c-driver@2.1.2", + "externalReferences": [ + { + "url": "https://github.com/mongodb/mongo-c-driver.git", + "type": "distribution" + } + ], + "scope": "required" + }, + { + "bom-ref": "pkg:github/troydhanson/uthash@v2.3.0", "type": "library", - "version": "2.1.2" + "author": "Troy D. Hanson", + "group": "troydhanson", + "name": "github.com/troydhanson/uthash", + "version": "2.3.0", + "description": "C macros for hash tables and more", + "licenses": [ + { + "license": { + "id": "BSD-1-Clause" + } + } + ], + "copyright": "Copyright (c) 2005-2025, Troy D. Hanson", + "purl": "pkg:github/troydhanson/uthash@v2.3.0", + "externalReferences": [ + { + "url": "https://github.com/troydhanson/uthash.git", + "type": "distribution" + } + ], + "scope": "required" + }, + { + "bom-ref": "pkg:github/jasonhills-mongodb/mongo-cxx-driver@sbom_github_action?package-id=0999c3478a09a2e2", + "name": "github.com/jasonhills-mongodb/mongo-cxx-driver", + "purl": "pkg:github/jasonhills-mongodb/mongo-cxx-driver@sbom_github_action", + "type": "library", + "version": "sbom_github_action", + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/mongodb/mongo-cxx-driver@r4.1.1?package-id=c2042e84ed2ed5ca", + "externalReferences": [ + { + "type": "distribution", + "url": "https://github.com/mongodb/mongo-cxx-driver.git" + } + ], + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ], + "name": "github.com/mongodb/mongo-cxx-driver", + "purl": "pkg:github/mongodb/mongo-cxx-driver@r4.1.1", + "type": "library", + "version": "r4.1.1", + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/mozilla/cubeb@471c7e209cf152414c0498f0c4015f9ff9b2f1c0?package-id=dd972228bb636626", + "externalReferences": [ + { + "type": "distribution", + "url": "https://github.com/mozilla/cubeb.git" + } + ], + "licenses": [ + { + "license": { + "id": "ISC" + } + } + ], + "name": "github.com/mozilla/cubeb", + "purl": "pkg:github/mozilla/cubeb@471c7e209cf152414c0498f0c4015f9ff9b2f1c0", + "type": "library", + "version": "471c7e209cf152414c0498f0c4015f9ff9b2f1c0", + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/zlib-ng/zlib-ng@343c4c549107d31f6eeabfb4b31bec4502a2ea0e?package-id=8369b3e53cb7a837", + "externalReferences": [ + { + "type": "distribution", + "url": "https://github.com/zlib-ng/zlib-ng.git" + } + ], + "licenses": [ + { + "license": { + "id": "Zlib" + } + } + ], + "name": "github.com/zlib-ng/zlib-ng", + "purl": "pkg:github/zlib-ng/zlib-ng@343c4c549107d31f6eeabfb4b31bec4502a2ea0e", + "type": "library", + "version": "343c4c549107d31f6eeabfb4b31bec4502a2ea0e", + "scope": "excluded" } ], "dependencies": [ { - "ref": "pkg:github/mongodb/mongo-c-driver@2.1.2" + "ref": "pkg:github/mongodb/mongo-cxx-driver@sbom_github_action", + "dependsOn": [ + "pkg:github/catchorg/catch2@v3.8.1", + "pkg:github/mongodb/mongo-c-driver@2.1.2" + ] + }, + { + "ref": "pkg:github/catchorg/catch2@v3.8.1", + "dependsOn": [ + "pkg:github/catchorg/clara@v1.1.5" + ] + }, + { + "ref": "pkg:github/mongodb/mongo-c-driver@2.1.2", + "dependsOn": [ + "pkg:github/madler/zlib@1.3.1", + "pkg:github/juliastrings/utf8proc@v2.8.0", + "pkg:github/mongodb/libmongocrypt@1.16.0", + "pkg:github/troydhanson/uthash@v2.3.0" + ] + }, + { + "ref": "pkg:github/catchorg/clara@v1.1.5", + "dependsOn": [] + }, + { + "ref": "pkg:github/juliastrings/utf8proc@v2.8.0", + "dependsOn": [] + }, + { + "ref": "pkg:github/madler/zlib@1.3.1", + "dependsOn": [] + }, + { + "ref": "pkg:github/mongodb/libmongocrypt@1.16.0", + "dependsOn": [] + }, + { + "ref": "pkg:github/troydhanson/uthash@v2.3.0", + "dependsOn": [] + }, + { + "ref": "pkg:github/jasonhills-mongodb/mongo-cxx-driver@sbom_github_action?package-id=0999c3478a09a2e2", + "dependsOn": [] + }, + { + "ref": "pkg:github/mongodb/mongo-cxx-driver@r4.1.1?package-id=c2042e84ed2ed5ca", + "dependsOn": [] + }, + { + "ref": "pkg:github/mozilla/cubeb@471c7e209cf152414c0498f0c4015f9ff9b2f1c0?package-id=dd972228bb636626", + "dependsOn": [] + }, + { + "ref": "pkg:github/zlib-ng/zlib-ng@343c4c549107d31f6eeabfb4b31bec4502a2ea0e?package-id=8369b3e53cb7a837", + "dependsOn": [] } - ], - "metadata": { - "timestamp": "2025-10-07T17:27:33.460808+00:00", - "tools": [ - { - "externalReferences": [ - { - "type": "build-system", - "url": "https://github.com/CycloneDX/cyclonedx-python-lib/actions" - }, - { - "type": "distribution", - "url": "https://pypi.org/project/cyclonedx-python-lib/" - }, - { - "type": "documentation", - "url": "https://cyclonedx-python-library.readthedocs.io/" - }, - { - "type": "issue-tracker", - "url": "https://github.com/CycloneDX/cyclonedx-python-lib/issues" - }, - { - "type": "license", - "url": "https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/LICENSE" - }, - { - "type": "release-notes", - "url": "https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/CHANGELOG.md" - }, - { - "type": "vcs", - "url": "https://github.com/CycloneDX/cyclonedx-python-lib" - }, - { - "type": "website", - "url": "https://github.com/CycloneDX/cyclonedx-python-lib/#readme" - } - ], - "name": "cyclonedx-python-lib", - "vendor": "CycloneDX", - "version": "6.4.4" - } - ] - }, - "serialNumber": "urn:uuid:837d822c-50e1-45de-bd89-fad61a9600c7", - "version": 1, - "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", - "bomFormat": "CycloneDX", - "specVersion": "1.5", - "vulnerabilities": [] + ] } From 0d874e64c206d973e5e007e5fe4070a3929c928d Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 26 Nov 2025 16:44:32 -0500 Subject: [PATCH 38/62] Fix endor_components_remove --- etc/sbom/config.py | 7 ++++--- etc/sbom/generate_sbom.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/etc/sbom/config.py b/etc/sbom/config.py index bc693ac47f..c9e979bf5c 100755 --- a/etc/sbom/config.py +++ b/etc/sbom/config.py @@ -12,7 +12,7 @@ # ################ Component Filters ################ # List of Endor Labs SBOM components that must be removed before processing -endor_components_remove = [ +components_remove = [ # A dependency erroneously matched in build/CMakeFiles 'mozilla/cubeb', # An incorrect match from parts of pkg:github/madler/zlib @@ -26,9 +26,10 @@ 'pkg:github/', ] -for component in endor_components_remove: +endor_components_remove=[] +for component in components_remove: for prefix in prefixes: - component = prefix + component + endor_components_remove.append(prefix + component) # ################ Component Renaming ################ # Endor does not have syntactically valid PURLs for C/C++ packages. diff --git a/etc/sbom/generate_sbom.py b/etc/sbom/generate_sbom.py index 18650c9c5e..4d6482569b 100755 --- a/etc/sbom/generate_sbom.py +++ b/etc/sbom/generate_sbom.py @@ -533,6 +533,7 @@ def main() -> None: logger.error('Check Endor Labs for status of the target monitoring scan.') sys.exit(1) + logger.info(f'Endor Labs SBOM exported with {len(endor_bom["components"])} components') # endregion export Endor Labs SBOM # region Pre-process Endor Labs SBOM From 524931e918d53fb7b78b34b8db367a6d7485d8c9 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Tue, 2 Dec 2025 12:57:32 -0500 Subject: [PATCH 39/62] Update PR body content --- .github/workflows/generate_sbom.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 6f99bf53e3..d390e44f93 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -63,17 +63,19 @@ jobs: enable-cache: true - name: Install dependencies - run: | - uv sync --group generate_sbom + run: uv sync --group generate_sbom - name: generate_sbom.py run: uv run etc/sbom/generate_sbom.py --enable-github-action-token --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json --save-warnings=${{runner.temp}}/warnings.txt + - name: Generate Pull Request Content + run: printf "SBOM updated after commit ${{ github.sha }}.\n\nThe following warnings were output when generating the SBOM:\n" | cat - ${{runner.temp}}/warnings.txt > ${{runner.temp}}/pr_body.txt + - name: Open Pull Request uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9 with: add-paths: sbom.json - body-path: ${{runner.temp}}/warnings.txt + body-path: ${{runner.temp}}/pr_body.txt branch: cxx-sbom-update commit-message: Update SBOM file(s) delete-branch: true From bab76ec2cbb2834259dea003ef59febc117e5f8e Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Tue, 2 Dec 2025 13:47:51 -0500 Subject: [PATCH 40/62] Add SBOM diff --- .github/workflows/generate_sbom.yml | 27 +++++++++++++++++++++++---- etc/sbom/generate_sbom.py | 6 +++--- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index d390e44f93..2106897323 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -65,13 +65,32 @@ jobs: - name: Install dependencies run: uv sync --group generate_sbom + - name: Save existing SBOM for comparison + run: | + # Strip out nondeterministic SBOM fields and save to temp file + jq 'del(.version, .metadata.timestamp, .metadata.tools.services[].version)' sbom.json > ${{runner.temp}}/sbom.existing.cdx.json + - name: generate_sbom.py - run: uv run etc/sbom/generate_sbom.py --enable-github-action-token --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json --save-warnings=${{runner.temp}}/warnings.txt + run: | + uv run etc/sbom/generate_sbom.py --enable-github-action-token --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json --save-warnings=${{runner.temp}}/warnings.txt + + - name: Check for SBOM changes + id: sbom_diff + run: | + # Strip out nondeterministic SBOM fields and save to temp file + jq 'del(.version, .metadata.timestamp, .metadata.tools.services[].version)' sbom.json > ${{runner.temp}}/sbom.generated.cdx.json + RESULT=$(diff ${{runner.temp}}/sbom.existing.cdx.json ${{runner.temp}}/sbom.generated.cdx.json) + # Set the output variable + echo "::set-output name=result::$RESULT" - - name: Generate Pull Request Content - run: printf "SBOM updated after commit ${{ github.sha }}.\n\nThe following warnings were output when generating the SBOM:\n" | cat - ${{runner.temp}}/warnings.txt > ${{runner.temp}}/pr_body.txt + - name: Generate pull request content and notice, if SBOM has changed + if: ${{ steps.sbom_diff.outputs.result }} + run: | + printf "SBOM updated after commit ${{ github.sha }}.\n\n" | cat - ${{runner.temp}}/warnings.txt > ${{runner.temp}}/pr_body.txt + echo "::notice SBOM has changed" - - name: Open Pull Request + - name: Open Pull Request, if SBOM has changed + if: ${{ steps.sbom_diff.outputs.result }} uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9 with: add-paths: sbom.json diff --git a/etc/sbom/generate_sbom.py b/etc/sbom/generate_sbom.py index 4d6482569b..5991d1f930 100755 --- a/etc/sbom/generate_sbom.py +++ b/etc/sbom/generate_sbom.py @@ -830,9 +830,9 @@ def main() -> None: # Access the collected warnings print_banner('CONSOLIDATED WARNINGS') - warnings = [] + warnings = ["The following warnings were output when generating the SBOM:\n"] for record in warning_handler.warnings: - warnings.append(record.getMessage()) + warnings.append(" - " + record.getMessage()) print('\n'.join(warnings)) @@ -840,7 +840,7 @@ def main() -> None: write_list_to_text_file(warnings, save_warnings) print_banner('COMPLETED') - if not os.getenv('CI'): + if not os.getenv('CI') and not os.getenv("GITHUB_ACTIONS"): print('Be sure to add the SBOM to your next commit if the file content has changed.') # endregion Finalize SBOM From 8ec256ff3e93aaa82620ddd1c554aca6d4b535ac Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Tue, 2 Dec 2025 13:55:59 -0500 Subject: [PATCH 41/62] diff brief and true --- .github/workflows/generate_sbom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 2106897323..3a9985be18 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -79,7 +79,7 @@ jobs: run: | # Strip out nondeterministic SBOM fields and save to temp file jq 'del(.version, .metadata.timestamp, .metadata.tools.services[].version)' sbom.json > ${{runner.temp}}/sbom.generated.cdx.json - RESULT=$(diff ${{runner.temp}}/sbom.existing.cdx.json ${{runner.temp}}/sbom.generated.cdx.json) + RESULT=$(diff --brief ${{runner.temp}}/sbom.existing.cdx.json ${{runner.temp}}/sbom.generated.cdx.json || true) # Set the output variable echo "::set-output name=result::$RESULT" From 9e8f7a0ef13164560cdb2fb901aa64f1a31168d5 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Tue, 2 Dec 2025 14:03:43 -0500 Subject: [PATCH 42/62] Remove set-output --- .github/workflows/generate_sbom.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 3a9985be18..d4d5129961 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -79,15 +79,16 @@ jobs: run: | # Strip out nondeterministic SBOM fields and save to temp file jq 'del(.version, .metadata.timestamp, .metadata.tools.services[].version)' sbom.json > ${{runner.temp}}/sbom.generated.cdx.json + # diff the temp SBOM files, save output to variable, supress exit code RESULT=$(diff --brief ${{runner.temp}}/sbom.existing.cdx.json ${{runner.temp}}/sbom.generated.cdx.json || true) # Set the output variable - echo "::set-output name=result::$RESULT" + echo "result=$RESULT" >> $GITHUB_OUTPUT - name: Generate pull request content and notice, if SBOM has changed if: ${{ steps.sbom_diff.outputs.result }} run: | printf "SBOM updated after commit ${{ github.sha }}.\n\n" | cat - ${{runner.temp}}/warnings.txt > ${{runner.temp}}/pr_body.txt - echo "::notice SBOM has changed" + echo "::notice title=SBOM-Diff::SBOM has changed" - name: Open Pull Request, if SBOM has changed if: ${{ steps.sbom_diff.outputs.result }} From bdfad45ca7bbf43d87c1055f1ddf0f6d5021515c Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Tue, 2 Dec 2025 14:07:20 -0500 Subject: [PATCH 43/62] Add warnings none --- etc/sbom/generate_sbom.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/etc/sbom/generate_sbom.py b/etc/sbom/generate_sbom.py index 5991d1f930..a527ffb9bc 100755 --- a/etc/sbom/generate_sbom.py +++ b/etc/sbom/generate_sbom.py @@ -831,8 +831,12 @@ def main() -> None: # Access the collected warnings print_banner('CONSOLIDATED WARNINGS') warnings = ["The following warnings were output when generating the SBOM:\n"] - for record in warning_handler.warnings: - warnings.append(" - " + record.getMessage()) + + if len(record): + for record in warning_handler.warnings: + warnings.append(" - " + record.getMessage()) + else: + warnings.append(" - None") print('\n'.join(warnings)) From 0eb87813328ab49cc30da2e24ec7fa18cc5cd189 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Tue, 2 Dec 2025 14:09:19 -0500 Subject: [PATCH 44/62] len(warning_handler.warnings) --- etc/sbom/generate_sbom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/sbom/generate_sbom.py b/etc/sbom/generate_sbom.py index a527ffb9bc..2dca17e893 100755 --- a/etc/sbom/generate_sbom.py +++ b/etc/sbom/generate_sbom.py @@ -832,7 +832,7 @@ def main() -> None: print_banner('CONSOLIDATED WARNINGS') warnings = ["The following warnings were output when generating the SBOM:\n"] - if len(record): + if len(warning_handler.warnings): for record in warning_handler.warnings: warnings.append(" - " + record.getMessage()) else: From 89c84d81a5c475e3fa92bd17e128e163b04118ef Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Tue, 2 Dec 2025 14:42:56 -0500 Subject: [PATCH 45/62] add back build and perform ruff format --- .github/workflows/generate_sbom.yml | 44 +++++++++++++---------------- etc/sbom/config.py | 21 ++++++++------ etc/sbom/generate_sbom.py | 34 ++++++++++++---------- 3 files changed, 50 insertions(+), 49 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index d4d5129961..d2615e37bc 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -9,7 +9,7 @@ on: # - 'CXX**' jobs: - configure-and-scan: + generate_sbom: strategy: fail-fast: false permissions: @@ -23,34 +23,28 @@ jobs: with: submodules: recursive - # - name: Configure CMake and fetch dependency sources - # env: - # BUILD_TYPE: Release - # BUILD: ${{github.workspace}}/build - # CXX_STANDARD: 17 - # working-directory: ${{env.BUILD}} - # run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON + - name: Configure CMake and fetch dependency sources + env: + BUILD_TYPE: Release + BUILD: ${{github.workspace}}/build + CXX_STANDARD: 17 + working-directory: ${{env.BUILD}} + run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON - # - name: Install endorctl and Scan with Endor Labs - # uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 - # with: - # additional_args: "--languages=c --exclude-path=\"build/CMakeFiles/**\"" - # log_level: info - # log_verbose: false - # namespace: mongodb.${{github.repository_owner}} - # pr: false - # scan_dependencies: true - # tags: github_action - # env: - # ENDOR_SCAN_EMBEDDINGS: true - - - name: Install endorctl - uses: endorlabs/github-action/setup@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 + - name: Install endorctl and Scan with Endor Labs + uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 with: + additional_args: "--languages=c --exclude-path=\"build/CMakeFiles/**\"" + log_level: info + log_verbose: false namespace: mongodb.${{github.repository_owner}} - enable_github_action_token: true + pr: false + scan_dependencies: true + tags: github_action + env: + ENDOR_SCAN_EMBEDDINGS: true - - name: Set up Python 3.10 + - name: Set up Python uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: '3.10' diff --git a/etc/sbom/config.py b/etc/sbom/config.py index c9e979bf5c..140074c854 100755 --- a/etc/sbom/config.py +++ b/etc/sbom/config.py @@ -3,9 +3,10 @@ import logging import re -import semver import subprocess +import semver + logger = logging.getLogger('generate_sbom') logger.setLevel(logging.NOTSET) @@ -26,7 +27,7 @@ 'pkg:github/', ] -endor_components_remove=[] +endor_components_remove = [] for component in components_remove: for prefix in prefixes: endor_components_remove.append(prefix + component) @@ -49,20 +50,22 @@ # ################ Primary Component Version ################ def get_primary_component_version() -> str: """Attempt to determine primary component version using repo script.""" - + # mongo-cxx-driver: etc/calc_release_version.py try: - result = subprocess.run(["python", "etc/calc_release_version.py"], capture_output=True, text=True) + result = subprocess.run(['python', 'etc/calc_release_version.py'], capture_output=True, text=True) version = semver.VersionInfo.parse(result.stdout) - if version.match("0.0.0"): + if version.match('0.0.0'): return None else: return version except Exception as e: - logger.warning("PRIMARY COMPONENT VERSION: Unable to parse output from etc/calc_release_version.py: %s", result.stdout) + logger.warning( + 'PRIMARY COMPONENT VERSION: Unable to parse output from etc/calc_release_version.py: %s', result.stdout + ) logger.warning(e) return None - + # ################ Version Transformation ################ @@ -75,8 +78,8 @@ def get_primary_component_version() -> str: regex_semver = re.compile(RE_SEMVER) # Release Naming Conventions -REGEX_RELEASE_BRANCH = rf'^releases/v{RE_SEMVER}$' # e.g., releases/v4.1 -REGEX_RELEASE_TAG = rf'^(r{RE_SEMVER})|(debian/{RE_SEMVER}-1)$' # e.g., r3.7.0-beta1, debian/4.1.4-1 +REGEX_RELEASE_BRANCH = rf'^releases/v{RE_SEMVER}$' # e.g., releases/v4.1 +REGEX_RELEASE_TAG = rf'^(r{RE_SEMVER})|(debian/{RE_SEMVER}-1)$' # e.g., r3.7.0-beta1, debian/4.1.4-1 VERSION_PATTERN_REPL = [ # 'debian/1.28.1-1' pkg:github/mongodb/mongo-c-driver (temporary workaround) diff --git a/etc/sbom/generate_sbom.py b/etc/sbom/generate_sbom.py index 2dca17e893..9b92ec4f9e 100755 --- a/etc/sbom/generate_sbom.py +++ b/etc/sbom/generate_sbom.py @@ -543,8 +543,8 @@ def main() -> None: ## remove uneeded components ## # [list]endor_components_remove is defined in config.py # Endor Labs includes the main component in 'components'. This is not standard, so we remove it. - config.endor_components_remove.append(f"{git_info.org}/{git_info.repo}") - + config.endor_components_remove.append(f'pkg:github/{git_info.org}/{git_info.repo}') + # Reverse iterate the SBOM components list to safely modify in situ for i in range(len(endor_bom['components']) - 1, -1, -1): component = endor_bom['components'][i] @@ -631,7 +631,7 @@ def main() -> None: # Attempt to determine the primary component version being scanned primary_component_version = config.get_primary_component_version() - + logger.debug( f'Available main component version options, repo script: {primary_component_version}, tag: {git_info.release_tag}, branch: {git_info.branch}, previous SBOM: {prev_bom["metadata"]["component"]["version"]}' ) @@ -639,10 +639,12 @@ def main() -> None: if primary_component_version: version = primary_component_version - purl_version = "r" + primary_component_version + purl_version = 'r' + primary_component_version cpe_version = primary_component_version - logger.info(f"PRIMARY COMPONENT VERSION: Using repo script output '{primary_component_version}' as primary component version") - + logger.info( + f"PRIMARY COMPONENT VERSION: Using repo script output '{primary_component_version}' as primary component version" + ) + # Project scan always set to 'master' or if using 'master' branch if target == 'project' or git_info.branch == 'master': version = 'master' @@ -655,14 +657,16 @@ def main() -> None: version = git_info.release_tag[1:] # remove leading 'r' purl_version = git_info.release_tag cpe_version = version # without leading 'r' - logger.info(f"PRIMARY COMPONENT VERSION: Using release_tag '{git_info.release_tag}' as primary component version") + logger.info( + f"PRIMARY COMPONENT VERSION: Using release_tag '{git_info.release_tag}' as primary component version" + ) # Release branch e.g., v7.0 or v8.2 elif target == 'branch' and re.fullmatch(config.REGEX_RELEASE_BRANCH, git_info.branch): version = git_info.branch purl_version = git_info.branch # remove leading 'v', add wildcard. e.g. 8.2.* - cpe_version = version.replace("releases/","")[1:] + '.*' + cpe_version = version.replace('releases/', '')[1:] + '.*' logger.info(f"PRIMARY COMPONENT VERSION: Using release branch '{git_info.branch}' as primary component version") # Previous SBOM app version, if all needed specifiers exist @@ -756,8 +760,8 @@ def main() -> None: print_banner('New Endor Labs components') if endor_components: - logger.info( - f'ENDOR SBOM: There are {len(endor_components)} unmatched components in the Endor Labs SBOM. Adding as-is. The applicable metadata should be added to the metadata SBOM for the next run.' + logger.warning( + f'ENDOR SBOM: There are {len(endor_components)} unmatched components in the Endor Labs SBOM. Adding as-is. The applicable metadata should be added to the metadata SBOM ({sbom_metadata_path}) for the next run.' ) for component in endor_components: # set scope to excluded by default until the component is evaluated @@ -830,13 +834,13 @@ def main() -> None: # Access the collected warnings print_banner('CONSOLIDATED WARNINGS') - warnings = ["The following warnings were output when generating the SBOM:\n"] - + warnings = ['The following warnings were output when generating the SBOM:\n'] + if len(warning_handler.warnings): for record in warning_handler.warnings: - warnings.append(" - " + record.getMessage()) + warnings.append(' - ' + record.getMessage()) else: - warnings.append(" - None") + warnings.append(' - None') print('\n'.join(warnings)) @@ -844,7 +848,7 @@ def main() -> None: write_list_to_text_file(warnings, save_warnings) print_banner('COMPLETED') - if not os.getenv('CI') and not os.getenv("GITHUB_ACTIONS"): + if not os.getenv('CI') and not os.getenv('GITHUB_ACTIONS'): print('Be sure to add the SBOM to your next commit if the file content has changed.') # endregion Finalize SBOM From 7833a2c34a9cfe8f879a1c9378b3016e0f6a595d Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Tue, 2 Dec 2025 14:53:21 -0500 Subject: [PATCH 46/62] Change tiggers --- .github/workflows/generate_sbom.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index d2615e37bc..fc08a50c41 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -1,12 +1,14 @@ name: Generate SBOM on: - workflow_dispatch: push: - # branches: - # - 'master' - # - 'releases/**' - # - 'CXX**' + branches: + - 'master' + - 'releases/v**' + - 'debian/*' + paths: + - '**/CMakeLists.txt' + - '**/*.cmake' jobs: generate_sbom: From a1090994a6ed8454318f6b6a55d6d46b28b75872 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Tue, 2 Dec 2025 15:47:07 -0500 Subject: [PATCH 47/62] Add pr scan --- .github/workflows/endor_labs_pr_scan.yml | 48 ++++++++++++++++++++++++ .github/workflows/generate_sbom.yml | 6 ++- 2 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/endor_labs_pr_scan.yml diff --git a/.github/workflows/endor_labs_pr_scan.yml b/.github/workflows/endor_labs_pr_scan.yml new file mode 100644 index 0000000000..f8c2e2f91a --- /dev/null +++ b/.github/workflows/endor_labs_pr_scan.yml @@ -0,0 +1,48 @@ +name: Endor Labs PR Scan + +on: + pull_request: + branches: + - 'master' + - 'releases/v*' + - 'debian/*' + paths: + - '**/CMakeLists.txt' + - '**/*.cmake' + +jobs: + configure_and_scan: + permissions: + id-token: write # Required to request a json web token (JWT) for keyless authentication with Endor Labs + pull-requests: write # Required for endorctl to write pr comments + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v6 + with: + fetch-tags: true + submodules: recursive + + - name: Configure CMake and fetch dependency sources + env: + BUILD_TYPE: Release + BUILD: ${{github.workspace}}/build + CXX_STANDARD: 17 + working-directory: ${{env.BUILD}} + run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON + + - name: Endor Labs - Pull Request Scan + uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 + with: + additional_args: "--languages=c --exclude-path=\"build/CMakeFiles/**\"" + enable_pr_comments: true + github_token: ${{ secrets.GITHUB_TOKEN }} # Required for endorctl to write pr comments + log_level: info + log_verbose: false + namespace: mongodb.${{github.repository_owner}} + pr: true + scan_dependencies: true + scan_summary_output_type: "table" + tags: github_action + env: + ENDOR_SCAN_EMBEDDINGS: true \ No newline at end of file diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index fc08a50c41..727cd19222 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -4,7 +4,7 @@ on: push: branches: - 'master' - - 'releases/v**' + - 'releases/v*' - 'debian/*' paths: - '**/CMakeLists.txt' @@ -23,6 +23,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@v6 with: + fetch-tags: true submodules: recursive - name: Configure CMake and fetch dependency sources @@ -33,7 +34,7 @@ jobs: working-directory: ${{env.BUILD}} run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON - - name: Install endorctl and Scan with Endor Labs + - name: Endor Labs - Monitoring Scan uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 with: additional_args: "--languages=c --exclude-path=\"build/CMakeFiles/**\"" @@ -42,6 +43,7 @@ jobs: namespace: mongodb.${{github.repository_owner}} pr: false scan_dependencies: true + scan_summary_output_type: "table" tags: github_action env: ENDOR_SCAN_EMBEDDINGS: true From ccd8b415e707e961abae19f5ba850ed42a8090e6 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Tue, 2 Dec 2025 15:50:04 -0500 Subject: [PATCH 48/62] Rename job --- .github/workflows/endor_labs_pr_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/endor_labs_pr_scan.yml b/.github/workflows/endor_labs_pr_scan.yml index f8c2e2f91a..61d47d53a3 100644 --- a/.github/workflows/endor_labs_pr_scan.yml +++ b/.github/workflows/endor_labs_pr_scan.yml @@ -11,7 +11,7 @@ on: - '**/*.cmake' jobs: - configure_and_scan: + endor_pr_scan: permissions: id-token: write # Required to request a json web token (JWT) for keyless authentication with Endor Labs pull-requests: write # Required for endorctl to write pr comments From b39034407cce0e57efdfab4d64144fcf82c01c6b Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Tue, 2 Dec 2025 16:41:33 -0500 Subject: [PATCH 49/62] Add warning --- etc/sbom/generate_sbom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/sbom/generate_sbom.py b/etc/sbom/generate_sbom.py index 9b92ec4f9e..e7b9f2d8f3 100755 --- a/etc/sbom/generate_sbom.py +++ b/etc/sbom/generate_sbom.py @@ -768,7 +768,7 @@ def main() -> None: endor_components[component]['scope'] = 'excluded' meta_bom['components'].append(endor_components[component]) meta_bom['dependencies'].append({'ref': endor_components[component]['bom-ref'], 'dependsOn': []}) - logger.info(f'SBOM AS-IS COMPONENT: Added {component}') + logger.warning(f'SBOM AS-IS COMPONENT: Added {component}') # endregion Parse unmatched Endor Labs components From 9b7c5d3a9b332ad6fb256e8342f1c449aa81f261 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 3 Dec 2025 10:02:20 -0500 Subject: [PATCH 50/62] update docs --- .../components/funcs/install_c_driver.py | 1 - .evergreen/scripts/sbom.sh | 8 +- .github/workflows/generate_sbom.yml | 6 +- etc/purls.txt | 9 -- etc/releasing.md | 105 +----------------- etc/third_party_vulnerabilities.md | 2 +- 6 files changed, 11 insertions(+), 120 deletions(-) delete mode 100644 etc/purls.txt diff --git a/.evergreen/config_generator/components/funcs/install_c_driver.py b/.evergreen/config_generator/components/funcs/install_c_driver.py index 2f0df245d4..da2e4e8413 100644 --- a/.evergreen/config_generator/components/funcs/install_c_driver.py +++ b/.evergreen/config_generator/components/funcs/install_c_driver.py @@ -8,7 +8,6 @@ # If updating mongoc_version_minimum to a new release (not pinning to an unreleased commit), also update: # - BSON_REQUIRED_VERSION and MONGOC_REQUIRED_VERSION in CMakeLists.txt -# - the version of pkg:github/mongodb/mongo-c-driver in etc/purls.txt # - the default value of --c-driver-build-ref in etc/make_release.py # If pinning to an unreleased commit, create a "Blocked" JIRA ticket with # a "depends on" link to the appropriate C Driver version release ticket. diff --git a/.evergreen/scripts/sbom.sh b/.evergreen/scripts/sbom.sh index f3949b44e0..24073da852 100755 --- a/.evergreen/scripts/sbom.sh +++ b/.evergreen/scripts/sbom.sh @@ -25,18 +25,14 @@ podman pull "${silkbomb:?}" silkbomb_augment_flags=( --repo mongodb/mongo-cxx-driver --branch "${branch_name:?}" - --sbom-in /pwd/etc/cyclonedx.sbom.json + --sbom-in /pwd/sbom.json --sbom-out /pwd/etc/augmented.sbom.json.new # Any notable updates to the Augmented SBOM version should be done manually after careful inspection. - # Otherwise, it should be equal to the SBOM Lite version, which should normally be `1`. + # Otherwise, it should be equal to the existing SBOM version. --no-update-sbom-version ) -# First validate the SBOM Lite. -podman run -it --rm -v "$(pwd):/pwd" "${silkbomb:?}" \ - validate --purls /pwd/etc/purls.txt --sbom-in /pwd/etc/cyclonedx.sbom.json --exclude jira - # Allow the timestamp to be updated in the Augmented SBOM for update purposes. podman run -it --rm -v "$(pwd):/pwd" --env 'KONDUKTO_TOKEN' "${silkbomb:?}" augment "${silkbomb_augment_flags[@]:?}" diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml index 727cd19222..8ffa345f56 100644 --- a/.github/workflows/generate_sbom.yml +++ b/.github/workflows/generate_sbom.yml @@ -19,6 +19,8 @@ jobs: contents: write # Required for commit pull-requests: write # Required for PR runs-on: ubuntu-latest + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} steps: - name: Checkout Repository uses: actions/checkout@v6 @@ -94,7 +96,7 @@ jobs: with: add-paths: sbom.json body-path: ${{runner.temp}}/pr_body.txt - branch: cxx-sbom-update + branch: cxx-sbom-update-${{ env.BRANCH_NAME }} commit-message: Update SBOM file(s) delete-branch: true - title: CXX Update SBOM action \ No newline at end of file + title: CXX Update SBOM action - ${{ env.BRANCH_NAME }} \ No newline at end of file diff --git a/etc/purls.txt b/etc/purls.txt deleted file mode 100644 index 2875338089..0000000000 --- a/etc/purls.txt +++ /dev/null @@ -1,9 +0,0 @@ -# These package URLs (purls) point to the versions (tags) of external dependencies -# that are committed to the project. Refer: https://github.com/package-url/purl-spec - -# This file is fed to silkbomb to generate the cyclonedx.sbom.json file. Edit this file -# instead of modifying the SBOM JSON directly. After modifying this file, be sure to -# re-generate the SBOM JSON file! - -# bson and mongoc may be obtained via cmake/FetchMongoC.cmake. -pkg:github/mongodb/mongo-c-driver@2.1.2 diff --git a/etc/releasing.md b/etc/releasing.md index be7db7f12f..e8b591d77a 100644 --- a/etc/releasing.md +++ b/etc/releasing.md @@ -75,12 +75,6 @@ Some release steps require one or more of the following secrets. GRS_CONFIG_USER1_USERNAME= GRS_CONFIG_USER1_PASSWORD= ``` -- Snyk credentials. - - Location: `~/.secrets/snyk-creds.txt` - - Format: - ```bash - SNYK_API_TOKEN= - ``` ## Pre-Release Steps @@ -118,22 +112,11 @@ All issues with an Impact level of "High" or greater must have a "MongoDB Final All issues with an Impact level of "Medium" or greater which do not have a "MongoDB Final Status" of "Fix Committed" must document rationale for its current status in the "Notes" field. -### SBOM Lite +### SBOM Ensure the container engine (e.g. `podman` or `docker`) is authenticated with the DevProd-provided Amazon ECR instance. -Ensure the list of bundled dependencies in `etc/purls.txt` is up-to-date. If not, update `etc/purls.txt`. - -If `etc/purls.txt` was updated, update the SBOM Lite document using the following command(s): - -```bash -# Ensure latest version of SilkBomb is being used. -podman pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 - -# Output: "... writing sbom to file" -podman run -it --rm -v "$(pwd):/pwd" 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 \ - update --refresh --no-update-sbom-version -p "/pwd/etc/purls.txt" -i "/pwd/etc/cyclonedx.sbom.json" -o "/pwd/etc/cyclonedx.sbom.json" -``` +Ensure that any `CXX Update SBOM action - $BRANCH_NAME` PRs are merged for the release branch. Run a patch build which executes the `sbom` task and download the "Augmented SBOM (Updated)" file as `etc/augmented.sbom.json`. Evergreen CLI may be used to schedule only the `sbom` task: @@ -154,12 +137,6 @@ Update `etc/third_party_vulnerabilities.md` with any updates to new or known vul Download the "Augmented SBOM (Updated)" file from the latest EVG commit build in the `sbom` task and commit it into the repo as `etc/augmented.sbom.json` (even if the only notable change is the timestamp field). -### Check Snyk - -Inspect the list of projects in the latest report for the `mongodb/mongo-cxx-driver` target in [Snyk](https://app.snyk.io/org/dev-prod/). - -Deactivate any projects that will not be relevant in the upcoming release. Remove any projects that are not relevant to the current release. - ### Check Jira Inspect the list of tickets assigned to the version to be released on [Jira](https://jira.mongodb.com/projects/CXX?selectedItem=com.atlassian.jira.jira-projects-plugin%3Arelease-page&status=unreleased). @@ -432,67 +409,7 @@ The new branch should be continuously tested on Evergreen. Update the "Display N ### Update SBOM serial number -Check out the release branch `releases/vX.Y`. - -Update `etc/cyclonedx.sbom.json` with a new unique serial number for the next upcoming patch release (e.g. for `1.3.1` following the release of `1.3.0`): - -```bash -# Ensure latest version of SilkBomb is being used. -podman pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 - -# Output: "... writing sbom to file" -podman run -it --rm -v "$(pwd):/pwd" 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 \ - update --refresh --generate-new-serial-number -p "/pwd/etc/purls.txt" -i "/pwd/etc/cyclonedx.sbom.json" -o "/pwd/etc/cyclonedx.sbom.json" -``` - -Update `etc/augmented.sbom.json` by running a patch build which executes the `sbom` task as described above in [SBOM Lite](#sbom-lite). - -Commit and push these changes to the `releases/vX.Y` branch. - -### Update Snyk - -> [!IMPORTANT] -> Run the Snyk commands in a fresh clone of the post-release repository to avoid existing build and release artifacts from affecting Snyk. - -Checkout the new release tag. - -Configure and build the CXX Driver (do not reuse an existing C Driver installation; use the auto-downloaded C Driver sources instead): - -```bash -cmake -S . -B build -cmake --build build -``` - -Then run: - -```bash -# Snyk credentials. Ask for these from a team member. -. ~/.secrets/snyk-creds.txt - -# The new release tag. Ensure this is correct! -release_tag="rX.Y.Z" - -# Authenticate with Snyk dev-prod organization. -snyk auth "${SNYK_API_TOKEN:?}" - -# Verify third party dependency sources listed in etc/purls.txt are detected by Snyk. -# If not, see: https://support.snyk.io/hc/en-us/requests/new -# Use --exclude=extras until CXX-3042 is resolved -snyk_args=( - --org=dev-prod - --remote-repo-url=https://github.com/mongodb/mongo-cxx-driver/ - --target-reference="${release_tag:?}" - --unmanaged - --all-projects - --exclude=extras -) -snyk test "${snyk_args[@]:?}" --print-deps - -# Create a new Snyk target reference for the new release tag. -snyk monitor "${snyk_args[@]:?}" -``` - -Verify the new Snyk target reference is present in the [Snyk project targets list](https://app.snyk.io/org/dev-prod/projects?groupBy=targets&before&after&searchQuery=mongo-cxx-driver&sortBy=highest+severity&filters[Show]=&filters[Integrations]=cli&filters[CollectionIds]=) for `mongodb/mongo-cxx-driver`. +A new SBOM serial number is automatically generated when an SBOM is generated on a new branch. ### Post-Release Changes @@ -512,21 +429,7 @@ For a patch release, in `etc/apidocmenu.md`, update the list of versions under " In `README.md`, sync the "Driver Development Status" table with the updated table from `etc/apidocmenu.md`. -Update `etc/cyclonedx.sbom.json` with a new unique serial number for the next upcoming non-patch release (e.g. for `1.4.0` following the release of `1.3.0`): - -```bash -# Ensure latest version of SilkBomb is being used. -podman pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 - -# Output: "... writing sbom to file" -podman run -it --rm -v "$(pwd):/pwd" 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 \ - update --refresh --generate-new-serial-number -p "/pwd/etc/purls.txt" -i "/pwd/etc/cyclonedx.sbom.json" -o "/pwd/etc/cyclonedx.sbom.json" - -git add etc/cyclonedx.sbom.json -git commit -m "update SBOM serial number" -``` - -Update `etc/augmented.sbom.json` by running a patch build which executes the `sbom` task as described above in [SBOM Lite](#sbom-lite). +Update `etc/augmented.sbom.json` by running a patch build which executes the `sbom` task as described above in [SBOM](#sbom). Commit these changes to the `post-release-changes` branch: diff --git a/etc/third_party_vulnerabilities.md b/etc/third_party_vulnerabilities.md index 17be84f7e4..8edda37d28 100644 --- a/etc/third_party_vulnerabilities.md +++ b/etc/third_party_vulnerabilities.md @@ -17,7 +17,7 @@ This section provides a template that may be used for actual vulnerability repor - **Date Detected:** YYYY-MM-DD - **Severity:** Low, Medium, High, or Critical -- **Detector:** Silk or Snyk +- **Detector:** Endor Labs or Dependency-Track - **Description:** A short vulnerability description. - **Dependency:** Name and version of the 3rd party dependency. - **Upstream Status:** False Positive, Won't Fix, Fix Pending, or Fix Available. This is the fix status for the 3rd party dependency, not the CXX Driver. "Fix Available" should include the version and/or date when the fix was released, e.g. "Fix Available (1.2.3, 1970-01-01)". From c6c40ea51c0e7406a5933a8a215896e1b0db16f2 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 3 Dec 2025 10:15:59 -0500 Subject: [PATCH 51/62] include-path build/_deps --- .github/workflows/endor_labs_pr_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/endor_labs_pr_scan.yml b/.github/workflows/endor_labs_pr_scan.yml index 61d47d53a3..c9a54ec8c2 100644 --- a/.github/workflows/endor_labs_pr_scan.yml +++ b/.github/workflows/endor_labs_pr_scan.yml @@ -34,7 +34,7 @@ jobs: - name: Endor Labs - Pull Request Scan uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 with: - additional_args: "--languages=c --exclude-path=\"build/CMakeFiles/**\"" + additional_args: "--languages=c --include-path=\"build/_deps/**\"" enable_pr_comments: true github_token: ${{ secrets.GITHUB_TOKEN }} # Required for endorctl to write pr comments log_level: info From 0bbabe17b86543899e84f35cf69590aad7041a79 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 3 Dec 2025 10:19:15 -0500 Subject: [PATCH 52/62] git add . --- .github/workflows/endor_labs_pr_scan.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/endor_labs_pr_scan.yml b/.github/workflows/endor_labs_pr_scan.yml index c9a54ec8c2..988598a94f 100644 --- a/.github/workflows/endor_labs_pr_scan.yml +++ b/.github/workflows/endor_labs_pr_scan.yml @@ -29,7 +29,9 @@ jobs: BUILD: ${{github.workspace}}/build CXX_STANDARD: 17 working-directory: ${{env.BUILD}} - run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON + run: | + cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON + git add . - name: Endor Labs - Pull Request Scan uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 From 2cf0e6827a1b17a60351b3f1df1444dbc82d70d8 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 3 Dec 2025 10:22:39 -0500 Subject: [PATCH 53/62] ignore-paths --- .github/workflows/endor_labs_pr_scan.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/endor_labs_pr_scan.yml b/.github/workflows/endor_labs_pr_scan.yml index 988598a94f..f3af5c9d21 100644 --- a/.github/workflows/endor_labs_pr_scan.yml +++ b/.github/workflows/endor_labs_pr_scan.yml @@ -29,14 +29,12 @@ jobs: BUILD: ${{github.workspace}}/build CXX_STANDARD: 17 working-directory: ${{env.BUILD}} - run: | - cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON - git add . + run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON - name: Endor Labs - Pull Request Scan uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 with: - additional_args: "--languages=c --include-path=\"build/_deps/**\"" + additional_args: "--languages=c --exclude-path=\"src/**\" --exclude-path=\"benchmark/**\" --exclude-path=\"build/CMakeFiles/**\"" enable_pr_comments: true github_token: ${{ secrets.GITHUB_TOKEN }} # Required for endorctl to write pr comments log_level: info From 13e06801a1569ea6df8d616cd8bfbd5a49b4a5d9 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 3 Dec 2025 10:50:47 -0500 Subject: [PATCH 54/62] cat config --- .github/workflows/endor_labs_pr_scan.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/endor_labs_pr_scan.yml b/.github/workflows/endor_labs_pr_scan.yml index f3af5c9d21..8115d755d7 100644 --- a/.github/workflows/endor_labs_pr_scan.yml +++ b/.github/workflows/endor_labs_pr_scan.yml @@ -45,4 +45,7 @@ jobs: scan_summary_output_type: "table" tags: github_action env: - ENDOR_SCAN_EMBEDDINGS: true \ No newline at end of file + ENDOR_SCAN_EMBEDDINGS: true + + - name: Display config.yml + run: cat $HOME/.endorctl/config.yaml \ No newline at end of file From 7ead74d059e6a39c470bc27d631fdbde8c3439b8 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 3 Dec 2025 10:57:54 -0500 Subject: [PATCH 55/62] which endorctl --- .github/workflows/endor_labs_pr_scan.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/endor_labs_pr_scan.yml b/.github/workflows/endor_labs_pr_scan.yml index 8115d755d7..33a448c6cf 100644 --- a/.github/workflows/endor_labs_pr_scan.yml +++ b/.github/workflows/endor_labs_pr_scan.yml @@ -48,4 +48,6 @@ jobs: ENDOR_SCAN_EMBEDDINGS: true - name: Display config.yml - run: cat $HOME/.endorctl/config.yaml \ No newline at end of file + run: | + which endorctl + #cat $HOME/.endorctl/config.yaml \ No newline at end of file From 63736a5e124fcf8231b1841b50c5b51d05ca44f3 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 3 Dec 2025 11:03:21 -0500 Subject: [PATCH 56/62] cat .endorctl/config.yaml --- .github/workflows/endor_labs_pr_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/endor_labs_pr_scan.yml b/.github/workflows/endor_labs_pr_scan.yml index 33a448c6cf..f1e783754f 100644 --- a/.github/workflows/endor_labs_pr_scan.yml +++ b/.github/workflows/endor_labs_pr_scan.yml @@ -50,4 +50,4 @@ jobs: - name: Display config.yml run: | which endorctl - #cat $HOME/.endorctl/config.yaml \ No newline at end of file + cat .endorctl/config.yaml \ No newline at end of file From cddf62c515941f4b092438eeae0e17d805ddcb04 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 3 Dec 2025 11:15:43 -0500 Subject: [PATCH 57/62] git add and commit --- .github/workflows/endor_labs_pr_scan.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/endor_labs_pr_scan.yml b/.github/workflows/endor_labs_pr_scan.yml index f1e783754f..b8ffa35657 100644 --- a/.github/workflows/endor_labs_pr_scan.yml +++ b/.github/workflows/endor_labs_pr_scan.yml @@ -29,12 +29,15 @@ jobs: BUILD: ${{github.workspace}}/build CXX_STANDARD: 17 working-directory: ${{env.BUILD}} - run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON + run: | + cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON + git add build/_deps + git commit -m "Add build/_deps" - name: Endor Labs - Pull Request Scan uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 with: - additional_args: "--languages=c --exclude-path=\"src/**\" --exclude-path=\"benchmark/**\" --exclude-path=\"build/CMakeFiles/**\"" + additional_args: "--languages=c" enable_pr_comments: true github_token: ${{ secrets.GITHUB_TOKEN }} # Required for endorctl to write pr comments log_level: info @@ -42,12 +45,8 @@ jobs: namespace: mongodb.${{github.repository_owner}} pr: true scan_dependencies: true + scan_path: build/_deps scan_summary_output_type: "table" tags: github_action env: - ENDOR_SCAN_EMBEDDINGS: true - - - name: Display config.yml - run: | - which endorctl - cat .endorctl/config.yaml \ No newline at end of file + ENDOR_SCAN_EMBEDDINGS: true \ No newline at end of file From 78f1f0bdb0c3b358bbeded94048321ee6a55648f Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 3 Dec 2025 11:20:26 -0500 Subject: [PATCH 58/62] git add _deps --- .github/workflows/endor_labs_pr_scan.yml | 2 +- etc/sbom/config.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/endor_labs_pr_scan.yml b/.github/workflows/endor_labs_pr_scan.yml index b8ffa35657..b7c17e86a6 100644 --- a/.github/workflows/endor_labs_pr_scan.yml +++ b/.github/workflows/endor_labs_pr_scan.yml @@ -31,7 +31,7 @@ jobs: working-directory: ${{env.BUILD}} run: | cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON - git add build/_deps + git add _deps git commit -m "Add build/_deps" - name: Endor Labs - Pull Request Scan diff --git a/etc/sbom/config.py b/etc/sbom/config.py index 140074c854..d4b2c3ebf0 100755 --- a/etc/sbom/config.py +++ b/etc/sbom/config.py @@ -40,7 +40,6 @@ # Run string replacements to correct for this: endor_components_rename = [ ['pkg:generic/zlib.net/zlib', 'pkg:github/madler/zlib'], - ['pkg:github/philsquared/clara', 'pkg:github/catchorg/clara'], # in case of regression ['pkg:generic/github.com/', 'pkg:github/'], ['pkg:c/github.com/', 'pkg:github/'], From da4af950ecffe67ab0dbd56694bb46183cdc96a8 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 3 Dec 2025 11:32:46 -0500 Subject: [PATCH 59/62] git rm .gitignore --- .github/workflows/endor_labs_pr_scan.yml | 3 +-- etc/sbom/metadata.cdx.json | 10 +++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/endor_labs_pr_scan.yml b/.github/workflows/endor_labs_pr_scan.yml index b7c17e86a6..c1b380f981 100644 --- a/.github/workflows/endor_labs_pr_scan.yml +++ b/.github/workflows/endor_labs_pr_scan.yml @@ -31,8 +31,7 @@ jobs: working-directory: ${{env.BUILD}} run: | cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON - git add _deps - git commit -m "Add build/_deps" + git rm .gitignore # prevent exclusion of build/_deps from endorctl scan - name: Endor Labs - Pull Request Scan uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 diff --git a/etc/sbom/metadata.cdx.json b/etc/sbom/metadata.cdx.json index c821b2816c..535b78813c 100644 --- a/etc/sbom/metadata.cdx.json +++ b/etc/sbom/metadata.cdx.json @@ -210,7 +210,7 @@ "scope": "required" }, { - "bom-ref": "pkg:github/catchorg/clara@{{VERSION}}", + "bom-ref": "pkg:github/philsquared/clara@{{VERSION}}", "type": "library", "author": "Phil Nash", "group": "catchorg", @@ -225,10 +225,10 @@ } ], "copyright": "Copyright 2017 Two Blue Cubes Ltd. All rights reserved.", - "purl": "pkg:github/catchorg/clara@{{VERSION}}", + "purl": "pkg:github/philsquared/clara@{{VERSION}}", "externalReferences": [ { - "url": "https://github.com/catchorg/clara.git", + "url": "https://github.com/philsquared/clara.git", "type": "distribution" } ], @@ -271,7 +271,7 @@ { "ref": "pkg:github/catchorg/catch2@{{VERSION}}", "dependsOn": [ - "pkg:github/catchorg/clara@{{VERSION}}" + "pkg:github/philsquared/clara@{{VERSION}}" ] }, { @@ -284,7 +284,7 @@ ] }, { - "ref": "pkg:github/catchorg/clara@{{VERSION}}", + "ref": "pkg:github/philsquared/clara@{{VERSION}}", "dependsOn": [] }, { From 21f381faf14d0fb7bf77325ffd42534f0a15fe11 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 3 Dec 2025 11:38:38 -0500 Subject: [PATCH 60/62] --include-path=\"build/_deps/**\" --- .github/workflows/endor_labs_pr_scan.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/endor_labs_pr_scan.yml b/.github/workflows/endor_labs_pr_scan.yml index c1b380f981..2e33ece754 100644 --- a/.github/workflows/endor_labs_pr_scan.yml +++ b/.github/workflows/endor_labs_pr_scan.yml @@ -36,7 +36,7 @@ jobs: - name: Endor Labs - Pull Request Scan uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 with: - additional_args: "--languages=c" + additional_args: "--languages=c --include-path=\"build/_deps/**\"" enable_pr_comments: true github_token: ${{ secrets.GITHUB_TOKEN }} # Required for endorctl to write pr comments log_level: info @@ -44,7 +44,6 @@ jobs: namespace: mongodb.${{github.repository_owner}} pr: true scan_dependencies: true - scan_path: build/_deps scan_summary_output_type: "table" tags: github_action env: From 72be23d395f3f1232d34fd051f126623c557b967 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 3 Dec 2025 13:18:38 -0500 Subject: [PATCH 61/62] Combine to single action --- .github/workflows/endor_labs_pr_scan.yml | 50 -------- .../endor_scan_and_generate_sbom.yml | 113 ++++++++++++++++++ .github/workflows/generate_sbom.yml | 102 ---------------- 3 files changed, 113 insertions(+), 152 deletions(-) delete mode 100644 .github/workflows/endor_labs_pr_scan.yml create mode 100644 .github/workflows/endor_scan_and_generate_sbom.yml delete mode 100644 .github/workflows/generate_sbom.yml diff --git a/.github/workflows/endor_labs_pr_scan.yml b/.github/workflows/endor_labs_pr_scan.yml deleted file mode 100644 index 2e33ece754..0000000000 --- a/.github/workflows/endor_labs_pr_scan.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Endor Labs PR Scan - -on: - pull_request: - branches: - - 'master' - - 'releases/v*' - - 'debian/*' - paths: - - '**/CMakeLists.txt' - - '**/*.cmake' - -jobs: - endor_pr_scan: - permissions: - id-token: write # Required to request a json web token (JWT) for keyless authentication with Endor Labs - pull-requests: write # Required for endorctl to write pr comments - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v6 - with: - fetch-tags: true - submodules: recursive - - - name: Configure CMake and fetch dependency sources - env: - BUILD_TYPE: Release - BUILD: ${{github.workspace}}/build - CXX_STANDARD: 17 - working-directory: ${{env.BUILD}} - run: | - cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON - git rm .gitignore # prevent exclusion of build/_deps from endorctl scan - - - name: Endor Labs - Pull Request Scan - uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 - with: - additional_args: "--languages=c --include-path=\"build/_deps/**\"" - enable_pr_comments: true - github_token: ${{ secrets.GITHUB_TOKEN }} # Required for endorctl to write pr comments - log_level: info - log_verbose: false - namespace: mongodb.${{github.repository_owner}} - pr: true - scan_dependencies: true - scan_summary_output_type: "table" - tags: github_action - env: - ENDOR_SCAN_EMBEDDINGS: true \ No newline at end of file diff --git a/.github/workflows/endor_scan_and_generate_sbom.yml b/.github/workflows/endor_scan_and_generate_sbom.yml new file mode 100644 index 0000000000..f2f9110842 --- /dev/null +++ b/.github/workflows/endor_scan_and_generate_sbom.yml @@ -0,0 +1,113 @@ +name: Generate SBOM + +on: + pull_request: + branches: + - "master" + - "releases/v*" + - "debian/*" + paths: + - "**/CMakeLists.txt" + - "**/*.cmake" + push: + branches: + - "master" + - "releases/v*" + - "debian/*" + paths: + - "**/CMakeLists.txt" + - "**/*.cmake" + +jobs: + endor_scan_and_generate_sbom: + permissions: + id-token: write # Required to request a json web token (JWT) for keyless authentication with Endor Labs + contents: write # Required for commit + pull-requests: write # Required for PR + runs-on: ubuntu-latest + env: + PR_SCAN: ${{ github.event_name == 'pull_request' }} + steps: + - name: Checkout Repository + uses: actions/checkout@v6 + with: + fetch-tags: true + submodules: recursive + + - name: Configure CMake and fetch dependency sources + env: + BUILD_TYPE: Release + BUILD: ${{github.workspace}}/build + CXX_STANDARD: 17 + working-directory: ${{env.BUILD}} + run: | + cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON + git rm .gitignore # prevent exclusion of build/_deps from endorctl scan + + - name: Endor Labs Scan (PR or Monitoring) + uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 + env: + ENDOR_SCAN_EMBEDDINGS: true + with: + additional_args: '--languages=c --include-path="build/_deps/**"' + enable_pr_comments: ${{ env.PR_SCAN }} + github_token: ${{ secrets.GITHUB_TOKEN }} # Required for endorctl to write pr comments + log_level: info + log_verbose: false + namespace: mongodb.${{github.repository_owner}} + pr: ${{ env.PR_SCAN }} + scan_dependencies: true + scan_summary_output_type: "table" + tags: github_action + + # - name: Set up Python + # if: env.PR_SCAN == false + # uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + # with: + # python-version: "3.10" + + - name: Install uv + if: env.PR_SCAN == false + uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4 + with: + python-version: "3.10" + activate-environment: true + enable-cache: true + + - name: Stash existing SBOM, generate new SBOM + if: env.PR_SCAN == false + run: | + # Existing SBOM: Strip out nondeterministic SBOM fields and save to temp file + jq 'del(.version, .metadata.timestamp, .metadata.tools.services[].version)' sbom.json > ${{runner.temp}}/sbom.existing.cdx.json + # etc/sbom/generate_sbom.py + uv run --group generate_sbom etc/sbom/generate_sbom.py --enable-github-action-token --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json --save-warnings=${{runner.temp}}/warnings.txt + # Generated SBOM: Strip out nondeterministic SBOM fields and save to temp file + jq 'del(.version, .metadata.timestamp, .metadata.tools.services[].version)' sbom.json > ${{runner.temp}}/sbom.generated.cdx.json + + - name: Check for SBOM changes + if: env.PR_SCAN == false + id: sbom_diff + run: | + # diff the temp SBOM files, save output to variable, supress exit code + RESULT=$(diff --brief ${{runner.temp}}/sbom.existing.cdx.json ${{runner.temp}}/sbom.generated.cdx.json) + # Set the output variable + echo "result=$RESULT" | tee -a $GITHUB_OUTPUT + + - name: Generate pull request content and notice message, if SBOM has changed + if: env.PR_SCAN == false && steps.sbom_diff.outputs.result + run: | + printf "SBOM updated after commit ${{ github.sha }}.\n\n" | cat - ${{runner.temp}}/warnings.txt > ${{runner.temp}}/pr_body.txt + echo "::notice title=SBOM-Diff::SBOM has changed" + + - name: Open Pull Request, if SBOM has changed + if: env.PR_SCAN == false && steps.sbom_diff.outputs.result + uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9 + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + with: + add-paths: sbom.json + body-path: ${{runner.temp}}/pr_body.txt + branch: cxx-sbom-update-${{ env.BRANCH_NAME }} + commit-message: Update SBOM file(s) + delete-branch: true + title: CXX Update SBOM action - ${{ env.BRANCH_NAME }} \ No newline at end of file diff --git a/.github/workflows/generate_sbom.yml b/.github/workflows/generate_sbom.yml deleted file mode 100644 index 8ffa345f56..0000000000 --- a/.github/workflows/generate_sbom.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: Generate SBOM - -on: - push: - branches: - - 'master' - - 'releases/v*' - - 'debian/*' - paths: - - '**/CMakeLists.txt' - - '**/*.cmake' - -jobs: - generate_sbom: - strategy: - fail-fast: false - permissions: - id-token: write # Required to request a json web token (JWT) for keyless authentication with Endor Labs - contents: write # Required for commit - pull-requests: write # Required for PR - runs-on: ubuntu-latest - env: - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - steps: - - name: Checkout Repository - uses: actions/checkout@v6 - with: - fetch-tags: true - submodules: recursive - - - name: Configure CMake and fetch dependency sources - env: - BUILD_TYPE: Release - BUILD: ${{github.workspace}}/build - CXX_STANDARD: 17 - working-directory: ${{env.BUILD}} - run: cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON - - - name: Endor Labs - Monitoring Scan - uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 - with: - additional_args: "--languages=c --exclude-path=\"build/CMakeFiles/**\"" - log_level: info - log_verbose: false - namespace: mongodb.${{github.repository_owner}} - pr: false - scan_dependencies: true - scan_summary_output_type: "table" - tags: github_action - env: - ENDOR_SCAN_EMBEDDINGS: true - - - name: Set up Python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 - with: - python-version: '3.10' - - - name: Install uv - uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4 - with: - python-version: "3.10" - activate-environment: true - enable-cache: true - - - name: Install dependencies - run: uv sync --group generate_sbom - - - name: Save existing SBOM for comparison - run: | - # Strip out nondeterministic SBOM fields and save to temp file - jq 'del(.version, .metadata.timestamp, .metadata.tools.services[].version)' sbom.json > ${{runner.temp}}/sbom.existing.cdx.json - - - name: generate_sbom.py - run: | - uv run etc/sbom/generate_sbom.py --enable-github-action-token --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json --save-warnings=${{runner.temp}}/warnings.txt - - - name: Check for SBOM changes - id: sbom_diff - run: | - # Strip out nondeterministic SBOM fields and save to temp file - jq 'del(.version, .metadata.timestamp, .metadata.tools.services[].version)' sbom.json > ${{runner.temp}}/sbom.generated.cdx.json - # diff the temp SBOM files, save output to variable, supress exit code - RESULT=$(diff --brief ${{runner.temp}}/sbom.existing.cdx.json ${{runner.temp}}/sbom.generated.cdx.json || true) - # Set the output variable - echo "result=$RESULT" >> $GITHUB_OUTPUT - - - name: Generate pull request content and notice, if SBOM has changed - if: ${{ steps.sbom_diff.outputs.result }} - run: | - printf "SBOM updated after commit ${{ github.sha }}.\n\n" | cat - ${{runner.temp}}/warnings.txt > ${{runner.temp}}/pr_body.txt - echo "::notice title=SBOM-Diff::SBOM has changed" - - - name: Open Pull Request, if SBOM has changed - if: ${{ steps.sbom_diff.outputs.result }} - uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9 - with: - add-paths: sbom.json - body-path: ${{runner.temp}}/pr_body.txt - branch: cxx-sbom-update-${{ env.BRANCH_NAME }} - commit-message: Update SBOM file(s) - delete-branch: true - title: CXX Update SBOM action - ${{ env.BRANCH_NAME }} \ No newline at end of file From acbcdd2fb4552890f54492a23360d92e4ce1b010 Mon Sep 17 00:00:00 2001 From: Jason Hills Date: Wed, 3 Dec 2025 13:25:05 -0500 Subject: [PATCH 62/62] Change names (push only) --- .github/workflows/endor_scan_and_generate_sbom.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/endor_scan_and_generate_sbom.yml b/.github/workflows/endor_scan_and_generate_sbom.yml index f2f9110842..79f53f38a7 100644 --- a/.github/workflows/endor_scan_and_generate_sbom.yml +++ b/.github/workflows/endor_scan_and_generate_sbom.yml @@ -66,7 +66,7 @@ jobs: # with: # python-version: "3.10" - - name: Install uv + - name: Install uv (push only) if: env.PR_SCAN == false uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4 with: @@ -74,7 +74,7 @@ jobs: activate-environment: true enable-cache: true - - name: Stash existing SBOM, generate new SBOM + - name: Stash existing SBOM, generate new SBOM (push only) if: env.PR_SCAN == false run: | # Existing SBOM: Strip out nondeterministic SBOM fields and save to temp file @@ -84,7 +84,7 @@ jobs: # Generated SBOM: Strip out nondeterministic SBOM fields and save to temp file jq 'del(.version, .metadata.timestamp, .metadata.tools.services[].version)' sbom.json > ${{runner.temp}}/sbom.generated.cdx.json - - name: Check for SBOM changes + - name: Check for SBOM changes (push only) if: env.PR_SCAN == false id: sbom_diff run: | @@ -93,13 +93,13 @@ jobs: # Set the output variable echo "result=$RESULT" | tee -a $GITHUB_OUTPUT - - name: Generate pull request content and notice message, if SBOM has changed + - name: Generate pull request content and notice message, if SBOM has changed (push only) if: env.PR_SCAN == false && steps.sbom_diff.outputs.result run: | printf "SBOM updated after commit ${{ github.sha }}.\n\n" | cat - ${{runner.temp}}/warnings.txt > ${{runner.temp}}/pr_body.txt echo "::notice title=SBOM-Diff::SBOM has changed" - - name: Open Pull Request, if SBOM has changed + - name: Open Pull Request, if SBOM has changed (push only) if: env.PR_SCAN == false && steps.sbom_diff.outputs.result uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9 env: