diff --git a/.evergreen/config_generator/components/funcs/install_c_driver.py b/.evergreen/config_generator/components/funcs/install_c_driver.py index 2f0df245d4..da2e4e8413 100644 --- a/.evergreen/config_generator/components/funcs/install_c_driver.py +++ b/.evergreen/config_generator/components/funcs/install_c_driver.py @@ -8,7 +8,6 @@ # If updating mongoc_version_minimum to a new release (not pinning to an unreleased commit), also update: # - BSON_REQUIRED_VERSION and MONGOC_REQUIRED_VERSION in CMakeLists.txt -# - the version of pkg:github/mongodb/mongo-c-driver in etc/purls.txt # - the default value of --c-driver-build-ref in etc/make_release.py # If pinning to an unreleased commit, create a "Blocked" JIRA ticket with # a "depends on" link to the appropriate C Driver version release ticket. diff --git a/.evergreen/scripts/sbom.sh b/.evergreen/scripts/sbom.sh index f3949b44e0..24073da852 100755 --- a/.evergreen/scripts/sbom.sh +++ b/.evergreen/scripts/sbom.sh @@ -25,18 +25,14 @@ podman pull "${silkbomb:?}" silkbomb_augment_flags=( --repo mongodb/mongo-cxx-driver --branch "${branch_name:?}" - --sbom-in /pwd/etc/cyclonedx.sbom.json + --sbom-in /pwd/sbom.json --sbom-out /pwd/etc/augmented.sbom.json.new # Any notable updates to the Augmented SBOM version should be done manually after careful inspection. - # Otherwise, it should be equal to the SBOM Lite version, which should normally be `1`. + # Otherwise, it should be equal to the existing SBOM version. --no-update-sbom-version ) -# First validate the SBOM Lite. -podman run -it --rm -v "$(pwd):/pwd" "${silkbomb:?}" \ - validate --purls /pwd/etc/purls.txt --sbom-in /pwd/etc/cyclonedx.sbom.json --exclude jira - # Allow the timestamp to be updated in the Augmented SBOM for update purposes. podman run -it --rm -v "$(pwd):/pwd" --env 'KONDUKTO_TOKEN' "${silkbomb:?}" augment "${silkbomb_augment_flags[@]:?}" diff --git a/.github/workflows/endor_scan_and_generate_sbom.yml b/.github/workflows/endor_scan_and_generate_sbom.yml new file mode 100644 index 0000000000..79f53f38a7 --- /dev/null +++ b/.github/workflows/endor_scan_and_generate_sbom.yml @@ -0,0 +1,113 @@ +name: Generate SBOM + +on: + pull_request: + branches: + - "master" + - "releases/v*" + - "debian/*" + paths: + - "**/CMakeLists.txt" + - "**/*.cmake" + push: + branches: + - "master" + - "releases/v*" + - "debian/*" + paths: + - "**/CMakeLists.txt" + - "**/*.cmake" + +jobs: + endor_scan_and_generate_sbom: + permissions: + id-token: write # Required to request a json web token (JWT) for keyless authentication with Endor Labs + contents: write # Required for commit + pull-requests: write # Required for PR + runs-on: ubuntu-latest + env: + PR_SCAN: ${{ github.event_name == 'pull_request' }} + steps: + - name: Checkout Repository + uses: actions/checkout@v6 + with: + fetch-tags: true + submodules: recursive + + - name: Configure CMake and fetch dependency sources + env: + BUILD_TYPE: Release + BUILD: ${{github.workspace}}/build + CXX_STANDARD: 17 + working-directory: ${{env.BUILD}} + run: | + cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_STANDARD=${{env.CXX_STANDARD}} -DENABLE_TESTS=ON + git rm .gitignore # prevent exclusion of build/_deps from endorctl scan + + - name: Endor Labs Scan (PR or Monitoring) + uses: endorlabs/github-action@519df81de5f68536c84ae05ebb2986d0bb1d19fc # v1.1.8 + env: + ENDOR_SCAN_EMBEDDINGS: true + with: + additional_args: '--languages=c --include-path="build/_deps/**"' + enable_pr_comments: ${{ env.PR_SCAN }} + github_token: ${{ secrets.GITHUB_TOKEN }} # Required for endorctl to write pr comments + log_level: info + log_verbose: false + namespace: mongodb.${{github.repository_owner}} + pr: ${{ env.PR_SCAN }} + scan_dependencies: true + scan_summary_output_type: "table" + tags: github_action + + # - name: Set up Python + # if: env.PR_SCAN == false + # uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + # with: + # python-version: "3.10" + + - name: Install uv (push only) + if: env.PR_SCAN == false + uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4 + with: + python-version: "3.10" + activate-environment: true + enable-cache: true + + - name: Stash existing SBOM, generate new SBOM (push only) + if: env.PR_SCAN == false + run: | + # Existing SBOM: Strip out nondeterministic SBOM fields and save to temp file + jq 'del(.version, .metadata.timestamp, .metadata.tools.services[].version)' sbom.json > ${{runner.temp}}/sbom.existing.cdx.json + # etc/sbom/generate_sbom.py + uv run --group generate_sbom etc/sbom/generate_sbom.py --enable-github-action-token --target=branch --sbom-metadata=etc/sbom/metadata.cdx.json --save-warnings=${{runner.temp}}/warnings.txt + # Generated SBOM: Strip out nondeterministic SBOM fields and save to temp file + jq 'del(.version, .metadata.timestamp, .metadata.tools.services[].version)' sbom.json > ${{runner.temp}}/sbom.generated.cdx.json + + - name: Check for SBOM changes (push only) + if: env.PR_SCAN == false + id: sbom_diff + run: | + # diff the temp SBOM files, save output to variable, supress exit code + RESULT=$(diff --brief ${{runner.temp}}/sbom.existing.cdx.json ${{runner.temp}}/sbom.generated.cdx.json) + # Set the output variable + echo "result=$RESULT" | tee -a $GITHUB_OUTPUT + + - name: Generate pull request content and notice message, if SBOM has changed (push only) + if: env.PR_SCAN == false && steps.sbom_diff.outputs.result + run: | + printf "SBOM updated after commit ${{ github.sha }}.\n\n" | cat - ${{runner.temp}}/warnings.txt > ${{runner.temp}}/pr_body.txt + echo "::notice title=SBOM-Diff::SBOM has changed" + + - name: Open Pull Request, if SBOM has changed (push only) + if: env.PR_SCAN == false && steps.sbom_diff.outputs.result + uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9 + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + with: + add-paths: sbom.json + body-path: ${{runner.temp}}/pr_body.txt + branch: cxx-sbom-update-${{ env.BRANCH_NAME }} + commit-message: Update SBOM file(s) + delete-branch: true + title: CXX Update SBOM action - ${{ env.BRANCH_NAME }} \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index c6188d4a02..826f996ae5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,7 +56,6 @@ else() message(WARNING "Unknown compiler... recklessly proceeding without a version check") endif() -# Also update etc/purls.txt. set(BSON_REQUIRED_VERSION 2.1.2) set(MONGOC_REQUIRED_VERSION 2.1.2) set(MONGOC_DOWNLOAD_VERSION 2.1.2) diff --git a/etc/purls.txt b/etc/purls.txt deleted file mode 100644 index 2875338089..0000000000 --- a/etc/purls.txt +++ /dev/null @@ -1,9 +0,0 @@ -# These package URLs (purls) point to the versions (tags) of external dependencies -# that are committed to the project. Refer: https://github.com/package-url/purl-spec - -# This file is fed to silkbomb to generate the cyclonedx.sbom.json file. Edit this file -# instead of modifying the SBOM JSON directly. After modifying this file, be sure to -# re-generate the SBOM JSON file! - -# bson and mongoc may be obtained via cmake/FetchMongoC.cmake. -pkg:github/mongodb/mongo-c-driver@2.1.2 diff --git a/etc/releasing.md b/etc/releasing.md index be7db7f12f..e8b591d77a 100644 --- a/etc/releasing.md +++ b/etc/releasing.md @@ -75,12 +75,6 @@ Some release steps require one or more of the following secrets. GRS_CONFIG_USER1_USERNAME= GRS_CONFIG_USER1_PASSWORD= ``` -- Snyk credentials. - - Location: `~/.secrets/snyk-creds.txt` - - Format: - ```bash - SNYK_API_TOKEN= - ``` ## Pre-Release Steps @@ -118,22 +112,11 @@ All issues with an Impact level of "High" or greater must have a "MongoDB Final All issues with an Impact level of "Medium" or greater which do not have a "MongoDB Final Status" of "Fix Committed" must document rationale for its current status in the "Notes" field. -### SBOM Lite +### SBOM Ensure the container engine (e.g. `podman` or `docker`) is authenticated with the DevProd-provided Amazon ECR instance. -Ensure the list of bundled dependencies in `etc/purls.txt` is up-to-date. If not, update `etc/purls.txt`. - -If `etc/purls.txt` was updated, update the SBOM Lite document using the following command(s): - -```bash -# Ensure latest version of SilkBomb is being used. -podman pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 - -# Output: "... writing sbom to file" -podman run -it --rm -v "$(pwd):/pwd" 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 \ - update --refresh --no-update-sbom-version -p "/pwd/etc/purls.txt" -i "/pwd/etc/cyclonedx.sbom.json" -o "/pwd/etc/cyclonedx.sbom.json" -``` +Ensure that any `CXX Update SBOM action - $BRANCH_NAME` PRs are merged for the release branch. Run a patch build which executes the `sbom` task and download the "Augmented SBOM (Updated)" file as `etc/augmented.sbom.json`. Evergreen CLI may be used to schedule only the `sbom` task: @@ -154,12 +137,6 @@ Update `etc/third_party_vulnerabilities.md` with any updates to new or known vul Download the "Augmented SBOM (Updated)" file from the latest EVG commit build in the `sbom` task and commit it into the repo as `etc/augmented.sbom.json` (even if the only notable change is the timestamp field). -### Check Snyk - -Inspect the list of projects in the latest report for the `mongodb/mongo-cxx-driver` target in [Snyk](https://app.snyk.io/org/dev-prod/). - -Deactivate any projects that will not be relevant in the upcoming release. Remove any projects that are not relevant to the current release. - ### Check Jira Inspect the list of tickets assigned to the version to be released on [Jira](https://jira.mongodb.com/projects/CXX?selectedItem=com.atlassian.jira.jira-projects-plugin%3Arelease-page&status=unreleased). @@ -432,67 +409,7 @@ The new branch should be continuously tested on Evergreen. Update the "Display N ### Update SBOM serial number -Check out the release branch `releases/vX.Y`. - -Update `etc/cyclonedx.sbom.json` with a new unique serial number for the next upcoming patch release (e.g. for `1.3.1` following the release of `1.3.0`): - -```bash -# Ensure latest version of SilkBomb is being used. -podman pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 - -# Output: "... writing sbom to file" -podman run -it --rm -v "$(pwd):/pwd" 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 \ - update --refresh --generate-new-serial-number -p "/pwd/etc/purls.txt" -i "/pwd/etc/cyclonedx.sbom.json" -o "/pwd/etc/cyclonedx.sbom.json" -``` - -Update `etc/augmented.sbom.json` by running a patch build which executes the `sbom` task as described above in [SBOM Lite](#sbom-lite). - -Commit and push these changes to the `releases/vX.Y` branch. - -### Update Snyk - -> [!IMPORTANT] -> Run the Snyk commands in a fresh clone of the post-release repository to avoid existing build and release artifacts from affecting Snyk. - -Checkout the new release tag. - -Configure and build the CXX Driver (do not reuse an existing C Driver installation; use the auto-downloaded C Driver sources instead): - -```bash -cmake -S . -B build -cmake --build build -``` - -Then run: - -```bash -# Snyk credentials. Ask for these from a team member. -. ~/.secrets/snyk-creds.txt - -# The new release tag. Ensure this is correct! -release_tag="rX.Y.Z" - -# Authenticate with Snyk dev-prod organization. -snyk auth "${SNYK_API_TOKEN:?}" - -# Verify third party dependency sources listed in etc/purls.txt are detected by Snyk. -# If not, see: https://support.snyk.io/hc/en-us/requests/new -# Use --exclude=extras until CXX-3042 is resolved -snyk_args=( - --org=dev-prod - --remote-repo-url=https://github.com/mongodb/mongo-cxx-driver/ - --target-reference="${release_tag:?}" - --unmanaged - --all-projects - --exclude=extras -) -snyk test "${snyk_args[@]:?}" --print-deps - -# Create a new Snyk target reference for the new release tag. -snyk monitor "${snyk_args[@]:?}" -``` - -Verify the new Snyk target reference is present in the [Snyk project targets list](https://app.snyk.io/org/dev-prod/projects?groupBy=targets&before&after&searchQuery=mongo-cxx-driver&sortBy=highest+severity&filters[Show]=&filters[Integrations]=cli&filters[CollectionIds]=) for `mongodb/mongo-cxx-driver`. +A new SBOM serial number is automatically generated when an SBOM is generated on a new branch. ### Post-Release Changes @@ -512,21 +429,7 @@ For a patch release, in `etc/apidocmenu.md`, update the list of versions under " In `README.md`, sync the "Driver Development Status" table with the updated table from `etc/apidocmenu.md`. -Update `etc/cyclonedx.sbom.json` with a new unique serial number for the next upcoming non-patch release (e.g. for `1.4.0` following the release of `1.3.0`): - -```bash -# Ensure latest version of SilkBomb is being used. -podman pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 - -# Output: "... writing sbom to file" -podman run -it --rm -v "$(pwd):/pwd" 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 \ - update --refresh --generate-new-serial-number -p "/pwd/etc/purls.txt" -i "/pwd/etc/cyclonedx.sbom.json" -o "/pwd/etc/cyclonedx.sbom.json" - -git add etc/cyclonedx.sbom.json -git commit -m "update SBOM serial number" -``` - -Update `etc/augmented.sbom.json` by running a patch build which executes the `sbom` task as described above in [SBOM Lite](#sbom-lite). +Update `etc/augmented.sbom.json` by running a patch build which executes the `sbom` task as described above in [SBOM](#sbom). Commit these changes to the `post-release-changes` branch: diff --git a/etc/sbom/config.py b/etc/sbom/config.py new file mode 100755 index 0000000000..d4b2c3ebf0 --- /dev/null +++ b/etc/sbom/config.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""generate_sbom.py config. Operational configuration values stored separately from the core code.""" + +import logging +import re +import subprocess + +import semver + +logger = logging.getLogger('generate_sbom') +logger.setLevel(logging.NOTSET) + +# ################ Component Filters ################ + +# List of Endor Labs SBOM components that must be removed before processing +components_remove = [ + # A dependency erroneously matched in build/CMakeFiles + 'mozilla/cubeb', + # An incorrect match from parts of pkg:github/madler/zlib + 'zlib-ng/zlib-ng', +] + +# bom-ref prefixes (Endor Labs has been changing them, so add all that we have seen) +prefixes = [ + 'pkg:c/github.com/', + 'pkg:generic/github.com/', + 'pkg:github/', +] + +endor_components_remove = [] +for component in components_remove: + for prefix in prefixes: + endor_components_remove.append(prefix + component) + +# ################ Component Renaming ################ +# Endor does not have syntactically valid PURLs for C/C++ packages. +# e.g., +# Invalid: pkg:c/github.com/abseil/abseil-cpp@20250512.1 +# Valid: pkg:github/abseil/abseil-cpp@20250512.1 +# Run string replacements to correct for this: +endor_components_rename = [ + ['pkg:generic/zlib.net/zlib', 'pkg:github/madler/zlib'], + # in case of regression + ['pkg:generic/github.com/', 'pkg:github/'], + ['pkg:c/github.com/', 'pkg:github/'], +] + + +# ################ Primary Component Version ################ +def get_primary_component_version() -> str: + """Attempt to determine primary component version using repo script.""" + + # mongo-cxx-driver: etc/calc_release_version.py + try: + result = subprocess.run(['python', 'etc/calc_release_version.py'], capture_output=True, text=True) + version = semver.VersionInfo.parse(result.stdout) + if version.match('0.0.0'): + return None + else: + return version + except Exception as e: + logger.warning( + 'PRIMARY COMPONENT VERSION: Unable to parse output from etc/calc_release_version.py: %s', result.stdout + ) + logger.warning(e) + return None + + +# ################ Version Transformation ################ + +# In some cases we need to transform the version string to strip out tag-related text +# It is unknown what patterns may appear in the future, so we have targeted (not broad) regex +# This a list of 'pattern' and 'repl' inputs to re.sub() +RE_VER_NUM = r'(0|[1-9]\d*)' +RE_VER_LBL = r'(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?' +RE_SEMVER = rf'{RE_VER_NUM}\.{RE_VER_NUM}\.{RE_VER_NUM}{RE_VER_LBL}' +regex_semver = re.compile(RE_SEMVER) + +# Release Naming Conventions +REGEX_RELEASE_BRANCH = rf'^releases/v{RE_SEMVER}$' # e.g., releases/v4.1 +REGEX_RELEASE_TAG = rf'^(r{RE_SEMVER})|(debian/{RE_SEMVER}-1)$' # e.g., r3.7.0-beta1, debian/4.1.4-1 + +VERSION_PATTERN_REPL = [ + # 'debian/1.28.1-1' pkg:github/mongodb/mongo-c-driver (temporary workaround) + [re.compile(rf'^debian/({RE_SEMVER})-\d$'), r'\1'], + # 'gperftools-2.9.1' pkg:github/gperftools/gperftools + # 'mongo/v1.5.2' pkg:github/google/benchmark + # 'mongodb-8.2.0-alpha2' pkg:github/wiredtiger/wiredtiger + # 'release-1.12.0' pkg:github/apache/avro + # 'yaml-cpp-0.6.3' pkg:github/jbeder/yaml-cpp + [re.compile(rf'^[-a-z]+[-/][vr]?({RE_SEMVER})$'), r'\1'], + # 'asio-1-34-2' pkg:github/chriskohlhoff/asio + # 'cares-1_27_0' pkg:github/c-ares/c-ares + [ + re.compile(rf'^[a-z]+-{RE_VER_NUM}[_-]{RE_VER_NUM}[_-]{RE_VER_NUM}{RE_VER_LBL}$'), + r'\1.\2.\3', + ], + # 'pcre2-10.40' pkg:github/pcre2project/pcre2 + [re.compile(rf'^[a-z0-9]+-({RE_VER_NUM}\.{RE_VER_NUM})$'), r'\1'], + # 'icu-release-57-1' pkg:github/unicode-org/icu + [re.compile(rf'^[a-z]+-?[a-z]+-{RE_VER_NUM}-{RE_VER_NUM}$'), r'\1.\2'], + # 'v2.6.0' pkg:github/confluentinc/librdkafka + # 'r2.5.1' + [re.compile(rf'^[rv]({RE_SEMVER})$'), r'\1'], + # 'v2025.04.21.00' pkg:github/facebook/folly + [re.compile(r'^v(\d+\.\d+\.\d+\.\d+)$'), r'\1'], +] + + +def get_semver_from_release_version(release_ver: str) -> semver: + """Extract the version number from string with tags or other annotations""" + if release_ver: + for re_obj, repl in VERSION_PATTERN_REPL: + if re_obj.match(release_ver): + return re_obj.sub(repl, release_ver) + return release_ver + + +# region special component use-case functions + + +def process_component_special_cases(component_key: str, component: dict, versions: dict, repo_root: str) -> None: + pass + + +# endregion special component use-case functions diff --git a/etc/sbom/endorctl_utils.py b/etc/sbom/endorctl_utils.py new file mode 100644 index 0000000000..0761edc538 --- /dev/null +++ b/etc/sbom/endorctl_utils.py @@ -0,0 +1,495 @@ +#!/usr/bin/env python3 +""" +Utility functions for the Endor Labs API via endorctl + +""" + +import json +import logging +import subprocess +import time +from datetime import datetime +from enum import Enum + +logger = logging.getLogger('generate_sbom') +logger.setLevel(logging.NOTSET) + +default_field_masks = { + 'PackageVersion': [ + 'context', + 'meta', + 'processing_status', + 'spec.package_name', + 'spec.resolved_dependencies.dependencies', + 'spec.source_code_reference', + ], + 'ScanResult': [ + 'context', + 'meta', + 'spec.end_time', + 'spec.logs', + 'spec.refs', + 'spec.start_time', + 'spec.status', + 'spec.versions', + ], +} + + +def _get_default_field_mask(kind): + default_field_mask = default_field_masks.get(kind, []) + return ','.join(default_field_mask) + + +class EndorResourceKind(Enum): + """Enumeration for Endor Labs API resource kinds""" + + PROJECT = 'Project' + REPOSITORY_VERSION = 'RepositoryVersion' + SCAN_RESULT = 'ScanResult' + PACKAGE_VERSION = 'PackageVersion' + + +class EndorContextType(Enum): + """Most objects include a common nested object called Context. Contexts keep objects from different scans separated. + https://docs.endorlabs.com/rest-api/using-the-rest-api/data-model/common-fields/#context""" + + # Objects from a scan of the default branch. All objects in the OSS namespace are in the main context. The context ID is always default. + MAIN = 'CONTEXT_TYPE_MAIN' + CONTEXT_TYPE_MAIN = 'CONTEXT_TYPE_MAIN' + # Objects from a scan of a specific branch. The context ID is the branch reference name. + REF = 'CONTEXT_TYPE_REF' + CONTEXT_TYPE_REF = 'CONTEXT_TYPE_REF' + # Objects from a PR scan. The context ID is the PR UUID. Objects in this context are deleted after 30 days. + CI_RUN = 'CONTEXT_TYPE_CI_RUN' + CONTEXT_TYPE_CI_RUN = 'CONTEXT_TYPE_CI_RUN' + # Objects from an SBOM scan. The context ID is the SBOM serial number or some other unique identifier. + SBOM = 'CONTEXT_TYPE_SBOM' + CONTEXT_TYPE_SBOM = 'CONTEXT_TYPE_SBOM' + # Indicates that this object is a copy/temporary value of an object in another project. Used for same-tenant dependencies. + # In source code reference this is equivalent to “vendor” folders. Package versions in the external context are only scanned for call + # graphs. No other operations are performed on them. + EXTERNAL = 'CONTEXT_TYPE_EXTERNAL' + CONTEXT_TYPE_EXTERNAL = 'CONTEXT_TYPE_EXTERNAL' + + +class EndorFilter: + """Provide standard filters for Endor Labs API resource kinds""" + + def __init__(self, context_id=None, context_type=None): + self.context_id = context_id + self.context_type = context_type + + def _base_filters(self): + base_filters = [] + if self.context_id: + base_filters.append(f'context.id=={self.context_id}') + if self.context_type: + base_filters.append(f'context.type=={self.context_type}') + + return base_filters + + def repository_version( + self, + project_uuid=None, + sha=None, + ref=None, + context_type: EndorContextType = None, + context_type_exclude: EndorContextType = None, + ): + filters = self._base_filters() + if context_type: + filters.append(f'context.type=={context_type.value}') + if context_type_exclude: + filters.append(f'context.type!={context_type_exclude.value}') + if project_uuid: + filters.append(f'meta.parent_uuid=={project_uuid}') + if sha: + filters.append(f'spec.version.sha=={sha}') + if ref: + filters.append(f'spec.version.ref=={ref}') + + return ' and '.join(filters) + + def package_version( + self, + context_type: EndorContextType = None, + context_id=None, + project_uuid=None, + name=None, + package_name=None, + ): + filters = self._base_filters() + if context_type: + filters.append(f'context.type=={context_type.value}') + if context_type: + filters.append(f'context.id=={context_id}') + if project_uuid: + filters.append(f'spec.project_uuid=={project_uuid}') + if name: + filters.append(f'spec.package_name=={name}') + if package_name: + filters.append(f'meta.name=={package_name}') + + return ' and '.join(filters) + + def scan_result( + self, + context_type: EndorContextType = None, + project_uuid=None, + ref=None, + sha=None, + status=None, + ): + filters = self._base_filters() + if context_type: + filters.append(f'context.type=={context_type.value}') + if project_uuid: + filters.append(f'meta.parent_uuid=={project_uuid}') + if ref: + filters.append(f"spec.versions.ref contains '{ref}'") + if sha: + filters.append(f"spec.versions.sha contains '{sha}'") + if status: + filters.append(f'spec.status=={status}') + + return ' and '.join(filters) + + +class EndorCtl: + """Interact with endorctl (Endor Labs CLI)""" + + # region internal functions + def __init__( + self, + namespace, + retry_limit=5, + sleep_duration=30, + endorctl_path='endorctl', + config_path=None, + enable_github_action_token=False, + ): + self.namespace = namespace + self.retry_limit = retry_limit + self.sleep_duration = sleep_duration + self.endorctl_path = endorctl_path + self.config_path = config_path + self.enable_github_action_token = enable_github_action_token + + def _call_endorctl(self, command, subcommand, **kwargs): + """https://docs.endorlabs.com/endorctl/""" + + try: + command = [self.endorctl_path, command, subcommand, f'--namespace={self.namespace}'] + if self.config_path: + command.append(f'--config-path={self.config_path}') + if self.enable_github_action_token: + command.append(f'--enable-github-action-token') + + # parse args into flags + for key, value in kwargs.items(): + # Handle endorctl flags with hyphens that are defined in the script with underscores + flag = key.replace('_', '-') + if value: + command.append(f'--{flag}={value}') + logger.info('Running: %s', ' '.join(command)) + + result = subprocess.run(command, capture_output=True, text=True, check=True) + + resource = json.loads(result.stdout) + + except subprocess.CalledProcessError as e: + logger.error(f'Error executing command: {e}') + logger.error(e.stderr) + except json.JSONDecodeError as e: + logger.error(f'Error decoding JSON: {e}') + logger.error(f'Stdout: {result.stdout}') + except FileNotFoundError as e: + logger.error(f'FileNotFoundError: {e}') + logger.error( + f"'endorctl' not found in path '{self.endorctl_path}'. Supply the correct path, run 'buildscripts/install_endorctl.sh' or visit https://docs.endorlabs.com/endorctl/install-and-configure/" + ) + except Exception as e: + logger.error(f'An unexpected error occurred: {e}') + else: + return resource + + def _api_get(self, resource, **kwargs): + """https://docs.endorlabs.com/endorctl/commands/api/""" + return self._call_endorctl('api', 'get', resource=resource, **kwargs) + + def _api_list(self, resource, filter=None, retry=True, **kwargs): + """https://docs.endorlabs.com/endorctl/commands/api/""" + # If this script is run immediately after making a commit, Endor Labs will likely not yet have created the assocaited ScanResult object. The wait/retry logic below handles this scenario. + tries = 0 + while True: + tries += 1 + result = self._call_endorctl('api', 'list', resource=resource, filter=filter, **kwargs) + + # The expected output of 'endorctl api list' is: { "list": { "objects": [...] } } + # We want to just return the objects. In case we get an empty list, return a list + # with a single None to avoid having to handle index errors downstream. + if result and result['list'].get('objects') and len(result['list']['objects']) > 0: + return result['list']['objects'] + elif retry: + logger.info( + f"API LIST: Resource not found: {resource} with filter '{filter}' in namespace '{self.namespace}'" + ) + if tries <= self.retry_limit: + logger.info( + f'API LIST: Waiting for {self.sleep_duration} seconds before retry attempt {tries} of {self.retry_limit}' + ) + time.sleep(self.sleep_duration) + else: + logger.warning( + f"API LIST: Maximum number of allowed retries {self.retry_limit} attempted with no {resource} found using filter '{filter}'" + ) + return [None] + else: + return [None] + + def _check_resource(self, resource, resource_description) -> None: + if not resource: + raise LookupError(f'Resource not found: {resource_description}') + logger.info(f'Retrieved: {resource_description}') + + # endregion internal functions + + # region resource functions + def get_resource(self, resource, uuid=None, name=None, field_mask=None, **kwargs): + """https://docs.endorlabs.com/rest-api/using-the-rest-api/data-model/resource-kinds/""" + if not field_mask: + field_mask = _get_default_field_mask(resource) + return self._api_get(resource=resource, uuid=uuid, name=name, field_mask=field_mask, **kwargs) + + def get_resources( + self, + resource, + filter=None, + field_mask=None, + sort_path='meta.create_time', + sort_order='descending', + retry=True, + **kwargs, + ): + """https://docs.endorlabs.com/rest-api/using-the-rest-api/data-model/resource-kinds/""" + if not field_mask: + field_mask = _get_default_field_mask(resource) + return self._api_list( + resource=resource, + filter=filter, + field_mask=field_mask, + sort_path=sort_path, + sort_order=sort_order, + retry=retry, + **kwargs, + ) + + def get_project(self, git_url): + resource_kind = EndorResourceKind.PROJECT.value + resource_description = f"{resource_kind} with name '{git_url}' in namespace '{self.namespace}'" + project = self.get_resource(resource_kind, name=git_url) + self._check_resource(project, resource_description) + return project + + def get_repository_version(self, filter=None, retry=True): + resource_kind = EndorResourceKind.REPOSITORY_VERSION.value + resource_description = f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'" + repository_version = self.get_resources(resource_kind, filter=filter, retry=retry, page_size=1)[0] + self._check_resource(repository_version, resource_description) + return repository_version + + def get_scan_result(self, filter=None, retry=True): + resource_kind = EndorResourceKind.SCAN_RESULT.value + resource_description = f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'" + scan_result = self.get_resources(resource_kind, filter=filter, retry=retry, page_size=1)[0] + self._check_resource(scan_result, resource_description) + uuid = scan_result.get('uuid') + start_time = scan_result['spec'].get('start_time') + refs = scan_result['spec'].get('refs') + polling_start_time = datetime.now() + while True: + status = scan_result['spec'].get('status') + end_time = scan_result['spec'].get('end_time') + if status == 'STATUS_SUCCESS': + logger.info( + f' Scan completed successfully. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}.' + ) + return scan_result + elif status == 'STATUS_RUNNING': + logger.info(f' Scan is running. ScanResult uuid {uuid} for refs {refs} started at {start_time}.') + logger.info( + f' Waiting {self.sleep_duration} seconds before checking status. Total wait time: {(datetime.now() - polling_start_time).total_seconds() / 60:.2f} minutes' + ) + time.sleep(self.sleep_duration) + scan_result = self.get_resources(resource_kind, filter=filter, retry=retry, page_size=1)[0] + elif status == 'STATUS_PARTIAL_SUCCESS': + scan_logs = scan_result['spec'].get('logs') + raise RuntimeError( + f' Scan completed, but with critical warnings or errors. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}. Scan logs: {scan_logs}' + ) + elif status == 'STATUS_FAILURE': + scan_logs = scan_result['spec'].get('logs') + raise RuntimeError( + f' Scan failed. ScanResult uuid {uuid} for refs {refs} started at {start_time}, ended at {end_time}. Scan logs: {scan_logs}' + ) + + def get_package_versions(self, filter): + resource_kind = EndorResourceKind.PACKAGE_VERSION.value + resource_description = f"{resource_kind} with filter '{filter}' in namespace '{self.namespace}'" + package_versions = self.get_resources(resource_kind, filter=filter) + self._check_resource(package_versions, resource_description) + return package_versions + + def export_sbom( + self, + package_version_uuid=None, + package_version_uuids=None, + package_version_name=None, + app_name=None, + project_name=None, + project_uuid=None, + ): + """Export an SBOM from Endor Labs + + Valid parameter sets (other combinations result in an error from 'endorctl'): + Single-Package SBOM: + package_version_uuid + package_version_name + Multi-Package SBOM: + package_version_uuids,app_name + project_uuid,app_name,app_name + project_name,app_name,app_name + + https://docs.endorlabs.com/endorctl/commands/sbom/export/ + """ + if package_version_uuids: + package_version_uuids = ','.join(package_version_uuids) + return self._call_endorctl( + 'sbom', + 'export', + package_version_uuid=package_version_uuid, + package_version_uuids=package_version_uuids, + package_version_name=package_version_name, + app_name=app_name, + project_name=project_name, + project_uuid=project_uuid, + ) + + # endregion resource functions + + # region workflow functions + def get_sbom_for_commit(self, git_url: str, commit_sha: str) -> dict: + """Export SBOM for the PR commit (sha)""" + + endor_filter = EndorFilter() + + try: + # Project: get uuid + project = self.get_project(git_url) + project_uuid = project['uuid'] + app_name = project['spec']['git']['full_name'] + + # RepositoryVersion: get the context for the PR scan + endor_filter.context_type = EndorContextType.CI_RUN.value + filter_str = endor_filter.repository_version(project_uuid, commit_sha) + repository_version = self.get_repository_version(filter_str) + context_id = repository_version['context']['id'] + + # ScanResult: wait for a completed scan + endor_filter.context_id = context_id + filter_str = endor_filter.scan_result(project_uuid) + self.get_scan_result(filter_str) + + # PackageVersions: get package versions for SBOM + filter_str = endor_filter.package_version(project_uuid) + package_versions = self.get_package_versions(filter_str) + package_version_uuids = [package_version['uuid'] for package_version in package_versions] + package_version_names = [package_version['meta']['name'] for package_version in package_versions] + + # Export SBOM + sbom = self.export_sbom(package_version_uuids=package_version_uuids, app_name=app_name) + print( + f'Retrieved: CycloneDX SBOM for PackageVersion(s), name: {package_version_names}, uuid: {package_version_uuids}' + ) + return sbom + + except Exception as e: + print(f'Exception: {e}') + return + + def get_sbom_for_branch(self, git_url: str, branch: str) -> dict: + """Export lastest SBOM for a monitored branch/ref""" + + endor_filter = EndorFilter() + + try: + # Project: get uuid + project = self.get_project(git_url) + project_uuid = project['uuid'] + app_name = project['spec']['git']['full_name'] + + # RepositoryVersion: get the context for the latest branch scan + filter_str = endor_filter.repository_version( + project_uuid, ref=branch, context_type_exclude=EndorContextType.CI_RUN + ) + repository_version = self.get_repository_version(filter_str) + repository_version_context_type = EndorContextType[repository_version['context']['type']] + repository_version_uuid = repository_version['uuid'] + repository_version_ref = repository_version['spec']['version']['ref'] + repository_version_sha = repository_version['spec']['version']['sha'] + repository_version_scan_object_status = repository_version['scan_object']['status'] + if repository_version_scan_object_status != 'STATUS_SCANNED': + logger.warning( + f"RepositoryVersion (uuid: {repository_version_uuid}, ref: {repository_version_ref}, sha: {repository_version_sha}) scan status is '{repository_version_scan_object_status}' (expected 'STATUS_SCANNED')" + ) + + # ScanResult: search for a completed scan + filter_str = endor_filter.scan_result( + repository_version_context_type, project_uuid, repository_version_ref, repository_version_sha + ) + scan_result = self.get_scan_result(filter_str, retry=False) + project_uuid = scan_result['meta']['parent_uuid'] + + # PackageVersions: get package versions for SBOM + if branch in ['master', 'main']: + context_type = EndorContextType.MAIN + context_id = 'default' + else: + context_type = EndorContextType.REF + context_id = branch + filter_str = endor_filter.package_version(context_type, context_id, project_uuid) + package_version = self.get_package_versions(filter_str)[0] + package_version_name = package_version['meta']['name'] + package_version_uuid = package_version['uuid'] + + # Export SBOM + sbom = self.export_sbom(package_version_uuid=package_version_uuid, app_name=app_name) + logger.info( + f'SBOM: Retrieved CycloneDX SBOM for PackageVersion, name: {package_version_name}, uuid {package_version_uuid}' + ) + return sbom + + except Exception as e: + print(f'Exception: {e}') + return + + def get_sbom_for_project(self, git_url: str) -> dict: + """Export latest SBOM for EndorCtl project default branch""" + + try: + # Project: get uuid + project = self.get_project(git_url) + project_uuid = project['uuid'] + app_name = project['spec']['git']['full_name'] + + # Export SBOM + sbom = self.export_sbom(project_uuid=project_uuid, app_name=app_name) + logger.info(f'Retrieved: CycloneDX SBOM for Project {app_name}') + return sbom + + except Exception as e: + print(f'Exception: {e}') + return + + # endregion workflow functions diff --git a/etc/sbom/generate_sbom.py b/etc/sbom/generate_sbom.py new file mode 100755 index 0000000000..e7b9f2d8f3 --- /dev/null +++ b/etc/sbom/generate_sbom.py @@ -0,0 +1,860 @@ +#!/usr/bin/env python3 +""" +Generate a CycloneDX SBOM using scan results from Endor Labs. +Schema validation of output is not performed. +Use 'buildscripts/sbom_linter.py' for validation. + +Invoke with ---help or -h for help message. +""" + +import argparse +import json +import logging +import os +import re +import subprocess +import sys +import urllib.parse +import uuid +from datetime import datetime, timezone +from pathlib import Path + +import config +from endorctl_utils import EndorCtl +from git import Commit, Repo + +# region init + + +class WarningListHandler(logging.Handler): + """Collect warnings""" + + def __init__(self): + super().__init__() + self.warnings = [] + + def emit(self, record): + if record.levelno >= logging.WARNING: + self.warnings.append(record) + + +logging.basicConfig(stream=sys.stdout) +logger = logging.getLogger('generate_sbom') +logger.setLevel(logging.INFO) + +# Create an instance of the custom handler +warning_handler = WarningListHandler() + +# Add the handler to the logger +logger.addHandler(warning_handler) + +# Get the absolute path of the script file and directory +script_path = Path(__file__).resolve() +script_directory = script_path.parent + +# Regex for validation +REGEX_COMMIT_SHA = r'^[0-9a-fA-F]{40}$' +REGEX_GIT_BRANCH = r'^[a-zA-Z0-9_.\-/]+$' +REGEX_GITHUB_URL = r'^(https://github.com/)([a-zA-Z0-9-]{1,39}/[a-zA-Z0-9-_.]{1,100})(\.git)$' + +# ################ PURL Validation ################ +REGEX_STR_PURL_OPTIONAL = ( # Optional Version (any chars except ? @ #) + r'(?:@[^?@#]*)?' + # Optional Qualifiers (any chars except @ #) + r'(?:\?[^@#]*)?' + # Optional Subpath (any chars) + r'(?:#.*)?$' +) + +REGEX_PURL = { + # deb PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/deb-definition.md + 'deb': re.compile( + r'^pkg:deb/' # Scheme and type + # Namespace (organization/user), letters must be lowercase + r'(debian|ubuntu)+' + r'/' + r'[a-z0-9._-]+' + REGEX_STR_PURL_OPTIONAL # Name + ), + # Generic PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/generic-definition.md + 'generic': re.compile( + r'^pkg:generic/' # Scheme and type + r'([a-zA-Z0-9._-]+/)?' # Optional namespace segment + r'[a-zA-Z0-9._-]+' + REGEX_STR_PURL_OPTIONAL # Name (required) + ), + # GitHub PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/github-definition.md + 'github': re.compile( + r'^pkg:github/' # Scheme and type + # Namespace (organization/user), letters must be lowercase + r'[a-z0-9-]+' + r'/' + r'[a-z0-9._-]+' + REGEX_STR_PURL_OPTIONAL # Name (repository) + ), + # PyPI PURL. https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md + 'pypi': re.compile( + r'^pkg:pypi/' # Scheme and type + r'[a-z0-9_-]+' + REGEX_STR_PURL_OPTIONAL # Name, letters must be lowercase, dashes, underscore + ), +} + + +# Metadata SBOM requirements +METADATA_FIELDS_REQUIRED = [ + 'type', + 'bom-ref', + 'group', + 'name', + 'version', + 'description', + 'licenses', + 'copyright', + 'externalReferences', + 'scope', +] +METADATA_FIELDS_ONE_OF = [ + ['author', 'supplier'], + ['purl', 'cpe'], +] + +# endregion init + + +# region functions and classes + + +class GitInfo: + """Get, set, format git info""" + + def __init__(self): + print_banner('Gathering git info') + try: + self.repo_root = Path( + subprocess.run( + 'git rev-parse --show-toplevel', + shell=True, + text=True, + capture_output=True, + check=True, + ).stdout.strip() + ) + self._repo = Repo(self.repo_root) + except Exception as e: + logger.warning('Unable to read git repo information. All necessary script arguments must be provided.') + logger.warning(e) + self._repo = None + else: + try: + self.project = self._repo.remotes.origin.config_reader.get('url') + if not self.project.endswith('.git'): + self.project += '.git' + org_repo = extract_repo_from_git_url(self.project) + self.org = org_repo['org'] + self.repo = org_repo['repo'] + self.commit = self._repo.head.commit.hexsha + self.branch = self._repo.active_branch.name + + # filter tags for latest release e.g., r8.2.1 + release_tags = [] + filtered_tags = [tag for tag in self._repo.tags if re.fullmatch(config.REGEX_RELEASE_TAG, tag.name)] + logging.info(f'GIT: Parsing {len(filtered_tags)} release tags for match to commit') + for tag in filtered_tags: + if tag.commit == self.commit: + release_tags.append(tag.name) + if len(release_tags) > 0: + self.release_tag = release_tags[-1] + else: + self.release_tag = None + logging.debug(f'GitInfo->release_tag(): {self.release_tag}') + + logging.debug(f'GitInfo->__init__: {self}') + except Exception as e: + logger.warning('Unable to fully parse git info.') + logger.warning(e) + + def close(self): + """Closes the underlying Git repo object to release resources.""" + if self._repo: + logger.debug('Closing Git repo object.') + self._repo.close() + self._repo = None + + def added_new_3p_folder(self, commit: Commit) -> bool: + """ + Checks if a given commit added a new third-party subfolder. + + Args: + commit: The GitPython Commit object to analyze. + + Returns: + True if the commit added a new subfolder, False otherwise. + """ + if not commit.parents: + # If it's the initial commit, all folders are "new" + # You might want to refine this logic based on your definition of "new" + # Check if there are any subfolders in the initial commit + return bool(commit.tree.trees) + + parent_commit = commit.parents[0] + diff_index = commit.diff(parent_commit) + + for diff in diff_index: + # Check for added items that are directories + if diff.change_type == 'A' and diff.b_is_dir: + return True + return False + + +def print_banner(text: str) -> None: + """print() a padded status message to stdout""" + print() + print(text.center(len(text) + 2, ' ').center(120, '=')) + + +def extract_repo_from_git_url(git_url: str) -> dict: + """Determine org/repo for a given git url""" + git_org = git_url.split('/')[-2].replace('.git', '') + git_repo = git_url.split('/')[-1].replace('.git', '') + return { + 'org': git_org, + 'repo': git_repo, + } + + +def is_valid_purl(purl: str) -> bool: + """Validate a GitHub or Generic PURL""" + for purl_type, regex in REGEX_PURL.items(): + if regex.match(purl): + logger.debug(f"PURL: {purl} matched PURL type '{purl_type}' regex '{regex.pattern}'") + return True + return False + + +def sbom_components_to_dict(sbom: dict, with_version: bool = False) -> dict: + """Create a dict of SBOM components with a version-less PURL as the key""" + components = sbom['components'] + if with_version: + components_dict = {urllib.parse.unquote(component['bom-ref']): component for component in components} + else: + components_dict = { + urllib.parse.unquote(component['bom-ref']).split('@')[0]: component for component in components + } + return components_dict + + +def check_metadata_sbom(meta_bom: dict) -> None: + for component in meta_bom['components']: + for field in METADATA_FIELDS_REQUIRED: + if field not in component: + logger.warning( + f"METADATA: '{component['bom-ref'] or component['name']} is missing required field '{field}'." + ) + for fields in METADATA_FIELDS_ONE_OF: + found = False + for field in fields: + found = found or field in component + if not found: + logger.warning( + f"METADATA: '{component['bom-ref'] or component['name']} is missing one of fields '{fields}'." + ) + + +def read_sbom_json_file(file_path: str) -> dict: + """Load a JSON SBOM file (schema is not validated)""" + try: + with open(file_path, 'r', encoding='utf-8') as input_json: + sbom_json = input_json.read() + result = json.loads(sbom_json) + except Exception as e: + logger.error(f'Error loading SBOM file from {file_path}') + logger.error(e) + else: + logger.info(f'SBOM loaded from {file_path} with {len(result["components"])} components') + return result + + +def write_sbom_json_file(sbom_dict: dict, file_path: str) -> None: + """Save a JSON SBOM file (schema is not validated)""" + try: + file_path = os.path.abspath(file_path) + with open(file_path, 'w', encoding='utf-8') as output_json: + formatted_sbom = json.dumps(sbom_dict, indent=2) + '\n' + output_json.write(formatted_sbom) + except Exception as e: + logger.error(f'Error writing SBOM file to {file_path}') + logger.error(e) + else: + logger.info(f'SBOM file saved to {file_path}') + + +def write_list_to_text_file(str_list: list, file_path: str) -> None: + """Save a list of strings to a text file""" + try: + file_path = os.path.abspath(file_path) + with open(file_path, 'w', encoding='utf-8') as output_txt: + for item in str_list: + output_txt.write(f'{item}\n') + except Exception as e: + logger.error(f'Error writing text file to {file_path}') + logger.error(e) + else: + logger.info(f'Text file saved to {file_path}') + + +def set_component_version(component: dict, version: str, purl_version: str = None, cpe_version: str = None) -> None: + """Update the appropriate version fields in a component from the metadata SBOM""" + if not purl_version: + purl_version = version + + if not cpe_version: + cpe_version = version + + component['bom-ref'] = component['bom-ref'].replace('{{VERSION}}', purl_version) + component['version'] = component['version'].replace('{{VERSION}}', version) + if component.get('purl'): + component['purl'] = component['purl'].replace('{{VERSION}}', purl_version) + if not is_valid_purl(component['purl']): + logger.warning(f'PURL: Invalid PURL ({component["purl"]})') + if component.get('cpe'): + component['cpe'] = component['cpe'].replace('{{VERSION}}', cpe_version) + + +def set_dependency_version(dependencies: list, meta_bom_ref: str, purl_version: str) -> None: + """Update the appropriate dependency version fields in the metadata SBOM""" + r = 0 + d = 0 + for dependency in dependencies: + if '{{VERSION}}' in dependency['ref'] and dependency['ref'] == meta_bom_ref: + dependency['ref'] = dependency['ref'].replace('{{VERSION}}', purl_version) + r += 1 + for i in range(len(dependency['dependsOn'])): + if dependency['dependsOn'][i] == meta_bom_ref: + dependency['dependsOn'][i] = dependency['dependsOn'][i].replace('{{VERSION}}', purl_version) + d += 1 + + logger.debug(f"set_dependency_version: '{meta_bom_ref}' updated {r} refs and {d} dependsOn") + + +def get_subfolders_dict(folder_path: str = '.') -> dict: + """Get list of all directories in the specified path""" + subfolders = [] + try: + # Get all entries (files and directories) in the specified path + entries = os.listdir(folder_path) + + # Filter for directories + for entry in entries: + full_path = os.path.join(folder_path, entry) + if os.path.isdir(full_path): + subfolders.append(entry) + except FileNotFoundError: + logger.error(f"Error: Directory '{folder_path}' not found.") + except Exception as e: + logger.error(f'An error occurred: {e}') + + subfolders.sort() + return {key: 0 for key in subfolders} + + +# endregion functions and classes + + +def main() -> None: + # region define args + + parser = argparse.ArgumentParser( + description="""Generate a CycloneDX v1.5 JSON SBOM file using a combination of scan results from Endor Labs, pre-defined SBOM metadata, and the existing SBOM. + Requires endorctl to be installed and configured, which can be done using 'buildscripts/sbom/install_endorctl.sh'. + For use in CI, script may be run with no arguments.""", + epilog='Note: The git-related default values are dynamically generated.', + formatter_class=argparse.MetavarTypeHelpFormatter, + ) + + endor = parser.add_argument_group("Endor Labs API (via 'endorctl')") + endor.add_argument( + '--endorctl-path', + help="Path to endorctl, the Endor Labs CLI (Default: 'endorctl')", + default='endorctl', + type=str, + ) + endor.add_argument( + '--config-path', + help="Path to endor config directory containing config.yaml (Default: '$HOME/.endorctl')", + default=None, + type=str, + ) + endor.add_argument( + '--enable-github-action-token', + help='Enable keyless authentication using Github action OIDC tokens', + action='store_true', + ) + endor.add_argument('--namespace', help='Endor Labs namespace (Default: mongodb.{git org})', type=str) + endor.add_argument( + '--target', + help="Target for generated SBOM. Commit: results from running/completed PR scan, Branch: results from latest monitoring scan, Project: results from latest monitoring scan of the 'default' branch (default: commit)", + choices=['commit', 'branch', 'project'], + default='commit', + type=str, + ) + endor.add_argument( + '--project', + help='Full GitHub git URL [e.g., https://github.com/10gen/mongo.git] (Default: current git URL)', + type=str, + ) + + target = parser.add_argument_group("Target values. Apply only if --target is not 'project'") + exclusive_target = target.add_mutually_exclusive_group() + exclusive_target.add_argument( + '--commit', + help='PR commit SHA [40-character hex string] (Default: current git commit)', + type=str, + ) + exclusive_target.add_argument( + '--branch', + help='Git repo branch monitored by Endor Labs [e.g., v8.0] (Default: current git org/repo)', + type=str, + ) + + files = parser.add_argument_group('SBOM files') + files.add_argument( + '--sbom-metadata', + help="Input path for template SBOM file with metadata (Default: './buildscripts/sbom/metadata.cdx.json')", + default='./buildscripts/sbom/metadata.cdx.json', + type=str, + ) + files.add_argument( + '--sbom-in', + help="Input path for previous SBOM file (Default: './sbom.json')", + default='./sbom.json', + type=str, + ) + files.add_argument( + '--sbom-out', + help="Output path for SBOM file (Default: './sbom.json')", + default='./sbom.json', + type=str, + ) + parser.add_argument( + '--retry-limit', + help='Maximum number of times to retry when a target PR scan has not started (Default: 5)', + default=5, + type=int, + ) + parser.add_argument( + '--sleep-duration', + help='Number of seconds to wait between retries (Default: 30)', + default=30, + type=int, + ) + parser.add_argument( + '--save-warnings', + help='Save warning messages to a specified file (Default: None)', + default=None, + type=str, + ) + parser.add_argument('--debug', help='Set logging level to DEBUG', action='store_true') + + # endregion define args + + # region parse args + + args = parser.parse_args() + + git_info = GitInfo() + + # endor + endorctl_path = args.endorctl_path + config_path = args.config_path + enable_github_action_token = args.enable_github_action_token + namespace = args.namespace if args.namespace else f'mongodb.{git_info.org}' + target = args.target + + # project + if args.project and args.project != git_info.project: + if not re.fullmatch(REGEX_GITHUB_URL, args.project): + parser.error(f'Invalid Git URL: {args.project}.') + git_info.project = args.project + git_info.org, git_info.repo = map(extract_repo_from_git_url(git_info.project).get, ('org', 'repo')) + git_info.release_tag = None + + # targets + # commit + if args.commit and args.commit != git_info.commit: + if not re.fullmatch(REGEX_COMMIT_SHA, args.commit): + parser.error(f'Invalid Git commit SHA: {args.commit}. Must be a 40-character hexadecimal string (SHA-1).') + git_info.commit = args.commit + + # branch + if args.branch and args.branch != git_info.branch: + if len(args.branch.encode('utf-8')) > 244 or not re.fullmatch(REGEX_GIT_BRANCH, args.branch): + parser.error( + f'Invalid Git branch name: {args.branch}. Limit is 244 bytes with allowed characters: [a-zA-Z0-9_.-/]' + ) + git_info.branch = args.branch + + # files + sbom_out_path = args.sbom_out + sbom_in_path = args.sbom_in + sbom_metadata_path = args.sbom_metadata + save_warnings = args.save_warnings + + # environment + retry_limit = args.retry_limit + sleep_duration = args.sleep_duration + + if args.debug: + logger.setLevel(logging.DEBUG) + + # endregion parse args + + # region export Endor Labs SBOM + + print_banner(f'Exporting Endor Labs SBOM for {target} {getattr(git_info, target)}') + endorctl = EndorCtl( + namespace, + retry_limit, + sleep_duration, + endorctl_path, + config_path, + enable_github_action_token=enable_github_action_token, + ) + if target == 'commit': + endor_bom = endorctl.get_sbom_for_commit(git_info.project, git_info.commit) + elif target == 'branch': + endor_bom = endorctl.get_sbom_for_branch(git_info.project, git_info.branch) + elif target == 'project': + endor_bom = endorctl.get_sbom_for_project(git_info.project) + else: + endor_bom = None + + if not endor_bom: + logger.error('Empty result for Endor SBOM!') + if target == 'commit': + logger.error('Check Endor Labs for any unanticipated issues with the target PR scan.') + else: + logger.error('Check Endor Labs for status of the target monitoring scan.') + sys.exit(1) + + logger.info(f'Endor Labs SBOM exported with {len(endor_bom["components"])} components') + # endregion export Endor Labs SBOM + + # region Pre-process Endor Labs SBOM + + print_banner('Pre-Processing Endor Labs SBOM') + + ## remove uneeded components ## + # [list]endor_components_remove is defined in config.py + # Endor Labs includes the main component in 'components'. This is not standard, so we remove it. + config.endor_components_remove.append(f'pkg:github/{git_info.org}/{git_info.repo}') + + # Reverse iterate the SBOM components list to safely modify in situ + for i in range(len(endor_bom['components']) - 1, -1, -1): + component = endor_bom['components'][i] + removed = False + for remove in config.endor_components_remove: + if component['bom-ref'].startswith(remove): + logger.info('ENDOR SBOM PRE-PROCESS: removing ' + component['bom-ref']) + del endor_bom['components'][i] + removed = True + break + if not removed: + for rename in config.endor_components_rename: + old = rename[0] + new = rename[1] + component['bom-ref'] = component['bom-ref'].replace(old, new) + component['purl'] = component['purl'].replace(old, new) + + logger.info(f'Endor Labs SBOM pre-processed with {len(endor_bom["components"])} components') + + # endregion Pre-process Endor Labs SBOM + + # region load metadata and previous SBOMs + + print_banner('Loading metadata SBOM and previous SBOM') + + meta_bom = read_sbom_json_file(sbom_metadata_path) + if not meta_bom: + logger.error('No SBOM metadata. This is fatal.') + sys.exit(1) + + prev_bom = read_sbom_json_file(sbom_in_path) + if not prev_bom: + logger.warning( + 'Unable to load previous SBOM data. The new SBOM will be generated without any previous context. This is unexpected, but not fatal.' + ) + # Create empty prev_bom to avoid downstream processing errors + prev_bom = { + 'bom-ref': None, + 'metadata': { + 'timestamp': endor_bom['metadata']['timestamp'], + 'component': { + 'version': None, + }, + }, + 'components': [], + } + else: + if 'metadata' not in prev_bom: + prev_bom['metadata'] = { + 'timestamp': endor_bom['metadata']['timestamp'], + 'component': { + 'version': None, + }, + } + else: + if 'timestamp' not in prev_bom['metadata']: + prev_bom['metadata']['timestamp'] = endor_bom['metadata']['timestamp'] + if 'component' not in prev_bom['metadata']: + prev_bom['metadata']['component'] = { + 'version': None, + } + + # endregion load metadata and previous SBOMs + + # region Build composite SBOM + # Note: No exception handling here. The most likely reason for an exception is missing data elements + # in SBOM files, which is fatal if it happens. Code is in place to handle the situation + # where there is no previous SBOM to include, but we want to fail if required data is absent. + print_banner('Building composite SBOM (metadata + endor + previous)') + + # Sort components by bom-ref + endor_bom['components'].sort(key=lambda c: c['bom-ref']) + meta_bom['components'].sort(key=lambda c: c['bom-ref']) + prev_bom['components'].sort(key=lambda c: c['bom-ref']) + + # Check metadata SBOM for completeness + check_metadata_sbom(meta_bom) + + # Create SBOM component lookup dicts + endor_components = sbom_components_to_dict(endor_bom) + prev_components = sbom_components_to_dict(prev_bom) + + # region primary component + + # Attempt to determine the primary component version being scanned + primary_component_version = config.get_primary_component_version() + + logger.debug( + f'Available main component version options, repo script: {primary_component_version}, tag: {git_info.release_tag}, branch: {git_info.branch}, previous SBOM: {prev_bom["metadata"]["component"]["version"]}' + ) + meta_bom_ref = meta_bom['metadata']['component']['bom-ref'] + + if primary_component_version: + version = primary_component_version + purl_version = 'r' + primary_component_version + cpe_version = primary_component_version + logger.info( + f"PRIMARY COMPONENT VERSION: Using repo script output '{primary_component_version}' as primary component version" + ) + + # Project scan always set to 'master' or if using 'master' branch + if target == 'project' or git_info.branch == 'master': + version = 'master' + purl_version = 'master' + cpe_version = 'master' + logger.info("PRIMARY COMPONENT VERSION: Using branch 'master' as primary component version") + + # tagged release. e.g., r8.1.0, r8.2.1-rc0 + elif git_info.release_tag: + version = git_info.release_tag[1:] # remove leading 'r' + purl_version = git_info.release_tag + cpe_version = version # without leading 'r' + logger.info( + f"PRIMARY COMPONENT VERSION: Using release_tag '{git_info.release_tag}' as primary component version" + ) + + # Release branch e.g., v7.0 or v8.2 + elif target == 'branch' and re.fullmatch(config.REGEX_RELEASE_BRANCH, git_info.branch): + version = git_info.branch + purl_version = git_info.branch + # remove leading 'v', add wildcard. e.g. 8.2.* + cpe_version = version.replace('releases/', '')[1:] + '.*' + logger.info(f"PRIMARY COMPONENT VERSION: Using release branch '{git_info.branch}' as primary component version") + + # Previous SBOM app version, if all needed specifiers exist + elif ( + prev_bom.get('metadata', {}).get('component', {}).get('version') + and prev_bom.get('metadata', {}).get('component', {}).get('purl') + and prev_bom.get('metadata', {}).get('component', {}).get('cpe') + ): + version = prev_bom['metadata']['component']['version'] + purl_version = prev_bom['metadata']['component']['purl'].split('@')[-1] + cpe_version = prev_bom['metadata']['component']['cpe'].split(':')[5] + logger.info(f"PRIMARY COMPONENT VERSION: Using previous SBOM version '{version}' as primary component version") + + else: + # Fall back to the version specified in the Endor SBOM + # This is unlikely to be accurate + version = endor_bom['metadata']['component']['version'] + purl_version = version + cpe_version = version + logger.warning( + f"PRIMARY COMPONENT VERSION: Using SBOM version '{version}' from Endor Labs scan. This is unlikely to be accurate and may specify a PR #." + ) + + # Set primary component version + set_component_version(meta_bom['metadata']['component'], version, purl_version, cpe_version) + # Run through 'dependency' objects to set main component version + set_dependency_version(meta_bom['dependencies'], meta_bom_ref, purl_version) + + # endregion primary component + + # region SBOM components + + # region Parse metadata SBOM components + + for component in meta_bom['components']: + versions = { + 'endor': None, + 'metadata': None, + } + + component_key = component['bom-ref'].split('@')[0] + + print_banner('Component: ' + component_key) + + ################ Endor Labs ################ + if component_key in endor_components: + # Pop component from dict so we are left with only unmatched components + endor_component = endor_components.pop(component_key) + versions['endor'] = endor_component.get('version') + logger.debug(f"VERSION ENDOR: {component_key}: Found version '{versions['endor']}' in Endor Labs results") + + ############## Metadata ############### + # Hard-coded metadata version, if exists + if '{{VERSION}}' not in component['version']: + versions['metadata'] = component.get('version') + + logger.info(f'VERSIONS: {component_key}: ' + str(versions)) + + ############## Component Special Cases ############### + config.process_component_special_cases(component_key, component, versions, git_info.repo_root.as_posix()) + + # For the standard workflow, we favor the Endor Labs version followed by hard coded + version = versions['endor'] or versions['metadata'] + + ############## Assign Version ############### + if version: + meta_bom_ref = component['bom-ref'] + + ## Special case for FireFox ## + # The CPE for FireFox ESR needs the 'esr' removed from the version, as it is specified in another section + if component['bom-ref'].startswith('pkg:deb/debian/firefox-esr@'): + set_component_version(component, version, cpe_version=version.replace('esr', '')) + else: + semver = config.get_semver_from_release_version(version) + set_component_version(component, semver, version, semver) + + set_dependency_version(meta_bom['dependencies'], meta_bom_ref, version) + else: + logger.warning( + f'VERSION NOT FOUND: Could not find a version for {component_key}! Removing from SBOM. Component may need to be removed from the {sbom_metadata_path} file.' + ) + del component + + # explicit cleanup to avoid gc race condition on script temination + git_info.close() + del git_info + + # endregion Parse metadata SBOM components + + # region Parse unmatched Endor Labs components + + print_banner('New Endor Labs components') + if endor_components: + logger.warning( + f'ENDOR SBOM: There are {len(endor_components)} unmatched components in the Endor Labs SBOM. Adding as-is. The applicable metadata should be added to the metadata SBOM ({sbom_metadata_path}) for the next run.' + ) + for component in endor_components: + # set scope to excluded by default until the component is evaluated + endor_components[component]['scope'] = 'excluded' + meta_bom['components'].append(endor_components[component]) + meta_bom['dependencies'].append({'ref': endor_components[component]['bom-ref'], 'dependsOn': []}) + logger.warning(f'SBOM AS-IS COMPONENT: Added {component}') + + # endregion Parse unmatched Endor Labs components + + # region Finalize SBOM + + # Have the SBOM app version changed? + sbom_app_version_changed = ( + prev_bom['metadata'].get('component', {}).get('version') != meta_bom['metadata']['component']['version'] + ) + logger.info(f'SUMMARY: Primary component version changed: {sbom_app_version_changed}') + + # Have the components changed? + prev_components = sbom_components_to_dict(prev_bom, with_version=True) + meta_components = sbom_components_to_dict(meta_bom, with_version=True) + sbom_components_changed = prev_components.keys() != meta_components.keys() + logger.info( + f'SBOM_DIFF: SBOM components changed (added, removed, or version): {sbom_components_changed}. Previous SBOM has {len(prev_components)} components; New SBOM has {len(meta_components)} components' + ) + + # Components in prev SBOM but not in generated SBOM + prev_components = sbom_components_to_dict(prev_bom, with_version=False) + meta_components = sbom_components_to_dict(meta_bom, with_version=False) + prev_components_diff = list(set(prev_components.keys()) - set(meta_components.keys())) + if prev_components_diff: + logger.info( + 'SBOM_DIFF: Components in previous SBOM and not in generated SBOM: ' + ','.join(prev_components_diff) + ) + + # Components in generated SBOM but not in prev SBOM + meta_components_diff = list(set(meta_components.keys()) - set(prev_components.keys())) + if meta_components_diff: + logger.info( + 'SBOM_DIFF: Components in generated SBOM and not in previous SBOM: ' + ','.join(meta_components_diff) + ) + + # serialNumber https://cyclonedx.org/docs/1.5/json/#serialNumber + # version (SBOM version) https://cyclonedx.org/docs/1.5/json/#version + if sbom_app_version_changed: + # New primary component version requires a unique serial number and version 1 + meta_bom['serialNumber'] = uuid.uuid4().urn + meta_bom['version'] = 1 + else: + # Primary component version is the same, so reuse the serial number and SBOM version + meta_bom['serialNumber'] = prev_bom['serialNumber'] + meta_bom['version'] = prev_bom['version'] + # If the components have changed, bump the SBOM version + if sbom_components_changed: + meta_bom['version'] += 1 + + # metadata.timestamp https://cyclonedx.org/docs/1.5/json/#metadata_timestamp + # Only update the timestamp if something has changed + if sbom_app_version_changed or sbom_components_changed: + meta_bom['metadata']['timestamp'] = ( + datetime.now(timezone.utc).isoformat(timespec='seconds').replace('+00:00', 'Z') + ) + else: + meta_bom['metadata']['timestamp'] = prev_bom['metadata']['timestamp'] + + # metadata.tools https://cyclonedx.org/docs/1.5/json/#metadata_tools + meta_bom['metadata']['tools'] = endor_bom['metadata']['tools'] + + write_sbom_json_file(meta_bom, sbom_out_path) + + # Access the collected warnings + print_banner('CONSOLIDATED WARNINGS') + warnings = ['The following warnings were output when generating the SBOM:\n'] + + if len(warning_handler.warnings): + for record in warning_handler.warnings: + warnings.append(' - ' + record.getMessage()) + else: + warnings.append(' - None') + + print('\n'.join(warnings)) + + if save_warnings: + write_list_to_text_file(warnings, save_warnings) + + print_banner('COMPLETED') + if not os.getenv('CI') and not os.getenv('GITHUB_ACTIONS'): + print('Be sure to add the SBOM to your next commit if the file content has changed.') + + # endregion Finalize SBOM + + # endregion Build composite SBOM + + +if __name__ == '__main__': + main() diff --git a/etc/sbom/metadata.cdx.json b/etc/sbom/metadata.cdx.json new file mode 100644 index 0000000000..535b78813c --- /dev/null +++ b/etc/sbom/metadata.cdx.json @@ -0,0 +1,307 @@ +{ + "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "serialNumber": "urn:uuid:89685c6d-f505-4943-a685-0fc532b179f8", + "version": 1, + "metadata": { + "timestamp": "2025-11-22T20:32:17Z", + "lifecycles": [ + { + "phase": "pre-build" + } + ], + "component": { + "type": "application", + "bom-ref": "pkg:github/mongodb/mongo-cxx-driver@{{VERSION}}", + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "author": "MongoDB, Inc.", + "publisher": "MongoDB, Inc.", + "group": "mongodb", + "name": "mongo-cxx-driver", + "description": "C++ Driver for MongoDB", + "version": "{{VERSION}}", + "cpe": "cpe:2.3:a:mongodb:c++:{{VERSION}}:*:*:*:*:*:*:*", + "purl": "pkg:github/mongodb/mongo-cxx-driver@{{VERSION}}", + "externalReferences": [ + { + "type": "license", + "url": "https://raw.githubusercontent.com/mongodb/mongo-cxx-driver/refs/heads/master/LICENSE", + "comment": "Apache License 2.0" + }, + { + "type": "website", + "url": "https://www.mongodb.com/docs/languages/cpp/cpp-driver/", + "comment": "Documentation site for the official MongoDB C++ Driver." + }, + { + "type": "release-notes", + "url": "https://www.mongodb.com/docs/languages/cpp/cpp-driver/current/whats-new/" + }, + { + "type": "vcs", + "url": "https://github.com/mongodb/mongo-cxx-driver" + } + ] + }, + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + } + }, + "components": [ + { + "type": "library", + "bom-ref": "pkg:github/madler/zlib@{{VERSION}}", + "supplier": { + "name": "zlib", + "url": [ + "https://zlib.net/" + ] + }, + "author": "Jean-loup Gailly, Mark Adler", + "group": "madler", + "name": "zlib", + "version": "{{VERSION}}", + "description": "zlib is a general purpose data compression library.", + "licenses": [ + { + "license": { + "id": "Zlib" + } + } + ], + "copyright": "Copyright \u00a9 1995-2024 Jean-loup Gailly and Mark Adler.", + "cpe": "cpe:2.3:a:zlib:zlib:{{VERSION}}:*:*:*:*:*:*:*", + "purl": "pkg:github/madler/zlib@{{VERSION}}", + "externalReferences": [ + { + "url": "https://zlib.net/fossils/", + "type": "distribution" + } + ], + "scope": "required" + }, + { + "bom-ref": "pkg:github/catchorg/catch2@{{VERSION}}", + "type": "library", + "author": "Martin Hořeňovský", + "group": "catchorg", + "name": "Catch2", + "version": "{{VERSION}}", + "description": "A modern, C++-native, test framework for unit-tests, TDD and BDD.", + "licenses": [ + { + "license": { + "id": "BSL-1.0" + } + } + ], + "copyright": "Copyright Catch2 Authors", + "purl": "pkg:github/catchorg/catch2@{{VERSION}}", + "externalReferences": [ + { + "url": "https://github.com/catchorg/catch2.git", + "type": "distribution" + } + ], + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/juliastrings/utf8proc@{{VERSION}}", + "type": "library", + "author": "Jan Behrens, the Public Software Group, the Julia-language developers", + "supplier": { + "name": "The Julia Programming Language", + "url": [ + "https://julialang.org/" + ] + }, + "group": "JuliaStrings", + "name": "utf8proc", + "version": "{{VERSION}}", + "description": "A clean C library for processing UTF-8 Unicode data", + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "copyright": "Copyright © 2014-2021 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.", + "purl": "pkg:github/juliastrings/utf8proc@{{VERSION}}", + "externalReferences": [ + { + "url": "https://github.com/juliastrings/utf8proc.git", + "type": "distribution" + } + ], + "scope": "required" + }, + { + "type": "library", + "bom-ref": "pkg:github/mongodb/libmongocrypt@{{VERSION}}", + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "author": "MongoDB, Inc.", + "group": "mongodb", + "name": "libmongocrypt", + "version": "{{VERSION}}", + "description": "C library for Client Side and Queryable Encryption in MongoDB", + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ], + "copyright": "Copyright 2019-present MongoDB, Inc.", + "cpe": "cpe:2.3:a:mongodb:libmongocrypt:{{VERSION}}:*:*:*:*:*:*:*", + "purl": "pkg:github/mongodb/libmongocrypt@{{VERSION}}", + "externalReferences": [ + { + "url": "https://github.com/mongodb/libmongocrypt.git", + "type": "distribution" + } + ], + "scope": "optional" + }, + { + "type": "library", + "bom-ref": "pkg:github/mongodb/mongo-c-driver@{{VERSION}}", + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "author": "MongoDB, Inc.", + "group": "mongodb", + "name": "MongoDB C Driver", + "version": "{{VERSION}}", + "description": "The Official MongoDB driver for C language", + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ], + "copyright": "2009-present, MongoDB, Inc.", + "cpe": "cpe:2.3:a:mongodb:c_driver:{{VERSION}}:*:*:*:*:*:*:*", + "purl": "pkg:github/mongodb/mongo-c-driver@{{VERSION}}", + "externalReferences": [ + { + "url": "https://github.com/mongodb/mongo-c-driver.git", + "type": "distribution" + } + ], + "scope": "required" + }, + { + "bom-ref": "pkg:github/philsquared/clara@{{VERSION}}", + "type": "library", + "author": "Phil Nash", + "group": "catchorg", + "name": "Clara", + "version": "{{VERSION}}", + "description": "A simple to use, composable, command line parser for C++ 11 and beyond", + "licenses": [ + { + "license": { + "id": "BSL-1.0" + } + } + ], + "copyright": "Copyright 2017 Two Blue Cubes Ltd. All rights reserved.", + "purl": "pkg:github/philsquared/clara@{{VERSION}}", + "externalReferences": [ + { + "url": "https://github.com/philsquared/clara.git", + "type": "distribution" + } + ], + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/troydhanson/uthash@{{VERSION}}", + "type": "library", + "author": "Troy D. Hanson", + "group": "troydhanson", + "name": "github.com/troydhanson/uthash", + "version": "{{VERSION}}", + "description": "C macros for hash tables and more", + "licenses": [ + { + "license": { + "id": "BSD-1-Clause" + } + } + ], + "copyright": "Copyright (c) 2005-2025, Troy D. Hanson", + "purl": "pkg:github/troydhanson/uthash@{{VERSION}}", + "externalReferences": [ + { + "url": "https://github.com/troydhanson/uthash.git", + "type": "distribution" + } + ], + "scope": "required" + } + ], + "dependencies": [ + { + "ref": "pkg:github/mongodb/mongo-cxx-driver@{{VERSION}}", + "dependsOn": [ + "pkg:github/catchorg/catch2@{{VERSION}}", + "pkg:github/mongodb/mongo-c-driver@{{VERSION}}" + ] + }, + { + "ref": "pkg:github/catchorg/catch2@{{VERSION}}", + "dependsOn": [ + "pkg:github/philsquared/clara@{{VERSION}}" + ] + }, + { + "ref": "pkg:github/mongodb/mongo-c-driver@{{VERSION}}", + "dependsOn": [ + "pkg:github/madler/zlib@{{VERSION}}", + "pkg:github/juliastrings/utf8proc@{{VERSION}}", + "pkg:github/mongodb/libmongocrypt@{{VERSION}}", + "pkg:github/troydhanson/uthash@{{VERSION}}" + ] + }, + { + "ref": "pkg:github/philsquared/clara@{{VERSION}}", + "dependsOn": [] + }, + { + "ref": "pkg:github/juliastrings/utf8proc@{{VERSION}}", + "dependsOn": [] + }, + { + "ref": "pkg:github/madler/zlib@{{VERSION}}", + "dependsOn": [] + }, + { + "ref": "pkg:github/mongodb/libmongocrypt@{{VERSION}}", + "dependsOn": [] + }, + { + "ref": "pkg:github/troydhanson/uthash@{{VERSION}}", + "dependsOn": [] + } + ] +} \ No newline at end of file diff --git a/etc/third_party_vulnerabilities.md b/etc/third_party_vulnerabilities.md index 17be84f7e4..8edda37d28 100644 --- a/etc/third_party_vulnerabilities.md +++ b/etc/third_party_vulnerabilities.md @@ -17,7 +17,7 @@ This section provides a template that may be used for actual vulnerability repor - **Date Detected:** YYYY-MM-DD - **Severity:** Low, Medium, High, or Critical -- **Detector:** Silk or Snyk +- **Detector:** Endor Labs or Dependency-Track - **Description:** A short vulnerability description. - **Dependency:** Name and version of the 3rd party dependency. - **Upstream Status:** False Positive, Won't Fix, Fix Pending, or Fix Available. This is the fix status for the 3rd party dependency, not the CXX Driver. "Fix Available" should include the version and/or date when the fix was released, e.g. "Fix Available (1.2.3, 1970-01-01)". diff --git a/pyproject.toml b/pyproject.toml index c239b1b24b..1e00caf00b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,13 @@ make_release = [ "pygithub>=2.1", ] +generate_sbom = [ + # etc/sbom/*.py + "gitpython>=3.1", + "pygithub>=2.1", + "semver>=3.0.0", +] + [tool.ruff] line-length = 120 src = [".evergreen", "etc"] diff --git a/sbom.json b/sbom.json new file mode 100644 index 0000000000..f6ebb1d8d7 --- /dev/null +++ b/sbom.json @@ -0,0 +1,402 @@ +{ + "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "serialNumber": "urn:uuid:14f3bbeb-851e-43f7-8e5c-26e087cc0970", + "version": 1, + "metadata": { + "timestamp": "2025-11-26T20:24:04Z", + "lifecycles": [ + { + "phase": "pre-build" + } + ], + "component": { + "type": "application", + "bom-ref": "pkg:github/mongodb/mongo-cxx-driver@sbom_github_action", + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "author": "MongoDB, Inc.", + "publisher": "MongoDB, Inc.", + "group": "mongodb", + "name": "mongo-cxx-driver", + "description": "C++ Driver for MongoDB", + "version": "sbom_github_action", + "cpe": "cpe:2.3:a:mongodb:c++:sbom_github_action:*:*:*:*:*:*:*", + "purl": "pkg:github/mongodb/mongo-cxx-driver@sbom_github_action", + "externalReferences": [ + { + "type": "license", + "url": "https://raw.githubusercontent.com/mongodb/mongo-cxx-driver/refs/heads/master/LICENSE", + "comment": "Apache License 2.0" + }, + { + "type": "website", + "url": "https://www.mongodb.com/docs/languages/cpp/cpp-driver/", + "comment": "Documentation site for the official MongoDB C++ Driver." + }, + { + "type": "release-notes", + "url": "https://www.mongodb.com/docs/languages/cpp/cpp-driver/current/whats-new/" + }, + { + "type": "vcs", + "url": "https://github.com/mongodb/mongo-cxx-driver" + } + ] + }, + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "tools": { + "services": [ + { + "name": "Endor Labs Inc", + "version": "v1.7.671" + } + ] + } + }, + "components": [ + { + "bom-ref": "pkg:github/catchorg/catch2@v3.8.1", + "type": "library", + "author": "Martin Ho\u0159e\u0148ovsk\u00fd", + "group": "catchorg", + "name": "Catch2", + "version": "3.8.1", + "description": "A modern, C++-native, test framework for unit-tests, TDD and BDD.", + "licenses": [ + { + "license": { + "id": "BSL-1.0" + } + } + ], + "copyright": "Copyright Catch2 Authors", + "purl": "pkg:github/catchorg/catch2@v3.8.1", + "externalReferences": [ + { + "url": "https://github.com/catchorg/catch2.git", + "type": "distribution" + } + ], + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/catchorg/clara@v1.1.5", + "type": "library", + "author": "Phil Nash", + "group": "catchorg", + "name": "Clara", + "version": "1.1.5", + "description": "A simple to use, composable, command line parser for C++ 11 and beyond", + "licenses": [ + { + "license": { + "id": "BSL-1.0" + } + } + ], + "copyright": "Copyright 2017 Two Blue Cubes Ltd. All rights reserved.", + "purl": "pkg:github/catchorg/clara@v1.1.5", + "externalReferences": [ + { + "url": "https://github.com/catchorg/clara.git", + "type": "distribution" + } + ], + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/juliastrings/utf8proc@v2.8.0", + "type": "library", + "author": "Jan Behrens, the Public Software Group, the Julia-language developers", + "supplier": { + "name": "The Julia Programming Language", + "url": [ + "https://julialang.org/" + ] + }, + "group": "JuliaStrings", + "name": "utf8proc", + "version": "2.8.0", + "description": "A clean C library for processing UTF-8 Unicode data", + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "copyright": "Copyright \u00a9 2014-2021 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.", + "purl": "pkg:github/juliastrings/utf8proc@v2.8.0", + "externalReferences": [ + { + "url": "https://github.com/juliastrings/utf8proc.git", + "type": "distribution" + } + ], + "scope": "required" + }, + { + "type": "library", + "bom-ref": "pkg:github/madler/zlib@1.3.1", + "supplier": { + "name": "zlib", + "url": [ + "https://zlib.net/" + ] + }, + "author": "Jean-loup Gailly, Mark Adler", + "group": "madler", + "name": "zlib", + "version": "1.3.1", + "description": "zlib is a general purpose data compression library.", + "licenses": [ + { + "license": { + "id": "Zlib" + } + } + ], + "copyright": "Copyright \u00a9 1995-2024 Jean-loup Gailly and Mark Adler.", + "cpe": "cpe:2.3:a:zlib:zlib:1.3.1:*:*:*:*:*:*:*", + "purl": "pkg:github/madler/zlib@1.3.1", + "externalReferences": [ + { + "url": "https://zlib.net/fossils/", + "type": "distribution" + } + ], + "scope": "required" + }, + { + "type": "library", + "bom-ref": "pkg:github/mongodb/libmongocrypt@1.16.0", + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "author": "MongoDB, Inc.", + "group": "mongodb", + "name": "libmongocrypt", + "version": "1.16.0", + "description": "C library for Client Side and Queryable Encryption in MongoDB", + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ], + "copyright": "Copyright 2019-present MongoDB, Inc.", + "cpe": "cpe:2.3:a:mongodb:libmongocrypt:1.16.0:*:*:*:*:*:*:*", + "purl": "pkg:github/mongodb/libmongocrypt@1.16.0", + "externalReferences": [ + { + "url": "https://github.com/mongodb/libmongocrypt.git", + "type": "distribution" + } + ], + "scope": "optional" + }, + { + "type": "library", + "bom-ref": "pkg:github/mongodb/mongo-c-driver@2.1.2", + "supplier": { + "name": "MongoDB, Inc.", + "url": [ + "https://mongodb.com" + ] + }, + "author": "MongoDB, Inc.", + "group": "mongodb", + "name": "MongoDB C Driver", + "version": "2.1.2", + "description": "The Official MongoDB driver for C language", + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ], + "copyright": "2009-present, MongoDB, Inc.", + "cpe": "cpe:2.3:a:mongodb:c_driver:2.1.2:*:*:*:*:*:*:*", + "purl": "pkg:github/mongodb/mongo-c-driver@2.1.2", + "externalReferences": [ + { + "url": "https://github.com/mongodb/mongo-c-driver.git", + "type": "distribution" + } + ], + "scope": "required" + }, + { + "bom-ref": "pkg:github/troydhanson/uthash@v2.3.0", + "type": "library", + "author": "Troy D. Hanson", + "group": "troydhanson", + "name": "github.com/troydhanson/uthash", + "version": "2.3.0", + "description": "C macros for hash tables and more", + "licenses": [ + { + "license": { + "id": "BSD-1-Clause" + } + } + ], + "copyright": "Copyright (c) 2005-2025, Troy D. Hanson", + "purl": "pkg:github/troydhanson/uthash@v2.3.0", + "externalReferences": [ + { + "url": "https://github.com/troydhanson/uthash.git", + "type": "distribution" + } + ], + "scope": "required" + }, + { + "bom-ref": "pkg:github/jasonhills-mongodb/mongo-cxx-driver@sbom_github_action?package-id=0999c3478a09a2e2", + "name": "github.com/jasonhills-mongodb/mongo-cxx-driver", + "purl": "pkg:github/jasonhills-mongodb/mongo-cxx-driver@sbom_github_action", + "type": "library", + "version": "sbom_github_action", + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/mongodb/mongo-cxx-driver@r4.1.1?package-id=c2042e84ed2ed5ca", + "externalReferences": [ + { + "type": "distribution", + "url": "https://github.com/mongodb/mongo-cxx-driver.git" + } + ], + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ], + "name": "github.com/mongodb/mongo-cxx-driver", + "purl": "pkg:github/mongodb/mongo-cxx-driver@r4.1.1", + "type": "library", + "version": "r4.1.1", + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/mozilla/cubeb@471c7e209cf152414c0498f0c4015f9ff9b2f1c0?package-id=dd972228bb636626", + "externalReferences": [ + { + "type": "distribution", + "url": "https://github.com/mozilla/cubeb.git" + } + ], + "licenses": [ + { + "license": { + "id": "ISC" + } + } + ], + "name": "github.com/mozilla/cubeb", + "purl": "pkg:github/mozilla/cubeb@471c7e209cf152414c0498f0c4015f9ff9b2f1c0", + "type": "library", + "version": "471c7e209cf152414c0498f0c4015f9ff9b2f1c0", + "scope": "excluded" + }, + { + "bom-ref": "pkg:github/zlib-ng/zlib-ng@343c4c549107d31f6eeabfb4b31bec4502a2ea0e?package-id=8369b3e53cb7a837", + "externalReferences": [ + { + "type": "distribution", + "url": "https://github.com/zlib-ng/zlib-ng.git" + } + ], + "licenses": [ + { + "license": { + "id": "Zlib" + } + } + ], + "name": "github.com/zlib-ng/zlib-ng", + "purl": "pkg:github/zlib-ng/zlib-ng@343c4c549107d31f6eeabfb4b31bec4502a2ea0e", + "type": "library", + "version": "343c4c549107d31f6eeabfb4b31bec4502a2ea0e", + "scope": "excluded" + } + ], + "dependencies": [ + { + "ref": "pkg:github/mongodb/mongo-cxx-driver@sbom_github_action", + "dependsOn": [ + "pkg:github/catchorg/catch2@v3.8.1", + "pkg:github/mongodb/mongo-c-driver@2.1.2" + ] + }, + { + "ref": "pkg:github/catchorg/catch2@v3.8.1", + "dependsOn": [ + "pkg:github/catchorg/clara@v1.1.5" + ] + }, + { + "ref": "pkg:github/mongodb/mongo-c-driver@2.1.2", + "dependsOn": [ + "pkg:github/madler/zlib@1.3.1", + "pkg:github/juliastrings/utf8proc@v2.8.0", + "pkg:github/mongodb/libmongocrypt@1.16.0", + "pkg:github/troydhanson/uthash@v2.3.0" + ] + }, + { + "ref": "pkg:github/catchorg/clara@v1.1.5", + "dependsOn": [] + }, + { + "ref": "pkg:github/juliastrings/utf8proc@v2.8.0", + "dependsOn": [] + }, + { + "ref": "pkg:github/madler/zlib@1.3.1", + "dependsOn": [] + }, + { + "ref": "pkg:github/mongodb/libmongocrypt@1.16.0", + "dependsOn": [] + }, + { + "ref": "pkg:github/troydhanson/uthash@v2.3.0", + "dependsOn": [] + }, + { + "ref": "pkg:github/jasonhills-mongodb/mongo-cxx-driver@sbom_github_action?package-id=0999c3478a09a2e2", + "dependsOn": [] + }, + { + "ref": "pkg:github/mongodb/mongo-cxx-driver@r4.1.1?package-id=c2042e84ed2ed5ca", + "dependsOn": [] + }, + { + "ref": "pkg:github/mozilla/cubeb@471c7e209cf152414c0498f0c4015f9ff9b2f1c0?package-id=dd972228bb636626", + "dependsOn": [] + }, + { + "ref": "pkg:github/zlib-ng/zlib-ng@343c4c549107d31f6eeabfb4b31bec4502a2ea0e?package-id=8369b3e53cb7a837", + "dependsOn": [] + } + ] +}