diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index adf8172d9..7bddaa315 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -515,16 +515,13 @@ functions: - command: subprocess.exec params: working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: scripts/dev/run_python.sh scripts/release/kubectl_mongodb/python/build_kubectl_plugin.py - - build_and_push_appdb_database: + binary: scripts/release/kubectl_mongodb/build_kubectl_plugin.sh + + download_multi_cluster_binary: - command: subprocess.exec params: - working_dir: src/github.com/mongodb/mongodb-kubernetes/docker/mongodb-kubernetes-appdb-database - binary: ./build_and_push_appdb_database_images.sh - add_to_path: - - ${workdir}/bin - - ${workdir} + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/release/kubectl_mongodb/download_kubectl_plugin.sh build_test_image_ibm: - command: subprocess.exec @@ -801,22 +798,6 @@ functions: script: | ./scripts/code_snippets/tests/${task_name} - # - # kubectl mongodb plugin release functions - # - install_goreleaser: - - command: shell.exec - type: setup - include_expansions_in_env: - - goreleaser_pro_tar_gz - params: - script: | - set -Eeu pipefail - curl -fL "${goreleaser_pro_tar_gz}" --output goreleaser_Linux_x86_64.tar.gz - tar -xf goreleaser_Linux_x86_64.tar.gz - chmod 755 ./goreleaser - sudo cp goreleaser /usr/local/bin/ - install_macos_notarization_service: - command: shell.exec type: setup @@ -851,7 +832,7 @@ functions: - command: github.generate_token params: expansion_name: GH_TOKEN - - command: shell.exec + - command: subprocess.exec type: setup params: working_dir: src/github.com/mongodb/mongodb-kubernetes @@ -868,10 +849,7 @@ functions: - triggered_by_git_tag - OPERATOR_VERSION env: - XDG_CONFIG_HOME: ${go_base_path}${workdir} - GO111MODULE: "on" - GOROOT: "/opt/golang/go1.24" MACOS_NOTARY_KEY: ${macos_notary_keyid} MACOS_NOTARY_SECRET: ${macos_notary_secret} GH_TOKEN: ${GH_TOKEN} - script: scripts/dev/run_python.sh scripts/release/kubectl_mongodb/python/promote_kubectl_plugin.py + binary: scripts/dev/run_python.sh scripts/release/kubectl_mongodb/promote_kubectl_plugin.py diff --git a/.evergreen-release.yml b/.evergreen-release.yml index e3efa6d23..b16960a21 100644 --- a/.evergreen-release.yml +++ b/.evergreen-release.yml @@ -9,7 +9,7 @@ tasks: commands: - func: clone - func: setup_building_host - - func: build_multi_cluster_binary + - func: download_multi_cluster_binary - func: pipeline vars: image_name: meko-tests @@ -123,9 +123,8 @@ tasks: tags: [ "binary_release" ] commands: - func: clone - - func: install_goreleaser - - func: install_macos_notarization_service - func: python_venv + - func: install_macos_notarization_service - func: release_kubectl_mongodb_plugin - name: create_chart_release_pr @@ -134,7 +133,7 @@ tasks: - func: clone - func: python_venv - func: create_chart_release_pr - + - name: release_chart_to_oci_registry tags: [ "release_chart_to_oci_registry" ] commands: @@ -224,6 +223,9 @@ buildvariants: allowed_requesters: [ "patch", "github_tag" ] run_on: - release-ubuntu2404-small # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 + depends_on: + - name: release_kubectl_mongodb_plugin + variant: release_kubectl_mongodb_plugin tasks: - name: build_test_image_for_smoke_tests @@ -250,6 +252,8 @@ buildvariants: depends_on: - name: "*" variant: release_images + - name: "*" + variant: init_smoke_tests tasks: - name: e2e_smoke_task_group diff --git a/.evergreen-snippets.yml b/.evergreen-snippets.yml index 5a61f6592..2446d08b8 100644 --- a/.evergreen-snippets.yml +++ b/.evergreen-snippets.yml @@ -7,7 +7,7 @@ variables: - func: setup_mongosh - func: download_kube_tools - func: switch_context - - func: build_multi_cluster_binary + - func: download_multi_cluster_binary teardown_task: - func: upload_e2e_logs - func: upload_code_snippets_logs diff --git a/.evergreen.yml b/.evergreen.yml index 41d5d257c..f7770deeb 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -82,8 +82,6 @@ variables: - func: clone - func: switch_context - func: setup_building_host_minikube - - func: install_goreleaser - - func: build_multi_cluster_binary - &setup_group_multi_cluster setup_group_can_fail_task: true @@ -92,8 +90,6 @@ variables: - func: download_kube_tools - func: switch_context - func: setup_building_host - - func: install_goreleaser - - func: build_multi_cluster_binary - &setup_and_teardown_task_cloudqa setup_task_can_fail_task: true @@ -348,12 +344,17 @@ tasks: image_name: agent flags: "--parallel --current-agents --skip-if-exists=false" - - name: build_test_image + - name: build_kubectl_mongodb_plugin commands: - func: clone - func: setup_building_host - - func: install_goreleaser - func: build_multi_cluster_binary + + - name: build_test_image + commands: + - func: clone + - func: setup_building_host + - func: download_multi_cluster_binary - func: pipeline vars: image_name: meko-tests @@ -362,8 +363,7 @@ tasks: commands: - func: clone - func: setup_building_host - - func: install_goreleaser - - func: build_multi_cluster_binary + - func: download_multi_cluster_binary - func: pipeline vars: image_name: meko-tests-arm64 @@ -380,8 +380,7 @@ tasks: add_to_path: - ${workdir}/bin command: scripts/evergreen/setup_minikube_host.sh - - func: install_goreleaser - - func: build_multi_cluster_binary + - func: download_multi_cluster_binary - func: build_test_image_ibm - name: build_mco_test_image @@ -1506,9 +1505,6 @@ buildvariants: variant: init_test_run - name: build_test_image_arm variant: init_test_run_arm - # TODO: Re-enable when staging is added to pipeline - # https://jira.mongodb.org/browse/CLOUDP-349096 - disable: true tasks: - name: e2e_smoke_arm_task_group @@ -1525,9 +1521,6 @@ buildvariants: variant: init_test_run - name: build_test_image_arm variant: init_test_run_arm - # TODO: Re-enable when staging is added to pipeline - # https://jira.mongodb.org/browse/CLOUDP-349096 - disable: true tasks: - name: e2e_smoke_arm_task_group @@ -1723,6 +1716,8 @@ buildvariants: - name: build_operator_ubi - name: build_operator_race_ubi - name: build_test_image + depends_on: + - name: build_kubectl_mongodb_plugin - name: build_mco_test_image - name: build_init_appdb_images_ubi - name: build_init_om_images_ubi @@ -1731,6 +1726,7 @@ buildvariants: - name: build_agent_images_ubi - name: build_readiness_probe_image - name: build_version_upgrade_hook_image + - name: build_kubectl_mongodb_plugin - name: prepare_aws - name: publish_helm_chart @@ -1744,6 +1740,9 @@ buildvariants: run_on: - rhel9-power-small - rhel9-power-large + depends_on: + - name: build_kubectl_mongodb_plugin + variant: init_test_run tasks: - name: build_test_image_ibm @@ -1757,6 +1756,9 @@ buildvariants: run_on: - rhel9-zseries-small - rhel9-zseries-large + depends_on: + - name: build_kubectl_mongodb_plugin + variant: init_test_run tasks: - name: build_test_image_ibm @@ -1766,6 +1768,9 @@ buildvariants: tags: [ "staging" ] run_on: - ubuntu2204-arm64-small + depends_on: + - name: build_kubectl_mongodb_plugin + variant: init_test_run tasks: - name: build_test_image_arm diff --git a/.gitignore b/.gitignore index 005d2179e..195315528 100644 --- a/.gitignore +++ b/.gitignore @@ -89,8 +89,6 @@ logs-debug/ docs/**/log/* docs/**/test.sh.run.log -# goreleaser generated files -dist logs *.run.log diff --git a/.goreleaser.yaml b/.goreleaser.yaml deleted file mode 100644 index 221b1f5bd..000000000 --- a/.goreleaser.yaml +++ /dev/null @@ -1,55 +0,0 @@ -project_name: kubectl-mongodb - -before: - hooks: - - go mod tidy - -builds: - - env: - - CGO_ENABLED=0 - main: ./cmd/kubectl-mongodb - goos: - - linux - - darwin - goarch: - - amd64 - - arm64 - - s390x - - ppc64le - hooks: - # This will notarize Apple binaries and replace goreleaser bins with the notarized ones - post: - - cmd: ./scripts/release/kubectl_mongodb/kubectl_mac_notarize.sh - output: true - - cmd: ./scripts/release/kubectl_mongodb/sign.sh {{ .Path }} - env: - - GRS_USERNAME={{ .Env.GRS_USERNAME }} - - GRS_PASSWORD={{ .Env.GRS_PASSWORD }} - - PKCS11_URI={{ .Env.PKCS11_URI }} - - ARTIFACTORY_URL={{ .Env.ARTIFACTORY_URL }} - - SIGNING_IMAGE_URI={{ .Env.SIGNING_IMAGE_URI }} - - ARTIFACTORY_USERNAME=mongodb-enterprise-kubernetes-operator - - ARTIFACTORY_PASSWORD={{ .Env.ARTIFACTORY_PASSWORD }} - - cmd: ./scripts/release/kubectl_mongodb/verify.sh {{ .Path }} && echo "VERIFIED OK" - -archives: - - format: tar.gz - name_template: "kubectl-mongodb_{{ .Version }}_{{ .Os }}_{{ .Arch }}" - files: - # Include signature files in each archive along with the binary, strip_parent avoid nested folders - - src: "./dist/kubectl-mongodb_{{ .Os }}_{{ .Arch }}*{{ .Amd64 }}/kubectl-mongodb.sig" - strip_parent: true -checksum: - name_template: 'checksums.txt' -snapshot: - name_template: "{{ incpatch .Version }}-next" -changelog: - skip: true - -release: - prerelease: auto - draft: true - name_template: "MongoDB Kubernetes Operator {{ .Version }}" - -git: - tag_sort: -version:creatordate diff --git a/Makefile b/Makefile index 42b428a98..42e37d7dc 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,7 @@ build-and-push-operator-image: aws_login build-and-push-database-image: aws_login @ scripts/dev/build_push_database_image -build-and-push-test-image: aws_login build-multi-cluster-binary +build-and-push-test-image: aws_login @ if [[ -z "$(local)" ]]; then \ scripts/dev/run_python.sh scripts/release/pipeline.py test; \ fi @@ -169,9 +169,6 @@ build-and-push-mco-test-image: aws_login scripts/dev/run_python.sh scripts/release/pipeline.py mco-test; \ fi -build-multi-cluster-binary: - scripts/evergreen/build_multi_cluster_kubeconfig_creator.sh - # builds all app images in parallel # note that we cannot build both appdb and database init images in parallel as they change the same docker file build-and-push-images: build-and-push-operator-image appdb-init-image om-init-image database operator-image database-init-image diff --git a/build_info.json b/build_info.json index 37ed1d8da..3bc34cff2 100644 --- a/build_info.json +++ b/build_info.json @@ -356,13 +356,14 @@ ] }, "staging": { - "sign": false, "s3-store": "mongodb-kubernetes-staging", "platforms": [ "darwin/amd64", "darwin/arm64", "linux/amd64", - "linux/arm64" + "linux/arm64", + "linux/s390x", + "linux/ppc64le" ] }, "release": { @@ -372,7 +373,9 @@ "darwin/amd64", "darwin/arm64", "linux/amd64", - "linux/arm64" + "linux/arm64", + "linux/s390x", + "linux/ppc64le" ] } } diff --git a/scripts/evergreen/build_multi_cluster_kubeconfig_creator.sh b/scripts/evergreen/build_multi_cluster_kubeconfig_creator.sh deleted file mode 100755 index fcf70f884..000000000 --- a/scripts/evergreen/build_multi_cluster_kubeconfig_creator.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -set -Eeou pipefail - -source scripts/dev/set_env_context.sh - -WORKDIR="${workdir}" -mkdir -p "${WORKDIR}/bin" - -OS=${OS:-$(uname -s | tr '[:upper:]' '[:lower:]')} -ARCH=${ARCH:-$(uname -m | tr '[:upper:]' '[:lower:]')} -if [[ "${ARCH}" == "x86_64" ]]; then - ARCH="amd64" -elif [[ "${ARCH}" == "aarch64" ]]; then - ARCH="arm64" -fi - -echo "Building multi cluster kube config creation tool." - -project_dir="$(pwd)" -pushd cmd/kubectl-mongodb -go mod download - -GOOS="linux" GOARCH="amd64" CGO_ENABLED=0 go build -buildvcs=false -o "${project_dir}/docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_amd64" main.go & -GOOS="linux" GOARCH="s390x" CGO_ENABLED=0 go build -buildvcs=false -o "${project_dir}/docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_s390x" main.go & -GOOS="linux" GOARCH="ppc64le" CGO_ENABLED=0 go build -buildvcs=false -o "${project_dir}/docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_ppc64le" main.go & -GOOS="linux" GOARCH="arm64" CGO_ENABLED=0 go build -buildvcs=false -o "${project_dir}/docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_arm64" main.go & -wait -popd - -# these are used in the dockerfile -chmod +x docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_amd64 -chmod +x docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_s390x -chmod +x docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_ppc64le -chmod +x docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_arm64 - -cp docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_"${ARCH}" docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator || true - -mkdir -p bin || true -cp docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator bin/kubectl-mongodb || true -cp bin/kubectl-mongodb "${WORKDIR}/bin/kubectl-mongodb" || true diff --git a/scripts/release/argparse_utils.py b/scripts/release/argparse_utils.py index 3d4ad4539..774b37ed4 100644 --- a/scripts/release/argparse_utils.py +++ b/scripts/release/argparse_utils.py @@ -1,5 +1,8 @@ import argparse +from scripts.release.build.build_scenario import BuildScenario +from scripts.release.build.image_build_configuration import SUPPORTED_PLATFORMS + def str2bool(v): if isinstance(v, bool): @@ -10,3 +13,22 @@ def str2bool(v): return False else: raise argparse.ArgumentTypeError("Boolean value expected.") + + +def get_scenario_from_arg(args_scenario: str) -> BuildScenario | None: + try: + return BuildScenario(args_scenario) + except ValueError as e: + raise ValueError(f"Invalid scenario '{args_scenario}': {e}") + + +def get_platforms_from_arg(args_platforms: str) -> list[str] | None: + if not args_platforms: + return None + + platforms = [p.strip() for p in args_platforms.split(",")] + if any(p not in SUPPORTED_PLATFORMS for p in platforms): + raise ValueError( + f"Unsupported platform in --platforms '{args_platforms}'. Supported platforms: {', '.join(SUPPORTED_PLATFORMS)}" + ) + return platforms diff --git a/scripts/release/build/build_info.py b/scripts/release/build/build_info.py index 739efe87d..45db5b290 100644 --- a/scripts/release/build/build_info.py +++ b/scripts/release/build/build_info.py @@ -18,6 +18,8 @@ INIT_OPS_MANAGER_IMAGE = "init-ops-manager" OPS_MANAGER_IMAGE = "ops-manager" +KUBECTL_PLUGIN_BINARY = "kubectl-mongodb" + @dataclass class ImageInfo: diff --git a/scripts/release/build/build_scenario.py b/scripts/release/build/build_scenario.py index 5b6f30df3..99760ea89 100644 --- a/scripts/release/build/build_scenario.py +++ b/scripts/release/build/build_scenario.py @@ -1,8 +1,9 @@ from enum import StrEnum - class BuildScenario(StrEnum): RELEASE = "release" # Official release triggered by a git tag or OM version bump PATCH = "patch" # CI build for a patch/pull request STAGING = "staging" # CI build from a merge to the master DEVELOPMENT = "development" # Local build on a developer machine + +SUPPORTED_SCENARIOS = supported_scenarios = list(BuildScenario) diff --git a/scripts/release/build/image_build_configuration.py b/scripts/release/build/image_build_configuration.py index d5b478a3a..f17feb2a3 100644 --- a/scripts/release/build/image_build_configuration.py +++ b/scripts/release/build/image_build_configuration.py @@ -3,7 +3,8 @@ from scripts.release.build.build_scenario import BuildScenario -SUPPORTED_PLATFORMS = ["linux/amd64", "linux/arm64"] +SUPPORTED_PLATFORMS = ["darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64", "linux/s390x", + "linux/ppc64le"] @dataclass diff --git a/scripts/release/helm_registry_login.py b/scripts/release/helm_registry_login.py index 4f2d6e0d2..3035d4197 100644 --- a/scripts/release/helm_registry_login.py +++ b/scripts/release/helm_registry_login.py @@ -7,7 +7,6 @@ from lib.base_logger import logger from scripts.release.build.build_info import load_build_info -BUILD_SCENARIO_RELEASE = "release" QUAY_USERNAME_ENV_VAR = "quay_prod_username" QUAY_PASSWORD_ENV_VAR = "quay_prod_robot_token" QUAY_REGISTRY = "quay.io" diff --git a/scripts/release/kubectl_mongodb/__init__.py b/scripts/release/kubectl_mongodb/__init__.py new file mode 100644 index 000000000..1045b03d5 --- /dev/null +++ b/scripts/release/kubectl_mongodb/__init__.py @@ -0,0 +1 @@ +# Makes 'kubectl_mongodb' a Python package. diff --git a/scripts/release/kubectl_mongodb/build_kubectl_plugin.py b/scripts/release/kubectl_mongodb/build_kubectl_plugin.py new file mode 100755 index 000000000..ac240b89e --- /dev/null +++ b/scripts/release/kubectl_mongodb/build_kubectl_plugin.py @@ -0,0 +1,139 @@ +import argparse +import os +import subprocess + +from lib.base_logger import logger +from scripts.release.argparse_utils import get_platforms_from_arg, get_scenario_from_arg +from scripts.release.build.build_info import ( + KUBECTL_PLUGIN_BINARY, + load_build_info, +) +from scripts.release.build.build_scenario import SUPPORTED_SCENARIOS, BuildScenario +from scripts.release.kubectl_mongodb.utils import ( + create_s3_client, + kubectl_plugin_name, + parse_platform, + s3_path, +) + + +def build_kubectl_plugin(local_dir: str, platforms: list[str]): + logger.info(f"Building kubectl-mongodb plugin for platforms: {platforms}") + + for platform in platforms: + os_name, arch_name = parse_platform(platform) + + os.makedirs(local_dir, exist_ok=True) + output_filename = kubectl_plugin_name(os_name, arch_name) + output_path = os.path.join(local_dir, output_filename) + + build_command = [ + "go", + "build", + "-o", + output_path, + "./cmd/kubectl-mongodb", + ] + + env = os.environ.copy() + env["GOOS"] = os_name + env["GOARCH"] = arch_name + env["CGO_ENABLED"] = "0" + + try: + process = subprocess.Popen( + build_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, env=env + ) + + for log in iter(process.stdout.readline, ""): + print(log, end="") + + process.stdout.close() + exit_code = process.wait() + + if exit_code != 0: + raise Exception(f"Build command failed with exit code {exit_code}: {process.stderr}") + + logger.debug(f"Successfully built kubectl-mongodb for {platform} at {output_path}") + except subprocess.CalledProcessError as e: + raise Exception(f"Build command failed with code {e.returncode}: {e.stderr}") + + logger.info("Building kubectl-mongodb plugin completed successfully!") + + +def upload_artifacts_to_s3(local_dir: str, platforms: list[str], s3_bucket: str, version: str): + """ + Uploads the artifacts that are generated to S3 bucket at a specific path. + The S3 bucket and version are figured out and passed to this function based on BuildScenario. + """ + if not os.path.isdir(local_dir): + raise Exception(f"Input directory '{local_dir}' not found.") + + s3_client = create_s3_client() + + for platform in platforms: + os_name, arch_name = parse_platform(platform) + filename = kubectl_plugin_name(os_name, arch_name) + filepath = os.path.join(local_dir, filename) + if not os.path.isfile(filepath): + raise Exception(f"Expected build artifact '{filename}' not found in '{local_dir}' directory.") + + s3_key = s3_path(filename, version) + logger.info(f"Uploading artifact {filepath} to s3://{s3_bucket}/{s3_key}") + + try: + s3_client.upload_file(filepath, s3_bucket, s3_key) + logger.info(f"Successfully uploaded the artifact {filepath}") + except Exception as e: + raise Exception(f"Failed to upload file {filepath}: {e}") + + +def main(): + parser = argparse.ArgumentParser( + description="Compile and upload kubectl-mongodb plugin binaries to S3 bucket based on the build scenario.", + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument( + "-b", + "--build-scenario", + metavar="", + action="store", + default=BuildScenario.DEVELOPMENT, + type=str, + choices=SUPPORTED_SCENARIOS, + help=f"""Build scenario when reading configuration from 'build_info.json'. +Options: {", ".join(SUPPORTED_SCENARIOS)}. For '{BuildScenario.DEVELOPMENT}' the '{BuildScenario.PATCH}' scenario is used to read values from 'build_info.json'""", + ) + parser.add_argument( + "-v", + "--version", + metavar="", + action="store", + required=True, + type=str, + help="Version to use when building kubectl-mongodb binary.", + ) + parser.add_argument( + "-p", + "--platform", + metavar="", + action="store", + type=str, + help="Override the platforms instead of resolving from build scenario. Multi-arch builds are comma-separated. Example: linux/amd64,linux/arm64", + ) + args = parser.parse_args() + + build_scenario = get_scenario_from_arg(args.build_scenario) + build_info = load_build_info(build_scenario).binaries[KUBECTL_PLUGIN_BINARY] + + platforms = get_platforms_from_arg(args.platform) or build_info.platforms + version = args.version + local_dir = "bin" + + build_kubectl_plugin(local_dir, platforms) + + upload_artifacts_to_s3(local_dir, platforms, build_info.s3_store, version) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/kubectl_mongodb/build_kubectl_plugin.sh b/scripts/release/kubectl_mongodb/build_kubectl_plugin.sh new file mode 100755 index 000000000..be66bfcff --- /dev/null +++ b/scripts/release/kubectl_mongodb/build_kubectl_plugin.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +scripts/dev/run_python.sh scripts/release/kubectl_mongodb/build_kubectl_plugin.py --build-scenario "${BUILD_SCENARIO}" --version "${OPERATOR_VERSION}" diff --git a/scripts/release/kubectl_mongodb/download_kubectl_plugin.py b/scripts/release/kubectl_mongodb/download_kubectl_plugin.py new file mode 100755 index 000000000..8d78f5d07 --- /dev/null +++ b/scripts/release/kubectl_mongodb/download_kubectl_plugin.py @@ -0,0 +1,102 @@ +import argparse +import os + +from botocore.exceptions import ClientError + +from lib.base_logger import logger +from scripts.release.argparse_utils import get_platforms_from_arg, get_scenario_from_arg +from scripts.release.build.build_info import ( + KUBECTL_PLUGIN_BINARY, + load_build_info, +) +from scripts.release.build.build_scenario import SUPPORTED_SCENARIOS, BuildScenario +from scripts.release.kubectl_mongodb.build_kubectl_plugin import ( + kubectl_plugin_name, + parse_platform, + s3_path, +) +from scripts.release.kubectl_mongodb.utils import create_s3_client + + +def local_tests_plugin_path(arch_name: str) -> str: + return f"docker/mongodb-kubernetes-tests/multi-cluster-kube-config-creator_{arch_name}" + + +def download_kubectl_plugin_from_s3(s3_bucket: str, s3_plugin_path: str, local_path: str): + """ + Downloads the plugin for provided platform and puts it in the path expected by the tests image + """ + s3_client = create_s3_client() + + logger.info(f"Downloading s3://{s3_bucket}/{s3_plugin_path} to {local_path}") + + try: + s3_client.download_file(s3_bucket, s3_plugin_path, local_path) + # change the file's permissions to make file executable + os.chmod(local_path, 0o755) + + logger.info(f"Successfully downloaded artifact to {local_path}") + except ClientError as e: + if e.response["Error"]["Code"] == "404": + raise Exception(f"Artifact not found at s3://{s3_bucket}/{s3_plugin_path}: {e}") + raise Exception(f"Failed to download artifact. S3 Client Error: {e}") + except Exception as e: + raise Exception(f"An unexpected error occurred during download: {e}") + + +def main(): + parser = argparse.ArgumentParser( + description="Download kubectl-mongodb plugin binaries from S3 bucket based on the build scenario.", + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument( + "-b", + "--build-scenario", + metavar="", + action="store", + default=BuildScenario.DEVELOPMENT, + type=str, + choices=SUPPORTED_SCENARIOS, + help=f"""Build scenario when reading configuration from 'build_info.json'. +Options: {", ".join(SUPPORTED_SCENARIOS)}. For '{BuildScenario.DEVELOPMENT}' the '{BuildScenario.PATCH}' scenario is used to read values from 'build_info.json'""", + ) + parser.add_argument( + "-v", + "--version", + metavar="", + action="store", + required=True, + type=str, + help="Version to use when building kubectl-mongodb binary.", + ) + parser.add_argument( + "-p", + "--platform", + metavar="", + action="store", + type=str, + help="Override the platforms instead of resolving from build scenario. Multi-arch builds are comma-separated. Example: linux/amd64,linux/arm64", + ) + args = parser.parse_args() + + build_scenario = get_scenario_from_arg(args.build_scenario) + build_info = load_build_info(build_scenario).binaries[KUBECTL_PLUGIN_BINARY] + + platforms = get_platforms_from_arg(args.platform) or build_info.platforms + version = args.version + + for platform in platforms: + os_name, arch_name = parse_platform(platform) + if os_name != "linux": + logger.debug(f"Skipping non-linux platform {platform}, not used in e2e tests in Evergreen CI") + continue + + filename = kubectl_plugin_name(os_name, arch_name) + s3_plugin_path = s3_path(filename, version) + local_path = local_tests_plugin_path(arch_name) + + download_kubectl_plugin_from_s3(build_info.s3_store, s3_plugin_path, local_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/kubectl_mongodb/download_kubectl_plugin.sh b/scripts/release/kubectl_mongodb/download_kubectl_plugin.sh new file mode 100755 index 000000000..2aaf5c5cb --- /dev/null +++ b/scripts/release/kubectl_mongodb/download_kubectl_plugin.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +scripts/dev/run_python.sh scripts/release/kubectl_mongodb/download_kubectl_plugin.py --build-scenario "${BUILD_SCENARIO}" --version "${OPERATOR_VERSION}" diff --git a/scripts/release/kubectl_mongodb/install_istio_separate_network.sh b/scripts/release/kubectl_mongodb/install_istio_separate_network.sh deleted file mode 100755 index adda0ff92..000000000 --- a/scripts/release/kubectl_mongodb/install_istio_separate_network.sh +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -# define here or provide the cluster names externally -export CTX_CLUSTER1=${CTX_CLUSTER1} -export CTX_CLUSTER2=${CTX_CLUSTER2} -export CTX_CLUSTER3=${CTX_CLUSTER3} -export ISTIO_VERSION=${ISTIO_VERSION} - -# download Istio under the path -curl -L https://istio.io/downloadIstio | sh - - -# checks if external IP has been assigned to a service object, in our case we are interested in east-west gateway -function_check_external_ip_assigned() { - while : ; do - ip=$(kubectl --context="$1" get svc istio-eastwestgateway -n istio-system --output jsonpath='{.status.loadBalancer.ingress[0].ip}') - if [ -n "${ip}" ] - then - echo "external ip assigned ${ip}" - break - else - echo "waiting for external ip to be assigned" - fi -done -} - -cd "istio-${ISTIO_VERSION}" -mkdir -p certs -pushd certs - -# create root trust for the clusters -make -f ../tools/certs/Makefile.selfsigned.mk root-ca -make -f ../tools/certs/Makefile.selfsigned.mk "${CTX_CLUSTER1}-cacerts" -make -f ../tools/certs/Makefile.selfsigned.mk "${CTX_CLUSTER2}-cacerts" -make -f ../tools/certs/Makefile.selfsigned.mk "${CTX_CLUSTER3}-cacerts" - -kubectl --context="${CTX_CLUSTER1}" create ns istio-system -kubectl --context="${CTX_CLUSTER1}" create secret generic cacerts -n istio-system \ - --from-file="${CTX_CLUSTER1}/ca-cert.pem" \ - --from-file="${CTX_CLUSTER1}/ca-key.pem" \ - --from-file="${CTX_CLUSTER1}/root-cert.pem" \ - --from-file="${CTX_CLUSTER1}/cert-chain.pem" - -kubectl --context="${CTX_CLUSTER2}" create ns istio-system -kubectl --context="${CTX_CLUSTER2}" create secret generic cacerts -n istio-system \ - --from-file="${CTX_CLUSTER2}/ca-cert.pem" \ - --from-file="${CTX_CLUSTER2}/ca-key.pem" \ - --from-file="${CTX_CLUSTER2}/root-cert.pem" \ - --from-file="${CTX_CLUSTER2}/cert-chain.pem" - -kubectl --context="${CTX_CLUSTER3}" create ns istio-system -kubectl --context="${CTX_CLUSTER3}" create secret generic cacerts -n istio-system \ - --from-file="${CTX_CLUSTER3}/ca-cert.pem" \ - --from-file="${CTX_CLUSTER3}/ca-key.pem" \ - --from-file="${CTX_CLUSTER3}/root-cert.pem" \ - --from-file="${CTX_CLUSTER3}/cert-chain.pem" -popd - -# label namespace in cluster1 -kubectl --context="${CTX_CLUSTER1}" get namespace istio-system && \ - kubectl --context="${CTX_CLUSTER1}" label namespace istio-system topology.istio.io/network=network1 - -cat < cluster1.yaml -apiVersion: install.istio.io/v1alpha1 -kind: IstioOperator -spec: - values: - global: - meshID: mesh1 - multiCluster: - clusterName: cluster1 - network: network1 -EOF -bin/istioctl install --context="${CTX_CLUSTER1}" -f cluster1.yaml -samples/multicluster/gen-eastwest-gateway.sh \ - --mesh mesh1 --cluster cluster1 --network network1 | \ - bin/istioctl --context="${CTX_CLUSTER1}" install -y -f - - - -# check if external IP is assigned to east-west gateway in cluster1 -function_check_external_ip_assigned "${CTX_CLUSTER1}" - - -# expose services in cluster1 -kubectl --context="${CTX_CLUSTER1}" apply -n istio-system -f \ - samples/multicluster/expose-services.yaml - - -kubectl --context="${CTX_CLUSTER2}" get namespace istio-system && \ - kubectl --context="${CTX_CLUSTER2}" label namespace istio-system topology.istio.io/network=network2 - - -cat < cluster2.yaml -apiVersion: install.istio.io/v1alpha1 -kind: IstioOperator -spec: - values: - global: - meshID: mesh1 - multiCluster: - clusterName: cluster2 - network: network2 -EOF - -bin/istioctl install --context="${CTX_CLUSTER2}" -f cluster2.yaml - -samples/multicluster/gen-eastwest-gateway.sh \ - --mesh mesh1 --cluster cluster2 --network network2 | \ - bin/istioctl --context="${CTX_CLUSTER2}" install -y -f - - -# check if external IP is assigned to east-west gateway in cluster2 -function_check_external_ip_assigned "${CTX_CLUSTER2}" - -kubectl --context="${CTX_CLUSTER2}" apply -n istio-system -f \ - samples/multicluster/expose-services.yaml - -# cluster3 -kubectl --context="${CTX_CLUSTER3}" get namespace istio-system && \ - kubectl --context="${CTX_CLUSTER3}" label namespace istio-system topology.istio.io/network=network3 - -cat < cluster3.yaml -apiVersion: install.istio.io/v1alpha1 -kind: IstioOperator -spec: - values: - global: - meshID: mesh1 - multiCluster: - clusterName: cluster3 - network: network3 -EOF - -bin/istioctl install --context="${CTX_CLUSTER3}" -f cluster3.yaml - -samples/multicluster/gen-eastwest-gateway.sh \ - --mesh mesh1 --cluster cluster3 --network network3 | \ - bin/istioctl --context="${CTX_CLUSTER3}" install -y -f - - - -# check if external IP is assigned to east-west gateway in cluster3 -function_check_external_ip_assigned "${CTX_CLUSTER3}" - -kubectl --context="${CTX_CLUSTER3}" apply -n istio-system -f \ - samples/multicluster/expose-services.yaml - - -# enable endpoint discovery -bin/istioctl x create-remote-secret \ - --context="${CTX_CLUSTER1}" \ - -n istio-system \ - --name=cluster1 | \ - kubectl apply -f - --context="${CTX_CLUSTER2}" - -bin/istioctl x create-remote-secret \ - --context="${CTX_CLUSTER1}" \ - -n istio-system \ - --name=cluster1 | \ - kubectl apply -f - --context="${CTX_CLUSTER3}" - -bin/istioctl x create-remote-secret \ - --context="${CTX_CLUSTER2}" \ - -n istio-system \ - --name=cluster2 | \ - kubectl apply -f - --context="${CTX_CLUSTER1}" - -bin/istioctl x create-remote-secret \ - --context="${CTX_CLUSTER2}" \ - -n istio-system \ - --name=cluster2 | \ - kubectl apply -f - --context="${CTX_CLUSTER3}" - -bin/istioctl x create-remote-secret \ - --context="${CTX_CLUSTER3}" \ - -n istio-system \ - --name=cluster3 | \ - kubectl apply -f - --context="${CTX_CLUSTER1}" - -bin/istioctl x create-remote-secret \ - --context="${CTX_CLUSTER3}" \ - -n istio-system \ - --name=cluster3 | \ - kubectl apply -f - --context="${CTX_CLUSTER2}" - - # cleanup: delete the istio repo at the end -cd .. -rm -r "istio-${ISTIO_VERSION}" -rm -f cluster1.yaml cluster2.yaml cluster3.yaml diff --git a/scripts/release/kubectl_mongodb/kubectl_mac_notarize.sh b/scripts/release/kubectl_mongodb/kubectl_mac_notarize.sh index 9f00c170b..1624d3456 100755 --- a/scripts/release/kubectl_mongodb/kubectl_mac_notarize.sh +++ b/scripts/release/kubectl_mongodb/kubectl_mac_notarize.sh @@ -17,8 +17,6 @@ set -Eeou pipefail # Notarize generated binaries with Apple and replace the original binary with the notarized one -# This depends on binaries being generated in a goreleaser manner and gon being set up. -# goreleaser should already take care of calling this script as a hook. if [ -z "${1-}" ]; then echo "Error: Missing required argument as first positional parameter to script" @@ -32,7 +30,7 @@ darwin_amd64_dir="./artifacts/kubectl-mongodb_${version}_darwin_amd64" darwin_arm64_dir="./artifacts/kubectl-mongodb_${version}_darwin_arm64" if [[ -f "${darwin_amd64_dir}/kubectl-mongodb" && -f "${darwin_arm64_dir}/kubectl-mongodb" && ! -f "./artifacts/kubectl-mongodb_macos_signed.zip" ]]; then - echo "notarizing macOs binaries" + echo "notarizing MacOS binaries" zip -r ./artifacts/kubectl-mongodb_amd64_arm64_bin.zip "${darwin_amd64_dir}/kubectl-mongodb" "${darwin_arm64_dir}/kubectl-mongodb" # The Notarization Service takes an archive as input "${workdir:-.}"/linux_amd64/macnotary \ -f ./artifacts/kubectl-mongodb_amd64_arm64_bin.zip \ diff --git a/scripts/release/kubectl_mongodb/promote_kubectl_plugin.py b/scripts/release/kubectl_mongodb/promote_kubectl_plugin.py new file mode 100644 index 000000000..bc7a02709 --- /dev/null +++ b/scripts/release/kubectl_mongodb/promote_kubectl_plugin.py @@ -0,0 +1,304 @@ +import hashlib +import os +import subprocess +import sys +import tarfile +from pathlib import Path + +from botocore.exceptions import ClientError +from github import Github, GithubException + +from lib.base_logger import logger +from scripts.release.build.build_info import ( + KUBECTL_PLUGIN_BINARY, + load_build_info, +) +from scripts.release.build.build_scenario import BuildScenario +from scripts.release.kubectl_mongodb.download_kubectl_plugin import ( + download_kubectl_plugin_from_s3, +) +from scripts.release.kubectl_mongodb.utils import ( + CHECKSUMS_PATH, + GITHUB_REPO, + LOCAL_ARTIFACTS_DIR, + create_s3_client, + kubectl_plugin_name, + parse_platform, + s3_path, +) + +GITHUB_TOKEN = os.environ.get("GH_TOKEN") + + +def main(): + release_version = os.environ.get("OPERATOR_VERSION") + + kubectl_plugin_release_info = load_build_info(BuildScenario.RELEASE).binaries[KUBECTL_PLUGIN_BINARY] + release_scenario_bucket_name = kubectl_plugin_release_info.s3_store + release_platforms = kubectl_plugin_release_info.platforms + + kubectl_plugin_staging_info = load_build_info(BuildScenario.STAGING).binaries[KUBECTL_PLUGIN_BINARY] + staging_scenario_bucket_name = kubectl_plugin_staging_info.s3_store + staging_version = get_commit_from_tag(release_version) + + artifacts_dict = download_artifacts_from_s3( + release_version, release_platforms, staging_version, staging_scenario_bucket_name + ) + + notarize_artifacts(release_version) + + sign_and_verify_artifacts(list(artifacts_dict.keys())) + + artifacts_tar = create_tarballs() + + checksum_file = generate_checksums(artifacts_tar) + + artifacts_tar_dict = {path: os.path.basename(path) for path in artifacts_tar} + checksum_file_dict = {checksum_file: os.path.basename(checksum_file)} + + s3_artifacts = artifacts_dict | artifacts_tar_dict | checksum_file_dict + promote_artifacts_to_s3(s3_artifacts, release_version, release_scenario_bucket_name) + + github_artifacts = artifacts_tar + [checksum_file] + upload_assets_to_github_release(github_artifacts, release_version) + + +# get_commit_from_tag gets the commit associated with a release tag, so that we can use that +# commit to pull the artifacts from staging bucket. +def get_commit_from_tag(tag: str) -> str: + try: + subprocess.run(["git", "fetch", "--tags"], capture_output=True, text=True, check=True) + + result = subprocess.run( + # using --short because that's how staging version is figured out for staging build scenario + # https://github.com/mongodb/mongodb-kubernetes/blob/1.5.0/scripts/dev/contexts/evg-private-context#L137 + ["git", "rev-parse", "--short", f"{tag}^{{commit}}"], # git rev-parse v1.1.1^{commit} + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + except subprocess.CalledProcessError as e: + logger.info(f"Failed to get commit for tag: {tag}, err: {e.stderr.strip()}") + sys.exit(1) + + +# generate_checksums generates checksums for the artifacts that we are going to upload to GitHub release as assets. +# It's formatted: checksum artifact_name +def generate_checksums(artifacts: list[str]): + checksums_path = Path(CHECKSUMS_PATH) + + with checksums_path.open("w") as out_file: + for artifact in artifacts: + artifact_path = Path(artifact) + if not artifact_path.is_file() or not artifact_path.name.endswith(".tar.gz"): + logger.info(f"skipping invalid tar file: {artifact_path}") + continue + + sha256 = hashlib.sha256() + with open(artifact_path, "rb") as f: + # read chunk of 8192 bites until end of file (b"") is received + for chunk in iter(lambda: f.read(8192), b""): + sha256.update(chunk) + + checksum_line = f"{sha256.hexdigest()} {artifact_path.name}" + out_file.write(checksum_line + "\n") + + logger.info(f"Checksums written to {checksums_path}") + return str(checksums_path.resolve()) + + +# promote_artifacts promotes (copies) the downloaded staging artifacts to release S3 bucket. +def promote_artifacts_to_s3(artifacts: dict[str, str], release_version: str, release_scenario_bucket_name: str): + s3_client = create_s3_client() + + for local_file_path, s3_file_name in artifacts.items(): + if not is_expected_artifact_for_promotion(local_file_path): + logger.warning(f"Skipping invalid or non-tar/checksum artifact: {local_file_path}") + continue + + s3_key = s3_path(s3_file_name, release_version) + + try: + s3_client.upload_file(local_file_path, release_scenario_bucket_name, s3_key) + logger.debug(f"{local_file_path} was promoted to s3://{release_scenario_bucket_name}/{s3_key} successfully") + except ClientError as e: + raise Exception(f"failed to upload the file {local_file_path}: {e}") + + logger.info("All artifacts were promoted to release bucket successfully") + + +def is_expected_artifact_for_promotion(file_path: str) -> bool: + if not os.path.isfile(file_path): + return False + + if file_path.endswith(".txt"): + logger.debug(f"Promoting checksum file: {file_path}") + elif file_path.endswith(".tar.gz"): + logger.debug(f"Promoting tarball file: {file_path}") + elif file_path.endswith(KUBECTL_PLUGIN_BINARY): + logger.debug(f"Promoting binary required for e2e tests: {file_path}") + else: + return False + + return True + + +# notarize_artifacts notarizes the darwin binaries in-place. +def notarize_artifacts(release_version: str): + notarize_result = subprocess.run( + ["scripts/release/kubectl_mongodb/kubectl_mac_notarize.sh", release_version], + capture_output=True, + text=True, + ) + if notarize_result.returncode == 0: + print(notarize_result.stdout) + logger.info("Notarization of artifacts was successful") + else: + logger.debug( + f"Notarization of artifacts failed. \nstdout: {notarize_result.stdout} \nstderr: {notarize_result.stderr}" + ) + sys.exit(1) + + +# sign_and_verify_artifacts iterates over the artifacts, that have been downloaded from S3, signs and verifies them. +def sign_and_verify_artifacts(artifacts: list[str]): + for file_path in artifacts: + # signing an already signed artifact fails with `Signature already exists. Displaying proof`. + sign_result = subprocess.run( + ["scripts/release/kubectl_mongodb/sign.sh", file_path], capture_output=True, text=True + ) + if sign_result.returncode == 0: + print(sign_result.stdout) + logger.info(f"Artifact {file_path} was signed successfully") + else: + logger.debug( + f"Signing the artifact {file_path} failed. \nstdout: {sign_result.stdout} \nstderr: {sign_result.stderr}" + ) + sys.exit(1) + + verify_result = subprocess.run( + ["scripts/release/kubectl_mongodb/verify.sh", file_path], capture_output=True, text=True + ) + if verify_result.returncode == 0: + print(verify_result.stdout) + logger.info(f"Artifact {file_path} was verified successfully") + else: + logger.debug( + f"Verification of the artifact {file_path} failed. \nstdout: {verify_result.stdout} \nstderr: {verify_result.stderr}" + ) + sys.exit(1) + + +# download_artifacts_from_s3 downloads the staging artifacts (only that ones that we would later promote) from S3 and puts +# them in the local temp dir. +def download_artifacts_from_s3( + release_version: str, + release_platforms: list[str], + staging_version: str, + staging_s3_bucket_name: str, +) -> dict[str, str]: + logger.info(f"Starting download of artifacts from staging S3 bucket: {staging_s3_bucket_name}") + + # Create the local temporary directory if it doesn't exist + os.makedirs(LOCAL_ARTIFACTS_DIR, exist_ok=True) + + artifacts = {} + for platform in release_platforms: + os_name, arch_name = parse_platform(platform) + + local_plugin_dir = f"{KUBECTL_PLUGIN_BINARY}_{release_version}_{os_name}_{arch_name}" + local_artifact_dir_path = os.path.join(LOCAL_ARTIFACTS_DIR, local_plugin_dir) + os.makedirs(local_artifact_dir_path, exist_ok=True) + local_artifact_file_path = os.path.join(local_artifact_dir_path, KUBECTL_PLUGIN_BINARY) + + s3_filename = kubectl_plugin_name(os_name, arch_name) + staging_s3_plugin_path = s3_path(s3_filename, staging_version) + + download_kubectl_plugin_from_s3(staging_s3_bucket_name, staging_s3_plugin_path, local_artifact_file_path) + artifacts[local_artifact_file_path] = s3_filename + + logger.info("All the artifacts have been downloaded successfully.") + return artifacts + + +def set_permissions_filter(tarinfo): + if tarinfo.name == "kubectl-mongodb": + # This is the binary, make it executable: rwxr-xr-x + tarinfo.mode = 0o755 + return tarinfo + + +# create_tarballs creates `.tar.gz` archives for the artifacts that before promoting them. +def create_tarballs(): + logger.info(f"Creating archives for subdirectories in {LOCAL_ARTIFACTS_DIR}") + created_archives = [] + original_cwd = os.getcwd() + try: + os.chdir(LOCAL_ARTIFACTS_DIR) + + for dir_name in os.listdir("."): + if os.path.isdir(dir_name): + archive_name = f"{dir_name}.tar.gz" + + with tarfile.open(archive_name, "w:gz") as tar: + # Iterate over the contents of the subdirectory (e.g., 'kubectl-mongodb_linux_s390x') + # and add them one by one. + for item_name in os.listdir(dir_name): + full_item_path = os.path.join(dir_name, item_name) + # Add just the binary (kubectl-mongodb_None_linux_s390x/kubectl-mongodb) to the tar + # instead of adding the dir. + # filter is passed to make the binary file executable + tar.add(full_item_path, arcname=item_name, filter=set_permissions_filter) + + full_archive_path = os.path.join(original_cwd, LOCAL_ARTIFACTS_DIR, archive_name) + logger.info(f"Successfully created archive at {full_archive_path}") + created_archives.append(full_archive_path) + + except Exception as e: + logger.debug(f"ERROR: Failed to create tar.gz archives: {e}") + sys.exit(1) + finally: + os.chdir(original_cwd) + + return created_archives + + +# upload_assets_to_github_release uploads the release artifacts (downloaded notarized/signed staging artifacts) to +# the GitHub release as assets. +def upload_assets_to_github_release(asset_paths: list[str], release_version: str): + if not GITHUB_TOKEN: + logger.info("ERROR: GITHUB_TOKEN environment variable not set.") + sys.exit(1) + + try: + g = Github(GITHUB_TOKEN) + repo = g.get_repo(GITHUB_REPO) + except GithubException as e: + logger.info(f"ERROR: Could not connect to GitHub or find repository '{GITHUB_REPO}', Error {e}.") + sys.exit(1) + + try: + release = repo.get_release(release_version) + except GithubException as e: + logger.debug( + f"ERROR: Could not find release with tag '{release_version}'. Please ensure release exists already. Error: {e}" + ) + sys.exit(2) + + for asset_path in asset_paths: + asset_name = os.path.basename(asset_path) + logger.info(f"Uploading artifact '{asset_name}' to github release as asset") + try: + release.upload_asset(path=asset_path, name=asset_name, content_type="application/gzip") + except GithubException as e: + logger.debug(f"ERROR: Failed to upload asset {asset_name}. Error: {e}") + sys.exit(2) + except Exception as e: + logger.debug(f"An unexpected error occurred during upload of {asset_name}: {e}") + sys.exit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/kubectl_mongodb/python/build_kubectl_plugin.py b/scripts/release/kubectl_mongodb/python/build_kubectl_plugin.py deleted file mode 100755 index bf63ea875..000000000 --- a/scripts/release/kubectl_mongodb/python/build_kubectl_plugin.py +++ /dev/null @@ -1,149 +0,0 @@ -import os -import subprocess -import sys - -import boto3 -from botocore.exceptions import ClientError, NoCredentialsError, PartialCredentialsError - -from lib.base_logger import logger -from scripts.release.build.build_info import ( - load_build_info, -) -from scripts.release.kubectl_mongodb.python.consts import * - -S3_BUCKET_KUBECTL_PLUGIN_SUBPATH = KUBECTL_PLUGIN_BINARY_NAME - - -def run_goreleaser(): - try: - command = ["goreleaser", "build", "--snapshot", "--clean", "--skip", "post-hooks"] - - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) - - for log in iter(process.stdout.readline, ""): - print(log, end="") - - process.stdout.close() - exit_code = process.wait() - - if exit_code != 0: - logger.debug(f"GoReleaser command failed with exit code {exit_code}.") - sys.exit(1) - - logger.info("GoReleaser build completed successfully!") - - except FileNotFoundError: - logger.debug( - "ERROR: 'goreleaser' command not found. Please ensure goreleaser is installed and in your system's PATH." - ) - sys.exit(1) - except Exception as e: - logger.debug(f"An unexpected error occurred while running `goreleaser build`: {e}") - sys.exit(1) - - -# upload_artifacts_to_s3 uploads the artifacts that are generated by goreleaser to S3 bucket at a specific path. -# The S3 bucket and version are figured out and passed to this function based on BuildScenario. -def upload_artifacts_to_s3(s3_bucket: str, version: str): - if not os.path.isdir(GORELEASER_DIST_DIR): - logger.info(f"ERROR: GoReleaser dist directory '{GORELEASER_DIST_DIR}' not found.") - sys.exit(1) - - try: - s3_client = boto3.client("s3", region_name=AWS_REGION) - except (NoCredentialsError, PartialCredentialsError): - logger.debug("ERROR: Failed to create S3 client. AWS credentials not found.") - sys.exit(1) - except Exception as e: - logger.debug(f"An error occurred connecting to S3: {e}") - sys.exit(1) - - uploaded_files = 0 - # iterate over all the files generated by goreleaser in the dist directory and upload them to S3 - for subdir in os.listdir(GORELEASER_DIST_DIR): - subdir_path = os.path.join(GORELEASER_DIST_DIR, subdir) - if not os.path.isdir(subdir_path): - continue # not a directory - - for filename in os.listdir(subdir_path): - local_file_path = os.path.join(subdir_path, filename) - if not os.path.isfile(local_file_path): - continue - - s3_key = s3_path(local_file_path, version) - logger.info(f"Uploading artifact {local_file_path} to s3://{s3_bucket}/{s3_key}") - try: - s3_client.upload_file(local_file_path, s3_bucket, s3_key) - logger.info(f"Successfully uploaded the artifact {filename}") - uploaded_files += 1 - except Exception as e: - logger.debug(f"ERROR: Failed to upload file {filename}: {e}") - sys.exit(1) - - if uploaded_files > 0: - logger.info(f"Successfully uploaded {uploaded_files} kubectl-mongodb plugin artifacts to S3.") - - -# s3_path returns the path where the artifacts should be uploaded to in S3 object store. -# For dev workflows it's going to be `kubectl-mongodb/{evg-patch-id}/{goreleaser-artifact}`, -# for staging workflows it would be `kubectl-mongodb/{commit-sha}/{goreleaser-artifact}`. -# The `version` string has the correct version (either patch id or commit sha), based on the BuildScenario. -def s3_path(local_path: str, version: str): - return f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{version}/{local_path}" - - -def s3_and_local_plugin_path(version: str) -> dict[str, str]: - s3_common_path = f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{version}/dist" - local_common_path = "docker/mongodb-kubernetes-tests" - # path in s3 : local path where tests image expects the binary - return { - f"{s3_common_path}/kubectl-mongodb_linux_amd64_v1/kubectl-mongodb": f"{local_common_path}/multi-cluster-kube-config-creator_amd64", - f"{s3_common_path}/kubectl-mongodb_linux_arm64/kubectl-mongodb": f"{local_common_path}/multi-cluster-kube-config-creator_arm64", - f"{s3_common_path}/kubectl-mongodb_linux_ppc64le/kubectl-mongodb": f"{local_common_path}/multi-cluster-kube-config-creator_ppc64le", - f"{s3_common_path}/kubectl-mongodb_linux_s390x/kubectl-mongodb": f"{local_common_path}/multi-cluster-kube-config-creator_s390x", - } - - -# download_plugin_for_tests_image downloads the plugin for all the architectures and places them to the paths configured in -# s3_and_local_plugin_path -def download_plugin_for_tests_image(s3_bucket: str, version: str): - try: - s3_client = boto3.client("s3", region_name=AWS_REGION) - except Exception as e: - logger.debug(f"An error occurred connecting to S3 to download kubectl plugin for tests image: {e}") - sys.exit(1) - - plugin_path = f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{version}/dist/kubectl-mongodb_linux_amd64_v1/kubectl-mongodb" - for plugin_path, local_path in s3_and_local_plugin_path(version).items(): - logger.info(f"Downloading s3://{s3_bucket}/{plugin_path} to {local_path}") - try: - s3_client.download_file(s3_bucket, plugin_path, local_path) - # change the file's permissions to make file executable - os.chmod(local_path, 0o755) - - logger.info(f"Successfully downloaded artifact to {local_path}") - except ClientError as e: - if e.response["Error"]["Code"] == "404": - logger.debug(f"ERROR: Artifact not found at s3://{s3_bucket}/{plugin_path} ") - else: - logger.debug(f"ERROR: Failed to download artifact. S3 Client Error: {e}") - sys.exit(1) - except Exception as e: - logger.debug(f"An unexpected error occurred during download: {e}") - sys.exit(1) - - -def main(): - build_scenario = os.environ.get("BUILD_SCENARIO") - kubectl_plugin_build_info = load_build_info(build_scenario).binaries[KUBECTL_PLUGIN_BINARY_NAME] - - run_goreleaser() - - version = os.environ.get("OPERATOR_VERSION") - upload_artifacts_to_s3(kubectl_plugin_build_info.s3_store, version) - - download_plugin_for_tests_image(kubectl_plugin_build_info.s3_store, version) - - -if __name__ == "__main__": - main() diff --git a/scripts/release/kubectl_mongodb/python/consts.py b/scripts/release/kubectl_mongodb/python/consts.py deleted file mode 100644 index 3e3efbcac..000000000 --- a/scripts/release/kubectl_mongodb/python/consts.py +++ /dev/null @@ -1,12 +0,0 @@ -AWS_REGION = "eu-north-1" -KUBECTL_PLUGIN_BINARY_NAME = "kubectl-mongodb" - -GITHUB_REPO = "mongodb/mongodb-kubernetes" - -LOCAL_ARTIFACTS_DIR = "artifacts" -CHECKSUMS_PATH = f"{LOCAL_ARTIFACTS_DIR}/checksums.txt" - -GORELEASER_DIST_DIR = "dist" - -BUILD_SCENARIO_RELEASE = "release" -BUILD_SCENARIO_STAGING = "staging" diff --git a/scripts/release/kubectl_mongodb/python/promote_kubectl_plugin.py b/scripts/release/kubectl_mongodb/python/promote_kubectl_plugin.py deleted file mode 100644 index efc2687c2..000000000 --- a/scripts/release/kubectl_mongodb/python/promote_kubectl_plugin.py +++ /dev/null @@ -1,324 +0,0 @@ -import argparse -import hashlib -import os -import subprocess -import sys -import tarfile -from pathlib import Path - -import boto3 -from botocore.exceptions import ClientError, NoCredentialsError, PartialCredentialsError -from github import Github, GithubException - -from lib.base_logger import logger -from scripts.release.build.build_info import ( - load_build_info, -) -from scripts.release.kubectl_mongodb.python.consts import * - -GITHUB_TOKEN = os.environ.get("GH_TOKEN") - -S3_BUCKET_KUBECTL_PLUGIN_SUBPATH = KUBECTL_PLUGIN_BINARY_NAME - - -def main(): - release_version = os.environ.get("OPERATOR_VERSION") - - kubectl_plugin_release_info = load_build_info(BUILD_SCENARIO_RELEASE).binaries[KUBECTL_PLUGIN_BINARY_NAME] - release_scenario_bucket_name = kubectl_plugin_release_info.s3_store - - kubectl_plugin_staging_info = load_build_info(BUILD_SCENARIO_STAGING).binaries[KUBECTL_PLUGIN_BINARY_NAME] - staging_scenario_bucket_name = kubectl_plugin_staging_info.s3_store - - download_artifacts_from_s3(release_version, get_commit_from_tag(release_version), staging_scenario_bucket_name) - - notarize_artifacts(release_version) - - sign_and_verify_artifacts() - - artifacts_tar = create_tarballs() - - artifacts = generate_checksums(artifacts_tar) - - promote_artifacts(artifacts, release_version, release_scenario_bucket_name) - - upload_assets_to_github_release(artifacts, release_version) - - -# get_commit_from_tag gets the commit associated with a release tag, so that we can use that -# commit to pull the artifacts from staging bucket. -def get_commit_from_tag(tag: str) -> str: - try: - subprocess.run(["git", "fetch", "--tags"], capture_output=True, text=True, check=True) - - result = subprocess.run( - # using --short because that's how staging version is figured out for staging build scenario - # https://github.com/mongodb/mongodb-kubernetes/blob/1.5.0/scripts/dev/contexts/evg-private-context#L137 - ["git", "rev-parse", "--short", f"{tag}^{{commit}}"], # git rev-parse v1.1.1^{commit} - capture_output=True, - text=True, - check=True, - ) - return result.stdout.strip() - - except subprocess.CalledProcessError as e: - logger.info(f"Failed to get commit for tag: {tag}, err: {e.stderr.strip()}") - sys.exit(1) - - -# generate_checksums generates checksums for the artifacts that we are going to upload to github release as assets. -# It's formatted: checksum artifact_name -def generate_checksums(artifacts: list[str]): - checksums_path = Path(CHECKSUMS_PATH) - - with checksums_path.open("w") as out_file: - for artifact in artifacts: - artifact_path = Path(artifact) - if not artifact_path.is_file() or not artifact_path.name.endswith(".tar.gz"): - logger.info(f"skipping invalid tar file: {artifact_path}") - continue - - sha256 = hashlib.sha256() - with open(artifact_path, "rb") as f: - # read chunk of 8192 bites until end of file (b"") is received - for chunk in iter(lambda: f.read(8192), b""): - sha256.update(chunk) - - checksum_line = f"{sha256.hexdigest()} {artifact_path.name}" - out_file.write(checksum_line + "\n") - - logger.info(f"Checksums written to {checksums_path}") - all_artifacts = list(artifacts) + [str(checksums_path.resolve())] - return all_artifacts - - -# promote_artifacts promotes (copies) the downloaded staging artifacts to release S3 bucket. -def promote_artifacts(artifacts: list[str], release_version: str, release_scenario_bucket_name: str): - s3_client = boto3.client("s3", region_name=AWS_REGION) - for file in artifacts: - if not os.path.isfile(file) or not file.endswith((".tar.gz", ".txt")): - logger.info(f"Skipping invalid or non-tar/checksum file: {file}") - continue - - file_name = os.path.basename(file) - s3_key = os.path.join(S3_BUCKET_KUBECTL_PLUGIN_SUBPATH, release_version, file_name) - - try: - s3_client.upload_file(file, release_scenario_bucket_name, s3_key) - logger.debug( - f"Plugin file {file} was promoted to release bucket {release_scenario_bucket_name}/{s3_key} successfully" - ) - except ClientError as e: - logger.debug(f"failed to upload the file {file}: {e}") - sys.exit(1) - - logger.info("Artifacts were promoted to release bucket successfully") - - -# notarize_artifacts notarizes the darwin goreleaser binaries in-place. -def notarize_artifacts(release_version: str): - notarize_result = subprocess.run( - ["scripts/release/kubectl_mongodb/kubectl_mac_notarize.sh", release_version], capture_output=True, text=True - ) - if notarize_result.returncode == 0: - logger.info("Notarization of artifacts was successful") - else: - logger.debug( - f"Notarization of artifacts failed. \nstdout: {notarize_result.stdout} \nstderr: {notarize_result.stderr}" - ) - sys.exit(1) - - -# sign_and_verify_artifacts iterates over the goreleaser artifacts, that have been downloaded from S3, and -# signs and verifies them. -def sign_and_verify_artifacts(): - cwd = os.getcwd() - artifacts_dir = os.path.join(cwd, LOCAL_ARTIFACTS_DIR) - - for subdir in os.listdir(artifacts_dir): - subdir_path = os.path.join(artifacts_dir, subdir) - - # just work on dirs and not files - if os.path.isdir(subdir_path): - for file in os.listdir(subdir_path): - file_path = os.path.join(subdir_path, file) - - if os.path.isfile(file_path): - # signing an already signed artifact fails with `Signature already exixts. Displaying proof`. - sign_result = subprocess.run( - ["scripts/release/kubectl_mongodb/sign.sh", file_path], capture_output=True, text=True - ) - if sign_result.returncode == 0: - logger.info(f"Artifact {file_path} was signed successfully") - else: - logger.debug( - f"Signing the artifact {file_path} failed. \nstdout: {sign_result.stdout} \nstderr: {sign_result.stderr}" - ) - sys.exit(1) - - verify_result = subprocess.run( - ["scripts/release/kubectl_mongodb/verify.sh", file_path], capture_output=True, text=True - ) - if verify_result.returncode == 0: - logger.info(f"Artifact {file_path} was verified successfully") - else: - logger.debug( - f"Verification of the artifact {file_path} failed. \nstdout: {verify_result.stdout} \nstderr: {verify_result.stderr}" - ) - sys.exit(1) - - -def s3_artifacts_path_to_local_path(release_version: str, commit_sha: str): - s3_common_path = f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{commit_sha}/dist" - return { - f"{s3_common_path}/kubectl-mongodb_darwin_amd64_v1/": f"kubectl-mongodb_{release_version}_darwin_amd64", - f"{s3_common_path}/kubectl-mongodb_darwin_arm64/": f"kubectl-mongodb_{release_version}_darwin_arm64", - f"{s3_common_path}/kubectl-mongodb_linux_amd64_v1/": f"kubectl-mongodb_{release_version}_linux_amd64", - f"{s3_common_path}/kubectl-mongodb_linux_arm64/": f"kubectl-mongodb_{release_version}_linux_arm64", - f"{s3_common_path}/kubectl-mongodb_linux_ppc64le/": f"kubectl-mongodb_{release_version}_linux_ppc64le", - f"{s3_common_path}/kubectl-mongodb_linux_s390x/": f"kubectl-mongodb_{release_version}_linux_s390x", - } - - -# download_artifacts_from_s3 downloads the staging artifacts (only that ones that we would later promote) from S3 and puts -# them in the local dir LOCAL_ARTIFACTS_DIR. -# ToDo: if the artifacts are not present at correct location, this is going to fail silently, we should instead fail this -def download_artifacts_from_s3(release_version: str, commit_sha: str, staging_s3_bucket_name: str): - logger.info(f"Starting download of artifacts from staging S3 bucket: {staging_s3_bucket_name}") - - try: - s3_client = boto3.client("s3", region_name=AWS_REGION) - except (NoCredentialsError, PartialCredentialsError): - logger.debug("ERROR: AWS credentials were not set.") - sys.exit(1) - except Exception as e: - logger.debug(f"An error occurred connecting to S3: {e}") - sys.exit(1) - - artifacts_to_promote = s3_artifacts_path_to_local_path(release_version, commit_sha) - - # Create the local temporary directory if it doesn't exist - os.makedirs(LOCAL_ARTIFACTS_DIR, exist_ok=True) - - download_count = 0 - for s3_artifact_dir, local_subdir in artifacts_to_promote.items(): - try: - paginator = s3_client.get_paginator("list_objects_v2") - pages = paginator.paginate(Bucket=staging_s3_bucket_name, Prefix=s3_artifact_dir) - for page in pages: - # "Contents" corresponds to the directory in the S3 bucket - if "Contents" not in page: - continue - for obj in page["Contents"]: - # obj is the S3 object in page["Contents"] directory - s3_key = obj["Key"] - if s3_key.endswith("/"): - # it's a directory - continue - - # Get the path of the file relative to its S3 prefix, this would mostly be the object name itself - # if s3_artifact_dir doesn't container directories and has just the objects. - relative_path = os.path.relpath(s3_key, s3_artifact_dir) - - final_local_path = os.path.join(LOCAL_ARTIFACTS_DIR, local_subdir, relative_path) - - # Create the local directory structure if it doesn't exist - os.makedirs(os.path.dirname(final_local_path), exist_ok=True) - - logger.info(f"Downloading staging artifact {s3_key} to {final_local_path}") - s3_client.download_file(staging_s3_bucket_name, s3_key, final_local_path) - download_count += 1 - - except ClientError as e: - logger.debug(f"ERROR: Failed to list or download from prefix '{s3_artifact_dir}'. S3 Client Error: {e}") - return False - - if download_count == 0: - logger.info( - f"Couldn't download artifacts from staging S3 bucket {staging_s3_bucket_name}, please verify that artifacts are available under dir: {commit_sha}" - ) - sys.exit(1) - - logger.info("All the artifacts have been downloaded successfully.") - return True - - -def set_permissions_filter(tarinfo): - if tarinfo.name == "kubectl-mongodb": - # This is the binary, make it executable: rwxr-xr-x - tarinfo.mode = 0o755 - return tarinfo - - -# create_tarballs creates `.tar.gz` archives for the artifacts that before promoting them. -def create_tarballs(): - logger.info(f"Creating archives for subdirectories in {LOCAL_ARTIFACTS_DIR}") - created_archives = [] - original_cwd = os.getcwd() - try: - os.chdir(LOCAL_ARTIFACTS_DIR) - - for dir_name in os.listdir("."): - if os.path.isdir(dir_name): - archive_name = f"{dir_name}.tar.gz" - - with tarfile.open(archive_name, "w:gz") as tar: - # Iterate over the contents of the subdirectory (e.g., 'kubectl-mongodb_linux_s390x') - # and add them one by one. - for item_name in os.listdir(dir_name): - full_item_path = os.path.join(dir_name, item_name) - # Add just the binary (kubectl-mongodb_None_linux_s390x/kubectl-mongodb) to the tar - # instead of adding the dir. - # filter is passed to make the binary file executable - tar.add(full_item_path, arcname=item_name, filter=set_permissions_filter) - - full_archive_path = os.path.join(original_cwd, LOCAL_ARTIFACTS_DIR, archive_name) - logger.info(f"Successfully created archive at {full_archive_path}") - created_archives.append(full_archive_path) - - except Exception as e: - logger.debug(f"ERROR: Failed to create tar.gz archives: {e}") - sys.exit(1) - finally: - os.chdir(original_cwd) - - return created_archives - - -# upload_assets_to_github_release uploads the release artifacts (downloaded notarized/signed staging artifacts) to -# the github release as assets. -def upload_assets_to_github_release(asset_paths, release_version: str): - if not GITHUB_TOKEN: - logger.info("ERROR: GITHUB_TOKEN environment variable not set.") - sys.exit(1) - - try: - g = Github(GITHUB_TOKEN) - repo = g.get_repo(GITHUB_REPO) - except GithubException as e: - logger.info(f"ERROR: Could not connect to GitHub or find repository '{GITHUB_REPO}', Error {e}.") - sys.exit(1) - - try: - release = repo.get_release(release_version) - except GithubException as e: - logger.debug( - f"ERROR: Could not find release with tag '{release_version}'. Please ensure release exists already. Error: {e}" - ) - sys.exit(2) - - for asset_path in asset_paths: - asset_name = os.path.basename(asset_path) - logger.info(f"Uploading artifact '{asset_name}' to github release as asset") - try: - release.upload_asset(path=asset_path, name=asset_name, content_type="application/gzip") - except GithubException as e: - logger.debug(f"ERROR: Failed to upload asset {asset_name}. Error: {e}") - sys.exit(2) - except Exception as e: - logger.debug(f"An unexpected error occurred during upload of {asset_name}: {e}") - sys.exit(2) - - -if __name__ == "__main__": - main() diff --git a/scripts/release/kubectl_mongodb/sign.sh b/scripts/release/kubectl_mongodb/sign.sh index adc26b35d..6a346689a 100755 --- a/scripts/release/kubectl_mongodb/sign.sh +++ b/scripts/release/kubectl_mongodb/sign.sh @@ -6,7 +6,6 @@ set -euo pipefail # Sign a binary using garasign credentials -# goreleaser takes care of calling this script as a hook. ARTIFACT=$1 SIGNATURE="${ARTIFACT}.sig" diff --git a/scripts/release/kubectl_mongodb/utils.py b/scripts/release/kubectl_mongodb/utils.py new file mode 100644 index 000000000..26d972bbc --- /dev/null +++ b/scripts/release/kubectl_mongodb/utils.py @@ -0,0 +1,36 @@ +import boto3 +from botocore.exceptions import NoCredentialsError, PartialCredentialsError + +from scripts.release.build.build_info import KUBECTL_PLUGIN_BINARY + +AWS_REGION = "eu-north-1" + +GITHUB_REPO = "mongodb/mongodb-kubernetes" + +LOCAL_ARTIFACTS_DIR = "artifacts" +CHECKSUMS_PATH = f"{LOCAL_ARTIFACTS_DIR}/checksums.txt" + + +def create_s3_client() -> boto3.client: + try: + return boto3.client("s3", region_name=AWS_REGION) + except (NoCredentialsError, PartialCredentialsError) as e: + raise Exception(f"Failed to create S3 client. AWS credentials not found: {e}") + except Exception as e: + raise Exception(f"An error occurred connecting to S3: {e}") + + +def parse_platform(platform) -> tuple[str, str]: + return platform.split("/") + + +def kubectl_plugin_name(os_name: str, arch_name: str) -> str: + return f"{KUBECTL_PLUGIN_BINARY}_{os_name}_{arch_name}" + + +# s3_path returns the path where the artifacts should be uploaded to in S3 object store. +# For dev workflows it's going to be `kubectl-mongodb/{evg-patch-id}/kubectl-mongodb_{goos}_{goarch}`, +# for staging workflows it would be `kubectl-mongodb/{commit-sha}/kubectl-mongodb_{goos}_{goarch}`. +# The `version` string has the correct version (either patch id or commit sha), based on the BuildScenario. +def s3_path(filename: str, version: str) -> str: + return f"{KUBECTL_PLUGIN_BINARY}/{version}/{filename}" diff --git a/scripts/release/kubectl_mongodb/verify.sh b/scripts/release/kubectl_mongodb/verify.sh index 1323b7b0f..91292a711 100755 --- a/scripts/release/kubectl_mongodb/verify.sh +++ b/scripts/release/kubectl_mongodb/verify.sh @@ -3,7 +3,6 @@ set -euo pipefail # Verify the signature of a binary with the operator's public key -# goreleaser takes care of calling this script as a hook. ARTIFACT=$1 SIGNATURE="${ARTIFACT}.sig" diff --git a/scripts/release/pipeline.py b/scripts/release/pipeline.py index b83091ac9..3af1bb164 100644 --- a/scripts/release/pipeline.py +++ b/scripts/release/pipeline.py @@ -16,7 +16,11 @@ from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags from lib.base_logger import logger -from scripts.release.argparse_utils import str2bool +from scripts.release.argparse_utils import ( + get_platforms_from_arg, + get_scenario_from_arg, + str2bool, +) from scripts.release.atomic_pipeline import ( build_agent, build_database_image, @@ -47,10 +51,10 @@ load_build_info, ) from scripts.release.build.build_scenario import ( + SUPPORTED_SCENARIOS, BuildScenario, ) from scripts.release.build.image_build_configuration import ( - SUPPORTED_PLATFORMS, ImageBuildConfiguration, ) from scripts.release.build.image_build_process import ( @@ -152,25 +156,6 @@ def image_build_config_from_args(args) -> ImageBuildConfiguration: ) -def get_scenario_from_arg(args_scenario: str) -> BuildScenario | None: - try: - return BuildScenario(args_scenario) - except ValueError as e: - raise ValueError(f"Invalid scenario '{args_scenario}': {e}") - - -def get_platforms_from_arg(args_platforms: str) -> list[str] | None: - if not args_platforms: - return None - - platforms = [p.strip() for p in args_platforms.split(",")] - if any(p not in SUPPORTED_PLATFORMS for p in platforms): - raise ValueError( - f"Unsupported platform in --platforms '{args_platforms}'. Supported platforms: {', '.join(SUPPORTED_PLATFORMS)}" - ) - return platforms - - def _setup_tracing(): trace_id = os.environ.get("otel_trace_id") parent_id = os.environ.get("otel_parent_id") @@ -205,7 +190,6 @@ def _setup_tracing(): def main(): _setup_tracing() supported_images = list(get_builder_function_for_image_name().keys()) - supported_scenarios = list(BuildScenario) parser = argparse.ArgumentParser( description="""Builder tool for container images. It allows to push and sign images with multiple architectures using Docker Buildx. @@ -226,9 +210,9 @@ def main(): action="store", required=True, type=str, - choices=supported_scenarios, + choices=SUPPORTED_SCENARIOS, help=f"""Build scenario when reading configuration from 'build_info.json'. -Options: {", ".join(supported_scenarios)}. For '{BuildScenario.DEVELOPMENT}' the '{BuildScenario.PATCH}' scenario is used to read values from 'build_info.json'""", +Options: {", ".join(SUPPORTED_SCENARIOS)}. For '{BuildScenario.DEVELOPMENT}' the '{BuildScenario.PATCH}' scenario is used to read values from 'build_info.json'""", ) parser.add_argument( "-p", diff --git a/scripts/release/publish_helm_chart.py b/scripts/release/publish_helm_chart.py index ea0e36945..cae99b19e 100644 --- a/scripts/release/publish_helm_chart.py +++ b/scripts/release/publish_helm_chart.py @@ -7,7 +7,6 @@ from lib.base_logger import logger from scripts.release.build.build_info import * -from scripts.release.helm_registry_login import BUILD_SCENARIO_RELEASE CHART_DIR = "helm_chart" @@ -54,7 +53,7 @@ def update_chart_and_get_metadata(chart_dir: str, build_scenario) -> tuple[str, raise Exception(f"Unable to load Chart.yaml from dir {chart_path}") # if build_scenario is release, the chart.yaml would already have correct chart version - if build_scenario == BUILD_SCENARIO_RELEASE: + if build_scenario == BuildScenario.RELEASE: return chart_name, version new_version = f"0.0.0+{version}" diff --git a/scripts/release/tests/build_info_test.py b/scripts/release/tests/build_info_test.py index 5a098cd29..e6d44e9df 100644 --- a/scripts/release/tests/build_info_test.py +++ b/scripts/release/tests/build_info_test.py @@ -295,7 +295,14 @@ def test_load_build_info_staging(): binaries={ "kubectl-mongodb": BinaryInfo( s3_store="mongodb-kubernetes-staging", - platforms=["darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64"], + platforms=[ + "darwin/amd64", + "darwin/arm64", + "linux/amd64", + "linux/arm64", + "linux/s390x", + "linux/ppc64le", + ], sign=False, ) }, @@ -399,7 +406,14 @@ def test_load_build_info_release(): binaries={ "kubectl-mongodb": BinaryInfo( s3_store="mongodb-kubernetes-release", - platforms=["darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64"], + platforms=[ + "darwin/amd64", + "darwin/arm64", + "linux/amd64", + "linux/arm64", + "linux/s390x", + "linux/ppc64le", + ], sign=True, ) }, diff --git a/scripts/release/tests/release_info_test.py b/scripts/release/tests/release_info_test.py index f08326b90..193cb38f9 100644 --- a/scripts/release/tests/release_info_test.py +++ b/scripts/release/tests/release_info_test.py @@ -44,7 +44,14 @@ def test_create_release_info_json(): }, "binaries": { "kubectl-mongodb": { - "platforms": ["darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64"], + "platforms": [ + "darwin/amd64", + "darwin/arm64", + "linux/amd64", + "linux/arm64", + "linux/s390x", + "linux/ppc64le", + ], "version": DUMMY_VERSION, } },