From 7be5618e6af9aaac26b631a890b8e8f16dd9475e Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 9 Jul 2025 10:56:33 +0200 Subject: [PATCH 1/3] [no-relnote] Move definitions from .common to .nvidia-ci Since we no longer maintain two pipeline definitions, it is simpler to maintain a single CI definition file. Signed-off-by: Evan Lezar --- .common-ci.yml | 56 -------------------------------------------------- .nvidia-ci.yml | 43 +++++++++++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 59 deletions(-) delete mode 100644 .common-ci.yml diff --git a/.common-ci.yml b/.common-ci.yml deleted file mode 100644 index 440262cc3..000000000 --- a/.common-ci.yml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -default: - image: docker - services: - - name: docker:dind - command: ["--experimental"] - -variables: - GIT_SUBMODULE_STRATEGY: recursive - BUILD_MULTI_ARCH_IMAGES: "true" - -stages: - - pull - - scan - - release - - sign - -.pipeline-trigger-rules: - rules: - # We trigger the pipeline if started manually - - if: $CI_PIPELINE_SOURCE == "web" - # We trigger the pipeline on the main branch - - if: $CI_COMMIT_BRANCH == "main" - # We trigger the pipeline on the release- branches - - if: $CI_COMMIT_BRANCH =~ /^release-.*$/ - # We trigger the pipeline on tags - - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != "" - -workflow: - rules: - # We trigger the pipeline on a merge request - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - # We then add all the regular triggers - - !reference [.pipeline-trigger-rules, rules] - -# Download the regctl binary for use in the release steps -.regctl-setup: - before_script: - - export REGCTL_VERSION=v0.4.5 - - apk add --no-cache curl - - mkdir -p bin - - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64 - - chmod a+x bin/regctl - - export PATH=$(pwd)/bin:${PATH} diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml index 198cf95a9..4808b53ec 100644 --- a/.nvidia-ci.yml +++ b/.nvidia-ci.yml @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -include: - - local: '.common-ci.yml' - default: + image: docker + services: + - name: docker:dind + command: ["--experimental"] tags: - cnt - container-dev @@ -25,6 +26,8 @@ default: - type/docker variables: + GIT_SUBMODULE_STRATEGY: recursive + DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "/certs" # Release "devel"-tagged images off the main branch @@ -39,6 +42,40 @@ variables: KITMAKER_RELEASE_FOLDER: "kitmaker" PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases" +stages: + - pull + - scan + - release + - sign + +.pipeline-trigger-rules: + rules: + # We trigger the pipeline if started manually + - if: $CI_PIPELINE_SOURCE == "web" + # We trigger the pipeline on the main branch + - if: $CI_COMMIT_BRANCH == "main" + # We trigger the pipeline on the release- branches + - if: $CI_COMMIT_BRANCH =~ /^release-.*$/ + # We trigger the pipeline on tags + - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != "" + +workflow: + rules: + # We trigger the pipeline on a merge request + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + # We then add all the regular triggers + - !reference [.pipeline-trigger-rules, rules] + +# Download the regctl binary for use in the release steps +.regctl-setup: + before_script: + - export REGCTL_VERSION=v0.4.5 + - apk add --no-cache curl + - mkdir -p bin + - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64 + - chmod a+x bin/regctl + - export PATH=$(pwd)/bin:${PATH} + # .copy-images copies the required application and packaging images from the # IN_IMAGE="${IN_IMAGE_NAME}:${IN_IMAGE_TAG}${TAG_SUFFIX}" # to From b923d6a8d0d97654b08ab25197f1ff4fab7f6eb5 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 9 Jul 2025 11:05:11 +0200 Subject: [PATCH 2/3] [no-relnote] Remove the signing of images Signed-off-by: Evan Lezar --- .nvidia-ci.yml | 62 -------------------------------------------------- 1 file changed, 62 deletions(-) diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml index 4808b53ec..c42fc22c7 100644 --- a/.nvidia-ci.yml +++ b/.nvidia-ci.yml @@ -46,7 +46,6 @@ stages: - pull - scan - release - - sign .pipeline-trigger-rules: rules: @@ -266,64 +265,3 @@ release-images-dummy: rules: - if: $CI_COMMIT_TAG == null || $CI_COMMIT_TAG == "" -# .sign-images forms the base of the jobs which sign images in the NGC registry. -.sign-images: - stage: sign - image: ubuntu:latest - parallel: - matrix: - - TAG_SUFFIX: ["", "-packaging"] - variables: - IMAGE_NAME: "${NGC_REGISTRY_IMAGE}" - IMAGE_TAG: "${CI_COMMIT_TAG}" - NGC_CLI: "ngc-cli/ngc" - before_script: - - !reference [.ngccli-setup, before_script] - script: - - | - # We ensure that the IMAGE_NAME and IMAGE_TAG is set - echo Image Name: ${IMAGE_NAME} && [[ -n "${IMAGE_NAME}" ]] || exit 1 - echo Image Tag: ${IMAGE_TAG} && [[ -n "${IMAGE_TAG}" ]] || exit 1 - - export IMAGE=${IMAGE_NAME}:${IMAGE_TAG}${TAG_SUFFIX} - echo "Signing the image ${IMAGE}" - ${NGC_CLI} registry image publish --source ${IMAGE} ${IMAGE} --public --discoverable --allow-guest --sign --org nvidia - -# Define the external image signing steps for NGC -# Download the ngc cli binary for use in the sign steps -.ngccli-setup: - before_script: - - apt-get update && apt-get install -y curl unzip jq - - | - if [ -z "${NGCCLI_VERSION}" ]; then - NGC_VERSION_URL="https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions" - # Extract the latest version from the JSON data using jq - export NGCCLI_VERSION=$(curl -s $NGC_VERSION_URL | jq -r '.recipe.latestVersionIdStr') - fi - echo "NGCCLI_VERSION ${NGCCLI_VERSION}" - - curl -sSLo ngccli_linux.zip https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/${NGCCLI_VERSION}/files/ngccli_linux.zip - - unzip ngccli_linux.zip - - chmod u+x ngc-cli/ngc - -sign-ngc-images: - extends: - - .sign-images - needs: - - release-images-to-ngc - rules: - - if: $CI_COMMIT_TAG - variables: - NGC_CLI_API_KEY: "${NGC_REGISTRY_TOKEN}" - retry: - max: 2 - -sign-images-dummy: - extends: - - .sign-images - needs: - - release-images-dummy - variables: - NGC_CLI: "echo [DUMMY] ngc-cli/ngc" - IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}" - rules: - - if: $CI_COMMIT_TAG == null || $CI_COMMIT_TAG == "" From 051fe94e32cc3fe9a2d1a60c4a76cd82246358f9 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 9 Jul 2025 11:27:52 +0200 Subject: [PATCH 3/3] [no-relnote] Add CI job to trigger image publishing Signed-off-by: Evan Lezar --- .nvidia-ci.yml | 59 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml index c42fc22c7..1bb33ac1a 100644 --- a/.nvidia-ci.yml +++ b/.nvidia-ci.yml @@ -46,6 +46,7 @@ stages: - pull - scan - release + - ngc-publish .pipeline-trigger-rules: rules: @@ -231,37 +232,59 @@ push-images-to-staging: OUT_IMAGE_NAME: "${NGC_STAGING_REGISTRY}/container-toolkit" OUT_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}" -.release-images: - extends: - - .copy-images - stage: release +.publish-images: + stage: ngc-publish needs: - scan-images - push-images-to-staging + image: + name: "${CNT_NGC_PUBLISH_IMAGE}" + pull_policy: always variables: - IN_REGISTRY: "${CI_REGISTRY}" - IN_REGISTRY_USER: "${CI_REGISTRY_USER}" - IN_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}" - IN_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit" - IN_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}" - OUT_REGISTRY: "${NGC_REGISTRY}" - OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}" - OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}" - OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}" + variables: + GITLAB_ACCESS_TOKEN: "${CNT_GITLAB_TOKEN}" + OUT_IMAGE_TAG: "${CI_COMMIT_TAG}" -release-images-to-ngc: + VERSION_FILE: "build-info-${CI_PIPELINE_ID}.txt" + PROJECT_NAME: "nvidia-container-toolkit" + before_script: + - | + if [ -z "$NGC_PUBLISHING_PROJECT_PATH" ]; then + echo "NGC_PUBLISHING_PROJECT_PATH not set" + exit 1 + else + echo "publishing to ${NGC_PUBLISHING_PROJECT_PATH}" + fi + + rm -f ${VERSION_FILE} + echo "${IN_IMAGE_TAG} ${OUT_IMAGE_TAG}" >> ${VERSION_FILE} + echo "${IN_IMAGE_TAG}-packaging ${OUT_IMAGE_TAG}-packaging" >> ${VERSION_FILE}" + cat ${VERSION_FILE} + script: + - cnt-ngc-publish render --project-name "${PROJECT_NAME}" --versions-file "${VERSION_FILE}" --output "${PROJECT_NAME}".yaml + - cnt-ngc-publish merge-request --files "${PROJECT_NAME}.yaml" + artifacts: + paths: + - "${VERSION_FILE}" + - "${PROJECT_NAME}.yaml" + + +publish-images-to-ngc: extends: - - .release-images + - .publish-images rules: - if: $CI_COMMIT_TAG -release-images-dummy: +# We create a dummy MR that exercises the publishing logic. +# TODO: This MR should be closed automatically. +publish-images-dummy: extends: - - .release-images + - .publish-images variables: - REGCTL: "echo [DUMMY] regctl" + NGC_PUBLISHING_PROJECT_PATH: dl/container-dev/ngc-automation + OUT_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}" rules: - if: $CI_COMMIT_TAG == null || $CI_COMMIT_TAG == ""