diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 099d1d25..f8becf52 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -233,7 +233,7 @@ tag_release: echo "Status of deleting the tag $code_tag: $status_del" if [ "$status_del" != "204" ]; then - + # if it is deleted, then a new one with the same tag name has to be created echo "Tag $code_tag couldn't been deleted. Exiting" echo "Result: $res_del" @@ -261,12 +261,21 @@ tag_release: fi for tag in $env_tags; do + if [ "$code_tag" == "prod" && "$tag" != "prod" ]; then + # if code_tag indicates prod and tag is 'v1...' + GH_USER = $(vault kv get -field=ghcr_pat_user firecrest/prod/)" + GH_PAT = $(vault kv get -field=ghcr_pat_token firecrest/prod/)" + crane auth login ghcr.io --username $GH_USER --password $GH_PAT + fi images="certificator compute reservations status storage tasks utilities"; for img in $images; do - jfrog rt copy --flat=true --url="https://${CI_REGISTRY}/artifactory" --user="${CI_REGISTRY_USER}" --password="${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY_GROUP}/$img/${CI_COMMIT_SHORT_SHA}/" "${CI_REGISTRY_GROUP}/$img/$tag/"; + jfrog rt copy --flat=true --url="https://${CI_REGISTRY}/artifactory" --user="${CI_REGISTRY_USER}" --password="${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY_GROUP}/$img/${CI_COMMIT_SHORT_SHA}/" "${CI_REGISTRY_GROUP}/$img/$tag/"; + if [ "$code_tag" == "prod" && "$tag" != "prod" && "$img" != "reservations" ]; then + # if code_tag indicates prod and tag is 'v1...', also publish to GitHub + crane copy ${CI_REGISTRY_PREFIX}/${img}:${tag} ghcr.io/eth-cscs/firecrest/f7t-${img}:${tag} + fi done done - else echo "The tag does not belong to master the branch"; exit 0 @@ -290,17 +299,17 @@ deploy_tds: - export VAULT_ADDR="https://vault.svc.cscs.ch" - export VAULT_TOKEN=$CI_VAULT_TOKEN - CI_K8S_TOKEN="$(vault kv get -field=firecrest-cicd-secret firecrest/dev)" - + - cd deploy/k8s - | echo -e "registry: ${CI_REGISTRY_PREFIX}" > values-tds.yaml echo -e "firecrest_version: ${CI_COMMIT_TAG}" >> values-tds.yaml - echo -e "namespace: ${CI_NAMESPACE_TDS}" >> values-tds.yaml + echo -e "namespace: ${CI_NAMESPACE_TDS}" >> values-tds.yaml echo -e "registry_secret_creds: regcred" >> values-tds.yaml echo -e "tag: tds" >> values-tds.yaml vault kv get -field=values_common firecrest/tds >> values-tds.yaml - + for app in config compute jaeger reservations status storage tasks utilities ; do helm uninstall -n ${CI_NAMESPACE_TDS} --kube-token=${CI_K8S_TOKEN} "$app" || true vault kv get -field=values_${app} firecrest/tds >> values-tds.yaml 2>/dev/null || true @@ -309,12 +318,12 @@ deploy_tds: done echo -e "firecrest_version: ${CI_COMMIT_TAG}" > values-tds-apps.yaml - echo -e "namespace: ${CI_NAMESPACE_TDS_APPS}" >> values-tds-apps.yaml + echo -e "namespace: ${CI_NAMESPACE_TDS_APPS}" >> values-tds-apps.yaml vault kv get -field=values_common firecrest/tds >> values-tds-apps.yaml - + helm uninstall -n ${CI_NAMESPACE_TDS_APPS} --kube-token=${CI_K8S_TOKEN} openapi || true helm install --wait --wait-for-jobs --timeout 180s -n ${CI_NAMESPACE_TDS_APPS} -f values-tds-apps.yaml --kube-token=${CI_K8S_TOKEN} openapi openapi; - + when: on_success diff --git a/deploy/docker/base/Dockerfile b/deploy/docker/base/Dockerfile index c0f411be..41f78644 100644 --- a/deploy/docker/base/Dockerfile +++ b/deploy/docker/base/Dockerfile @@ -6,7 +6,7 @@ ## FROM python:3.8.12-slim -RUN pip3 install --upgrade pip +LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest ADD deploy/docker/base/requirements.txt base/requirements.txt -RUN pip3 install -r base/requirements.txt +RUN pip3 install --upgrade pip && pip3 install -r base/requirements.txt diff --git a/deploy/docker/certificator/Dockerfile b/deploy/docker/certificator/Dockerfile index cc03b093..022ca97c 100644 --- a/deploy/docker/certificator/Dockerfile +++ b/deploy/docker/certificator/Dockerfile @@ -7,8 +7,9 @@ ARG BASE_IMAGE=f7t-base FROM $BASE_IMAGE -RUN apt-get update -RUN apt-get -y install openssh-client +LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest + +RUN apt-get update && apt-get -y install openssh-client ADD deploy/docker/base/requirements.txt base/requirements.txt ADD deploy/docker/certificator/requirements.txt deps/requirements.txt diff --git a/deploy/docker/ci-util/Dockerfile b/deploy/docker/ci-util/Dockerfile index 167b9bc5..9b7acb33 100644 --- a/deploy/docker/ci-util/Dockerfile +++ b/deploy/docker/ci-util/Dockerfile @@ -7,22 +7,26 @@ ARG BASE_IMAGE=ubuntu FROM $BASE_IMAGE +ARG CRANE_ARCH=x86_64 +ARG CRANE_VERSION="v0.16.1" ARG HELM_VERSION=3.9.0 +ARG VAULT_VERSION=1.14.3 -RUN apt update; apt-get install -y curl gnupg jq lsb-release software-properties-common; \ - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -; \ - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"; \ - apt update; \ - apt -y upgrade; \ - apt install -y vault; \ - apt-get install --reinstall -y vault; - -RUN curl -s -O https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz; \ - tar -xvf helm-v${HELM_VERSION}-linux-amd64.tar.gz; \ - mv linux-amd64/helm /usr/bin/ ; \ - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"; \ - chmod +x kubectl; \ - mv kubectl /usr/bin/; \ - curl -fL https://getcli.jfrog.io | sh ; \ - mv jfrog /usr/bin +RUN apt update && apt-get install -y curl jq unzip && \ + curl -s -O https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip && \ + unzip vault_${VAULT_VERSION}_linux_amd64.zip && \ + mv vault /usr/bin && \ + rm vault_${VAULT_VERSION}_linux_amd64.zip && \ + curl -s -O https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ + tar -xvf helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ + mv linux-amd64/helm /usr/bin/ && \ + rm -fr linux-amd64 helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \ + chmod +x kubectl && \ + mv kubectl /usr/bin/ && \ + curl -fL https://getcli.jfrog.io | sh && \ + mv jfrog /usr/bin && \ + curl -sL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_${CRANE_ARCH}.tar.gz" > go-containerregistry.tar.gz && \ + tar -zxvf go-containerregistry.tar.gz -C /usr/bin/ crane && \ + rm go-containerregistry.tar.gz diff --git a/deploy/docker/compute/Dockerfile b/deploy/docker/compute/Dockerfile index c8b79ab5..0ef06590 100644 --- a/deploy/docker/compute/Dockerfile +++ b/deploy/docker/compute/Dockerfile @@ -7,6 +7,8 @@ ARG BASE_IMAGE=f7t-base from $BASE_IMAGE +LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest + ADD deploy/docker/compute/requirements.txt deps/requirements.txt RUN pip3 install -r deps/requirements.txt diff --git a/deploy/docker/reservations/Dockerfile b/deploy/docker/reservations/Dockerfile index 148131ff..ea676d54 100644 --- a/deploy/docker/reservations/Dockerfile +++ b/deploy/docker/reservations/Dockerfile @@ -1,6 +1,8 @@ ARG BASE_IMAGE=f7t-base from $BASE_IMAGE +LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest + ADD deploy/docker/reservations/requirements.txt deps/requirements.txt RUN pip3 install -r deps/requirements.txt diff --git a/deploy/docker/status/Dockerfile b/deploy/docker/status/Dockerfile index f1a71359..0799ffcd 100644 --- a/deploy/docker/status/Dockerfile +++ b/deploy/docker/status/Dockerfile @@ -5,7 +5,9 @@ ## SPDX-License-Identifier: BSD-3-Clause ## ARG BASE_IMAGE=f7t-base -from $BASE_IMAGE +FROM $BASE_IMAGE + +LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest ADD deploy/docker/status/requirements.txt deps/requirements.txt RUN pip3 install -r deps/requirements.txt diff --git a/deploy/docker/storage/Dockerfile b/deploy/docker/storage/Dockerfile index 67edbfce..36997c77 100644 --- a/deploy/docker/storage/Dockerfile +++ b/deploy/docker/storage/Dockerfile @@ -5,10 +5,11 @@ ## SPDX-License-Identifier: BSD-3-Clause ## ARG BASE_IMAGE=f7t-base -from $BASE_IMAGE +FROM $BASE_IMAGE -RUN apt-get update -RUN apt-get -y install gcc +LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest + +RUN apt-get update && apt-get -y install gcc ADD deploy/docker/storage/requirements.txt deps/requirements.txt RUN pip3 install -r deps/requirements.txt diff --git a/deploy/docker/tasks/Dockerfile b/deploy/docker/tasks/Dockerfile index 1f0c6d39..73df2151 100644 --- a/deploy/docker/tasks/Dockerfile +++ b/deploy/docker/tasks/Dockerfile @@ -5,7 +5,9 @@ ## SPDX-License-Identifier: BSD-3-Clause ## ARG BASE_IMAGE=f7t-base -from $BASE_IMAGE +FROM $BASE_IMAGE + +LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest ADD deploy/docker/tasks/requirements.txt deps/requirements.txt RUN pip3 install -r deps/requirements.txt diff --git a/deploy/docker/utilities/Dockerfile b/deploy/docker/utilities/Dockerfile index c38518d0..e088ddc1 100644 --- a/deploy/docker/utilities/Dockerfile +++ b/deploy/docker/utilities/Dockerfile @@ -5,7 +5,9 @@ ## SPDX-License-Identifier: BSD-3-Clause ## ARG BASE_IMAGE=f7t-base -from $BASE_IMAGE +FROM $BASE_IMAGE + +LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest ADD deploy/docker/utilities/requirements.txt deps/requirements.txt RUN pip3 install -r deps/requirements.txt diff --git a/src/tests/template_client/Dockerfile b/src/tests/template_client/Dockerfile index ded9e4eb..add0f75f 100644 --- a/src/tests/template_client/Dockerfile +++ b/src/tests/template_client/Dockerfile @@ -8,7 +8,7 @@ FROM python:3.8.12-slim RUN pip3 install --upgrade pip -RUN pip3 install markupsafe==2.1.2 Flask==2.0.0 itsdangerous==2.0.1 flask-WTF flask-bootstrap flask-oidc flask_sslify requests==2.22.0 +RUN pip3 install markupsafe==2.1.2 Flask==2.0.0 itsdangerous==2.0.1 flask-WTF flask-bootstrap flask-oidc flask_sslify requests==2.22.0 Werkzeug==2.2 ADD ./ app