Merge branch 'RESTAPI-868-investigate-how-to-expose-images-using-gith…

…ub-container-registry' into 'master' Restapi 868 investigate how to expose images using github container registry See merge request firecrest/firecrest!243
eth-cscs · Oct 9, 2023 · 12c0c2f · 12c0c2f
2 parents b7f1d26 + 4eccac5
commit 12c0c2f
Show file tree

Hide file tree

Showing 11 changed files with 61 additions and 36 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -233,7 +233,7 @@ tag_release:
             echo "Status of deleting the tag $code_tag: $status_del"
 
             if [ "$status_del" != "204" ]; then
-              
+
               # if it is deleted, then a new one with the same tag name has to be created
               echo "Tag $code_tag couldn't been deleted. Exiting"
               echo "Result: $res_del"
@@ -261,12 +261,21 @@ tag_release:
           fi
 
           for tag in $env_tags; do
+            if [ "$code_tag" == "prod" && "$tag" != "prod" ]; then
+              #  if code_tag indicates prod and tag is 'v1...'
+              GH_USER = $(vault kv get -field=ghcr_pat_user firecrest/prod/)"
+              GH_PAT = $(vault kv get -field=ghcr_pat_token firecrest/prod/)"
+              crane auth login ghcr.io --username $GH_USER --password $GH_PAT
+            fi
             images="certificator compute reservations status storage tasks utilities";
             for img in $images; do
-                jfrog rt copy --flat=true --url="https://${CI_REGISTRY}/artifactory" --user="${CI_REGISTRY_USER}" --password="${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY_GROUP}/$img/${CI_COMMIT_SHORT_SHA}/" "${CI_REGISTRY_GROUP}/$img/$tag/";
+              jfrog rt copy --flat=true --url="https://${CI_REGISTRY}/artifactory" --user="${CI_REGISTRY_USER}" --password="${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY_GROUP}/$img/${CI_COMMIT_SHORT_SHA}/" "${CI_REGISTRY_GROUP}/$img/$tag/";
+              if [ "$code_tag" == "prod" && "$tag" != "prod" && "$img" != "reservations" ]; then
+                # if code_tag indicates prod and tag is 'v1...', also publish to GitHub
+                crane copy ${CI_REGISTRY_PREFIX}/${img}:${tag} ghcr.io/eth-cscs/firecrest/f7t-${img}:${tag}
+              fi
             done
           done
-
         else
           echo "The tag does not belong to master the branch";
           exit 0
@@ -290,17 +299,17 @@ deploy_tds:
     - export VAULT_ADDR="https://vault.svc.cscs.ch"
     - export VAULT_TOKEN=$CI_VAULT_TOKEN
     - CI_K8S_TOKEN="$(vault kv get -field=firecrest-cicd-secret firecrest/dev)"
-    
+
     - cd deploy/k8s
     - |
       echo -e "registry: ${CI_REGISTRY_PREFIX}" > values-tds.yaml
       echo -e "firecrest_version: ${CI_COMMIT_TAG}" >> values-tds.yaml
-      echo -e "namespace: ${CI_NAMESPACE_TDS}" >> values-tds.yaml 
+      echo -e "namespace: ${CI_NAMESPACE_TDS}" >> values-tds.yaml
       echo -e "registry_secret_creds: regcred" >> values-tds.yaml
       echo -e "tag: tds" >> values-tds.yaml
 
       vault kv get -field=values_common firecrest/tds >> values-tds.yaml
-      
+
       for app in config compute jaeger reservations status storage tasks utilities ; do
         helm uninstall -n ${CI_NAMESPACE_TDS} --kube-token=${CI_K8S_TOKEN} "$app" || true
         vault kv get -field=values_${app} firecrest/tds >> values-tds.yaml 2>/dev/null || true
@@ -309,12 +318,12 @@ deploy_tds:
       done
 
       echo -e "firecrest_version: ${CI_COMMIT_TAG}" > values-tds-apps.yaml
-      echo -e "namespace: ${CI_NAMESPACE_TDS_APPS}" >> values-tds-apps.yaml 
+      echo -e "namespace: ${CI_NAMESPACE_TDS_APPS}" >> values-tds-apps.yaml
       vault kv get -field=values_common firecrest/tds >> values-tds-apps.yaml
-      
+
       helm uninstall -n ${CI_NAMESPACE_TDS_APPS} --kube-token=${CI_K8S_TOKEN} openapi || true
       helm install  --wait --wait-for-jobs --timeout 180s -n ${CI_NAMESPACE_TDS_APPS} -f values-tds-apps.yaml --kube-token=${CI_K8S_TOKEN} openapi openapi;
-      
+
 
   when: on_success
 

diff --git a/deploy/docker/base/Dockerfile b/deploy/docker/base/Dockerfile
@@ -6,7 +6,7 @@
 ##
 FROM python:3.8.12-slim
 
-RUN pip3 install --upgrade pip
+LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest
 
 ADD deploy/docker/base/requirements.txt base/requirements.txt
-RUN pip3 install -r base/requirements.txt
+RUN pip3 install --upgrade pip && pip3 install -r base/requirements.txt
diff --git a/deploy/docker/certificator/Dockerfile b/deploy/docker/certificator/Dockerfile
@@ -7,8 +7,9 @@
 ARG BASE_IMAGE=f7t-base
 FROM $BASE_IMAGE
 
-RUN apt-get update
-RUN apt-get -y install openssh-client
+LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest
+
+RUN apt-get update && apt-get -y install openssh-client
 
 ADD deploy/docker/base/requirements.txt base/requirements.txt
 ADD deploy/docker/certificator/requirements.txt deps/requirements.txt

diff --git a/deploy/docker/ci-util/Dockerfile b/deploy/docker/ci-util/Dockerfile
@@ -7,22 +7,26 @@
 ARG BASE_IMAGE=ubuntu
 FROM $BASE_IMAGE
 
+ARG CRANE_ARCH=x86_64
+ARG CRANE_VERSION="v0.16.1"
 ARG HELM_VERSION=3.9.0
+ARG VAULT_VERSION=1.14.3
 
-RUN apt update; apt-get install -y curl gnupg jq lsb-release software-properties-common; \
-      curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -;  \
-      apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"; \
-      apt update; \
-      apt -y upgrade; \
-      apt install -y vault; \
-      apt-get install --reinstall -y vault;
-
-RUN curl -s -O https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz; \
-      tar -xvf helm-v${HELM_VERSION}-linux-amd64.tar.gz; \
-      mv linux-amd64/helm /usr/bin/ ; \
-      curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"; \
-      chmod +x kubectl; \
-      mv kubectl /usr/bin/; \
-      curl -fL https://getcli.jfrog.io | sh ; \
-      mv jfrog /usr/bin
+RUN apt update && apt-get install -y curl jq unzip && \
+      curl -s -O https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip && \
+      unzip vault_${VAULT_VERSION}_linux_amd64.zip && \
+      mv vault /usr/bin && \
+      rm vault_${VAULT_VERSION}_linux_amd64.zip && \
+      curl -s -O https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \
+      tar -xvf helm-v${HELM_VERSION}-linux-amd64.tar.gz && \
+      mv linux-amd64/helm /usr/bin/ && \
+      rm -fr linux-amd64 helm-v${HELM_VERSION}-linux-amd64.tar.gz && \
+      curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \
+      chmod +x kubectl && \
+      mv kubectl /usr/bin/  &&  \
+      curl -fL https://getcli.jfrog.io | sh  && \
+      mv jfrog /usr/bin &&  \
+      curl -sL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_${CRANE_ARCH}.tar.gz" > go-containerregistry.tar.gz && \
+      tar -zxvf go-containerregistry.tar.gz -C /usr/bin/ crane && \
+      rm go-containerregistry.tar.gz
 
diff --git a/deploy/docker/compute/Dockerfile b/deploy/docker/compute/Dockerfile
@@ -7,6 +7,8 @@
 ARG BASE_IMAGE=f7t-base
 from $BASE_IMAGE
 
+LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest
+
 ADD deploy/docker/compute/requirements.txt deps/requirements.txt
 RUN pip3 install -r deps/requirements.txt
 

diff --git a/deploy/docker/reservations/Dockerfile b/deploy/docker/reservations/Dockerfile
@@ -1,6 +1,8 @@
 ARG BASE_IMAGE=f7t-base
 from $BASE_IMAGE
 
+LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest
+
 ADD deploy/docker/reservations/requirements.txt deps/requirements.txt
 RUN pip3 install -r deps/requirements.txt
 

diff --git a/deploy/docker/status/Dockerfile b/deploy/docker/status/Dockerfile
@@ -5,7 +5,9 @@
 ##  SPDX-License-Identifier: BSD-3-Clause
 ##
 ARG BASE_IMAGE=f7t-base
-from $BASE_IMAGE
+FROM $BASE_IMAGE
+
+LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest
 
 ADD deploy/docker/status/requirements.txt deps/requirements.txt
 RUN pip3 install -r deps/requirements.txt

diff --git a/deploy/docker/storage/Dockerfile b/deploy/docker/storage/Dockerfile
@@ -5,10 +5,11 @@
 ##  SPDX-License-Identifier: BSD-3-Clause
 ##
 ARG BASE_IMAGE=f7t-base
-from $BASE_IMAGE
+FROM $BASE_IMAGE
 
-RUN apt-get update
-RUN apt-get -y install gcc
+LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest
+
+RUN apt-get update && apt-get -y install gcc
 
 ADD deploy/docker/storage/requirements.txt deps/requirements.txt
 RUN pip3 install -r deps/requirements.txt

diff --git a/deploy/docker/tasks/Dockerfile b/deploy/docker/tasks/Dockerfile
@@ -5,7 +5,9 @@
 ##  SPDX-License-Identifier: BSD-3-Clause
 ##
 ARG BASE_IMAGE=f7t-base
-from $BASE_IMAGE
+FROM $BASE_IMAGE
+
+LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest
 
 ADD deploy/docker/tasks/requirements.txt deps/requirements.txt
 RUN pip3 install -r deps/requirements.txt

diff --git a/deploy/docker/utilities/Dockerfile b/deploy/docker/utilities/Dockerfile
@@ -5,7 +5,9 @@
 ##  SPDX-License-Identifier: BSD-3-Clause
 ##
 ARG BASE_IMAGE=f7t-base
-from $BASE_IMAGE
+FROM $BASE_IMAGE
+
+LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest
 
 ADD deploy/docker/utilities/requirements.txt deps/requirements.txt
 RUN pip3 install -r deps/requirements.txt

diff --git a/src/tests/template_client/Dockerfile b/src/tests/template_client/Dockerfile
@@ -8,7 +8,7 @@ FROM python:3.8.12-slim
 
 RUN pip3 install --upgrade pip
 
-RUN pip3 install markupsafe==2.1.2 Flask==2.0.0 itsdangerous==2.0.1 flask-WTF flask-bootstrap flask-oidc flask_sslify requests==2.22.0
+RUN pip3 install markupsafe==2.1.2 Flask==2.0.0 itsdangerous==2.0.1 flask-WTF flask-bootstrap flask-oidc flask_sslify requests==2.22.0 Werkzeug==2.2
 
 ADD ./ app