From 71bb50c197f746fa9dfb1257ffe0a43e288944c9 Mon Sep 17 00:00:00 2001
From: niladic <git@nil.choron.cc>
Date: Wed, 2 Jun 2021 21:29:45 +0200
Subject: [PATCH 1/7] Add kubernetes config

---
 kubernetes/aplus-app.yaml           | 127 ++++++++++++++++++++++++++++
 kubernetes/aplus-db-backup-job.yaml | 112 ++++++++++++++++++++++++
 kubernetes/aplus-db.yaml            |  65 ++++++++++++++
 3 files changed, 304 insertions(+)
 create mode 100644 kubernetes/aplus-app.yaml
 create mode 100644 kubernetes/aplus-db-backup-job.yaml
 create mode 100644 kubernetes/aplus-db.yaml

diff --git a/kubernetes/aplus-app.yaml b/kubernetes/aplus-app.yaml
new file mode 100644
index 000000000..6f696ecb9
--- /dev/null
+++ b/kubernetes/aplus-app.yaml
@@ -0,0 +1,127 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: aplus-app
+  labels:
+    app: aplus
+spec:
+  type: NodePort
+  ports:
+  - port: 80
+    targetPort: 9000
+  selector:     # Should be a Pod selector
+    app: aplus
+    tier: frontend
+  externalIPs:
+    - 54.38.254.141
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: aplus-files-pvc
+  labels:
+    app: aplus
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: csi-cinder-classic
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: aplus-app-deployment
+  labels:
+    app: aplus
+spec:
+  replicas: 1
+  selector:       # Should match .spec.template
+    matchLabels:
+      app: aplus
+      tier: frontend
+  strategy:
+    type: Recreate  # here RollingUpdate is not well handled by app sql queries
+  template:
+    metadata:
+      labels:
+        app: aplus
+        tier: frontend
+    spec:
+      containers:
+      - image: administrationplus.azurecr.io/aplus:master-20210526-084710z-f9c361b67edc2536f89e0cf1d1190a7467b6dba7
+        name: aplus-app
+        env:
+        - name: APP_HOST
+          value: aplus.beta.gouv.fr
+        - name: APP_HTTPS
+          value: "false"
+        - name: APPLICATION_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: aplus-application-secret
+              key: APPLICATION_SECRET
+        - name: DATABASE_URL
+          valueFrom:
+            secretKeyRef:
+              name: aplus-db-app-secret
+              key: DATABASE_URL
+        - name: EVOLUTIONS_AUTOAPPLY
+          value: "true"
+        - name: FEATURE_AUTO_ADD_EXPERT
+          value: "true"
+        - name: FEATURE_SEND_APPLICATIONS_ANYWHERE
+          value: "true"
+        - name: FEATURE_SMS_MANDAT
+          value: "false"
+        - name: FEATURE_WEEKLY_EMAILS
+          value: "true"
+        - name: FILES_EXPIRATION_IN_DAYS
+          value: "15"
+        - name: FILES_PATH
+          value: "/app/files"
+        - name: GROUPS_WHICH_CANNOT_HAVE_INSTRUCTORS
+          value: "f32d20bf-a201-4875-9c69-16a5a4ad2f9c,ecb83438-b78b-4fbc-b0cd-880ce55562df"
+        - name: MAIL_HOST
+          valueFrom:
+            secretKeyRef:
+              name: aplus-email-secret
+              key: MAIL_HOST
+        - name: MAIL_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: aplus-email-secret
+              key: MAIL_PASSWORD
+        - name: MAIL_PORT
+          valueFrom:
+            secretKeyRef:
+              name: aplus-email-secret
+              key: MAIL_PORT
+        - name: MAIL_USER
+          valueFrom:
+            secretKeyRef:
+              name: aplus-email-secret
+              key: MAIL_USER
+        - name: NOTIFICATION_EMAIL_BLACKLIST
+          value: "daniel.balmy@beta.gouv.fr"
+        - name: SMS_USE_LIVE_API
+          value: "false"
+        - name: WEEKLY_EMAILS_DAY_OF_WEEK
+          value: tuesday
+        - name: WEEKLY_EMAILS_HOUR_OF_DAY
+          value: "10"
+        - name: WEEKLY_EMAILS_MAX_NUMBER
+          value: "1000"
+
+        ports:
+        - containerPort: 9000
+        volumeMounts:
+        - name: aplus-files-pv
+          mountPath: /app/files
+      imagePullSecrets:
+      - name: acr-docker-creds
+      volumes:
+      - name: aplus-files-pv
+        persistentVolumeClaim:
+          claimName: aplus-files-pvc
diff --git a/kubernetes/aplus-db-backup-job.yaml b/kubernetes/aplus-db-backup-job.yaml
new file mode 100644
index 000000000..17a2025bc
--- /dev/null
+++ b/kubernetes/aplus-db-backup-job.yaml
@@ -0,0 +1,112 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: aplus-db-backup-script
+data:
+  backup-script: |
+    set -e
+    apt-get update
+    apt-get install -y jq postgresql-client-11
+    pip install python-swiftclient==3.12.0
+    pip install python-keystoneclient==4.2.0
+
+    if [ ! -s /key/key.pub ]
+    then
+      echo "Stopping: no public key"
+      exit 0
+    fi
+    gpg --no-tty --import /key/key.pub
+
+    # Create new backup
+    NOW="$(date +"%Y-%m-%d-%s")"
+    FILENAME="$DATABASE_BACKUP_PREFIX.$NOW.pgdump.gz"
+    pg_dump -Fc $DATABASE_NAME | gzip > $FILENAME
+    gpg --batch --trust-model always --output "${FILENAME}.gpg" --recipient ${RECIPIENT_PUBLIC_KEY_EMAIL} --encrypt ${FILENAME}
+    swift --os-auth-token $AUTH_TOKEN --os-storage-url $STORAGE_URL upload $STORAGE_CONTAINER "${FILENAME}.gpg"
+
+    # Cleanup old backups
+    while read line
+    do
+      date=$(echo "$line" | jq -r '.last_modified')
+      if [ $(date -d "$date" +%s) -le $(date +%s -d "$RETENTION_NUM_OF_DAYS days ago") ]
+      then
+        OLD=$(echo "$line" | jq -r '.name')
+        echo WILL DELETE OLD BACKUP $OLD
+        swift --os-auth-token $AUTH_TOKEN --os-storage-url $STORAGE_URL delete $STORAGE_CONTAINER "$OLD"
+      fi
+    done < <(swift --os-auth-token $AUTH_TOKEN --os-storage-url $STORAGE_URL list --json $STORAGE_CONTAINER | jq -c '.[]')
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: aplus-db-backup-job
+spec:
+  schedule: "0 * * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: aplus-db-backup
+            image: python:3
+            env:
+            - name: AUTH_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: ovh-storage-creds
+                  key: AUTH_TOKEN
+            - name: DATABASE_BACKUP_PREFIX
+              value: "aplus-db"
+            - name: DATABASE_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: aplus-db-app-secret
+                  key: DATABASE_NAME
+            - name: PGHOST
+              valueFrom:
+                secretKeyRef:
+                  name: aplus-db-app-secret
+                  key: DATABASE_HOST
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: aplus-db-postgres-secret
+                  key: POSTGRES_PASSWORD
+            - name: PGUSER
+              value: postgres
+            - name: RECIPIENT_PUBLIC_KEY_EMAIL
+              valueFrom:
+                secretKeyRef:
+                  name: aplus-backup-pub-key
+                  key: RECIPIENT_PUBLIC_KEY_EMAIL
+            - name: RETENTION_NUM_OF_DAYS
+              value: "2"
+            - name: STORAGE_CONTAINER
+              value: "aplus-test-storage"
+            - name: STORAGE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: ovh-storage-creds
+                  key: STORAGE_URL
+            volumeMounts:
+            - name: script
+              mountPath: "/script"
+            - name: public-key
+              mountPath: "/key"
+            command: ["bash",  "/script/backup.sh"]
+          volumes:
+          - name: script
+            configMap:
+              name: aplus-db-backup-script
+              items:
+              - key: backup-script
+                path: "backup.sh"
+          - name: public-key
+            secret:
+              secretName: aplus-backup-pub-key
+              items:
+              - key: RECIPIENT_PUBLIC_KEY
+                path: "key.pub"
+          restartPolicy: Never
+      backoffLimit: 4
+      # ttlSecondsAfterFinished is not activated on the cluster
diff --git a/kubernetes/aplus-db.yaml b/kubernetes/aplus-db.yaml
new file mode 100644
index 000000000..d1af14ad7
--- /dev/null
+++ b/kubernetes/aplus-db.yaml
@@ -0,0 +1,65 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: aplus-db
+  labels:
+    app: aplus
+spec:
+  clusterIP: None
+  ports:
+  - port: 5432
+  selector:
+    app: aplus
+    tier: postgres
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: aplus-db-pvc
+  labels:
+    app: aplus
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: csi-cinder-classic
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: aplus-db-statefulset
+  labels:
+    app: aplus
+spec:
+  selector:
+    matchLabels:
+      app: aplus
+      tier: postgres
+  serviceName: aplus-db
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: aplus
+        tier: postgres
+    spec:
+      containers:
+        - name: postgres-database
+          image: postgres:11
+          volumeMounts:
+            - name: aplus-db-pv
+              mountPath: /var/lib/postgresql/data
+          env:
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: aplus-db-postgres-secret
+                  key: DATABASE_URL
+            - name: PGDATA
+              value: /var/lib/postgresql/data/pgdata
+      volumes:
+        - name: aplus-db-pv
+          persistentVolumeClaim:
+            claimName: aplus-db-pvc

From ae8dbb459b4421b4e19c97caeaa758833258321c Mon Sep 17 00:00:00 2001
From: niladic <git@nil.choron.cc>
Date: Tue, 15 Jun 2021 08:49:04 +0200
Subject: [PATCH 2/7] Fix incorrect env var

---
 kubernetes/aplus-db.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kubernetes/aplus-db.yaml b/kubernetes/aplus-db.yaml
index d1af14ad7..e5db5fbfa 100644
--- a/kubernetes/aplus-db.yaml
+++ b/kubernetes/aplus-db.yaml
@@ -56,7 +56,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: aplus-db-postgres-secret
-                  key: DATABASE_URL
+                  key: POSTGRES_PASSWORD
             - name: PGDATA
               value: /var/lib/postgresql/data/pgdata
       volumes:

From f4f057867b5fdc99587582aa9d9d3732a9b0fa8c Mon Sep 17 00:00:00 2001
From: niladic <git@nil.choron.cc>
Date: Tue, 15 Jun 2021 08:49:50 +0200
Subject: [PATCH 3/7] Document beta api usage

---
 kubernetes/aplus-db-backup-job.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/kubernetes/aplus-db-backup-job.yaml b/kubernetes/aplus-db-backup-job.yaml
index 17a2025bc..a8508308a 100644
--- a/kubernetes/aplus-db-backup-job.yaml
+++ b/kubernetes/aplus-db-backup-job.yaml
@@ -36,6 +36,7 @@ data:
       fi
     done < <(swift --os-auth-token $AUTH_TOKEN --os-storage-url $STORAGE_URL list --json $STORAGE_CONTAINER | jq -c '.[]')
 ---
+# TODO: change for `batch/v1` when upgrading to kubernetes v1.21
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:

From ea3d62d8b78a8dd7b46570aef07c5708a3cbd371 Mon Sep 17 00:00:00 2001
From: niladic <git@nil.choron.cc>
Date: Tue, 15 Jun 2021 08:51:24 +0200
Subject: [PATCH 4/7] Fix cronjob

---
 kubernetes/aplus-db-backup-job.yaml | 42 +++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/kubernetes/aplus-db-backup-job.yaml b/kubernetes/aplus-db-backup-job.yaml
index a8508308a..67eda2e72 100644
--- a/kubernetes/aplus-db-backup-job.yaml
+++ b/kubernetes/aplus-db-backup-job.yaml
@@ -7,6 +7,7 @@ data:
     set -e
     apt-get update
     apt-get install -y jq postgresql-client-11
+    pip install python-openstackclient==5.5.0
     pip install python-swiftclient==3.12.0
     pip install python-keystoneclient==4.2.0
 
@@ -17,6 +18,9 @@ data:
     fi
     gpg --no-tty --import /key/key.pub
 
+    # Create token here because they are valid only for a short period of time
+    AUTH_TOKEN=$(openstack token issue -c id -f value)
+
     # Create new backup
     NOW="$(date +"%Y-%m-%d-%s")"
     FILENAME="$DATABASE_BACKUP_PREFIX.$NOW.pgdump.gz"
@@ -36,6 +40,10 @@ data:
       fi
     done < <(swift --os-auth-token $AUTH_TOKEN --os-storage-url $STORAGE_URL list --json $STORAGE_CONTAINER | jq -c '.[]')
 ---
+# Note: The CronJob default history is 3 completed and 1 failed
+#       this means having in general 4 jobs (kubectl get jobs) together with
+#       the corresponding pods (many for the failed job, and mostly one by completed job)
+#
 # TODO: change for `batch/v1` when upgrading to kubernetes v1.21
 apiVersion: batch/v1beta1
 kind: CronJob
@@ -51,11 +59,41 @@ spec:
           - name: aplus-db-backup
             image: python:3
             env:
-            - name: AUTH_TOKEN
+            - name: OS_AUTH_URL
+              valueFrom:
+                secretKeyRef:
+                  name: ovh-storage-creds
+                  key: OS_AUTH_URL
+            - name: OS_IDENTITY_API_VERSION
+              valueFrom:
+                secretKeyRef:
+                  name: ovh-storage-creds
+                  key: OS_IDENTITY_API_VERSION
+            - name: OS_TENANT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: ovh-storage-creds
+                  key: OS_TENANT_ID
+            - name: OS_TENANT_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ovh-storage-creds
+                  key: OS_TENANT_NAME
+            - name: OS_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: ovh-storage-creds
+                  key: OS_USERNAME
+            - name: OS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: ovh-storage-creds
+                  key: OS_PASSWORD
+            - name: OS_REGION_NAME
               valueFrom:
                 secretKeyRef:
                   name: ovh-storage-creds
-                  key: AUTH_TOKEN
+                  key: OS_REGION_NAME
             - name: DATABASE_BACKUP_PREFIX
               value: "aplus-db"
             - name: DATABASE_NAME

From 31d7a2d843135a715722bdd36eabeaa076dcfc28 Mon Sep 17 00:00:00 2001
From: niladic <git@nil.choron.cc>
Date: Wed, 23 Jun 2021 21:01:44 +0200
Subject: [PATCH 5/7] Add ingress, preprod and metabase

---
 kubernetes/aplus-acme.yaml               |  29 +++++
 kubernetes/aplus-app.yaml                |  72 +++++++++--
 kubernetes/aplus-config-preprod.yaml     |   9 ++
 kubernetes/aplus-config.yaml             |   9 ++
 kubernetes/aplus-db-backup-job.yaml      |  31 ++---
 kubernetes/aplus-ingress-preprod.yaml    |  30 +++++
 kubernetes/aplus-ingress.yaml            |  29 +++++
 kubernetes/aplus-restore-job.yaml        | 149 +++++++++++++++++++++++
 kubernetes/ingress-nginx/values.yml      |   9 ++
 kubernetes/metabase-app.yaml             |  46 +++++++
 kubernetes/metabase-db.yaml              |  65 ++++++++++
 kubernetes/metabase-ingress-preprod.yaml |  23 ++++
 kubernetes/metabase-ingress.yaml         |  23 ++++
 13 files changed, 497 insertions(+), 27 deletions(-)
 create mode 100644 kubernetes/aplus-acme.yaml
 create mode 100644 kubernetes/aplus-config-preprod.yaml
 create mode 100644 kubernetes/aplus-config.yaml
 create mode 100644 kubernetes/aplus-ingress-preprod.yaml
 create mode 100644 kubernetes/aplus-ingress.yaml
 create mode 100644 kubernetes/aplus-restore-job.yaml
 create mode 100644 kubernetes/ingress-nginx/values.yml
 create mode 100644 kubernetes/metabase-app.yaml
 create mode 100644 kubernetes/metabase-db.yaml
 create mode 100644 kubernetes/metabase-ingress-preprod.yaml
 create mode 100644 kubernetes/metabase-ingress.yaml

diff --git a/kubernetes/aplus-acme.yaml b/kubernetes/aplus-acme.yaml
new file mode 100644
index 000000000..042a90cc2
--- /dev/null
+++ b/kubernetes/aplus-acme.yaml
@@ -0,0 +1,29 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: letsencrypt-staging
+spec:
+  acme:
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    email: alexandre.grimault@beta.gouv.fr
+    privateKeySecretRef:
+      name: letsencrypt-staging
+    solvers:
+    - http01:
+        ingress:
+          class: nginx
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: letsencrypt-prod
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: alexandre.grimault@beta.gouv.fr
+    privateKeySecretRef:
+      name: letsencrypt-prod
+    solvers:
+    - http01:
+        ingress:
+          class: nginx
diff --git a/kubernetes/aplus-app.yaml b/kubernetes/aplus-app.yaml
index 6f696ecb9..1147b7194 100644
--- a/kubernetes/aplus-app.yaml
+++ b/kubernetes/aplus-app.yaml
@@ -5,15 +5,13 @@ metadata:
   labels:
     app: aplus
 spec:
-  type: NodePort
+  # Note: this is the default type = ClusterIP
   ports:
-  - port: 80
+  - port: 9000
     targetPort: 9000
   selector:     # Should be a Pod selector
     app: aplus
     tier: frontend
-  externalIPs:
-    - 54.38.254.141
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -50,13 +48,16 @@ spec:
         tier: frontend
     spec:
       containers:
-      - image: administrationplus.azurecr.io/aplus:master-20210526-084710z-f9c361b67edc2536f89e0cf1d1190a7467b6dba7
+      - image: betagouv/aplus:0.16.1
         name: aplus-app
         env:
         - name: APP_HOST
-          value: aplus.beta.gouv.fr
+          valueFrom:
+            configMapKeyRef:
+              name: aplus-config
+              key: APP_HOST
         - name: APP_HTTPS
-          value: "false"
+          value: "true"
         - name: APPLICATION_SECRET
           valueFrom:
             secretKeyRef:
@@ -70,15 +71,24 @@ spec:
         - name: EVOLUTIONS_AUTOAPPLY
           value: "true"
         - name: FEATURE_AUTO_ADD_EXPERT
-          value: "true"
+          valueFrom:
+            configMapKeyRef:
+              name: aplus-config
+              key: FEATURE_AUTO_ADD_EXPERT
         - name: FEATURE_SEND_APPLICATIONS_ANYWHERE
           value: "true"
         - name: FEATURE_SMS_MANDAT
-          value: "false"
-        - name: FEATURE_WEEKLY_EMAILS
           value: "true"
+        - name: FEATURE_WEEKLY_EMAILS
+          valueFrom:
+            configMapKeyRef:
+              name: aplus-config
+              key: FEATURE_WEEKLY_EMAILS
         - name: FILES_EXPIRATION_IN_DAYS
           value: "15"
+        - name: FILES_SECOND_INSTANCE_HOST
+          value: "https://aplusprodapp.azurewebsites.net/"
+
         - name: FILES_PATH
           value: "/app/files"
         - name: GROUPS_WHICH_CANNOT_HAVE_INSTRUCTORS
@@ -98,6 +108,8 @@ spec:
             secretKeyRef:
               name: aplus-email-secret
               key: MAIL_PORT
+        - name: MAIL_SSL
+          value: "yes"
         - name: MAIL_USER
           valueFrom:
             secretKeyRef:
@@ -105,8 +117,44 @@ spec:
               key: MAIL_USER
         - name: NOTIFICATION_EMAIL_BLACKLIST
           value: "daniel.balmy@beta.gouv.fr"
+
+        - name: OVH_APPLICATION_KEY
+          valueFrom:
+            secretKeyRef:
+              name: aplus-ovh-sms-secret
+              key: OVH_APPLICATION_KEY
+        - name: OVH_APPLICATION_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: aplus-ovh-sms-secret
+              key: OVH_APPLICATION_SECRET
+        - name: OVH_CONSUMER_KEY
+          valueFrom:
+            secretKeyRef:
+              name: aplus-ovh-sms-secret
+              key: OVH_CONSUMER_KEY
+        - name: OVH_SERVICE_NAME
+          valueFrom:
+            secretKeyRef:
+              name: aplus-ovh-sms-secret
+              key: OVH_SERVICE_NAME
+
+        - name: PERSONAL_DATA_RETENTION_IN_MONTHS
+          valueFrom:
+            configMapKeyRef:
+              name: aplus-config
+              key: PERSONAL_DATA_RETENTION_IN_MONTHS
+
+        - name: SENTRY_DSN
+          valueFrom:
+            secretKeyRef:
+              name: aplus-sentry-secret
+              key: SENTRY_DSN
+        - name: SENTRY_TRACES_SAMPLE_RATE
+          value: "0.01"
+
         - name: SMS_USE_LIVE_API
-          value: "false"
+          value: "true"
         - name: WEEKLY_EMAILS_DAY_OF_WEEK
           value: tuesday
         - name: WEEKLY_EMAILS_HOUR_OF_DAY
@@ -119,8 +167,6 @@ spec:
         volumeMounts:
         - name: aplus-files-pv
           mountPath: /app/files
-      imagePullSecrets:
-      - name: acr-docker-creds
       volumes:
       - name: aplus-files-pv
         persistentVolumeClaim:
diff --git a/kubernetes/aplus-config-preprod.yaml b/kubernetes/aplus-config-preprod.yaml
new file mode 100644
index 000000000..49c086316
--- /dev/null
+++ b/kubernetes/aplus-config-preprod.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: aplus-config
+data:
+  APP_HOST: preprod.aplus.incubateur.net
+  FEATURE_AUTO_ADD_EXPERT: "false"
+  FEATURE_WEEKLY_EMAILS: "false"
+  PERSONAL_DATA_RETENTION_IN_MONTHS: "1"
diff --git a/kubernetes/aplus-config.yaml b/kubernetes/aplus-config.yaml
new file mode 100644
index 000000000..70c062044
--- /dev/null
+++ b/kubernetes/aplus-config.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: aplus-config
+data:
+  APP_HOST: aplus.beta.gouv.fr
+  FEATURE_AUTO_ADD_EXPERT: "true"
+  FEATURE_WEEKLY_EMAILS: "true"
+  PERSONAL_DATA_RETENTION_IN_MONTHS: "3"
diff --git a/kubernetes/aplus-db-backup-job.yaml b/kubernetes/aplus-db-backup-job.yaml
index 67eda2e72..ffa5ba1ea 100644
--- a/kubernetes/aplus-db-backup-job.yaml
+++ b/kubernetes/aplus-db-backup-job.yaml
@@ -22,11 +22,11 @@ data:
     AUTH_TOKEN=$(openstack token issue -c id -f value)
 
     # Create new backup
-    NOW="$(date +"%Y-%m-%d-%s")"
-    FILENAME="$DATABASE_BACKUP_PREFIX.$NOW.pgdump.gz"
-    pg_dump -Fc $DATABASE_NAME | gzip > $FILENAME
-    gpg --batch --trust-model always --output "${FILENAME}.gpg" --recipient ${RECIPIENT_PUBLIC_KEY_EMAIL} --encrypt ${FILENAME}
-    swift --os-auth-token $AUTH_TOKEN --os-storage-url $STORAGE_URL upload $STORAGE_CONTAINER "${FILENAME}.gpg"
+    NOW="$(date +"%Y-%m-%d-%H%M%S")"
+    FILENAME="$DATABASE_BACKUP_PREFIX.$NOW.pgdump.gz.gpg"
+    # Pipe directly into gpg to avoid creating an unencrypted file
+    pg_dump -Fc | gzip | gpg --batch --trust-model always --output "$FILENAME" --recipient "$GPG_RECIPIENT" --encrypt
+    swift --os-auth-token $AUTH_TOKEN --os-storage-url $STORAGE_URL upload $STORAGE_CONTAINER "$FILENAME"
 
     # Cleanup old backups
     while read line
@@ -57,8 +57,9 @@ spec:
         spec:
           containers:
           - name: aplus-db-backup
-            image: python:3
+            image: python:3-buster
             env:
+
             - name: OS_AUTH_URL
               valueFrom:
                 secretKeyRef:
@@ -94,9 +95,8 @@ spec:
                 secretKeyRef:
                   name: ovh-storage-creds
                   key: OS_REGION_NAME
-            - name: DATABASE_BACKUP_PREFIX
-              value: "aplus-db"
-            - name: DATABASE_NAME
+
+            - name: PGDATABASE
               valueFrom:
                 secretKeyRef:
                   name: aplus-db-app-secret
@@ -113,15 +113,18 @@ spec:
                   key: POSTGRES_PASSWORD
             - name: PGUSER
               value: postgres
-            - name: RECIPIENT_PUBLIC_KEY_EMAIL
+
+            - name: DATABASE_BACKUP_PREFIX
+              value: "aplus-db"
+            - name: GPG_RECIPIENT
               valueFrom:
                 secretKeyRef:
                   name: aplus-backup-pub-key
-                  key: RECIPIENT_PUBLIC_KEY_EMAIL
+                  key: GPG_RECIPIENT
             - name: RETENTION_NUM_OF_DAYS
-              value: "2"
+              value: "30"
             - name: STORAGE_CONTAINER
-              value: "aplus-test-storage"
+              value: "aplusprodbackup"
             - name: STORAGE_URL
               valueFrom:
                 secretKeyRef:
@@ -144,7 +147,7 @@ spec:
             secret:
               secretName: aplus-backup-pub-key
               items:
-              - key: RECIPIENT_PUBLIC_KEY
+              - key: GPG_PUBLIC_KEY
                 path: "key.pub"
           restartPolicy: Never
       backoffLimit: 4
diff --git a/kubernetes/aplus-ingress-preprod.yaml b/kubernetes/aplus-ingress-preprod.yaml
new file mode 100644
index 000000000..27a6e272b
--- /dev/null
+++ b/kubernetes/aplus-ingress-preprod.yaml
@@ -0,0 +1,30 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    cert-manager.io/issuer: "letsencrypt-prod"
+    nginx.ingress.kubernetes.io/proxy-body-size: "10m"
+    nginx.ingress.kubernetes.io/ssl-ciphers: "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA"
+  name: aplus-ingress
+spec:
+  tls:
+  - hosts:
+    - preprod.aplus.incubateur.net
+    secretName: preprod-tls
+  defaultBackend:
+    service:
+      name: aplus-app
+      port:
+        number: 9000
+  rules:
+  - host: preprod.aplus.incubateur.net
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: aplus-app
+            port:
+              number: 9000
diff --git a/kubernetes/aplus-ingress.yaml b/kubernetes/aplus-ingress.yaml
new file mode 100644
index 000000000..0c625f597
--- /dev/null
+++ b/kubernetes/aplus-ingress.yaml
@@ -0,0 +1,29 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    nginx.ingress.kubernetes.io/proxy-body-size: "10m"
+    nginx.ingress.kubernetes.io/ssl-ciphers: "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA"
+  name: aplus-ingress
+spec:
+  tls:
+  - hosts:
+    - aplus.beta.gouv.fr
+    secretName: tls-aplus-prod
+  defaultBackend:
+    service:
+      name: aplus-app
+      port:
+        number: 9000
+  rules:
+  - host: aplus.beta.gouv.fr
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: aplus-app
+            port:
+              number: 9000
diff --git a/kubernetes/aplus-restore-job.yaml b/kubernetes/aplus-restore-job.yaml
new file mode 100644
index 000000000..086796320
--- /dev/null
+++ b/kubernetes/aplus-restore-job.yaml
@@ -0,0 +1,149 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: aplus-restore-script
+data:
+  restore-script: |
+    set -e
+    apt-get update
+    apt-get install -y postgresql-client-11
+    pip install python-openstackclient==5.5.0
+    pip install python-swiftclient==3.12.0
+    pip install python-keystoneclient==4.2.0
+
+    if [ ! -s /key/key.pub ]
+    then
+      echo "Stopping: no public key"
+      exit 0
+    fi
+    gpg --no-tty --batch --import /key/key.pub
+    gpg --no-tty --batch --allow-secret-key-import --import /private/key.gpg
+
+    # Create token here because they are valid only for a short period of time
+    AUTH_TOKEN=$(openstack token issue -c id -f value)
+
+    swift --os-auth-token $AUTH_TOKEN --os-storage-url $STORAGE_URL download $STORAGE_CONTAINER "$BACKUP_FILENAME"
+
+    # Note: here --pinentry-mode loopback is necessary to pass the passphrase
+    # Note 2: pg_restore does not read PGDATABASE, it is necessary to put a -d switch
+    gpg --no-tty --batch --pinentry-mode loopback --passphrase "$GPG_PRIVATE_KEY_PASSWORD" --recipient "$GPG_RECIPIENT" --decrypt "$BACKUP_FILENAME" | gzip -d | pg_restore -d "$PGDATABASE"
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: aplus-restore-job
+spec:
+  template:
+    spec:
+      containers:
+      - name: aplus-restore
+        image: python:3-buster
+        env:
+
+        - name: OS_AUTH_URL
+          valueFrom:
+            secretKeyRef:
+              name: ovh-storage-creds
+              key: OS_AUTH_URL
+        - name: OS_IDENTITY_API_VERSION
+          valueFrom:
+            secretKeyRef:
+              name: ovh-storage-creds
+              key: OS_IDENTITY_API_VERSION
+        - name: OS_TENANT_ID
+          valueFrom:
+            secretKeyRef:
+              name: ovh-storage-creds
+              key: OS_TENANT_ID
+        - name: OS_TENANT_NAME
+          valueFrom:
+            secretKeyRef:
+              name: ovh-storage-creds
+              key: OS_TENANT_NAME
+        - name: OS_USERNAME
+          valueFrom:
+            secretKeyRef:
+              name: ovh-storage-creds
+              key: OS_USERNAME
+        - name: OS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: ovh-storage-creds
+              key: OS_PASSWORD
+        - name: OS_REGION_NAME
+          valueFrom:
+            secretKeyRef:
+              name: ovh-storage-creds
+              key: OS_REGION_NAME
+
+        - name: PGDATABASE
+          valueFrom:
+            secretKeyRef:
+              name: aplus-db-app-secret
+              key: DATABASE_NAME
+        - name: PGHOST
+          valueFrom:
+            secretKeyRef:
+              name: aplus-db-app-secret
+              key: DATABASE_HOST
+        - name: PGPASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: aplus-db-app-secret
+              key: DATABASE_PASSWORD
+        - name: PGUSER
+          valueFrom:
+            secretKeyRef:
+              name: aplus-db-app-secret
+              key: DATABASE_USER
+
+        - name: BACKUP_FILENAME
+          value: "CHANGEME.pgdump.gz.gpg"
+        - name: GPG_RECIPIENT
+          valueFrom:
+            secretKeyRef:
+              name: aplus-backup-pub-key
+              key: GPG_RECIPIENT
+        - name: GPG_PRIVATE_KEY_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: aplus-backup-priv-key
+              key: GPG_PRIVATE_KEY_PASSWORD
+        - name: STORAGE_CONTAINER
+          value: "aplusprodbackup"
+        - name: STORAGE_URL
+          valueFrom:
+            secretKeyRef:
+              name: ovh-storage-creds
+              key: STORAGE_URL
+
+        volumeMounts:
+        - name: script
+          mountPath: "/script"
+        - name: public-key
+          mountPath: "/key"
+        - name: private-key
+          mountPath: "/private"
+        command: ["bash",  "/script/run.sh"]
+      volumes:
+      - name: script
+        configMap:
+          name: aplus-restore-script
+          items:
+          - key: restore-script
+            path: "run.sh"
+      - name: public-key
+        secret:
+          secretName: aplus-backup-pub-key
+          items:
+          - key: GPG_PUBLIC_KEY
+            path: "key.pub"
+      - name: private-key
+        secret:
+          secretName: aplus-backup-priv-key
+          items:
+          - key: GPG_PRIVATE_KEY
+            path: "key.gpg"
+      restartPolicy: Never
+  backoffLimit: 0
diff --git a/kubernetes/ingress-nginx/values.yml b/kubernetes/ingress-nginx/values.yml
new file mode 100644
index 000000000..e1329b940
--- /dev/null
+++ b/kubernetes/ingress-nginx/values.yml
@@ -0,0 +1,9 @@
+# Default values here:
+# https://github.com/kubernetes/ingress-nginx/blob/master/charts/ingress-nginx/values.yaml
+controller:
+  service:
+    type: NodePort
+    externalIPs:
+      - 141.94.167.220
+      - 141.94.166.107
+      - 141.94.164.120
diff --git a/kubernetes/metabase-app.yaml b/kubernetes/metabase-app.yaml
new file mode 100644
index 000000000..ed2ce5e7e
--- /dev/null
+++ b/kubernetes/metabase-app.yaml
@@ -0,0 +1,46 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: metabase-app
+  labels:
+    app: metabase
+spec:
+  # Note: this is the default type = ClusterIP
+  ports:
+  - port: 3000
+    targetPort: 3000
+  selector:     # Should be a Pod selector
+    app: metabase
+    tier: frontend
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metabase-app-deployment
+  labels:
+    app: metabase
+spec:
+  replicas: 1
+  selector:       # Should match .spec.template
+    matchLabels:
+      app: metabase
+      tier: frontend
+  strategy:
+    type: Recreate  # here RollingUpdate is not well handled by app sql queries
+  template:
+    metadata:
+      labels:
+        app: metabase
+        tier: frontend
+    spec:
+      containers:
+      - image: metabase/metabase:v0.39.1
+        name: metabase-app
+        env:
+        - name: MB_DB_CONNECTION_URI
+          valueFrom:
+            secretKeyRef:
+              name: metabase-app-secret
+              key: MB_DB_CONNECTION_URI
+        ports:
+        - containerPort: 3000
diff --git a/kubernetes/metabase-db.yaml b/kubernetes/metabase-db.yaml
new file mode 100644
index 000000000..fe3959bcf
--- /dev/null
+++ b/kubernetes/metabase-db.yaml
@@ -0,0 +1,65 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: metabase-db
+  labels:
+    app: metabase
+spec:
+  clusterIP: None
+  ports:
+  - port: 5432
+  selector:
+    app: metabase
+    tier: postgres
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: metabase-db-pvc
+  labels:
+    app: metabase
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 7Gi
+  storageClassName: csi-cinder-classic
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: metabase-db-statefulset
+  labels:
+    app: metabase
+spec:
+  selector:
+    matchLabels:
+      app: metabase
+      tier: postgres
+  serviceName: metabase-db
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: metabase
+        tier: postgres
+    spec:
+      containers:
+        - name: metabase-postgres-database
+          image: postgres:11
+          volumeMounts:
+            - name: metabase-db-pv
+              mountPath: /var/lib/postgresql/data
+          env:
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: metabase-db-postgres-secret
+                  key: POSTGRES_PASSWORD
+            - name: PGDATA
+              value: /var/lib/postgresql/data/pgdata
+      volumes:
+        - name: metabase-db-pv
+          persistentVolumeClaim:
+            claimName: metabase-db-pvc
diff --git a/kubernetes/metabase-ingress-preprod.yaml b/kubernetes/metabase-ingress-preprod.yaml
new file mode 100644
index 000000000..1102f47f8
--- /dev/null
+++ b/kubernetes/metabase-ingress-preprod.yaml
@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    cert-manager.io/issuer: "letsencrypt-prod"
+  name: metabase-ingress
+spec:
+  tls:
+  - hosts:
+    - metabase-preprod.aplus.incubateur.net
+    secretName: preprod-metabase-tls
+  rules:
+  - host: metabase-preprod.aplus.incubateur.net
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: metabase-app
+            port:
+              number: 3000
diff --git a/kubernetes/metabase-ingress.yaml b/kubernetes/metabase-ingress.yaml
new file mode 100644
index 000000000..5780106c2
--- /dev/null
+++ b/kubernetes/metabase-ingress.yaml
@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    cert-manager.io/issuer: "letsencrypt-prod"
+  name: metabase-ingress
+spec:
+  tls:
+  - hosts:
+    - metabase-prod.aplus.incubateur.net
+    secretName: metabase-prod-tls
+  rules:
+  - host: metabase-prod.aplus.incubateur.net
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: metabase-app
+            port:
+              number: 3000

From d861fe8073f9985f5c045a965e5c68b5001d17e9 Mon Sep 17 00:00:00 2001
From: niladic <git@nil.choron.cc>
Date: Mon, 27 Sep 2021 18:39:22 +0200
Subject: [PATCH 6/7] Update metabase

---
 kubernetes/metabase-app.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kubernetes/metabase-app.yaml b/kubernetes/metabase-app.yaml
index ed2ce5e7e..e0a70d544 100644
--- a/kubernetes/metabase-app.yaml
+++ b/kubernetes/metabase-app.yaml
@@ -34,7 +34,7 @@ spec:
         tier: frontend
     spec:
       containers:
-      - image: metabase/metabase:v0.39.1
+      - image: metabase/metabase:v0.40.5
         name: metabase-app
         env:
         - name: MB_DB_CONNECTION_URI

From b6921258352a3cb1fd050c8f83d6dd434b4374f2 Mon Sep 17 00:00:00 2001
From: niladic <git@nil.choron.cc>
Date: Tue, 26 Oct 2021 21:07:19 +0200
Subject: [PATCH 7/7] Supprime Azure

---
 kubernetes/aplus-app.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/kubernetes/aplus-app.yaml b/kubernetes/aplus-app.yaml
index 1147b7194..08ffd799d 100644
--- a/kubernetes/aplus-app.yaml
+++ b/kubernetes/aplus-app.yaml
@@ -86,8 +86,6 @@ spec:
               key: FEATURE_WEEKLY_EMAILS
         - name: FILES_EXPIRATION_IN_DAYS
           value: "15"
-        - name: FILES_SECOND_INSTANCE_HOST
-          value: "https://aplusprodapp.azurewebsites.net/"
 
         - name: FILES_PATH
           value: "/app/files"