chore: use jobs subfolder (#297)

* chore(ci): upgrade kosko jobs * chore(ci): use jobs subfolder * chore: updates
SocialGouv · Feb 5, 2021 · 912f483 · 912f483
1 parent c45bba6
commit 912f483
Show file tree

Hide file tree

Showing 14 changed files with 72 additions and 222 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -142,7 +142,7 @@ Restore container:
     BACKUP_DB_OWNER: "user_${CI_COMMIT_SHORT_SHA}"
     BACKUP_DB_FILE: "hasura_prod_db.psql.gz"
     # kosko options
-    KOSKO_GENERATE_ARGS: --env prod restore
+    KOSKO_GENERATE_ARGS: --env prod jobs/restore
 
 Generate sitemap (prod):
   stage: Install
@@ -163,7 +163,7 @@ Generate sitemap (prod):
     DESTINATION_CONTAINER: sitemap
     DESTINATION_NAME: sitemap.xml
     SECRET_NAME: azure-cdtnadminprod-volume
-    KOSKO_GENERATE_ARGS: --env prod sitemap-uploader
+    KOSKO_GENERATE_ARGS: --env prod jobs/sitemap-uploader
 
 Generate sitemap (dev):
   stage: Install
@@ -184,4 +184,4 @@ Generate sitemap (dev):
     DESTINATION_CONTAINER: sitemap
     DESTINATION_NAME: sitemap.xml
     SECRET_NAME: azure-cdtnadmindev-volume
-    KOSKO_GENERATE_ARGS: --env dev sitemap-uploader
+    KOSKO_GENERATE_ARGS: --env dev jobs/sitemap-uploader
diff --git a/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap b/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap
@@ -1,7 +1,9 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`kosko generate --prod restore 1`] = `
+exports[`kosko generate --prod jobs/restore 1`] = `
 "---
+apiVersion: batch/v1
+kind: Job
 metadata:
   name: restore-container-8843083e
   namespace: cdtn-admin-secret
@@ -69,29 +71,29 @@ spec:
             - name: SOURCE_ACCOUNT_NAME
               valueFrom:
                 secretKeyRef:
-                  name: azure-cdtnadminprod-volume
                   key: azurestorageaccountname
+                  name: azure-cdtnadminprod-volume
             - name: SOURCE_ACCOUNT_KEY
               valueFrom:
                 secretKeyRef:
-                  name: azure-cdtnadminprod-volume
                   key: azurestorageaccountkey
+                  name: azure-cdtnadminprod-volume
             - name: DESTINATION_ACCOUNT_NAME
               valueFrom:
                 secretKeyRef:
-                  name: azure-cdtnadmindev-volume
                   key: azurestorageaccountname
+                  name: azure-cdtnadmindev-volume
             - name: DESTINATION_ACCOUNT_KEY
               valueFrom:
                 secretKeyRef:
-                  name: azure-cdtnadmindev-volume
                   key: azurestorageaccountkey
+                  name: azure-cdtnadmindev-volume
             - name: SOURCE_CONTAINER
               value: source-container
             - name: DESTINATION_CONTAINER
               value: destination-container
           envFrom: []
-          image: 'mcr.microsoft.com/azure-cli:2.15.1'
+          image: mcr.microsoft.com/azure-cli:2.15.1
           imagePullPolicy: IfNotPresent
           name: restore-db
           resources:
@@ -102,13 +104,20 @@ spec:
               cpu: 50m
               memory: 64Mi
       restartPolicy: Never
-apiVersion: batch/v1
-kind: Job
+  ttlSecondsAfterFinished: 86400
 ---
+apiVersion: v1
+kind: ConfigMap
 data:
   post-restore.sql: |
     TRUNCATE TABLE \\"auth\\".\\"users\\" CASCADE;
 
+    --
+    -- DISABLE TRIGGERS
+    --
+
+    ALTER TABLE auth.users DISABLE TRIGGER USER;
+
     WITH admin_row AS (
     INSERT INTO auth.users (email, PASSWORD, name, default_role, active)
         VALUES ('codedutravailnumerique@travail.gouv.fr', '$argon2i$v=19$m=4096,t=3,p=1$n9eoWSv+5sCgc7SjB5hLig$iBQ7NzrHHLkJSku/dCetNs+n/JI1CMdkWaoZsUekLU8', 'Administrateur', 'admin', TRUE)
@@ -132,12 +141,18 @@ data:
         id
       FROM
         admin_row;
+
+    --
+    -- ENABLE TRIGGERS
+    --
+
+    ALTER TABLE auth.users ENABLE TRIGGER USER;
 metadata:
   name: post-restore-script-configmap-8843083e
   namespace: cdtn-admin-secret
-apiVersion: v1
-kind: ConfigMap
 ---
+apiVersion: batch/v1
+kind: Job
 metadata:
   name: restore-db-8843083e
   namespace: cdtn-admin-secret
@@ -225,7 +240,30 @@ spec:
         - configMap:
             name: post-restore-script-configmap-8843083e
           name: scripts
-apiVersion: batch/v1
-kind: Job
+      initContainers:
+        - env:
+            - name: PGHOST
+              value: cdtnadmindevserver.postgres.database.azure.com
+            - name: PGDATABASE
+              value: autodevops_8843083e
+            - name: PGPASSWORD
+              value: password_8843083e
+            - name: PGUSER
+              value: user_8843083e@cdtnadmindevserver.postgres.database.azure.com
+            - name: PGSSLMODE
+              value: require
+          envFrom: []
+          image: >-
+            registry.gitlab.factory.social.gouv.fr/socialgouv/docker/wait-for-postgres:4.3.1
+          imagePullPolicy: Always
+          name: wait-for-postgres
+          resources:
+            limits:
+              cpu: 20m
+              memory: 32Mi
+            requests:
+              cpu: 5m
+              memory: 16Mi
+  ttlSecondsAfterFinished: 86400
 "
 `;
diff --git a/.k8s/__tests__/__snapshots__/generate-prod-sitemap-uploader.ts.snap b/.k8s/__tests__/__snapshots__/generate-prod-sitemap-uploader.ts.snap
@@ -1,7 +1,9 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`kosko generate --prod sitemap-uploader 1`] = `
-"metadata:
+exports[`kosko generate --prod jobs/sitemap-uploader 1`] = `
+"apiVersion: batch/v1
+kind: Job
+metadata:
   name: sitemap-uploader-424242
   namespace: cdtn-admin-secret
 spec:
@@ -14,7 +16,7 @@ spec:
       restartPolicy: OnFailure
       containers:
         - name: az-sitemap-uploader
-          image: 'mcr.microsoft.com/azure-cli:2.9.1'
+          image: mcr.microsoft.com/azure-cli:2.9.1
           command:
             - bash
           args:
@@ -42,17 +44,15 @@ spec:
             - name: DESTINATION_NAME
               value: sitemap.xml
             - name: SITEMAP_ENDPOINT
-              value: 'https://path/to/sitemap'
+              value: https://path/to/sitemap
             - name: BASE_URL
-              value: 'https://host.tmp'
+              value: https://host.tmp
             - name: AZ_ACCOUNT_NAME
               value: $(azurestorageaccountname)
             - name: AZ_ACCOUNT_KEY
               value: $(azurestorageaccountkey)
           envFrom:
             - secretRef:
                 name: azure-volume-dev-secret
-apiVersion: batch/v1
-kind: Job
 "
 `;
diff --git a/.k8s/__tests__/generate-prod-restore.ts b/.k8s/__tests__/generate-prod-restore.ts
@@ -4,9 +4,9 @@ import { getEnvManifests } from "@socialgouv/kosko-charts/testing";
 import { project } from "@socialgouv/kosko-charts/testing/fake/gitlab-ci.env";
 
 jest.setTimeout(1000 * 60);
-test("kosko generate --prod restore", async () => {
+test("kosko generate --prod jobs/restore", async () => {
   expect(
-    await getEnvManifests("prod", "restore", {
+    await getEnvManifests("prod", "jobs/restore", {
       ...project("cdtn-admin").prod,
       SOURCE_CONTAINER: "source-container",
       SOURCE_SERVER: "prod",

diff --git a/.k8s/__tests__/generate-prod-sitemap-uploader.ts b/.k8s/__tests__/generate-prod-sitemap-uploader.ts
@@ -4,9 +4,9 @@ import { getEnvManifests } from "@socialgouv/kosko-charts/testing";
 import { project } from "@socialgouv/kosko-charts/testing/fake/gitlab-ci.env";
 
 jest.setTimeout(1000 * 60);
-test("kosko generate --prod sitemap-uploader", async () => {
+test("kosko generate --prod jobs/sitemap-uploader", async () => {
   expect(
-    await getEnvManifests("prod", "sitemap-uploader", {
+    await getEnvManifests("prod", "jobs/sitemap-uploader", {
       ...project("cdtn-admin").prod,
       SITEMAP_ENDPOINT: "https://path/to/sitemap",
       DESTINATION_CONTAINER: "destination-container",

diff --git a/.k8s/components/restore/container.ts → .k8s/components/jobs/restore/container.ts b/.k8s/components/restore/container.ts → .k8s/components/jobs/restore/container.ts
diff --git a/.k8s/components/restore/db.ts → .k8s/components/jobs/restore/db.ts b/.k8s/components/restore/db.ts → .k8s/components/jobs/restore/db.ts
@@ -4,7 +4,7 @@ import env from "@kosko/env";
 import { ok } from "assert";
 
 import { EnvVar } from "kubernetes-models/v1/EnvVar";
-import { restoreDbJob } from "../../restore-db.job";
+import { restoreDbJob } from "@socialgouv/kosko-charts/components/azure-pg/restore-db.job";
 
 ok(process.env.BACKUP_DB_NAME);
 ok(process.env.BACKUP_DB_OWNER);

diff --git a/.k8s/components/restore/index.ts → .k8s/components/jobs/restore/index.ts b/.k8s/components/restore/index.ts → .k8s/components/jobs/restore/index.ts
diff --git a/.k8s/components/restore/post-restore.sql → .../components/jobs/restore/post-restore.sql b/.k8s/components/restore/post-restore.sql → .../components/jobs/restore/post-restore.sql
diff --git a/.k8s/components/sitemap-uploader/index.ts → ...components/jobs/sitemap-uploader/index.ts b/.k8s/components/sitemap-uploader/index.ts → ...components/jobs/sitemap-uploader/index.ts
diff --git a/.k8s/kosko.toml b/.k8s/kosko.toml
@@ -1,6 +1,6 @@
 
-# dont include restore jobs on deploys
-components = ["!restore", "!sitemap-uploader","*"]
+# dont include jobs on deploys
+components = ["!jobs", "*"]
 require = ["ts-node/register"]
 
 # dont include underscores-prefix components

diff --git a/.k8s/package.json b/.k8s/package.json
@@ -9,7 +9,7 @@
   "dependencies": {
     "@kosko/env": "^1.0.3",
     "@kubernetes-models/sealed-secrets": "^1.0.3",
-    "@socialgouv/kosko-charts": "^4.7.1",
+    "@socialgouv/kosko-charts": "^4.7.4",
     "@types/node": "^14.14.22",
     "kosko": "^1.0.3",
     "kubernetes-models": "^1.0.3",