From 05438afe22a6c55831291bd1145b45ca1207489f Mon Sep 17 00:00:00 2001 From: Julien Bouquillon Date: Sun, 31 Jan 2021 02:51:02 +0100 Subject: [PATCH 1/3] chore(ci): upgrade kosko jobs --- .../generate-prod-restore.ts.snap | 60 +++++- .../generate-prod-sitemap-uploader.ts.snap | 12 +- .k8s/components/restore/db.ts | 2 +- .k8s/restore-db.job.ts | 188 ------------------ 4 files changed, 56 insertions(+), 206 deletions(-) delete mode 100644 .k8s/restore-db.job.ts diff --git a/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap b/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap index c3abe23c0..9215c3963 100644 --- a/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap +++ b/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap @@ -2,6 +2,8 @@ exports[`kosko generate --prod restore 1`] = ` "--- +apiVersion: batch/v1 +kind: Job metadata: name: restore-container-8843083e namespace: cdtn-admin-secret @@ -69,29 +71,29 @@ spec: - name: SOURCE_ACCOUNT_NAME valueFrom: secretKeyRef: - name: azure-cdtnadminprod-volume key: azurestorageaccountname + name: azure-cdtnadminprod-volume - name: SOURCE_ACCOUNT_KEY valueFrom: secretKeyRef: - name: azure-cdtnadminprod-volume key: azurestorageaccountkey + name: azure-cdtnadminprod-volume - name: DESTINATION_ACCOUNT_NAME valueFrom: secretKeyRef: - name: azure-cdtnadmindev-volume key: azurestorageaccountname + name: azure-cdtnadmindev-volume - name: DESTINATION_ACCOUNT_KEY valueFrom: secretKeyRef: - name: azure-cdtnadmindev-volume key: azurestorageaccountkey + name: azure-cdtnadmindev-volume - name: SOURCE_CONTAINER value: source-container - name: DESTINATION_CONTAINER value: destination-container envFrom: [] - image: 'mcr.microsoft.com/azure-cli:2.15.1' + image: mcr.microsoft.com/azure-cli:2.15.1 imagePullPolicy: IfNotPresent name: restore-db resources: @@ -102,13 +104,20 @@ spec: cpu: 50m memory: 64Mi restartPolicy: Never -apiVersion: batch/v1 -kind: Job + ttlSecondsAfterFinished: 86400 --- +apiVersion: v1 +kind: ConfigMap data: post-restore.sql: | TRUNCATE TABLE \\"auth\\".\\"users\\" CASCADE; + -- + -- DISABLE TRIGGERS + -- + + ALTER TABLE auth.users DISABLE TRIGGER USER; + WITH admin_row AS ( INSERT INTO auth.users (email, PASSWORD, name, default_role, active) VALUES ('codedutravailnumerique@travail.gouv.fr', '$argon2i$v=19$m=4096,t=3,p=1$n9eoWSv+5sCgc7SjB5hLig$iBQ7NzrHHLkJSku/dCetNs+n/JI1CMdkWaoZsUekLU8', 'Administrateur', 'admin', TRUE) @@ -132,12 +141,18 @@ data: id FROM admin_row; + + -- + -- ENABLE TRIGGERS + -- + + ALTER TABLE auth.users ENABLE TRIGGER USER; metadata: name: post-restore-script-configmap-8843083e namespace: cdtn-admin-secret -apiVersion: v1 -kind: ConfigMap --- +apiVersion: batch/v1 +kind: Job metadata: name: restore-db-8843083e namespace: cdtn-admin-secret @@ -225,7 +240,30 @@ spec: - configMap: name: post-restore-script-configmap-8843083e name: scripts -apiVersion: batch/v1 -kind: Job + initContainers: + - env: + - name: PGHOST + value: cdtnadmindevserver.postgres.database.azure.com + - name: PGDATABASE + value: autodevops_8843083e + - name: PGPASSWORD + value: password_8843083e + - name: PGUSER + value: user_8843083e + - name: PGSSLMODE + value: require + envFrom: [] + image: >- + registry.gitlab.factory.social.gouv.fr/socialgouv/docker/wait-for-postgres:2.6.1 + imagePullPolicy: Always + name: wait-for-postgres + resources: + limits: + cpu: 20m + memory: 32Mi + requests: + cpu: 5m + memory: 16Mi + ttlSecondsAfterFinished: 86400 " `; diff --git a/.k8s/__tests__/__snapshots__/generate-prod-sitemap-uploader.ts.snap b/.k8s/__tests__/__snapshots__/generate-prod-sitemap-uploader.ts.snap index 3a09e28bf..ce55b3718 100644 --- a/.k8s/__tests__/__snapshots__/generate-prod-sitemap-uploader.ts.snap +++ b/.k8s/__tests__/__snapshots__/generate-prod-sitemap-uploader.ts.snap @@ -1,7 +1,9 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`kosko generate --prod sitemap-uploader 1`] = ` -"metadata: +"apiVersion: batch/v1 +kind: Job +metadata: name: sitemap-uploader-424242 namespace: cdtn-admin-secret spec: @@ -14,7 +16,7 @@ spec: restartPolicy: OnFailure containers: - name: az-sitemap-uploader - image: 'mcr.microsoft.com/azure-cli:2.9.1' + image: mcr.microsoft.com/azure-cli:2.9.1 command: - bash args: @@ -42,9 +44,9 @@ spec: - name: DESTINATION_NAME value: sitemap.xml - name: SITEMAP_ENDPOINT - value: 'https://path/to/sitemap' + value: https://path/to/sitemap - name: BASE_URL - value: 'https://host.tmp' + value: https://host.tmp - name: AZ_ACCOUNT_NAME value: $(azurestorageaccountname) - name: AZ_ACCOUNT_KEY @@ -52,7 +54,5 @@ spec: envFrom: - secretRef: name: azure-volume-dev-secret -apiVersion: batch/v1 -kind: Job " `; diff --git a/.k8s/components/restore/db.ts b/.k8s/components/restore/db.ts index 007ccfa0b..9e942127c 100644 --- a/.k8s/components/restore/db.ts +++ b/.k8s/components/restore/db.ts @@ -4,7 +4,7 @@ import env from "@kosko/env"; import { ok } from "assert"; import { EnvVar } from "kubernetes-models/v1/EnvVar"; -import { restoreDbJob } from "../../restore-db.job"; +import { restoreDbJob } from "@socialgouv/kosko-charts/components/azure-pg/restore-db.job"; ok(process.env.BACKUP_DB_NAME); ok(process.env.BACKUP_DB_OWNER); diff --git a/.k8s/restore-db.job.ts b/.k8s/restore-db.job.ts deleted file mode 100644 index a95633b79..000000000 --- a/.k8s/restore-db.job.ts +++ /dev/null @@ -1,188 +0,0 @@ -import ok from "assert"; -import { ConfigMap } from "kubernetes-models/_definitions/IoK8sApiCoreV1ConfigMap"; -import { Job } from "kubernetes-models/batch/v1/Job"; -import { EnvFromSource } from "kubernetes-models/v1/EnvFromSource"; -import type { EnvVar } from "kubernetes-models/v1/EnvVar"; - -//import { addInitContainer } from "@socialgouv/kosko-charts/utils/addInitContainer"; -//import { waitForPostgres } from "@socialgouv/kosko-charts/utils/waitForPostgres"; - -import type { IIoK8sApiCoreV1Container } from "kubernetes-models/_definitions/IoK8sApiCoreV1Container"; -import type { Deployment } from "kubernetes-models/apps/v1/Deployment"; -import type { Job as JobType } from "kubernetes-models/batch/v1/Job"; - -//type Manifest = Deployment | JobType; - -// export const addInitContainer = ( -// deployment: Manifest, -// initContainer: IIoK8sApiCoreV1Container -// ): Manifest => { -// if (!deployment.spec?.template) { -// return deployment; -// } - -// deployment.spec.template.spec = deployment.spec.template.spec ?? { -// containers: [], -// initContainers: [], -// }; -// const containers = deployment.spec.template.spec.initContainers ?? []; -// containers.push(initContainer); -// deployment.spec.template.spec.initContainers = containers; - -// return deployment; -// }; - -interface RestoreDbJobArgs { - project: string; - env: EnvVar[]; - envFrom?: EnvFromSource[]; - postRestoreScript?: string; -} - -// renovate: datasource=docker depName=registry.gitlab.factory.social.gouv.fr/socialgouv/docker/azure-db versioning=2.6.1 -const SOCIALGOUV_DOCKER_AZURE_DB = "2.6.1"; - -const restoreScript = ` - -echo "starting restore into $PGHOST/$PGDATABASE" - -[ ! -z $PGDATABASE ] || (echo "No PGDATABASE"; exit 1) -[ ! -z $PGHOST ] || (echo "No PGHOST"; exit 1) -[ ! -z $PGUSER ] || (echo "No PGUSER"; exit 1) -[ ! -z $PGPASSWORD ] || (echo "No PGPASSWORD"; exit 1) -[ ! -z $OWNER ] || (echo "No OWNER"; exit 1) - -# get latest backup folder -LATEST=$(ls -1Fr /mnt/data | head -n 1); -DUMP="/mnt/data/\${LATEST}\${FILE}" -echo "Restore \${DUMP} into \${PGDATABASE}"; - -pg_isready; - -pg_restore \ - --dbname \${PGDATABASE} \ - --clean --if-exists \ - --no-owner \ - --role \${OWNER} \ - --no-acl \ - --verbose \ - \${DUMP}; - -psql -v ON_ERROR_STOP=1 \${PGDATABASE} -c "ALTER SCHEMA public owner to \${OWNER};" - -[ -f "/mnt/scripts/post-restore.sql" ] && psql -v ON_ERROR_STOP=1 -a < /mnt/scripts/post-restore.sql -`; - -const getProjectSecretNamespace = (project: string) => `${project}-secret`; - -const getAzureProdVolumeSecretName = (project: string) => - `azure-${project.replace(/-/g, "")}prod-volume`; - -const getAzureBackupShareName = (project: string) => - `${project}-backup-restore`; - -type ReturnManifest = Job | ConfigMap; - -export const restoreDbJob = ({ - project, - env = [], - envFrom = [], - postRestoreScript, -}: RestoreDbJobArgs): ReturnManifest[] => { - ok(process.env.CI_COMMIT_SHORT_SHA); - const secretNamespace = getProjectSecretNamespace(project); - const azureSecretName = getAzureProdVolumeSecretName(project); - const azureShareName = getAzureBackupShareName(project); - - const manifests = []; - - const jobSpec = { - containers: [ - { - command: ["sh", "-c", restoreScript], - env, - envFrom: [ - new EnvFromSource({ - secretRef: { - name: "azure-pg-admin-user-dev", - }, - }), - ...envFrom, - ], - image: `registry.gitlab.factory.social.gouv.fr/socialgouv/docker/azure-db:${SOCIALGOUV_DOCKER_AZURE_DB}`, - imagePullPolicy: "IfNotPresent", - name: "restore-db", - resources: { - limits: { - cpu: "300m", - memory: "512Mi", - }, - requests: { - cpu: "100m", - memory: "64Mi", - }, - }, - volumeMounts: [ - { - mountPath: "/mnt/data", - name: "backups", - }, - ], - }, - ], - restartPolicy: "OnFailure", - volumes: [ - { - azureFile: { - readOnly: true, - secretName: azureSecretName, - shareName: azureShareName, - }, - name: "backups", - }, - ], - }; - - if (postRestoreScript) { - jobSpec.containers[0].volumeMounts.push({ - mountPath: "/mnt/scripts", - name: "scripts", - }); - jobSpec.volumes.push({ - //@ts-expect-error - configMap: { - name: `post-restore-script-configmap-${process.env.CI_COMMIT_SHORT_SHA}`, - }, - - name: "scripts", - }); - const configMap = new ConfigMap({ - data: { - "post-restore.sql": postRestoreScript, - }, - metadata: { - name: `post-restore-script-configmap-${process.env.CI_COMMIT_SHORT_SHA}`, - namespace: secretNamespace, - }, - }); - manifests.push(configMap); - } - - const job = new Job({ - metadata: { - name: `restore-db-${process.env.CI_COMMIT_SHORT_SHA}`, - namespace: secretNamespace, - }, - spec: { - backoffLimit: 0, - template: { - metadata: {}, - spec: jobSpec, - }, - }, - }); - - manifests.push(job); - - return manifests; -}; From 2296c0dd46d662f402ff0cc5eaf766c4036d5ff1 Mon Sep 17 00:00:00 2001 From: Julien Bouquillon Date: Sun, 31 Jan 2021 02:58:00 +0100 Subject: [PATCH 2/3] chore(ci): use jobs subfolder --- .gitlab-ci.yml | 6 +++--- .k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap | 2 +- .../__snapshots__/generate-prod-sitemap-uploader.ts.snap | 2 +- .k8s/__tests__/generate-prod-restore.ts | 4 ++-- .k8s/__tests__/generate-prod-sitemap-uploader.ts | 4 ++-- .k8s/components/{ => jobs}/restore/container.ts | 0 .k8s/components/{ => jobs}/restore/db.ts | 0 .k8s/components/{ => jobs}/restore/index.ts | 0 .k8s/components/{ => jobs}/restore/post-restore.sql | 0 .k8s/components/{ => jobs}/sitemap-uploader/index.ts | 0 .k8s/kosko.toml | 4 ++-- 11 files changed, 11 insertions(+), 11 deletions(-) rename .k8s/components/{ => jobs}/restore/container.ts (100%) rename .k8s/components/{ => jobs}/restore/db.ts (100%) rename .k8s/components/{ => jobs}/restore/index.ts (100%) rename .k8s/components/{ => jobs}/restore/post-restore.sql (100%) rename .k8s/components/{ => jobs}/sitemap-uploader/index.ts (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8abb0f9bd..08189e214 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -142,7 +142,7 @@ Restore container: BACKUP_DB_OWNER: "user_${CI_COMMIT_SHORT_SHA}" BACKUP_DB_FILE: "hasura_prod_db.psql.gz" # kosko options - KOSKO_GENERATE_ARGS: --env prod restore + KOSKO_GENERATE_ARGS: --env prod jobs/restore Generate sitemap (prod): stage: Install @@ -163,7 +163,7 @@ Generate sitemap (prod): DESTINATION_CONTAINER: sitemap DESTINATION_NAME: sitemap.xml SECRET_NAME: azure-cdtnadminprod-volume - KOSKO_GENERATE_ARGS: --env prod sitemap-uploader + KOSKO_GENERATE_ARGS: --env prod jobs/sitemap-uploader Generate sitemap (dev): stage: Install @@ -184,4 +184,4 @@ Generate sitemap (dev): DESTINATION_CONTAINER: sitemap DESTINATION_NAME: sitemap.xml SECRET_NAME: azure-cdtnadmindev-volume - KOSKO_GENERATE_ARGS: --env dev sitemap-uploader + KOSKO_GENERATE_ARGS: --env dev jobs/sitemap-uploader diff --git a/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap b/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap index 9215c3963..79e9b4e56 100644 --- a/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap +++ b/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`kosko generate --prod restore 1`] = ` +exports[`kosko generate --prod jobs/restore 1`] = ` "--- apiVersion: batch/v1 kind: Job diff --git a/.k8s/__tests__/__snapshots__/generate-prod-sitemap-uploader.ts.snap b/.k8s/__tests__/__snapshots__/generate-prod-sitemap-uploader.ts.snap index ce55b3718..5c5e94d5d 100644 --- a/.k8s/__tests__/__snapshots__/generate-prod-sitemap-uploader.ts.snap +++ b/.k8s/__tests__/__snapshots__/generate-prod-sitemap-uploader.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`kosko generate --prod sitemap-uploader 1`] = ` +exports[`kosko generate --prod jobs/sitemap-uploader 1`] = ` "apiVersion: batch/v1 kind: Job metadata: diff --git a/.k8s/__tests__/generate-prod-restore.ts b/.k8s/__tests__/generate-prod-restore.ts index ff6677666..968b77fbd 100644 --- a/.k8s/__tests__/generate-prod-restore.ts +++ b/.k8s/__tests__/generate-prod-restore.ts @@ -4,9 +4,9 @@ import { getEnvManifests } from "@socialgouv/kosko-charts/testing"; import { project } from "@socialgouv/kosko-charts/testing/fake/gitlab-ci.env"; jest.setTimeout(1000 * 60); -test("kosko generate --prod restore", async () => { +test("kosko generate --prod jobs/restore", async () => { expect( - await getEnvManifests("prod", "restore", { + await getEnvManifests("prod", "jobs/restore", { ...project("cdtn-admin").prod, SOURCE_CONTAINER: "source-container", SOURCE_SERVER: "prod", diff --git a/.k8s/__tests__/generate-prod-sitemap-uploader.ts b/.k8s/__tests__/generate-prod-sitemap-uploader.ts index d35ed2df7..409f1bb64 100644 --- a/.k8s/__tests__/generate-prod-sitemap-uploader.ts +++ b/.k8s/__tests__/generate-prod-sitemap-uploader.ts @@ -4,9 +4,9 @@ import { getEnvManifests } from "@socialgouv/kosko-charts/testing"; import { project } from "@socialgouv/kosko-charts/testing/fake/gitlab-ci.env"; jest.setTimeout(1000 * 60); -test("kosko generate --prod sitemap-uploader", async () => { +test("kosko generate --prod jobs/sitemap-uploader", async () => { expect( - await getEnvManifests("prod", "sitemap-uploader", { + await getEnvManifests("prod", "jobs/sitemap-uploader", { ...project("cdtn-admin").prod, SITEMAP_ENDPOINT: "https://path/to/sitemap", DESTINATION_CONTAINER: "destination-container", diff --git a/.k8s/components/restore/container.ts b/.k8s/components/jobs/restore/container.ts similarity index 100% rename from .k8s/components/restore/container.ts rename to .k8s/components/jobs/restore/container.ts diff --git a/.k8s/components/restore/db.ts b/.k8s/components/jobs/restore/db.ts similarity index 100% rename from .k8s/components/restore/db.ts rename to .k8s/components/jobs/restore/db.ts diff --git a/.k8s/components/restore/index.ts b/.k8s/components/jobs/restore/index.ts similarity index 100% rename from .k8s/components/restore/index.ts rename to .k8s/components/jobs/restore/index.ts diff --git a/.k8s/components/restore/post-restore.sql b/.k8s/components/jobs/restore/post-restore.sql similarity index 100% rename from .k8s/components/restore/post-restore.sql rename to .k8s/components/jobs/restore/post-restore.sql diff --git a/.k8s/components/sitemap-uploader/index.ts b/.k8s/components/jobs/sitemap-uploader/index.ts similarity index 100% rename from .k8s/components/sitemap-uploader/index.ts rename to .k8s/components/jobs/sitemap-uploader/index.ts diff --git a/.k8s/kosko.toml b/.k8s/kosko.toml index 230f77ec9..962a5fa57 100644 --- a/.k8s/kosko.toml +++ b/.k8s/kosko.toml @@ -1,6 +1,6 @@ -# dont include restore jobs on deploys -components = ["!restore", "!sitemap-uploader","*"] +# dont include jobs on deploys +components = ["!jobs", "*"] require = ["ts-node/register"] # dont include underscores-prefix components From 81fd3cb230bff41303c7ec949c1c4e0c93aa5b9a Mon Sep 17 00:00:00 2001 From: Julien Bouquillon Date: Thu, 4 Feb 2021 13:42:13 +0100 Subject: [PATCH 3/3] chore: updates --- .../__tests__/__snapshots__/generate-prod-restore.ts.snap | 4 ++-- .k8s/package.json | 2 +- .k8s/yarn.lock | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap b/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap index 79e9b4e56..d1aee0494 100644 --- a/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap +++ b/.k8s/__tests__/__snapshots__/generate-prod-restore.ts.snap @@ -249,12 +249,12 @@ spec: - name: PGPASSWORD value: password_8843083e - name: PGUSER - value: user_8843083e + value: user_8843083e@cdtnadmindevserver.postgres.database.azure.com - name: PGSSLMODE value: require envFrom: [] image: >- - registry.gitlab.factory.social.gouv.fr/socialgouv/docker/wait-for-postgres:2.6.1 + registry.gitlab.factory.social.gouv.fr/socialgouv/docker/wait-for-postgres:4.3.1 imagePullPolicy: Always name: wait-for-postgres resources: diff --git a/.k8s/package.json b/.k8s/package.json index ed3a482c7..fb7c1cca4 100644 --- a/.k8s/package.json +++ b/.k8s/package.json @@ -9,7 +9,7 @@ "dependencies": { "@kosko/env": "^1.0.3", "@kubernetes-models/sealed-secrets": "^1.0.3", - "@socialgouv/kosko-charts": "^4.7.1", + "@socialgouv/kosko-charts": "^4.7.4", "@types/node": "^14.14.22", "kosko": "^1.0.3", "kubernetes-models": "^1.0.3", diff --git a/.k8s/yarn.lock b/.k8s/yarn.lock index 8351812f7..0c75bf7f8 100644 --- a/.k8s/yarn.lock +++ b/.k8s/yarn.lock @@ -628,10 +628,10 @@ dependencies: "@sinonjs/commons" "^1.7.0" -"@socialgouv/kosko-charts@^4.7.1": - version "4.7.1" - resolved "https://registry.yarnpkg.com/@socialgouv/kosko-charts/-/kosko-charts-4.7.1.tgz#2383f518ec89a93fb638293bc666a08f00e86828" - integrity sha512-soSOKpDzuDyF2Hini94EFy/tFqHsu2i47rfE6v6xezT2dXTE6AZM9ZBHSuVSJQ6crc6Qg1S+zwzdCfUMxNozYg== +"@socialgouv/kosko-charts@^4.7.4": + version "4.7.4" + resolved "https://registry.yarnpkg.com/@socialgouv/kosko-charts/-/kosko-charts-4.7.4.tgz#196378b8e3e00bb3c9ae6d0c517d2454791d8ac2" + integrity sha512-XtEP24RBKmm8f6k1mPAxr9KgyZN9OTvxbUv8oVQ4oGCsiy96XpbhXNbhvJyUB3E6mIRnd9uA7JgRim5iEA28uw== dependencies: "@kosko/env" "^1.0.3" "@kubernetes-models/sealed-secrets" "^1.0.3"