From d1001eaa4f245610546e627fb04be7bfcab5929e Mon Sep 17 00:00:00 2001 From: Douglas DUTEIL Date: Thu, 25 Mar 2021 12:11:51 +0100 Subject: [PATCH] build(k8s): deploy ingester-elasticsearch job --- .../generate ingester-elasticsearch/--dev.ts | 14 ++++ .../__snapshots__/--dev.ts.snap | 66 +++++++++++++++ .../jobs/ingester-elasticsearch/index.ts | 81 +++++++++++++++++++ .../dev/ingester-elasticsearch.configmap.yaml | 8 ++ .../ingester-elasticsearch.sealed-secret.yaml | 17 ++++ .k8s/kosko.toml | 2 +- .k8s/utils/ES_INDEX_PREFIX.ts | 17 ++++ targets/ingester-elasticsearch/.gitlab-ci.yml | 28 +++++++ 8 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 .k8s/__tests__/generate ingester-elasticsearch/--dev.ts create mode 100644 .k8s/__tests__/generate ingester-elasticsearch/__snapshots__/--dev.ts.snap create mode 100644 .k8s/components/jobs/ingester-elasticsearch/index.ts create mode 100644 .k8s/environments/dev/ingester-elasticsearch.configmap.yaml create mode 100644 .k8s/environments/dev/ingester-elasticsearch.sealed-secret.yaml create mode 100644 .k8s/utils/ES_INDEX_PREFIX.ts diff --git a/.k8s/__tests__/generate ingester-elasticsearch/--dev.ts b/.k8s/__tests__/generate ingester-elasticsearch/--dev.ts new file mode 100644 index 000000000..020e1326a --- /dev/null +++ b/.k8s/__tests__/generate ingester-elasticsearch/--dev.ts @@ -0,0 +1,14 @@ +// + +import { getEnvManifests } from "@socialgouv/kosko-charts/testing"; +import { project } from "@socialgouv/kosko-charts/testing/fake/gitlab-ci.env"; + +jest.setTimeout(1000 * 60); +test("kosko generate ingester-elasticsearch --dev", async () => { + expect( + await getEnvManifests("dev", "jobs/ingester-elasticsearch", { + ...project("code-du-travail-numerique").dev, + RANCHER_PROJECT_ID: "c-bar:p-foo", + }) + ).toMatchSnapshot(); +}); diff --git a/.k8s/__tests__/generate ingester-elasticsearch/__snapshots__/--dev.ts.snap b/.k8s/__tests__/generate ingester-elasticsearch/__snapshots__/--dev.ts.snap new file mode 100644 index 000000000..565d9a3d6 --- /dev/null +++ b/.k8s/__tests__/generate ingester-elasticsearch/__snapshots__/--dev.ts.snap @@ -0,0 +1,66 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`kosko generate ingester-elasticsearch --dev 1`] = ` +"--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ingester-elasticsearch +data: + CDTN_ADMIN_ENDPOINT: https://cdtn-admin.fabrique.social.gouv.fr/api/graphql + NLP_URL: https://preprod-serving-ml.dev2.fabrique.social.gouv.fr + NODE_ENV: production +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + annotations: + sealedsecrets.bitnami.com/cluster-wide: 'true' + name: ingester-elasticsearch + namespace: null +spec: + encryptedData: + ELASTICSEARCH_DATA_TOKEN: >- + AgA+5TqL1kYeeCl8R7j0JE6dgEQ+9gNtGvD2eGyNHYveWA9N0l3UD+EF3OUNzcoeBseMVK/7tgPC1PBXTluABbIMZe8v0oKf7McgJZEkXpzUh/wL+icjGKIDb1zFCYV+Ox2m1L0fsMVRcOEBAh0RpgRcY8CEpLpszoEvlgTKnFljkqQfDUTwofr96Q6FYolD1cjikx+jzBvhxAkbf8mrrwz775UUY3u9OPFk3TQSqVVyd0kzwJWRTRlb1g3mYtfC79FM1j/44LkBYdeAQfNDTx12LNg+n23RQLXlGfEu2oTMfkVQgVnQ+Xqxgzc4wvvglgOP+yJe0S01i49uBXIh3ZniBM99rVtwRONu6Q1RTN2lzGwdMPT1WhiCk8nJDqrLFYs2yg++qWYo/HMPVDYCbzpH+7j1NbEvzUSH4+KSb8Qsk1YIIMskOACWNpvmAraT6KzxsK/QnEAQckex5Y03xSFB1L4boAlVkmhOkJXaz92Ug/ab6tMWTKoalz0zxBYHjVovJeN0RhKV9kw48S12B3aSkqgO3QTHN7CY/FsebcUZgE2eup+LDyPRwM1IMwMsU+t8QL73YBdZx6Fk01nBBuT3D5Ba46xA4dWm4tLH04niErwhuSJalaHQA3AwUNdAiUx9tf48lkAeinALlcGG9Hj1o+EqhF9dXFad+4QQUN7nXIkSLRpEu9kWQZKW2FnlfJ6KADkokMmm77eMFAL8ifToozRn7rgjopyaR9hVxT91iNmEhWpKopMBfRjzP/bP9uWMr7cqIwLQlKAUgwg= + ELASTICSEARCH_URL: >- + AgBxkNsQpkk6V5GIA8WuYBXE1iTLglrzRoR5HQ9QtBzsjDb9siCHQc+U9UFI72hDKi3yNO2g1lk+Nh7ooKHpqzzV0PSB0gyUJwo4MmqfxUDw3ef2BHezK7DK0z1YB/ZU8BLK5LenLchoKsKejv1YC1OQmG6EUHPW/3hm2qiS56anl6AIyUqXsokx+5QScjmHE3KybThaBkva/7PEQn/vks9wbY2mePvBPanSnVSLuSUY0yn0UXrmJnyW87oXVaJSQU/+5oDPdnb4pN8wivmjQEBLyVvjL2xSFz2druu4wjDpArup981GA7jCTarCzxjomGKnfxu7U98YKho1OUbuGevL1TXZaOkXaSIpls5mViZjUB2WaUVqQhHr9K1f6Hmhidh6S7v8NX4NbPKgKA7PTJ1dzi7EeKUCiIJPXiSI76H/rNlEx55TlroXQxEPjmQ91yEcCtRNa6MS2X6Flu539+t2m0+nm+Wqx7SmbGROjvCbSaYMShdsYQgnRj3VMgzoToLcnRPO0MdIop2L6GN5ocyFPk/QRUry+A/TH5eYkuMGu/FhctdrNEMo2kcYgIiTfqQ9tuQQqD3UCJ74DkUOod8OJoTu2EQ6RVBXqVCb9cM0AmoYgUDx1JW0l1124UYzpqIPhaRdPe6M8JH9zXuQAI93R7Paezn8GEMaXfiBRc/fVRUdTbAdbGvdmEZVeiczEK+JsKcdvMuqud15UmcFF0N7rfuia1u6TjCvaF/gvFIlLnMccFYmKACJnf1SJ/5Cx6L0SajjyBhxNP8DMVHhmMpbIqfQBPJho0nVQk7ZoU5Heg== + template: + metadata: + annotations: + sealedsecrets.bitnami.com/cluster-wide: 'true' + name: ingester-elasticsearch + type: Opaque +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ingester-elasticsearch + namespace: cdtn-admin-24-branch-42 +spec: + backoffLimit: 0 + template: + spec: + containers: + - name: ingester-elasticsearch + image: >- + registry.gitlab.factory.social.gouv.fr/socialgouv/cdtn-admin/ingester-elasticsearch:8843083edb7f873cad1d1420731a60773594ffae + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: '2' + memory: 1Gi + requests: + cpu: '1' + memory: 512Mi + envFrom: + - configMapRef: + name: ingester-elasticsearch + - secretRef: + name: ingester-elasticsearch + env: + - name: ES_INDEX_PREFIX + value: cdtn-master + restartPolicy: Never + ttlSecondsAfterFinished: 86400 +" +`; diff --git a/.k8s/components/jobs/ingester-elasticsearch/index.ts b/.k8s/components/jobs/ingester-elasticsearch/index.ts new file mode 100644 index 000000000..0292b7471 --- /dev/null +++ b/.k8s/components/jobs/ingester-elasticsearch/index.ts @@ -0,0 +1,81 @@ +import { ok } from "assert"; +import env from "@kosko/env"; +import type { SealedSecret } from "@kubernetes-models/sealed-secrets/bitnami.com/v1alpha1/SealedSecret"; +import { Job } from "kubernetes-models/batch/v1/Job"; +import gitlab from "@socialgouv/kosko-charts/environments/gitlab"; +import { loadYaml } from "@socialgouv/kosko-charts/utils/getEnvironmentComponent"; +import { merge } from "@socialgouv/kosko-charts/utils/@kosko/env/merge"; +import { ConfigMap } from "kubernetes-models/_definitions/IoK8sApiCoreV1ConfigMap"; + +import { ES_INDEX_PREFIX } from "../../../utils/ES_INDEX_PREFIX"; + +ok(process.env.CI_REGISTRY_IMAGE, "Missing CI_REGISTRY_IMAGE"); + +const configMap = loadYaml( + env, + `ingester-elasticsearch.configmap.yaml` +); +ok(configMap, "Missing ingester-elasticsearch.configmap.yaml"); +const secret = loadYaml( + env, + "ingester-elasticsearch.sealed-secret.yaml" +); +ok(secret, "Missing ingester-elasticsearch.sealed-secret.yaml"); + +const ingester = () => { + const envParams = merge(gitlab(process.env), {}); + + const job = new Job({ + metadata: { + name: "ingester-elasticsearch", + namespace: envParams.namespace.name, + }, + spec: { + backoffLimit: 0, + template: { + spec: { + containers: [ + { + name: "ingester-elasticsearch", + image: `${process.env.CI_REGISTRY_IMAGE}/ingester-elasticsearch:${process.env.CI_COMMIT_SHA}`, + imagePullPolicy: "IfNotPresent", + resources: { + limits: { + cpu: "2", + memory: "1Gi", + }, + requests: { + cpu: "1", + memory: "512Mi", + }, + }, + envFrom: [ + { + configMapRef: { + name: configMap?.metadata?.name, + }, + }, + { + secretRef: { + name: secret?.metadata?.name, + }, + }, + ], + env: [ + { + name: "ES_INDEX_PREFIX", + value: ES_INDEX_PREFIX, + }, + ], + }, + ], + restartPolicy: "Never", + }, + }, + ttlSecondsAfterFinished: 86400, + }, + }); + return job; +}; + +export default [configMap, secret, ingester()]; diff --git a/.k8s/environments/dev/ingester-elasticsearch.configmap.yaml b/.k8s/environments/dev/ingester-elasticsearch.configmap.yaml new file mode 100644 index 000000000..0d91f0ae0 --- /dev/null +++ b/.k8s/environments/dev/ingester-elasticsearch.configmap.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: ingester-elasticsearch +data: + CDTN_ADMIN_ENDPOINT: "https://cdtn-admin.fabrique.social.gouv.fr/api/graphql" + NLP_URL: "https://preprod-serving-ml.dev2.fabrique.social.gouv.fr" + NODE_ENV: "production" diff --git a/.k8s/environments/dev/ingester-elasticsearch.sealed-secret.yaml b/.k8s/environments/dev/ingester-elasticsearch.sealed-secret.yaml new file mode 100644 index 000000000..ddfc6520f --- /dev/null +++ b/.k8s/environments/dev/ingester-elasticsearch.sealed-secret.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: "bitnami.com/v1alpha1" +kind: "SealedSecret" +metadata: + annotations: &a1 + sealedsecrets.bitnami.com/cluster-wide: "true" + name: "ingester-elasticsearch" + namespace: null +spec: + encryptedData: + ELASTICSEARCH_DATA_TOKEN: AgA+5TqL1kYeeCl8R7j0JE6dgEQ+9gNtGvD2eGyNHYveWA9N0l3UD+EF3OUNzcoeBseMVK/7tgPC1PBXTluABbIMZe8v0oKf7McgJZEkXpzUh/wL+icjGKIDb1zFCYV+Ox2m1L0fsMVRcOEBAh0RpgRcY8CEpLpszoEvlgTKnFljkqQfDUTwofr96Q6FYolD1cjikx+jzBvhxAkbf8mrrwz775UUY3u9OPFk3TQSqVVyd0kzwJWRTRlb1g3mYtfC79FM1j/44LkBYdeAQfNDTx12LNg+n23RQLXlGfEu2oTMfkVQgVnQ+Xqxgzc4wvvglgOP+yJe0S01i49uBXIh3ZniBM99rVtwRONu6Q1RTN2lzGwdMPT1WhiCk8nJDqrLFYs2yg++qWYo/HMPVDYCbzpH+7j1NbEvzUSH4+KSb8Qsk1YIIMskOACWNpvmAraT6KzxsK/QnEAQckex5Y03xSFB1L4boAlVkmhOkJXaz92Ug/ab6tMWTKoalz0zxBYHjVovJeN0RhKV9kw48S12B3aSkqgO3QTHN7CY/FsebcUZgE2eup+LDyPRwM1IMwMsU+t8QL73YBdZx6Fk01nBBuT3D5Ba46xA4dWm4tLH04niErwhuSJalaHQA3AwUNdAiUx9tf48lkAeinALlcGG9Hj1o+EqhF9dXFad+4QQUN7nXIkSLRpEu9kWQZKW2FnlfJ6KADkokMmm77eMFAL8ifToozRn7rgjopyaR9hVxT91iNmEhWpKopMBfRjzP/bP9uWMr7cqIwLQlKAUgwg= + ELASTICSEARCH_URL: AgBxkNsQpkk6V5GIA8WuYBXE1iTLglrzRoR5HQ9QtBzsjDb9siCHQc+U9UFI72hDKi3yNO2g1lk+Nh7ooKHpqzzV0PSB0gyUJwo4MmqfxUDw3ef2BHezK7DK0z1YB/ZU8BLK5LenLchoKsKejv1YC1OQmG6EUHPW/3hm2qiS56anl6AIyUqXsokx+5QScjmHE3KybThaBkva/7PEQn/vks9wbY2mePvBPanSnVSLuSUY0yn0UXrmJnyW87oXVaJSQU/+5oDPdnb4pN8wivmjQEBLyVvjL2xSFz2druu4wjDpArup981GA7jCTarCzxjomGKnfxu7U98YKho1OUbuGevL1TXZaOkXaSIpls5mViZjUB2WaUVqQhHr9K1f6Hmhidh6S7v8NX4NbPKgKA7PTJ1dzi7EeKUCiIJPXiSI76H/rNlEx55TlroXQxEPjmQ91yEcCtRNa6MS2X6Flu539+t2m0+nm+Wqx7SmbGROjvCbSaYMShdsYQgnRj3VMgzoToLcnRPO0MdIop2L6GN5ocyFPk/QRUry+A/TH5eYkuMGu/FhctdrNEMo2kcYgIiTfqQ9tuQQqD3UCJ74DkUOod8OJoTu2EQ6RVBXqVCb9cM0AmoYgUDx1JW0l1124UYzpqIPhaRdPe6M8JH9zXuQAI93R7Paezn8GEMaXfiBRc/fVRUdTbAdbGvdmEZVeiczEK+JsKcdvMuqud15UmcFF0N7rfuia1u6TjCvaF/gvFIlLnMccFYmKACJnf1SJ/5Cx6L0SajjyBhxNP8DMVHhmMpbIqfQBPJho0nVQk7ZoU5Heg== + template: + metadata: + annotations: *a1 + name: "ingester-elasticsearch" + type: "Opaque" diff --git a/.k8s/kosko.toml b/.k8s/kosko.toml index 962a5fa57..1fdc93a72 100644 --- a/.k8s/kosko.toml +++ b/.k8s/kosko.toml @@ -5,4 +5,4 @@ require = ["ts-node/register"] # dont include underscores-prefix components [environments.prod] -components = ["!(_*)"] +components = ["!_*"] diff --git a/.k8s/utils/ES_INDEX_PREFIX.ts b/.k8s/utils/ES_INDEX_PREFIX.ts new file mode 100644 index 000000000..1b713f091 --- /dev/null +++ b/.k8s/utils/ES_INDEX_PREFIX.ts @@ -0,0 +1,17 @@ +let ES_INDEX_PREFIX = + process.env.ES_INDEX_PREFIX ?? + `cdtn-feature-${process.env.CI_ENVIRONMENT_SLUG}`; + +if (process.env.CI_COMMIT_REF_SLUG === "master") { + ES_INDEX_PREFIX = "cdtn-master"; +} + +if (process.env.CI_COMMIT_TAG) { + ES_INDEX_PREFIX = "cdtn-preprod"; +} + +if (process.env.PRODUCTION) { + ES_INDEX_PREFIX = "cdtn-prod"; +} + +export { ES_INDEX_PREFIX }; diff --git a/targets/ingester-elasticsearch/.gitlab-ci.yml b/targets/ingester-elasticsearch/.gitlab-ci.yml index 57d7e5a83..fbe79b784 100644 --- a/targets/ingester-elasticsearch/.gitlab-ci.yml +++ b/targets/ingester-elasticsearch/.gitlab-ci.yml @@ -55,6 +55,8 @@ Build ingester-elasticsearch: artifacts: true - job: Build module/ingester artifacts: true + - job: Build logger + artifacts: true before_script: - cd targets/ingester-elasticsearch script: @@ -79,3 +81,29 @@ Register ingester-elasticsearch: variables: CONTEXT: . IMAGE_NAME: $CI_REGISTRY_IMAGE/ingester-elasticsearch + +# +# +# + +ingester-elasticsearch (dev): + extends: + - .base_deploy_kosko_stage + stage: Deploy + rules: + - if: "$PRODUCTION || $RELEASE || $CI_COMMIT_TAG" + when: never + # - when: on_success + - when: manual + allow_failure: true + environment: + name: ${CI_COMMIT_REF_SLUG}-dev2 + needs: + - job: K8S Test + dependencies: null + before_script: + - kubectl config set-context --current --namespace=${KUBE_NAMESPACE} + - kubectl delete job ingester-elasticsearch || true + variables: + # kosko options + KOSKO_GENERATE_ARGS: --env dev jobs/ingester-elasticsearch