build(k8s): deploy ingester-elasticsearch job

SocialGouv · Mar 25, 2021 · d1001ea · d1001ea
1 parent 96118cc
commit d1001ea
Show file tree

Hide file tree

Showing 8 changed files with 232 additions and 1 deletion.
diff --git a/.k8s/__tests__/generate ingester-elasticsearch/--dev.ts b/.k8s/__tests__/generate ingester-elasticsearch/--dev.ts
@@ -0,0 +1,14 @@
+//
+
+import { getEnvManifests } from "@socialgouv/kosko-charts/testing";
+import { project } from "@socialgouv/kosko-charts/testing/fake/gitlab-ci.env";
+
+jest.setTimeout(1000 * 60);
+test("kosko generate ingester-elasticsearch --dev", async () => {
+  expect(
+    await getEnvManifests("dev", "jobs/ingester-elasticsearch", {
+      ...project("code-du-travail-numerique").dev,
+      RANCHER_PROJECT_ID: "c-bar:p-foo",
+    })
+  ).toMatchSnapshot();
+});
diff --git a/.k8s/__tests__/generate ingester-elasticsearch/__snapshots__/--dev.ts.snap b/.k8s/__tests__/generate ingester-elasticsearch/__snapshots__/--dev.ts.snap
@@ -0,0 +1,66 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`kosko generate ingester-elasticsearch --dev 1`] = `
+"---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ingester-elasticsearch
+data:
+  CDTN_ADMIN_ENDPOINT: https://cdtn-admin.fabrique.social.gouv.fr/api/graphql
+  NLP_URL: https://preprod-serving-ml.dev2.fabrique.social.gouv.fr
+  NODE_ENV: production
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  annotations:
+    sealedsecrets.bitnami.com/cluster-wide: 'true'
+  name: ingester-elasticsearch
+  namespace: null
+spec:
+  encryptedData:
+    ELASTICSEARCH_DATA_TOKEN: >-
+      AgA+5TqL1kYeeCl8R7j0JE6dgEQ+9gNtGvD2eGyNHYveWA9N0l3UD+EF3OUNzcoeBseMVK/7tgPC1PBXTluABbIMZe8v0oKf7McgJZEkXpzUh/wL+icjGKIDb1zFCYV+Ox2m1L0fsMVRcOEBAh0RpgRcY8CEpLpszoEvlgTKnFljkqQfDUTwofr96Q6FYolD1cjikx+jzBvhxAkbf8mrrwz775UUY3u9OPFk3TQSqVVyd0kzwJWRTRlb1g3mYtfC79FM1j/44LkBYdeAQfNDTx12LNg+n23RQLXlGfEu2oTMfkVQgVnQ+Xqxgzc4wvvglgOP+yJe0S01i49uBXIh3ZniBM99rVtwRONu6Q1RTN2lzGwdMPT1WhiCk8nJDqrLFYs2yg++qWYo/HMPVDYCbzpH+7j1NbEvzUSH4+KSb8Qsk1YIIMskOACWNpvmAraT6KzxsK/QnEAQckex5Y03xSFB1L4boAlVkmhOkJXaz92Ug/ab6tMWTKoalz0zxBYHjVovJeN0RhKV9kw48S12B3aSkqgO3QTHN7CY/FsebcUZgE2eup+LDyPRwM1IMwMsU+t8QL73YBdZx6Fk01nBBuT3D5Ba46xA4dWm4tLH04niErwhuSJalaHQA3AwUNdAiUx9tf48lkAeinALlcGG9Hj1o+EqhF9dXFad+4QQUN7nXIkSLRpEu9kWQZKW2FnlfJ6KADkokMmm77eMFAL8ifToozRn7rgjopyaR9hVxT91iNmEhWpKopMBfRjzP/bP9uWMr7cqIwLQlKAUgwg=
+    ELASTICSEARCH_URL: >-
+      AgBxkNsQpkk6V5GIA8WuYBXE1iTLglrzRoR5HQ9QtBzsjDb9siCHQc+U9UFI72hDKi3yNO2g1lk+Nh7ooKHpqzzV0PSB0gyUJwo4MmqfxUDw3ef2BHezK7DK0z1YB/ZU8BLK5LenLchoKsKejv1YC1OQmG6EUHPW/3hm2qiS56anl6AIyUqXsokx+5QScjmHE3KybThaBkva/7PEQn/vks9wbY2mePvBPanSnVSLuSUY0yn0UXrmJnyW87oXVaJSQU/+5oDPdnb4pN8wivmjQEBLyVvjL2xSFz2druu4wjDpArup981GA7jCTarCzxjomGKnfxu7U98YKho1OUbuGevL1TXZaOkXaSIpls5mViZjUB2WaUVqQhHr9K1f6Hmhidh6S7v8NX4NbPKgKA7PTJ1dzi7EeKUCiIJPXiSI76H/rNlEx55TlroXQxEPjmQ91yEcCtRNa6MS2X6Flu539+t2m0+nm+Wqx7SmbGROjvCbSaYMShdsYQgnRj3VMgzoToLcnRPO0MdIop2L6GN5ocyFPk/QRUry+A/TH5eYkuMGu/FhctdrNEMo2kcYgIiTfqQ9tuQQqD3UCJ74DkUOod8OJoTu2EQ6RVBXqVCb9cM0AmoYgUDx1JW0l1124UYzpqIPhaRdPe6M8JH9zXuQAI93R7Paezn8GEMaXfiBRc/fVRUdTbAdbGvdmEZVeiczEK+JsKcdvMuqud15UmcFF0N7rfuia1u6TjCvaF/gvFIlLnMccFYmKACJnf1SJ/5Cx6L0SajjyBhxNP8DMVHhmMpbIqfQBPJho0nVQk7ZoU5Heg==
+  template:
+    metadata:
+      annotations:
+        sealedsecrets.bitnami.com/cluster-wide: 'true'
+      name: ingester-elasticsearch
+    type: Opaque
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ingester-elasticsearch
+  namespace: cdtn-admin-24-branch-42
+spec:
+  backoffLimit: 0
+  template:
+    spec:
+      containers:
+        - name: ingester-elasticsearch
+          image: >-
+            registry.gitlab.factory.social.gouv.fr/socialgouv/cdtn-admin/ingester-elasticsearch:8843083edb7f873cad1d1420731a60773594ffae
+          imagePullPolicy: IfNotPresent
+          resources:
+            limits:
+              cpu: '2'
+              memory: 1Gi
+            requests:
+              cpu: '1'
+              memory: 512Mi
+          envFrom:
+            - configMapRef:
+                name: ingester-elasticsearch
+            - secretRef:
+                name: ingester-elasticsearch
+          env:
+            - name: ES_INDEX_PREFIX
+              value: cdtn-master
+      restartPolicy: Never
+  ttlSecondsAfterFinished: 86400
+"
+`;
diff --git a/.k8s/components/jobs/ingester-elasticsearch/index.ts b/.k8s/components/jobs/ingester-elasticsearch/index.ts
@@ -0,0 +1,81 @@
+import { ok } from "assert";
+import env from "@kosko/env";
+import type { SealedSecret } from "@kubernetes-models/sealed-secrets/bitnami.com/v1alpha1/SealedSecret";
+import { Job } from "kubernetes-models/batch/v1/Job";
+import gitlab from "@socialgouv/kosko-charts/environments/gitlab";
+import { loadYaml } from "@socialgouv/kosko-charts/utils/getEnvironmentComponent";
+import { merge } from "@socialgouv/kosko-charts/utils/@kosko/env/merge";
+import { ConfigMap } from "kubernetes-models/_definitions/IoK8sApiCoreV1ConfigMap";
+
+import { ES_INDEX_PREFIX } from "../../../utils/ES_INDEX_PREFIX";
+
+ok(process.env.CI_REGISTRY_IMAGE, "Missing CI_REGISTRY_IMAGE");
+
+const configMap = loadYaml<ConfigMap>(
+  env,
+  `ingester-elasticsearch.configmap.yaml`
+);
+ok(configMap, "Missing ingester-elasticsearch.configmap.yaml");
+const secret = loadYaml<SealedSecret>(
+  env,
+  "ingester-elasticsearch.sealed-secret.yaml"
+);
+ok(secret, "Missing ingester-elasticsearch.sealed-secret.yaml");
+
+const ingester = () => {
+  const envParams = merge(gitlab(process.env), {});
+
+  const job = new Job({
+    metadata: {
+      name: "ingester-elasticsearch",
+      namespace: envParams.namespace.name,
+    },
+    spec: {
+      backoffLimit: 0,
+      template: {
+        spec: {
+          containers: [
+            {
+              name: "ingester-elasticsearch",
+              image: `${process.env.CI_REGISTRY_IMAGE}/ingester-elasticsearch:${process.env.CI_COMMIT_SHA}`,
+              imagePullPolicy: "IfNotPresent",
+              resources: {
+                limits: {
+                  cpu: "2",
+                  memory: "1Gi",
+                },
+                requests: {
+                  cpu: "1",
+                  memory: "512Mi",
+                },
+              },
+              envFrom: [
+                {
+                  configMapRef: {
+                    name: configMap?.metadata?.name,
+                  },
+                },
+                {
+                  secretRef: {
+                    name: secret?.metadata?.name,
+                  },
+                },
+              ],
+              env: [
+                {
+                  name: "ES_INDEX_PREFIX",
+                  value: ES_INDEX_PREFIX,
+                },
+              ],
+            },
+          ],
+          restartPolicy: "Never",
+        },
+      },
+      ttlSecondsAfterFinished: 86400,
+    },
+  });
+  return job;
+};
+
+export default [configMap, secret, ingester()];
diff --git a/.k8s/environments/dev/ingester-elasticsearch.configmap.yaml b/.k8s/environments/dev/ingester-elasticsearch.configmap.yaml
@@ -0,0 +1,8 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: ingester-elasticsearch
+data:
+  CDTN_ADMIN_ENDPOINT: "https://cdtn-admin.fabrique.social.gouv.fr/api/graphql"
+  NLP_URL: "https://preprod-serving-ml.dev2.fabrique.social.gouv.fr"
+  NODE_ENV: "production"
diff --git a/.k8s/environments/dev/ingester-elasticsearch.sealed-secret.yaml b/.k8s/environments/dev/ingester-elasticsearch.sealed-secret.yaml
@@ -0,0 +1,17 @@
+---
+apiVersion: "bitnami.com/v1alpha1"
+kind: "SealedSecret"
+metadata:
+  annotations: &a1
+    sealedsecrets.bitnami.com/cluster-wide: "true"
+  name: "ingester-elasticsearch"
+  namespace: null
+spec:
+  encryptedData:
+    ELASTICSEARCH_DATA_TOKEN: AgA+5TqL1kYeeCl8R7j0JE6dgEQ+9gNtGvD2eGyNHYveWA9N0l3UD+EF3OUNzcoeBseMVK/7tgPC1PBXTluABbIMZe8v0oKf7McgJZEkXpzUh/wL+icjGKIDb1zFCYV+Ox2m1L0fsMVRcOEBAh0RpgRcY8CEpLpszoEvlgTKnFljkqQfDUTwofr96Q6FYolD1cjikx+jzBvhxAkbf8mrrwz775UUY3u9OPFk3TQSqVVyd0kzwJWRTRlb1g3mYtfC79FM1j/44LkBYdeAQfNDTx12LNg+n23RQLXlGfEu2oTMfkVQgVnQ+Xqxgzc4wvvglgOP+yJe0S01i49uBXIh3ZniBM99rVtwRONu6Q1RTN2lzGwdMPT1WhiCk8nJDqrLFYs2yg++qWYo/HMPVDYCbzpH+7j1NbEvzUSH4+KSb8Qsk1YIIMskOACWNpvmAraT6KzxsK/QnEAQckex5Y03xSFB1L4boAlVkmhOkJXaz92Ug/ab6tMWTKoalz0zxBYHjVovJeN0RhKV9kw48S12B3aSkqgO3QTHN7CY/FsebcUZgE2eup+LDyPRwM1IMwMsU+t8QL73YBdZx6Fk01nBBuT3D5Ba46xA4dWm4tLH04niErwhuSJalaHQA3AwUNdAiUx9tf48lkAeinALlcGG9Hj1o+EqhF9dXFad+4QQUN7nXIkSLRpEu9kWQZKW2FnlfJ6KADkokMmm77eMFAL8ifToozRn7rgjopyaR9hVxT91iNmEhWpKopMBfRjzP/bP9uWMr7cqIwLQlKAUgwg=
+    ELASTICSEARCH_URL: AgBxkNsQpkk6V5GIA8WuYBXE1iTLglrzRoR5HQ9QtBzsjDb9siCHQc+U9UFI72hDKi3yNO2g1lk+Nh7ooKHpqzzV0PSB0gyUJwo4MmqfxUDw3ef2BHezK7DK0z1YB/ZU8BLK5LenLchoKsKejv1YC1OQmG6EUHPW/3hm2qiS56anl6AIyUqXsokx+5QScjmHE3KybThaBkva/7PEQn/vks9wbY2mePvBPanSnVSLuSUY0yn0UXrmJnyW87oXVaJSQU/+5oDPdnb4pN8wivmjQEBLyVvjL2xSFz2druu4wjDpArup981GA7jCTarCzxjomGKnfxu7U98YKho1OUbuGevL1TXZaOkXaSIpls5mViZjUB2WaUVqQhHr9K1f6Hmhidh6S7v8NX4NbPKgKA7PTJ1dzi7EeKUCiIJPXiSI76H/rNlEx55TlroXQxEPjmQ91yEcCtRNa6MS2X6Flu539+t2m0+nm+Wqx7SmbGROjvCbSaYMShdsYQgnRj3VMgzoToLcnRPO0MdIop2L6GN5ocyFPk/QRUry+A/TH5eYkuMGu/FhctdrNEMo2kcYgIiTfqQ9tuQQqD3UCJ74DkUOod8OJoTu2EQ6RVBXqVCb9cM0AmoYgUDx1JW0l1124UYzpqIPhaRdPe6M8JH9zXuQAI93R7Paezn8GEMaXfiBRc/fVRUdTbAdbGvdmEZVeiczEK+JsKcdvMuqud15UmcFF0N7rfuia1u6TjCvaF/gvFIlLnMccFYmKACJnf1SJ/5Cx6L0SajjyBhxNP8DMVHhmMpbIqfQBPJho0nVQk7ZoU5Heg==
+  template:
+    metadata:
+      annotations: *a1
+      name: "ingester-elasticsearch"
+    type: "Opaque"
diff --git a/.k8s/kosko.toml b/.k8s/kosko.toml
@@ -5,4 +5,4 @@ require = ["ts-node/register"]
 
 # dont include underscores-prefix components
 [environments.prod]
-components = ["!(_*)"]
+components = ["!_*"]
diff --git a/.k8s/utils/ES_INDEX_PREFIX.ts b/.k8s/utils/ES_INDEX_PREFIX.ts
@@ -0,0 +1,17 @@
+let ES_INDEX_PREFIX =
+  process.env.ES_INDEX_PREFIX ??
+  `cdtn-feature-${process.env.CI_ENVIRONMENT_SLUG}`;
+
+if (process.env.CI_COMMIT_REF_SLUG === "master") {
+  ES_INDEX_PREFIX = "cdtn-master";
+}
+
+if (process.env.CI_COMMIT_TAG) {
+  ES_INDEX_PREFIX = "cdtn-preprod";
+}
+
+if (process.env.PRODUCTION) {
+  ES_INDEX_PREFIX = "cdtn-prod";
+}
+
+export { ES_INDEX_PREFIX };
diff --git a/targets/ingester-elasticsearch/.gitlab-ci.yml b/targets/ingester-elasticsearch/.gitlab-ci.yml
@@ -55,6 +55,8 @@ Build ingester-elasticsearch:
       artifacts: true
     - job: Build module/ingester
       artifacts: true
+    - job: Build logger
+      artifacts: true
   before_script:
     - cd targets/ingester-elasticsearch
   script:
@@ -79,3 +81,29 @@ Register ingester-elasticsearch:
   variables:
     CONTEXT: .
     IMAGE_NAME: $CI_REGISTRY_IMAGE/ingester-elasticsearch
+
+#
+#
+#
+
+ingester-elasticsearch (dev):
+  extends:
+    - .base_deploy_kosko_stage
+  stage: Deploy
+  rules:
+    - if: "$PRODUCTION || $RELEASE || $CI_COMMIT_TAG"
+      when: never
+    # - when: on_success
+    - when: manual
+      allow_failure: true
+  environment:
+    name: ${CI_COMMIT_REF_SLUG}-dev2
+  needs:
+    - job: K8S Test
+  dependencies: null
+  before_script:
+    - kubectl config set-context --current --namespace=${KUBE_NAMESPACE}
+    - kubectl delete job ingester-elasticsearch || true
+  variables:
+    # kosko options
+    KOSKO_GENERATE_ARGS: --env dev jobs/ingester-elasticsearch