feat(alerts): add kali / legi alerts (#24)

Co-authored-by: Douglas Duteil <douglasduteil@gmail.com>

.dockerignore

@@ -4,3 +4,4 @@
 *.md
 **/node_modules
 **/.next/cache
+data/*

.gitignore

@@ -2,3 +2,4 @@
 *.DS_Store
 node_modules
 .env.production
+data/*

.gitlab-ci.yml

@@ -17,6 +17,10 @@ variables:
   ENABLE_AZURE_POSTGRES: 1
   VALUES_FILE: ./.k8s/app.values.yml
 
+Install:
+  extends: .autodevops_install
+  image: node:12.18.0-alpine3.11
+
 Build:
   extends: .autodevops_build
   variables:

Dockerfile

@@ -1,14 +1,18 @@
-FROM node:14.4-alpine3.11
+FROM node:12.18.0-alpine3.11
 
 WORKDIR /app
 
 COPY package.json yarn.lock ./
 
-RUN yarn --frozen-lockfile
+RUN apk add --no-cache build-base python --virtual .build-deps \
+  && yarn --production --frozen-lockfile \
+  && apk del .build-deps
 
 COPY next.config.js  ./
 COPY .env  ./.env
 COPY .next/ ./.next
+COPY scripts/ ./scripts
+COPY data/ ./data
 COPY public/ ./public
 
 USER node

hasura/Dockerfile

@@ -1,4 +1,4 @@
-FROM hasura/graphql-engine:v1.2.1.cli-migrations-v2
+FROM hasura/graphql-engine:v1.2.2.cli-migrations-v2
 ENV HASURA_GRAPHQL_ENABLE_TELEMETRY false
 COPY ./migrations /hasura-migrations
 COPY ./metadata /hasura-metadata

hasura/metadata/tables.yaml

@@ -98,6 +98,24 @@
     headers:
     - name: email-secret
       value_from_env: ACCOUNT_EMAIL_SECRET
+- table:
+    schema: public
+    name: alert_status
+  array_relationships:
+  - name: alerts
+    using:
+      foreign_key_constraint_on:
+        column: status
+        table:
+          schema: public
+          name: alerts
+- table:
+    schema: public
+    name: alerts
+  object_relationships:
+  - name: source
+    using:
+      foreign_key_constraint_on: repository
 - table:
     schema: public
     name: roles
@@ -122,3 +140,14 @@
       columns:
       - role
       filter: {}
+- table:
+    schema: public
+    name: sources
+  array_relationships:
+  - name: alerts
+    using:
+      foreign_key_constraint_on:
+        column: repository
+        table:
+          schema: public
+          name: alerts

hasura/migrations/1591618615701_alerts/down.sql

@@ -0,0 +1,6 @@
+
+DROP TABLE "public"."alerts";
+
+DROP TABLE "public"."sources";
+
+DROP TABLE "public"."alert_status";

hasura/migrations/1591618615701_alerts/up.sql

@@ -0,0 +1,45 @@
+
+CREATE TABLE "public"."alert_status"("name" text NOT NULL DEFAULT 'new', PRIMARY KEY ("name") );
+COMMENT ON TABLE "public"."alert_status" IS E'alert statuses';
+
+INSERT INTO public.alert_status (name) VALUES ('todo');
+INSERT INTO public.alert_status (name) VALUES ('doing');
+INSERT INTO public.alert_status (name) VALUES ('done');
+INSERT INTO public.alert_status (name) VALUES ('rejected');
+
+CREATE TABLE "public"."sources"(
+  "repository" text NOT NULL,
+  "label" text NOT NULL,
+  "tag" text NOT NULL,
+  "created_at" timestamptz NOT NULL DEFAULT now(),
+  PRIMARY KEY ("repository")
+);
+
+COMMENT ON TABLE "public"."sources" IS E'sources are git repository that acts  as data sources to track changes';
+
+INSERT INTO public.sources (repository, label, tag) VALUES ('socialgouv/legi-data', 'code du travail', 'v1.12.0');
+INSERT INTO public.sources (repository, label, tag) VALUES ('socialgouv/kali-data', 'conventions collectives', 'v1.64.0');
+
+CREATE TABLE "public"."alerts"(
+  "id" uuid NOT NULL DEFAULT gen_random_uuid(),
+  "info" jsonb NOT NULL,
+  "status" text NOT NULL DEFAULT 'todo',
+  "repository" text NOT NULL,
+  "ref" text NOT NULL,
+  "changes" jsonb NOT NULL,
+  "created_at" timestamptz NULL DEFAULT now(),
+  "updated_at" timestamptz NULL DEFAULT now(),
+  PRIMARY KEY ("id") ,
+  FOREIGN KEY ("status") REFERENCES "public"."alert_status"("name") ON UPDATE restrict ON DELETE restrict,
+  FOREIGN KEY ("repository") REFERENCES "public"."sources"("repository") ON UPDATE restrict ON DELETE cascade);
+
+COMMENT ON TABLE "public"."alerts" IS
+  E'alerts reprensent a change in a text from a source';
+
+CREATE TRIGGER "set_public_alerts_updated_at"
+  BEFORE UPDATE ON public.alerts
+  FOR EACH ROW
+  EXECUTE PROCEDURE trigger_set_timestamp();
+
+COMMENT ON TRIGGER "set_public_alerts_updated_at" ON public.alerts
+  IS 'trigger to set value of column "updated_at" to current timestamp on row update';

package.json

@@ -6,6 +6,7 @@
   "dependencies": {
     "@hapi/boom": "^9.1.0",
     "@hapi/joi": "^17.1.1",
+    "@reach/accordion": "^0.10.3",
     "@reach/dialog": "^0.10.3",
     "@reach/menu-button": "^0.10.3",
     "@reach/visually-hidden": "^0.10.2",
@@ -17,21 +18,26 @@
     "@zeit/next-source-maps": "0.0.4-canary.1",
     "argon2": "^0.26.2",
     "cookie": "^0.4.1",
-    "dotenv": "^8.2.0",
+    "diff": "^4.0.2",
     "graphql": "^15.0.0",
     "http-proxy-middleware": "^1.0.4",
+    "isomorphic-unfetch": "^3.0.0",
     "jsonwebtoken": "^8.5.1",
     "next": "^9.4.4",
     "next-urql": "^0.3.8",
+    "nodegit": "^0.26.5",
     "nodemailer": "^6.4.8",
     "polished": "^3.6.5",
     "react": "^16.13.1",
     "react-dom": "^16.13.1",
     "react-hook-form": "^5.7.2",
     "react-icons": "^3.10.0",
     "react-is": "^16.13.1",
+    "semver": "^7.3.2",
     "sentry-testkit": "^3.2.1",
     "theme-ui": "^0.3.1",
+    "unist-util-parents": "^1.0.3",
+    "unist-util-select": "^3.0.1",
     "urql": "^1.9.8",
     "uuid": "^8.1.0",
     "wonka": "^4.0.14"
@@ -50,7 +56,12 @@
   "scripts": {
     "dev": "next dev",
     "build": "next build",
+    "prestart": "node scripts/update-alerts.js",
     "start": "next start",
+    "alert": " node scripts/update-alerts.js",
+    "alert:dev": "GRAPHQL_ENDPOINT=http://localhost:8080/v1/graphql HASURA_GRAPHQL_ADMIN_SECRET=admin1 node scripts/update-alerts.js",
+    "alert:dump": "GRAPHQL_ENDPOINT=http://localhost:8080/v1/graphql HASURA_GRAPHQL_ADMIN_SECRET=admin1 DUMP=true node scripts/update-alerts.js > data/dump.json",
+    "alert:populate": "GRAPHQL_ENDPOINT=http://localhost:8080/v1/graphql HASURA_GRAPHQL_ADMIN_SECRET=admin1 node scripts/add-alerts.js",
     "lint": "eslint src/*",
     "test": "jest"
   },

scripts/add-alerts.js

@@ -0,0 +1,30 @@
+const { promises: fs } = require("fs");
+const path = require("path");
+const filename =
+  process.env.DUMP_FILE || path.join(__dirname, "..", "data", "dump.json");
+const { updateSource, insertAlert } = require("./update-alerts");
+
+async function main() {
+  console.log(filename);
+  const fileContent = await fs.readFile(filename);
+  const data = JSON.parse(fileContent);
+
+  for (const result of data) {
+    if (result.changes.length === 0) {
+      console.log(`no update for ${result.repository}`);
+      continue;
+    }
+    const inserts = await Promise.all(
+      result.changes.map((diff) => insertAlert(result.repository, diff))
+    );
+    inserts.forEach((insert) => {
+      const { ref, repository, info } = insert.returning[0];
+      console.log(`insert alert for ${ref} on ${repository} (${info.file})`);
+    });
+    console.log(`create ${inserts.length} alert for ${result.repository}`);
+    const update = await updateSource(result.repository, result.newRef);
+    console.log(`update source ${update.repository} to ${update.tag}`);
+  }
+}
+
+main().catch(console.error);

scripts/lib/ccn-list.js

@@ -0,0 +1,55 @@
+const ccns = [
+  { id: "KALICONT000005635624", num: 16 },
+  { id: "KALICONT000005635234", num: 29 },
+  { id: "KALICONT000005635613", num: 44 },
+  { id: "KALICONT000005635630", num: 86 },
+  { id: "KALICONT000005635184", num: 176 },
+  { id: "KALICONT000005635872", num: 275 },
+  { id: "KALICONT000005635856", num: 292 },
+  { id: "KALICONT000005635407", num: 413 },
+  { id: "KALICONT000005635373", num: 573 },
+  { id: "KALICONT000005635842", num: 650 },
+  { id: "KALICONT000005635617", num: 675 },
+  { id: "KALICONT000005635826", num: 787 },
+  { id: "KALICONT000005635886", num: 843 },
+  { id: "KALICONT000005635953", num: 1043 },
+  { id: "KALICONT000005635191", num: 1090 },
+  { id: "KALICONT000005635409", num: 1147 },
+  { id: "KALICONT000005635418", num: 1266 },
+  { id: "KALICONT000005635405", num: 1351 },
+  { id: "KALICONT000005635653", num: 1404 },
+  { id: "KALICONT000005635444", num: 1480 },
+  { id: "KALICONT000005635594", num: 1483 },
+  { id: "KALICONT000005635173", num: 1486 },
+  { id: "KALICONT000005635596", num: 1501 },
+  { id: "KALICONT000005635421", num: 1505 },
+  { id: "KALICONT000005635435", num: 1516 },
+  { id: "KALICONT000005635870", num: 1517 },
+  { id: "KALICONT000005635177", num: 1518 },
+  { id: "KALICONT000005635413", num: 1527 },
+  { id: "KALICONT000005635221", num: 1596 },
+  { id: "KALICONT000005635220", num: 1597 },
+  { id: "KALICONT000005635871", num: 1606 },
+  { id: "KALICONT000005635918", num: 1672 },
+  { id: "KALICONT000005635467", num: 1702 },
+  { id: "KALICONT000005635685", num: 1740 },
+  { id: "KALICONT000005635534", num: 1979 },
+  { id: "KALICONT000005635528", num: 1996 },
+  { id: "KALICONT000005635550", num: 2098 },
+  { id: "KALICONT000005635792", num: 2111 },
+  { id: "KALICONT000005635780", num: 2120 },
+  { id: "KALICONT000005635557", num: 2148 },
+  { id: "KALICONT000005635085", num: 2216 },
+  { id: "KALICONT000005635813", num: 2264 },
+  { id: "KALICONT000005635807", num: 2395 },
+  { id: "KALICONT000017941839", num: 2420 },
+  { id: "KALICONT000017577652", num: 2511 },
+  { id: "KALICONT000018563755", num: 2596 },
+  { id: "KALICONT000018773893", num: 2609 },
+  { id: "KALICONT000018926209", num: 2614 },
+  { id: "KALICONT000025805800", num: 2941 },
+  { id: "KALICONT000027172335", num: 3043 },
+  { id: "KALICONT000027084096", num: 3127 },
+];
+
+module.exports = { ccns };