fix(ingester): handle legifrance new url style (#334)

* fix(ingester): handle legifrance new url style * fix: review
SocialGouv · Feb 23, 2021 · 7ecf74b · 7ecf74b
1 parent 4dafe99
commit 7ecf74b
Show file tree

Hide file tree

Showing 4 changed files with 455 additions and 163 deletions.
diff --git a/targets/ingester/src/index.d.ts b/targets/ingester/src/index.d.ts
@@ -1,122 +1,134 @@
-import type { SourceValues } from "@socialgouv/cdtn-sources"
-import type { Answer, Question, DilaRef, GenericAnswer } from "@socialgouv/contributions-data-types"
+import type { SourceValues } from "@socialgouv/cdtn-sources";
+import type {
+  Answer,
+  Question,
+  DilaRef,
+  GenericAnswer,
+} from "@socialgouv/contributions-data-types";
 
-export as namespace ingester
+export as namespace ingester;
 
 type Document = {
-  id: string
-  description: string
-  title: string
-  source: SourceValues
-  text: string
-  slug: string
-  is_searchable: Boolean
-}
+  id: string;
+  description: string;
+  title: string;
+  source: SourceValues;
+  text: string;
+  slug: string;
+  is_searchable: Boolean;
+};
 
 type ExternalDocument = Document & {
-  url: string
-}
+  url: string;
+};
 
 type ExtendedQuestion = Omit<Question, "answers"> & {
   answers: {
-    generic: GenericAnswer,
+    generic: GenericAnswer;
     conventionAnswer?: Answer & {
-      shortName: string
-    }
-    conventions?: Answer[]
-  }
-}
+      shortName: string;
+    };
+    conventions?: Answer[];
+  };
+};
 
-type Contribution = Document & ExtendedQuestion
+type Contribution = Document & ExtendedQuestion;
 
 type LegiArticle = ExternalDocument & {
-  dateDebut: number
-  html: string
-}
+  dateDebut: number;
+  html: string;
+};
 
 type FicheServicePublic = ExternalDocument & {
-  date: string //"O1/01/2021"
-  raw: string
-  referencedTexts: ReferencedTexts[]
-  legalReferences: LegalReference[]
-}
+  date: string; //"O1/01/2021"
+  raw: string;
+  referencedTexts: ReferencedTexts[];
+};
 
 type FicheTravailEmploi = ExternalDocument & {
-  date: string // "30/09/2020",
-  intro: string,
-  sections: TravailEmploiSection[]
-}
+  date: string; // "30/09/2020",
+  intro: string;
+  sections: TravailEmploiSection[];
+};
 
 type TravailEmploiSection = {
-  title: string
-  anchor: string
-  html: string
-  text: string
-  description: string
-  references: LegalReference[]
-}
+  title: string;
+  anchor: string;
+  html: string;
+  text: string;
+  description: string;
+  references: LegalReference[];
+};
 
 type AgreementPage = Document & {
-  num: Number
-  date_publi?: string
-  effectif?: Number
-  mtime?: Number
-  shortTitle: string
-  answers: AgreementAnswer[]
-  url?: string,
-  articlesByTheme: AgreementArticleByBlock[]
-  synonymes?: string[]
-}
+  num: Number;
+  date_publi?: string;
+  effectif?: Number;
+  mtime?: Number;
+  shortTitle: string;
+  answers: AgreementAnswer[];
+  url?: string;
+  articlesByTheme: AgreementArticleByBlock[];
+  synonymes?: string[];
+};
 
 type AgreementAnswer = {
-  index: Number
-  references: DilaRef[]
-  slug: string
-  question: string
-  answer: string
-}
+  index: Number;
+  references: DilaRef[];
+  slug: string;
+  question: string;
+  answer: string;
+};
 
 type AgreementArticleByBlock = {
   bloc: string;
   articles: {
-    cid: string
-    id: string
-    section: string
-    title: string
+    cid: string;
+    id: string;
+    section: string;
+    title: string;
   }[];
 };
 
-
 type InternalReference = {
-  title: string
-  slug: string
-  type: Exclude<cdtnSources.SourceRoute, "external">
-}
+  title: string;
+  slug: string;
+  type: Exclude<cdtnSources.SourceRoute, "external">;
+};
 
 type ExternalReference = {
-  title: string
-  url: string
-  type: "external"
-}
+  title: string;
+  url: string;
+  type: "external";
+};
 
 type LegalReference = {
-  id: string
-  cid: string
-  title: string
-  type: "conventions_collectives" | "code_du_travail"
-  url: string
-  slug: string
-}
-
-type ReferencedTexts = ExternalReference | InternalReference
+  id: string;
+  cid: string;
+  title: string;
+  type: "conventions_collectives" | "code_du_travail";
+  url: string;
+  slug: string;
+};
 
+type ReferencedTexts = ExternalReference | InternalReference;
 
 /** Document type */
-type CdtnDocument = Contribution | LegiArticle | AgreementPage | FicheServicePublic | FicheTravailEmploi
-
-type referenceResolver = (id: string) => (import("@socialgouv/legi-data-types").CodeSection | import("@socialgouv/legi-data-types").CodeArticle | import("@socialgouv/kali-data").AgreementSection | import("@socialgouv/kali-data").AgreementArticle)[]
-
-
+type CdtnDocument =
+  | Contribution
+  | LegiArticle
+  | AgreementPage
+  | FicheServicePublic
+  | FicheTravailEmploi;
+
+type referenceResolver = (
+  id: string
+) => (
+  | import("@socialgouv/legi-data-types").CodeSection
+  | import("@socialgouv/legi-data-types").CodeArticle
+  | import("@socialgouv/kali-data").AgreementSection
+  | import("@socialgouv/kali-data").AgreementArticle
+)[];
 
 // type EditorialDocument = Document & {
 //   date: string

diff --git a/targets/ingester/src/transform/fichesServicePublic/__tests__/parseReferences.test.js b/targets/ingester/src/transform/fichesServicePublic/__tests__/parseReferences.test.js
@@ -0,0 +1,139 @@
+import { describe, expect, test } from "@jest/globals";
+import { SOURCES } from "@socialgouv/cdtn-sources";
+
+import { extractNewReference, extractOldReference } from "../parseReference";
+
+/** @type{import("@socialgouv/kali-data-types").IndexedAgreement[]} */
+const agreements = [
+  {
+    active: true,
+    date_publi: "2019-02-15T00:00:00.000Z",
+    etat: "VIGUEUR_NON_ETEN",
+    id: "KALICONT123",
+    mtime: 1561146467,
+    nature: "IDCC",
+    num: 123,
+    shortTitle: "convention 123",
+    texte_de_base: "KALITEXT123",
+    title: "Convention collective nationale des 123",
+    url:
+      "https://www.legifrance.gouv.fr/affichIDCC.do?idConvention=KALICONT123",
+  },
+];
+const referenceResolver = jest.fn().mockImplementation((id) => {
+  if (id === "LEGISCTA42")
+    return [
+      {
+        children: [
+          {
+            data: {
+              id: "LEGIARTI42",
+              num: "L42-1",
+            },
+            type: "article",
+          },
+          {
+            data: {
+              id: "LEGIARTI43",
+              num: "L43-1",
+            },
+            type: "article",
+          },
+        ],
+        data: {
+          id: "LEGISCTA42",
+          num: "L42",
+        },
+        type: "section",
+      },
+    ];
+  if (id === "LEGIARTI42")
+    return [
+      {
+        data: {
+          id: "LEGIARTI42",
+          num: "L42-1",
+        },
+        type: "article",
+      },
+    ];
+  return [];
+});
+
+const external = {
+  title: "external",
+  type: SOURCES.EXTERNALS,
+  url: "https://legi.url/affichJuriJudi.do?idTexte=JURITEXT000007023096",
+};
+const external2 = {
+  title: "external",
+  type: SOURCES.EXTERNALS,
+  url: "https://legi.url/circulaire/id/39848",
+};
+
+const article42 = {
+  slug: "l42-1",
+  title: "L42-1",
+  type: SOURCES.CDT,
+};
+const article43 = {
+  slug: "l43-1",
+  title: "L43-1",
+  type: SOURCES.CDT,
+};
+const kali = {
+  slug: "123-convention-123",
+  title: "convention 123",
+  type: SOURCES.CCN,
+};
+const section42 = [article42, article43];
+
+describe("extractNewReference", () => {
+  test.each([
+    ["https://legi.url/codes/id/LEGISCTA42", "L42", section42],
+    ["/codes/section_lc/LEGITEXT000006072050/LEGISCTA42", "L42", section42],
+    ["https://legi.url/codes/article_lc/LEGIARTI42/", "L42-1", [article42]],
+    ["https://legi.url/codes/id/LEGIARTI42/", "L42-1", [article42]],
+    ["https://legi.url/conv_coll/id/KALICONT123", "convention 123", [kali]],
+    ["https://legi.url/circulaire/id/39848", "external", [external2]],
+  ])(`it should extract new reference from %p`, (url, label, reference) => {
+    expect(
+      extractNewReference(url, label, referenceResolver, agreements)
+    ).toEqual(reference);
+  });
+});
+
+describe("extractOldReference", () => {
+  test.each([
+    [
+      "https://legi.url/affichCode.do?cidTexte=LEGITEXT000006072050&idSectionTA=LEGISCTA42",
+      "L42",
+      section42,
+    ],
+
+    [
+      "https://legi.url/affichCode.do?idArticle=LEGIARTI42&idSectionTA=LEGISCTA42&cidTexte=LEGITEXT000006072050",
+      "L42-1",
+      [article42],
+    ],
+    [
+      "https://legi.url/affichCodeArticle.do;?cidTexte=LEGITEXT000006072050&idArticle=LEGIARTI42",
+      "L42-1",
+      [article42],
+    ],
+    [
+      "https://legi.url/affichIDCC.do?cidTexte=KALITEXT000005670044&idSectionTA=KALISCTA000005747382&idConvention=KALICONT123",
+      "convention 123",
+      [kali],
+    ],
+    [
+      "https://legi.url/affichJuriJudi.do?idTexte=JURITEXT000007023096",
+      "external",
+      [external],
+    ],
+  ])(`it should extract old reference for %s %s`, (url, label, reference) => {
+    expect(
+      extractOldReference(url, label, referenceResolver, agreements)
+    ).toEqual(reference);
+  });
+});
diff --git a/targets/ingester/src/transform/fichesServicePublic/format.js b/targets/ingester/src/transform/fichesServicePublic/format.js
@@ -64,7 +64,7 @@ export function format(fiche, resolveCdtReference, agreements) {
     (el) => el.name === "Reference"
   );
 
-  const [legalReferences, referencedTexts] = parseReferences(
+  const referencedTexts = parseReferences(
     references,
     resolveCdtReference,
     agreements
@@ -74,7 +74,6 @@ export function format(fiche, resolveCdtReference, agreements) {
     date,
     description,
     id,
-    legalReferences,
     raw: JSON.stringify(publication),
     referencedTexts,
     source: SOURCES.SHEET_SP,