Merge pull request #402 from Open-Earth-Foundation/fix/source-info-di…

…splay Adjust CC to new data catalogue schema and use new columns on add data page
Open-Earth-Foundation · Mar 29, 2024 · 83f7657 · 83f7657
2 parents 3a52995 + 8f0aaf3
commit 83f7657
Show file tree

Hide file tree

Showing 12 changed files with 155 additions and 101 deletions.
diff --git a/app/migrations/20240329160302-new-catalog.cjs b/app/migrations/20240329160302-new-catalog.cjs
@@ -0,0 +1,79 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    return queryInterface.sequelize.transaction(async (transaction) => {
+      await queryInterface.addColumn("DataSource", "methodology_description", {
+        type: Sequelize.TEXT,
+        transaction,
+      });
+      await queryInterface.addColumn(
+        "DataSource",
+        "transformation_description",
+        {
+          type: Sequelize.TEXT,
+          transaction,
+        },
+      );
+      await queryInterface.addColumn("DataSource", "dataset_name", {
+        type: Sequelize.TEXT,
+        transaction,
+      });
+      await queryInterface.renameColumn(
+        "DataSource",
+        "name",
+        "datasource_name",
+        {
+          transaction,
+        },
+      );
+      await queryInterface.renameColumn(
+        "DataSource",
+        "description",
+        "dataset_description",
+        {
+          transaction,
+        },
+      );
+    });
+  },
+
+  async down(queryInterface) {
+    return queryInterface.sequelize.transaction(async (transaction) => {
+      await queryInterface.removeColumn(
+        "DataSource",
+        "methodology_description",
+        {
+          transaction,
+        },
+      );
+      await queryInterface.removeColumn(
+        "DataSource",
+        "transformation_description",
+        {
+          transaction,
+        },
+      );
+      await queryInterface.removeColumn("DataSource", "dataset_name", {
+        transaction,
+      });
+      await queryInterface.renameColumn(
+        "DataSource",
+        "datasource_name",
+        "name",
+        {
+          transaction,
+        },
+      );
+      await queryInterface.renameColumn(
+        "DataSource",
+        "dataset_description",
+        "description",
+        {
+          transaction,
+        },
+      );
+    });
+  },
+};
diff --git a/app/scripts/catalogue-sync.ts b/app/scripts/catalogue-sync.ts
@@ -6,10 +6,11 @@ import { logger } from "@/services/logger";
 
 interface Source {
   datasource_id: string;
-  name: string;
+  datasource_name: string;
+  dataset_name: string;
   source_type: string;
   url: string;
-  description: string;
+  dataset_description: string;
   access_type: string;
   geographical_location: string;
   start_year: number;
@@ -23,6 +24,8 @@ interface Source {
   notes: string;
   units: string;
   methodology_url: string;
+  methodology_description: string;
+  transformation_description: string;
   publisher_id: string;
   retrieval_method: string;
   api_endpoint: string;
@@ -116,7 +119,7 @@ async function syncDataCatalogue() {
 
     if (!source.notes) {
       // publisher_id is still a name at this stage
-      source.notes = `${source.name} by ${source.publisher_id}. For more details see ${source.url}`;
+      source.notes = `${source.datasource_name} by ${source.publisher_id}. For more details see ${source.url}`;
     }
 
     if (source.geographical_location === "global") {
@@ -171,48 +174,15 @@ async function syncDataCatalogue() {
   console.dir(sources);
   logger.debug("Saving sources...");
 
+  /*
+   * TODO switch to single query when this issue is fixed:
+   * https://github.com/sequelize/sequelize/issues/15221
+   * https://github.com/sequelize/sequelize/issues/13545
+   */
   for (const source of sources) {
     await db.models.DataSource.upsert(source);
   }
 
-  /* TODO switch to single query when this issue is fixed:
-  // https://github.com/sequelize/sequelize/issues/15221
-  // https://github.com/sequelize/sequelize/issues/13545
-  await db.models.DataSource.bulkCreate(
-    sources,
-    {
-      updateOnDuplicate: [
-        "name",
-        "sourceType",
-        "url",
-        "description",
-        "accessType",
-        "geographicalLocation",
-        "startYear",
-        "endYear",
-        "latestAccountingYear",
-        "frequencyOfUpdate",
-        "spatialResolution",
-        "language",
-        "accessibility",
-        "dataQuality",
-        "notes",
-        "units",
-        "methodologyUrl",
-        "publisherId",
-        "retrievalMethod",
-        "apiEndpoint",
-        "sectorId",
-        "subsectorId",
-        "subcategoryId",
-        "created",
-        "lastUpdated",
-      ],
-      // ignoreDuplicates: true,
-    },
-  );
-  */
-
   await catalogue.update({ lastUpdate: new Date(lastUpdate) });
   logger.debug("Updated Catalogue, done!");
 

diff --git a/app/src/app/[lng]/[inventory]/data/[step]/ActivityDataTab.tsx b/app/src/app/[lng]/[inventory]/data/[step]/ActivityDataTab.tsx
@@ -50,7 +50,7 @@ const activityDataUnits: Record<string, string[]> = {
 
 export function determineEmissionsFactorType(factor: EmissionsFactorData) {
   let sourceName = factor.dataSources
-    ? factor.dataSources[0].name || "Unknown data source"
+    ? factor.dataSources[0].datasetName || "Unknown data source"
     : "Unknown data source";
   if (sourceName.includes("IPCC") && sourceName.includes("US")) {
     return "National (US)";
@@ -108,7 +108,8 @@ export function ActivityDataTab({
       : Object.keys(factorsByUnit);
 
   // TODO this should happen in default form value, as the form still contains null/ undefined here
-  const selectedUnit = watch(prefix + "activityDataUnit") ?? scopeUnits[0] ?? "";
+  const selectedUnit =
+    watch(prefix + "activityDataUnit") ?? scopeUnits[0] ?? "";
   const selectedUnitShort = selectedUnit.split(" ")[0];
 
   useEffect(() => {

diff --git a/app/src/app/[lng]/[inventory]/data/[step]/SourceDrawer.tsx b/app/src/app/[lng]/[inventory]/data/[step]/SourceDrawer.tsx
@@ -111,7 +111,7 @@ export function SourceDrawer({
                 lineHeight="40px"
                 textTransform="capitalize"
               >
-                {source.name}
+                {source.datasetName}
               </Heading>
 
               <Heading size="sm">
@@ -245,7 +245,9 @@ export function SourceDrawer({
 
               <Stack className="space-y-4">
                 <Heading size="sm">{t("inside-dataset")}</Heading>
-                <Text color="content.tertiary">{source.description}</Text>
+                <Text color="content.tertiary">
+                  {source.datasetDescription}
+                </Text>
                 <chakra.hr borderColor="border.neutral" />
                 <Heading
                   size="sm"
@@ -268,9 +270,9 @@ export function SourceDrawer({
                     />
                   </Link>
                 </Heading>
-                {/*<Text color="content.tertiary">
-                  // TODO add methodology description to data source model/ data catalog
-                </Text>*/}
+                <Text color="content.tertiary">
+                  {source.methodologyDescription}
+                </Text>
                 <Heading
                   size="sm"
                   verticalAlign="baseline"
@@ -293,7 +295,7 @@ export function SourceDrawer({
                   </Link>
                 </Heading>
                 <Text color="content.tertiary">
-                  {t("transform-data-description")}
+                  {source.transformationDescription}
                 </Text>
               </Stack>
             </DrawerBody>

diff --git a/app/src/app/[lng]/[inventory]/data/[step]/page.tsx b/app/src/app/[lng]/[inventory]/data/[step]/page.tsx
@@ -60,7 +60,6 @@ import {
   MdOutlineEdit,
   MdOutlineHomeWork,
   MdOutlineSkipNext,
-  MdPlaylistAddCheck,
   MdRefresh,
 } from "react-icons/md";
 import { useDispatch, useSelector } from "react-redux";
@@ -73,9 +72,7 @@ import type {
 } from "./types";
 
 import AddFileDataModal from "@/components/Modals/add-file-data-modal";
-import { v4 as uuidv4 } from "uuid";
 import { InventoryValueAttributes } from "@/models/InventoryValue";
-import { DataSource } from "@/models/DataSource";
 
 function getMailURI(locode?: string, sector?: string, year?: number): string {
   const emails =
@@ -700,7 +697,7 @@ export default function AddDataSteps({
                     {/* TODO add icon to DataSource */}
                     <Icon as={MdHomeWork} boxSize={9} mb={6} />
                     <Heading size="sm" noOfLines={2} minHeight={10}>
-                      {source.name}
+                      {source.datasetName}
                     </Heading>
                     <Flex direction="row" my={4} wrap="wrap" gap={2}>
                       <Tag>
@@ -732,7 +729,8 @@ export default function AddDataSteps({
                       noOfLines={5}
                       minHeight={120}
                     >
-                      {source.description}
+                      {source.datasetDescription ||
+                        source.methodologyDescription}
                     </Text>
                     <Link
                       className="underline"

diff --git a/app/src/app/api/v0/inventory/[inventory]/route.ts b/app/src/app/api/v0/inventory/[inventory]/route.ts
@@ -192,7 +192,9 @@ async function inventoryXLS(inventory: Inventory): Promise<Buffer> {
 
       // TODO get these as separate values from emissions factor seeder
       const activityUnit = inventoryValue.activityUnits?.split("/")[0];
-      const emissionsFactorUnit = inventoryValue.activityUnits?.split(" ")[0].split("/")[1];
+      const emissionsFactorUnit = inventoryValue.activityUnits
+        ?.split(" ")[0]
+        .split("/")[1];
       const activityType = inventoryValue.activityUnits?.split(" ")[1];
 
       row.getCell("G").value = activityType;
@@ -241,7 +243,7 @@ async function inventoryXLS(inventory: Inventory): Promise<Buffer> {
       row.getCell("AD").value = inventoryValue.dataSource.dataQuality
         ?.slice(0, 1)
         .toUpperCase();
-      row.getCell("AP").value = inventoryValue.dataSource.name; // TODO add source to Data sources sheet (ID 20)
+      row.getCell("AP").value = inventoryValue.dataSource.datasetName; // TODO add source to Data sources sheet (ID 20)
 
       row.commit();
     }

diff --git a/app/src/i18n/locales/de/data.json b/app/src/i18n/locales/de/data.json
@@ -118,7 +118,6 @@
   "files-uploaded": "Dateien hochgeladen",
   "add-custom": "Benutzerdefiniert",
   "save-missing-scope-info": "Speichern oder fehlende Informationen ausfüllen, bevor Sie fortfahren.",
-  "transform-data-description": "Lorem ipsum dolor sit amet, qui minim labore adipisicing minim sint cillum sint consectetur cupidatat.",
   "file-context": "Erzählen Sie uns mehr über diese Datei",
   "file-data-subtitle": "Welche Art von Daten enthält sie?",
   "file-data-description": "Wählen Sie aus den folgenden Optionen. Dies wird uns helfen, diese Informationen besser zu identifizieren und in Ihre Bestandsaufnahme aufzunehmen.",

diff --git a/app/src/i18n/locales/en/data.json b/app/src/i18n/locales/en/data.json
@@ -168,7 +168,6 @@
   "city": "City",
   "inside-dataset": "What's inside this dataset",
   "transform-data-heading": "How do we transform this data?",
-  "transform-data-description": "Lorem ipsum dolor sit amet, qui minim labore adipisicing minim sint cillum sint consectetur cupidatat.",
   "file-context": "Tell Us More About This File",
   "file-data-subtitle": "What type of data is contained in it?",
   "file-data-description": " Choose from the options below. This will help us to better identify and include this information in your inventory.",

diff --git a/app/src/i18n/locales/es/data.json b/app/src/i18n/locales/es/data.json
@@ -114,7 +114,6 @@
   "files-uploaded": "Archivos subidos",
   "add-custom": "Agregar personalizado",
   "save-missing-scope-info": "Guarde o complete la información faltante antes de continuar.",
-  "transform-data-description": "Lorem ipsum dolor sit amet, qui minim labore adipisicing minim sint cillum sint consectetur cupidatat.",
   "file-context": "Cuéntanos más sobre este archivo",
   "file-data-subtitle": "¿Qué tipo de datos contiene?",
   "file-data-description": "Elija entre las opciones a continuación. Esto nos ayudará a identificar e incluir mejor esta información en su inventario.",