diff --git a/app/migrations/20240329160302-new-catalog.cjs b/app/migrations/20240329160302-new-catalog.cjs new file mode 100644 index 000000000..298f6ca9b --- /dev/null +++ b/app/migrations/20240329160302-new-catalog.cjs @@ -0,0 +1,79 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + return queryInterface.sequelize.transaction(async (transaction) => { + await queryInterface.addColumn("DataSource", "methodology_description", { + type: Sequelize.TEXT, + transaction, + }); + await queryInterface.addColumn( + "DataSource", + "transformation_description", + { + type: Sequelize.TEXT, + transaction, + }, + ); + await queryInterface.addColumn("DataSource", "dataset_name", { + type: Sequelize.TEXT, + transaction, + }); + await queryInterface.renameColumn( + "DataSource", + "name", + "datasource_name", + { + transaction, + }, + ); + await queryInterface.renameColumn( + "DataSource", + "description", + "dataset_description", + { + transaction, + }, + ); + }); + }, + + async down(queryInterface) { + return queryInterface.sequelize.transaction(async (transaction) => { + await queryInterface.removeColumn( + "DataSource", + "methodology_description", + { + transaction, + }, + ); + await queryInterface.removeColumn( + "DataSource", + "transformation_description", + { + transaction, + }, + ); + await queryInterface.removeColumn("DataSource", "dataset_name", { + transaction, + }); + await queryInterface.renameColumn( + "DataSource", + "datasource_name", + "name", + { + transaction, + }, + ); + await queryInterface.renameColumn( + "DataSource", + "dataset_description", + "description", + { + transaction, + }, + ); + }); + }, +}; diff --git a/app/scripts/catalogue-sync.ts b/app/scripts/catalogue-sync.ts index c36627874..d1b3e4809 100644 --- a/app/scripts/catalogue-sync.ts +++ b/app/scripts/catalogue-sync.ts @@ -6,10 +6,11 @@ import { logger } from "@/services/logger"; interface Source { datasource_id: string; - name: string; + datasource_name: string; + dataset_name: string; source_type: string; url: string; - description: string; + dataset_description: string; access_type: string; geographical_location: string; start_year: number; @@ -23,6 +24,8 @@ interface Source { notes: string; units: string; methodology_url: string; + methodology_description: string; + transformation_description: string; publisher_id: string; retrieval_method: string; api_endpoint: string; @@ -116,7 +119,7 @@ async function syncDataCatalogue() { if (!source.notes) { // publisher_id is still a name at this stage - source.notes = `${source.name} by ${source.publisher_id}. For more details see ${source.url}`; + source.notes = `${source.datasource_name} by ${source.publisher_id}. For more details see ${source.url}`; } if (source.geographical_location === "global") { @@ -171,48 +174,15 @@ async function syncDataCatalogue() { console.dir(sources); logger.debug("Saving sources..."); + /* + * TODO switch to single query when this issue is fixed: + * https://github.com/sequelize/sequelize/issues/15221 + * https://github.com/sequelize/sequelize/issues/13545 + */ for (const source of sources) { await db.models.DataSource.upsert(source); } - /* TODO switch to single query when this issue is fixed: - // https://github.com/sequelize/sequelize/issues/15221 - // https://github.com/sequelize/sequelize/issues/13545 - await db.models.DataSource.bulkCreate( - sources, - { - updateOnDuplicate: [ - "name", - "sourceType", - "url", - "description", - "accessType", - "geographicalLocation", - "startYear", - "endYear", - "latestAccountingYear", - "frequencyOfUpdate", - "spatialResolution", - "language", - "accessibility", - "dataQuality", - "notes", - "units", - "methodologyUrl", - "publisherId", - "retrievalMethod", - "apiEndpoint", - "sectorId", - "subsectorId", - "subcategoryId", - "created", - "lastUpdated", - ], - // ignoreDuplicates: true, - }, - ); - */ - await catalogue.update({ lastUpdate: new Date(lastUpdate) }); logger.debug("Updated Catalogue, done!"); diff --git a/app/src/app/[lng]/[inventory]/data/[step]/ActivityDataTab.tsx b/app/src/app/[lng]/[inventory]/data/[step]/ActivityDataTab.tsx index 170faa23e..4694909fe 100644 --- a/app/src/app/[lng]/[inventory]/data/[step]/ActivityDataTab.tsx +++ b/app/src/app/[lng]/[inventory]/data/[step]/ActivityDataTab.tsx @@ -50,7 +50,7 @@ const activityDataUnits: Record = { export function determineEmissionsFactorType(factor: EmissionsFactorData) { let sourceName = factor.dataSources - ? factor.dataSources[0].name || "Unknown data source" + ? factor.dataSources[0].datasetName || "Unknown data source" : "Unknown data source"; if (sourceName.includes("IPCC") && sourceName.includes("US")) { return "National (US)"; @@ -108,7 +108,8 @@ export function ActivityDataTab({ : Object.keys(factorsByUnit); // TODO this should happen in default form value, as the form still contains null/ undefined here - const selectedUnit = watch(prefix + "activityDataUnit") ?? scopeUnits[0] ?? ""; + const selectedUnit = + watch(prefix + "activityDataUnit") ?? scopeUnits[0] ?? ""; const selectedUnitShort = selectedUnit.split(" ")[0]; useEffect(() => { diff --git a/app/src/app/[lng]/[inventory]/data/[step]/SourceDrawer.tsx b/app/src/app/[lng]/[inventory]/data/[step]/SourceDrawer.tsx index 071611e2e..27fff5f31 100644 --- a/app/src/app/[lng]/[inventory]/data/[step]/SourceDrawer.tsx +++ b/app/src/app/[lng]/[inventory]/data/[step]/SourceDrawer.tsx @@ -111,7 +111,7 @@ export function SourceDrawer({ lineHeight="40px" textTransform="capitalize" > - {source.name} + {source.datasetName} @@ -245,7 +245,9 @@ export function SourceDrawer({ {t("inside-dataset")} - {source.description} + + {source.datasetDescription} + - {/* - // TODO add methodology description to data source model/ data catalog - */} + + {source.methodologyDescription} + - {t("transform-data-description")} + {source.transformationDescription} diff --git a/app/src/app/[lng]/[inventory]/data/[step]/page.tsx b/app/src/app/[lng]/[inventory]/data/[step]/page.tsx index 9020a50c3..2ac9c5b0b 100644 --- a/app/src/app/[lng]/[inventory]/data/[step]/page.tsx +++ b/app/src/app/[lng]/[inventory]/data/[step]/page.tsx @@ -60,7 +60,6 @@ import { MdOutlineEdit, MdOutlineHomeWork, MdOutlineSkipNext, - MdPlaylistAddCheck, MdRefresh, } from "react-icons/md"; import { useDispatch, useSelector } from "react-redux"; @@ -73,9 +72,7 @@ import type { } from "./types"; import AddFileDataModal from "@/components/Modals/add-file-data-modal"; -import { v4 as uuidv4 } from "uuid"; import { InventoryValueAttributes } from "@/models/InventoryValue"; -import { DataSource } from "@/models/DataSource"; function getMailURI(locode?: string, sector?: string, year?: number): string { const emails = @@ -700,7 +697,7 @@ export default function AddDataSteps({ {/* TODO add icon to DataSource */} - {source.name} + {source.datasetName} @@ -732,7 +729,8 @@ export default function AddDataSteps({ noOfLines={5} minHeight={120} > - {source.description} + {source.datasetDescription || + source.methodologyDescription} { // TODO get these as separate values from emissions factor seeder const activityUnit = inventoryValue.activityUnits?.split("/")[0]; - const emissionsFactorUnit = inventoryValue.activityUnits?.split(" ")[0].split("/")[1]; + const emissionsFactorUnit = inventoryValue.activityUnits + ?.split(" ")[0] + .split("/")[1]; const activityType = inventoryValue.activityUnits?.split(" ")[1]; row.getCell("G").value = activityType; @@ -241,7 +243,7 @@ async function inventoryXLS(inventory: Inventory): Promise { row.getCell("AD").value = inventoryValue.dataSource.dataQuality ?.slice(0, 1) .toUpperCase(); - row.getCell("AP").value = inventoryValue.dataSource.name; // TODO add source to Data sources sheet (ID 20) + row.getCell("AP").value = inventoryValue.dataSource.datasetName; // TODO add source to Data sources sheet (ID 20) row.commit(); } diff --git a/app/src/i18n/locales/de/data.json b/app/src/i18n/locales/de/data.json index ec1ece653..e0d0bd78c 100644 --- a/app/src/i18n/locales/de/data.json +++ b/app/src/i18n/locales/de/data.json @@ -118,7 +118,6 @@ "files-uploaded": "Dateien hochgeladen", "add-custom": "Benutzerdefiniert", "save-missing-scope-info": "Speichern oder fehlende Informationen ausfüllen, bevor Sie fortfahren.", - "transform-data-description": "Lorem ipsum dolor sit amet, qui minim labore adipisicing minim sint cillum sint consectetur cupidatat.", "file-context": "Erzählen Sie uns mehr über diese Datei", "file-data-subtitle": "Welche Art von Daten enthält sie?", "file-data-description": "Wählen Sie aus den folgenden Optionen. Dies wird uns helfen, diese Informationen besser zu identifizieren und in Ihre Bestandsaufnahme aufzunehmen.", diff --git a/app/src/i18n/locales/en/data.json b/app/src/i18n/locales/en/data.json index 3a9891862..f2d09914a 100644 --- a/app/src/i18n/locales/en/data.json +++ b/app/src/i18n/locales/en/data.json @@ -168,7 +168,6 @@ "city": "City", "inside-dataset": "What's inside this dataset", "transform-data-heading": "How do we transform this data?", - "transform-data-description": "Lorem ipsum dolor sit amet, qui minim labore adipisicing minim sint cillum sint consectetur cupidatat.", "file-context": "Tell Us More About This File", "file-data-subtitle": "What type of data is contained in it?", "file-data-description": " Choose from the options below. This will help us to better identify and include this information in your inventory.", diff --git a/app/src/i18n/locales/es/data.json b/app/src/i18n/locales/es/data.json index 1e1acdcd3..01f82f95c 100644 --- a/app/src/i18n/locales/es/data.json +++ b/app/src/i18n/locales/es/data.json @@ -114,7 +114,6 @@ "files-uploaded": "Archivos subidos", "add-custom": "Agregar personalizado", "save-missing-scope-info": "Guarde o complete la información faltante antes de continuar.", - "transform-data-description": "Lorem ipsum dolor sit amet, qui minim labore adipisicing minim sint cillum sint consectetur cupidatat.", "file-context": "Cuéntanos más sobre este archivo", "file-data-subtitle": "¿Qué tipo de datos contiene?", "file-data-description": "Elija entre las opciones a continuación. Esto nos ayudará a identificar e incluir mejor esta información en su inventario.", diff --git a/app/src/models/DataSource.ts b/app/src/models/DataSource.ts index f060a1d78..58f970a90 100644 --- a/app/src/models/DataSource.ts +++ b/app/src/models/DataSource.ts @@ -34,10 +34,11 @@ import { InventoryValue, InventoryValueId } from "./InventoryValue"; export interface DataSourceAttributes { datasourceId: string; - name?: string; + datasetName?: string; + datasourceName?: string; sourceType?: string; url?: string; - description?: string; + datasetDescription?: string; accessType?: string; geographicalLocation?: string; // comma separated list of locodes for either EARTH, country, region or city startYear?: number; // inclusive @@ -51,6 +52,8 @@ export interface DataSourceAttributes { notes?: string; units?: string; methodologyUrl?: string; + methodologyDescription?: string; + transformationDescription?: string; publisherId?: string; retrievalMethod?: string; apiEndpoint?: string; @@ -64,10 +67,11 @@ export interface DataSourceAttributes { export type DataSourcePk = "datasourceId"; export type DataSourceId = DataSource[DataSourcePk]; export type DataSourceOptionalAttributes = - | "name" + | "datasetName" + | "datasourceName" | "sourceType" | "url" - | "description" + | "datasetDescription" | "accessType" | "geographicalLocation" | "startYear" @@ -81,6 +85,8 @@ export type DataSourceOptionalAttributes = | "notes" | "units" | "methodologyUrl" + | "methodologyDescription" + | "transformationDescription" | "publisherId" | "retrievalMethod" | "apiEndpoint" @@ -99,10 +105,11 @@ export class DataSource implements DataSourceAttributes { datasourceId!: string; - name?: string; + datasetName?: string; + datasourceName?: string; sourceType?: string; url?: string; - description?: string; + datasetDescription?: string; accessType?: string; geographicalLocation?: string; startYear?: number; // inclusive @@ -116,6 +123,8 @@ export class DataSource notes?: string; units?: string; methodologyUrl?: string; + methodologyDescription?: string; + transformationDescription?: string; publisherId?: string; retrievalMethod?: string; apiEndpoint?: string; @@ -359,10 +368,7 @@ export class DataSource // DataSource hasOne Sector via sectorId sector!: Sector; getSector!: Sequelize.HasOneGetAssociationMixin; - setSector!: Sequelize.HasOneSetAssociationMixin< - Sector, - SectorId - >; + setSector!: Sequelize.HasOneSetAssociationMixin; createSector!: Sequelize.HasOneCreateAssociationMixin; // DataSource hasOne SubCategory via subCategoryId subCategory!: SubCategory; @@ -375,10 +381,7 @@ export class DataSource // DataSource hasOneSubSector via subSectorId subSector!: SubSector; getSubSector!: Sequelize.HasOneGetAssociationMixin; - setSubSector!: Sequelize.HasOneSetAssociationMixin< - SubSector, - SubSectorId - >; + setSubSector!: Sequelize.HasOneSetAssociationMixin; createSubSector!: Sequelize.HasOneCreateAssociationMixin; // DataSource belongsToMany EmissionsFactor via datasourceId and emissionsFactorId emissionsFactorIdEmissionsFactors!: EmissionsFactor[]; @@ -575,29 +578,14 @@ export class DataSource // DataSource belongsToMany Scope via datasourceId and scopeId scopes!: Scope[]; getScopes!: Sequelize.BelongsToManyGetAssociationsMixin; - setScopes!: Sequelize.BelongsToManySetAssociationsMixin< - Scope, - ScopeId - >; + setScopes!: Sequelize.BelongsToManySetAssociationsMixin; addScope!: Sequelize.BelongsToManyAddAssociationMixin; - addScopes!: Sequelize.BelongsToManyAddAssociationsMixin< - Scope, - ScopeId - >; + addScopes!: Sequelize.BelongsToManyAddAssociationsMixin; createScope!: Sequelize.BelongsToManyCreateAssociationMixin; - removeScope!: Sequelize.BelongsToManyRemoveAssociationMixin< - Scope, - ScopeId - >; - removeScopes!: Sequelize.BelongsToManyRemoveAssociationsMixin< - Scope, - ScopeId - >; + removeScope!: Sequelize.BelongsToManyRemoveAssociationMixin; + removeScopes!: Sequelize.BelongsToManyRemoveAssociationsMixin; hasScope!: Sequelize.BelongsToManyHasAssociationMixin; - hasScopes!: Sequelize.BelongsToManyHasAssociationsMixin< - Scope, - ScopeId - >; + hasScopes!: Sequelize.BelongsToManyHasAssociationsMixin; countScopes!: Sequelize.BelongsToManyCountAssociationsMixin; // DataSource belongsToMany Sector via datasourceId and sectorId sectorIdSectors!: Sector[]; @@ -745,9 +733,15 @@ export class DataSource primaryKey: true, field: "datasource_id", }, - name: { + datasetName: { type: DataTypes.STRING(255), allowNull: true, + field: "dataset_name", + }, + datasourceName: { + type: DataTypes.STRING(255), + allowNull: true, + field: "datasource_name", }, sourceType: { type: DataTypes.STRING(255), @@ -759,9 +753,10 @@ export class DataSource allowNull: true, field: "URL", }, - description: { + datasetDescription: { type: DataTypes.TEXT, allowNull: true, + field: "dataset_description", }, accessType: { type: DataTypes.STRING(255), @@ -824,6 +819,16 @@ export class DataSource allowNull: true, field: "methodology_url", }, + methodologyDescription: { + type: DataTypes.TEXT, + allowNull: true, + field: "methodology_description", + }, + transformationDescription: { + type: DataTypes.TEXT, + allowNull: true, + field: "transformation_description", + }, publisherId: { type: DataTypes.UUID, allowNull: true, diff --git a/app/tests/api/datasource.test.ts b/app/tests/api/datasource.test.ts index 0e604c1e8..47402c5a6 100644 --- a/app/tests/api/datasource.test.ts +++ b/app/tests/api/datasource.test.ts @@ -66,7 +66,7 @@ describe("DataSource API", () => { where: { year: inventoryData.year }, }); await db.models.DataSource.destroy({ - where: { name: { [Op.like]: "XX_DATASOURCE_TEST%" } }, + where: { datasetName: { [Op.like]: "XX_DATASOURCE_TEST%" } }, }); await db.models.City.destroy({ where: { locode } }); city = await db.models.City.create({ @@ -105,7 +105,7 @@ describe("DataSource API", () => { for (let i = 0; i < 3; i++) { const source = await db.models.DataSource.create({ datasourceId: randomUUID(), - name: "XX_DATASOURCE_TEST_" + i, + datasetName: "XX_DATASOURCE_TEST_" + i, sectorId: sector.sectorId, apiEndpoint, startYear: 4000 + i, @@ -134,7 +134,7 @@ describe("DataSource API", () => { const { data } = await res.json(); assert.equal(data.length, 1); const { source } = data[0]; - assert.equal(source.name, "XX_DATASOURCE_TEST_0"); + assert.equal(source.datasetName, "XX_DATASOURCE_TEST_0"); assert.equal(source.sectorId, sector.sectorId); assert.equal(source.apiEndpoint, apiEndpoint); assert.equal(source.geographicalLocation, "EARTH"); diff --git a/app/tests/api/inventory.test.ts b/app/tests/api/inventory.test.ts index c426890cc..f0d168f37 100644 --- a/app/tests/api/inventory.test.ts +++ b/app/tests/api/inventory.test.ts @@ -66,7 +66,7 @@ describe("Inventory API", () => { where: { inventoryName }, }); await db.models.DataSource.destroy({ - where: { name: { [Op.like]: "XX_INVENTORY_TEST_%" } }, + where: { datasetName: { [Op.like]: "XX_INVENTORY_TEST_%" } }, }); await db.models.Sector.destroy({ where: { sectorName: { [Op.like]: "XX_INVENTORY_TEST%" } }, @@ -280,12 +280,12 @@ describe("Inventory API", () => { const userSource = await db.models.DataSource.create({ datasourceId: randomUUID(), sourceType: "user", - name: "XX_INVENTORY_TEST_USER", + datasetName: "XX_INVENTORY_TEST_USER", }); const thirdPartySource = await db.models.DataSource.create({ datasourceId: randomUUID(), sourceType: "third_party", - name: "XX_INVENTORY_TEST_THIRD_PARTY", + datasetName: "XX_INVENTORY_TEST_THIRD_PARTY", }); const sources = [userSource, thirdPartySource, null];