Make district postal code route asynchronous

BaseAdresseNationale · Feb 23, 2024 · 2f9e081 · 2f9e081
1 parent a48ece8
commit 2f9e081
Show file tree

Hide file tree

Showing 3 changed files with 85 additions and 70 deletions.
diff --git a/lib/api/consumers/api-consumer.js b/lib/api/consumers/api-consumer.js
@@ -5,7 +5,7 @@ import {checkAddressesRequest, checkAddressesIDsRequest} from '../address/utils.
 import {setCommonToponyms, updateCommonToponyms, patchCommonToponyms, deleteCommonToponyms, getAllDistrictIDsFromCommonToponyms} from '../common-toponym/models.js'
 import {checkCommonToponymsRequest, checkCommonToponymsIDsRequest} from '../common-toponym/utils.js'
 import {setDistricts, updateDistricts, patchDistricts, deleteDistricts} from '../district/models.js'
-import {checkDistrictsRequest, checkDistrictsIDsRequest} from '../district/utils.js'
+import {checkDistrictsRequest, checkDistrictsIDsRequest, formatDataNova} from '../district/utils.js'
 import {dataValidationReportFrom, formatObjectWithDefaults, addOrUpdateJob, formatPayloadDates} from '../helper.js'
 import {addressDefaultOptionalValues} from '../address/schema.js'
 import {commonToponymDefaultOptionalValues} from '../common-toponym/schema.js'
@@ -216,6 +216,8 @@ const districtConsumer = async (jobType, payload, statusID) => {
         return checkDistrictsRequest(payload, jobType)
       case 'delete':
         return checkDistrictsIDsRequest(payload, jobType)
+      case 'updatePostalCode':
+        return dataValidationReportFrom(true)
       default:
         return dataValidationReportFrom(false, 'Unknown action type', {actionType: jobType, payload})
     }
@@ -229,6 +231,8 @@ const districtConsumer = async (jobType, payload, statusID) => {
         return formatPayloadDates(payload, jobType)
       case 'delete':
         return payload
+      case 'updatePostalCode':
+        return formatDataNova(payload)
       default:
         console.warn(`District Consumer Warn: Unknown job type : '${jobType}'`)
     }
@@ -255,6 +259,7 @@ const districtConsumer = async (jobType, payload, statusID) => {
       }
 
       case 'patch':
+      case 'updatePostalCode':
         await patchDistricts(formattedPayload)
         break
       case 'delete':

diff --git a/lib/api/district/routes.js b/lib/api/district/routes.js
@@ -1,12 +1,11 @@
 import 'dotenv/config.js' // eslint-disable-line import/no-unassigned-import
 import {customAlphabet} from 'nanoid'
 import express from 'express'
-import Papa from 'papaparse'
 import queue from '../../util/queue.cjs'
 import auth from '../../middleware/auth.js'
 import analyticsMiddleware from '../../middleware/analytics.js'
 import fetch from '../../util/fetch.cjs'
-import {getDistrict, getDistrictFromCog, getDistrictsFromCog, deleteDistrict, patchDistricts} from './models.js'
+import {getDistrict, getDistrictFromCog, deleteDistrict} from './models.js'
 
 const apiQueue = queue('api')
 
@@ -217,70 +216,6 @@ app.get('/cog/:cog', analyticsMiddleware, async (req, res) => {
   res.send(response)
 })
 
-async function updateDbFromDataNova(postalFile) {
-  /* eslint-disable camelcase */
-  const headers = {
-    '#Code_commune_INSEE': 'codeInsee',
-    Nom_de_la_commune: 'nomCommune',
-    Code_postal: 'codePostal',
-    Libellé_d_acheminement: 'libelleAcheminement',
-    'Libell�_d_acheminement': 'libelleAcheminement', // Postal file url returns wrong header charset code (UTF-8 instead of ISO-8859-1)
-    Ligne_5: 'ligne5',
-  }
-  /* eslint-enable camelcase */
-
-  const dataRaw = await Papa.parse(postalFile, {
-    header: true,
-    transformHeader: name => headers[name] || name,
-    skipEmptyLines: true,
-  })
-
-  const districts = await getDistrictsFromCog(
-    dataRaw.data.map(({codeInsee}) => codeInsee)
-  )
-
-  const districtsByInseeCode = districts.reduce(
-    (acc, district) => ({
-      ...acc,
-      ...(!district?.meta || acc[district.meta?.insee?.cog] ? {} : {[district.meta.insee.cog]: district}),
-    }), {})
-
-  const banDistricts = Object.values(
-    (dataRaw?.data || []).map(({codeInsee, codePostal, libelleAcheminement}) => {
-      const {id} = districtsByInseeCode[codeInsee] || {}
-      return {id, codePostal: [codePostal], libelleAcheminement}
-    })
-      .reduce(
-        (acc, district) => {
-          if (district.id) {
-            if (acc[district.id]) {
-              acc[district.id].codePostal = [...acc[district.id].codePostal, ...district.codePostal]
-            } else {
-              acc[district.id] = district
-            }
-          }
-
-          return acc
-        }, {}
-      )
-  )
-
-  const patchBulkOperations = banDistricts.map(({id, codePostal, libelleAcheminement}) => ({
-    id,
-    meta: {
-      laPoste: {
-        codePostal,
-        libelleAcheminement,
-        source: 'La Poste - dataNOVA',
-      }
-    }
-  }))
-
-  patchDistricts(patchBulkOperations)
-
-  return banDistricts
-}
-
 app.route('/codePostal')
   .get(async (req, res) => {
     let response
@@ -289,8 +224,12 @@ app.route('/codePostal')
       // https://datanova.laposte.fr/data-fair/api/v1/datasets/laposte-hexasmal/raw
       const {url} = req.query
       const postalFile = await fetch(url)
+      const statusID = nanoid()
 
-      response = await updateDbFromDataNova(postalFile)
+      await apiQueue.add(
+        {dataType: 'district', jobType: 'updatePostalCode', data: postalFile, statusID},
+        {jobId: statusID, removeOnComplete: true}
+      )
     } catch (error) {
       const {message} = error
       response = {
@@ -307,8 +246,18 @@ app.route('/codePostal')
     let response
     try {
       const postalFile = req.body
+      const statusID = nanoid()
 
-      response = await updateDbFromDataNova(postalFile)
+      await apiQueue.add(
+        {dataType: 'district', jobType: 'updatePostalCode', data: postalFile, statusID},
+        {jobId: statusID, removeOnComplete: true}
+      )
+      response = {
+        date: new Date(),
+        status: 'success',
+        message: `Check the status of your request : ${BAN_API_URL}/job-status/${statusID}`,
+        response: {statusID},
+      }
     } catch (error) {
       const {message} = error
       response = {

diff --git a/lib/api/district/utils.js b/lib/api/district/utils.js
@@ -1,6 +1,7 @@
+import Papa from 'papaparse'
 import {checkDataFormat, dataValidationReportFrom, checkIdsIsUniq, checkIdsIsVacant, checkIdsIsAvailable, checkDataShema, checkIdsShema} from '../helper.js'
 import {banID} from '../schema.js'
-import {getDistricts} from './models.js'
+import {getDistricts, getDistrictsFromCog} from './models.js'
 import {banDistrictSchema} from './schema.js'
 
 const getExistingDistrictIDs = async districtIDs => {
@@ -66,3 +67,63 @@ export const checkDistrictsRequest = async (districts, actionType) => {
 
   return report
 }
+
+export async function formatDataNova(postalFile) {
+  /* eslint-disable camelcase */
+  const headers = {
+    '#Code_commune_INSEE': 'codeInsee',
+    Nom_de_la_commune: 'nomCommune',
+    Code_postal: 'codePostal',
+    Libellé_d_acheminement: 'libelleAcheminement',
+    'Libell�_d_acheminement': 'libelleAcheminement', // Postal file url returns wrong header charset code (UTF-8 instead of ISO-8859-1)
+    Ligne_5: 'ligne5',
+  }
+  /* eslint-enable camelcase */
+
+  const dataRaw = await Papa.parse(postalFile, {
+    header: true,
+    transformHeader: name => headers[name] || name,
+    skipEmptyLines: true,
+  })
+
+  const districts = await getDistrictsFromCog(
+    dataRaw.data.map(({codeInsee}) => codeInsee)
+  )
+
+  const districtsByInseeCode = districts.reduce(
+    (acc, district) => ({
+      ...acc,
+      ...(!district?.meta || acc[district.meta?.insee?.cog] ? {} : {[district.meta.insee.cog]: district}),
+    }), {})
+
+  const banDistricts = Object.values(
+    (dataRaw?.data || []).map(({codeInsee, codePostal, libelleAcheminement}) => {
+      const {id} = districtsByInseeCode[codeInsee] || {}
+      return {id, codePostal: [codePostal], libelleAcheminement}
+    })
+      .reduce(
+        (acc, district) => {
+          if (district.id) {
+            if (acc[district.id]) {
+              acc[district.id].codePostal = [...acc[district.id].codePostal, ...district.codePostal]
+            } else {
+              acc[district.id] = district
+            }
+          }
+
+          return acc
+        }, {}
+      )
+  )
+
+  return banDistricts.map(({id, codePostal, libelleAcheminement}) => ({
+    id,
+    meta: {
+      laPoste: {
+        codePostal,
+        libelleAcheminement,
+        source: 'La Poste - dataNOVA',
+      }
+    }
+  }))
+}