v1.0.4

README.md

@@ -131,5 +131,9 @@ Normalize and upload the new cropping calendar data.
 
 Normalize and upload the new municipalities and other details attached to the crop recommendations.
 
+### `npm run find:mismatching`
+
+Writes a list of municipalities from the cropping calendar that have mismatching names from the 10-day weather forecast excel file into a `municipalities.csv` CSV file.
+
 @ciatph<br>
 20221205

package.json

@@ -8,6 +8,7 @@
     "process:recommendations": "node src/01_recommendations",
     "process:calendar": "node src/02_crop_calendar",
     "process:details": "node src/03_recommendations_detail",
+    "find:mismatching": "node src/04_find_mismatching_municipalities",
     "lint": "eslint src",
     "lint:fix": "eslint src --fix"
   },

src/02_crop_calendar/constants.js

@@ -5,6 +5,19 @@ const CROP_STAGE_LABELS = {
   lprep: 'Preparation Stage'
 }
 
+// Uncommon municipality names detected from /04_find_mismatching_municipalities
+// Replace the following with municipality names from the 10-day weather forecast file
+const MUNICIPALITIES_TO_REPLACE_WITH_PAGASA = [
+  { province: 'Albay', municipality: 'Sto.Domingo', replace: 'Santo Domingo' },
+  { province: 'Albay', municipality: 'Pioduran', replace: 'Pio Duran' },
+  { province: 'Camarines Norte', municipality: 'Sta. Elena', replace: 'Santa Elena' },
+  { province: 'Camarines Norte', municipality: 'SL Ruiz', replace: 'San Lorenzo Ruiz' },
+  { province: 'Camarines Sur', municipality: 'Sangay', replace: 'Sagnay' },
+  { province: 'Sorsogon', municipality: 'Pto. Diaz', replace: 'Prieto Diaz' },
+  { province: 'Sorsogon', municipality: 'Sta. Magdalena', replace: 'Santa Magdalena' }
+]
+
 module.exports = {
-  CROP_STAGE_LABELS
+  CROP_STAGE_LABELS,
+  MUNICIPALITIES_TO_REPLACE_WITH_PAGASA
 }

src/02_crop_calendar/cropping_calendar.js

@@ -1,5 +1,5 @@
 const { CsvToFireStore } = require('csv-firestore')
-const { CROP_STAGE_LABELS } = require('./constants')
+const { CROP_STAGE_LABELS, MUNICIPALITIES_TO_REPLACE_WITH_PAGASA } = require('./constants')
 
 class CroppingCalendar extends CsvToFireStore {
   constructor (csvFilePath) {
@@ -18,6 +18,9 @@ class CroppingCalendar extends CsvToFireStore {
     this.crop_stages = []
 
     this.count = 0
+
+    // Municipalities that have different naming convention from the 10-day weather forecast files
+    this.susMunicipalities = MUNICIPALITIES_TO_REPLACE_WITH_PAGASA.map(item => item.municipality)
   }
 
   /**
@@ -84,6 +87,7 @@ class CroppingCalendar extends CsvToFireStore {
       }
 
       const value = row[item].trim()
+      let tempMunicipality = null
 
       // Extract unique provinces
       if (key === 'province' && !this.itemExists('province', value) && value !== '') {
@@ -95,13 +99,20 @@ class CroppingCalendar extends CsvToFireStore {
 
       // Extract unique municipalities
       if (key === 'municipality' && value !== '') {
-        const combo = `${row.prov.trim()}|${value}`
+        tempMunicipality = value.trim()
+
+        if (this.susMunicipalities.includes(tempMunicipality)) {
+          tempMunicipality = MUNICIPALITIES_TO_REPLACE_WITH_PAGASA.find(x =>
+            x.province === row.prov.trim() && x.municipality === value.trim())?.replace ?? value.trim()
+        }
+
+        const combo = `${row.prov.trim()}|${tempMunicipality}`
 
         if (!this.itemExists('municipality', combo)) {
           this.municipalities.push({
             id: this.municipalities.length + 1,
             province: row.prov.trim(),
-            name: value,
+            name: tempMunicipality,
             unique: combo
           })
         }
@@ -130,7 +141,9 @@ class CroppingCalendar extends CsvToFireStore {
       }
 
       if (include && ['province', 'municipality', 'crop'].includes(key)) {
-        obj[key] = value
+        obj[key] = (key === 'municipality')
+          ? tempMunicipality
+          : value
       }
     })
 

src/04_find_mismatching_municipalities/index.js

@@ -0,0 +1,38 @@
+const path = require('path')
+const { CsvToFireStore } = require('csv-firestore')
+const { municipalitiesFromCalendar } = require('./src/calendar')
+const { municipalitiesFromForecast } = require('./src/forecast')
+
+/**
+ * Creates a list of mismatching municipality names from the cropping calendar and PAGASA 10-day weather forecast excel file.
+ * Requires the cropping calendar CSV file and 1 of PAGASA's 10-day weather forecast EXCEL file.
+ */
+const main = async () => {
+  // Read the cropping calendar file
+  const { municipalities: calendarMunicipalities } = await municipalitiesFromCalendar(path.join(__dirname, '..', '02_crop_calendar', process.env.CSV_FILENAME))
+
+  // PAGASA municipalities
+  const { municipalities: forecastMunicipalities } = municipalitiesFromForecast(path.join(__dirname, process.env.EXCEL_FILENAME))
+
+  const handler = new CsvToFireStore()
+
+  try {
+    // Find municipalities in calendarMunicipalities that are not available in forecastMunicipalities
+    const mismatchingNames = calendarMunicipalities.reduce((list, item) => {
+      if (!forecastMunicipalities.find(x => x.province === item.province &&
+        x.municipality === item.municipality)
+      ) {
+        list.push(item)
+      }
+
+      return list
+    }, [])
+
+    console.log(mismatchingNames)
+    handler.write(mismatchingNames, path.join(__dirname, 'municipalities.csv'))
+  } catch (err) {
+    console.log(err)
+  }
+}
+
+main()

src/04_find_mismatching_municipalities/src/calendar.js

@@ -0,0 +1,20 @@
+const CroppingCalendar = require('../../02_crop_calendar/cropping_calendar')
+
+/**
+ * Extracts the municipalities with province list from the cropping calendar
+ * @param {String} filepath - Full file path to a CSV file
+ * @returns {Object[]} List of municipalities
+ */
+module.exports.municipalitiesFromCalendar = async (filepath) => {
+  try {
+    // Read the cropping calendar file
+    const cropCalendarData = new CroppingCalendar(filepath)
+
+    await cropCalendarData.readCSV()
+    const municipalities = cropCalendarData.municipalities.map(x => ({ id: x.id, province: x.province, municipality: x.name }))
+
+    return { municipalities }
+  } catch (err) {
+    throw new Error(err.message)
+  }
+}

src/04_find_mismatching_municipalities/src/forecast.js

@@ -0,0 +1,26 @@
+const XLSXWrapper = require('../../lib/xlsxwrapper')
+
+/**
+ * Extracts the municipalities with province list from a 10-day weather forecast excel file
+ * @param {String} filepath - Full file path to an EXCEL file
+ * @returns {Object[]} List of municipalities
+ */
+module.exports.municipalitiesFromForecast = (filepath) => {
+  // Read the excel file
+  const tendayForecastFile = new XLSXWrapper(filepath)
+  const tendayForecastData = tendayForecastFile.getDataSheet(0)
+  const bicolProvinces = ['Albay', 'Camarines Norte', 'Camarines Sur', 'Catanduanes', 'Masbate', 'Sorsogon']
+
+  const municipalities = tendayForecastData
+    .filter(x => (x.__EMPTY !== undefined && bicolProvinces.find(y => x.__EMPTY.toString().includes(y))))
+    .map((x, id) => {
+      const province = bicolProvinces.find(y => x.__EMPTY.toString().includes(y))
+      return {
+        id,
+        province,
+        municipality: x.__EMPTY.toString().split(`(${province})`)[0].trim()
+      }
+    })
+
+  return { municipalities }
+}