diff --git a/README.md b/README.md index a6e6535..1870cd3 100644 --- a/README.md +++ b/README.md @@ -131,5 +131,9 @@ Normalize and upload the new cropping calendar data. Normalize and upload the new municipalities and other details attached to the crop recommendations. +### `npm run find:mismatching` + +Writes a list of municipalities from the cropping calendar that have mismatching names from the 10-day weather forecast excel file into a `municipalities.csv` CSV file. + @ciatph
20221205 diff --git a/package.json b/package.json index 0326ac2..3e76ba7 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "process:recommendations": "node src/01_recommendations", "process:calendar": "node src/02_crop_calendar", "process:details": "node src/03_recommendations_detail", + "find:mismatching": "node src/04_find_mismatching_municipalities", "lint": "eslint src", "lint:fix": "eslint src --fix" }, diff --git a/src/02_crop_calendar/constants.js b/src/02_crop_calendar/constants.js index 1ce24ce..1e2257a 100644 --- a/src/02_crop_calendar/constants.js +++ b/src/02_crop_calendar/constants.js @@ -5,6 +5,19 @@ const CROP_STAGE_LABELS = { lprep: 'Preparation Stage' } +// Uncommon municipality names detected from /04_find_mismatching_municipalities +// Replace the following with municipality names from the 10-day weather forecast file +const MUNICIPALITIES_TO_REPLACE_WITH_PAGASA = [ + { province: 'Albay', municipality: 'Sto.Domingo', replace: 'Santo Domingo' }, + { province: 'Albay', municipality: 'Pioduran', replace: 'Pio Duran' }, + { province: 'Camarines Norte', municipality: 'Sta. Elena', replace: 'Santa Elena' }, + { province: 'Camarines Norte', municipality: 'SL Ruiz', replace: 'San Lorenzo Ruiz' }, + { province: 'Camarines Sur', municipality: 'Sangay', replace: 'Sagnay' }, + { province: 'Sorsogon', municipality: 'Pto. Diaz', replace: 'Prieto Diaz' }, + { province: 'Sorsogon', municipality: 'Sta. Magdalena', replace: 'Santa Magdalena' } +] + module.exports = { - CROP_STAGE_LABELS + CROP_STAGE_LABELS, + MUNICIPALITIES_TO_REPLACE_WITH_PAGASA } diff --git a/src/02_crop_calendar/cropping_calendar.js b/src/02_crop_calendar/cropping_calendar.js index 5df64e4..b403fc9 100644 --- a/src/02_crop_calendar/cropping_calendar.js +++ b/src/02_crop_calendar/cropping_calendar.js @@ -1,5 +1,5 @@ const { CsvToFireStore } = require('csv-firestore') -const { CROP_STAGE_LABELS } = require('./constants') +const { CROP_STAGE_LABELS, MUNICIPALITIES_TO_REPLACE_WITH_PAGASA } = require('./constants') class CroppingCalendar extends CsvToFireStore { constructor (csvFilePath) { @@ -18,6 +18,9 @@ class CroppingCalendar extends CsvToFireStore { this.crop_stages = [] this.count = 0 + + // Municipalities that have different naming convention from the 10-day weather forecast files + this.susMunicipalities = MUNICIPALITIES_TO_REPLACE_WITH_PAGASA.map(item => item.municipality) } /** @@ -84,6 +87,7 @@ class CroppingCalendar extends CsvToFireStore { } const value = row[item].trim() + let tempMunicipality = null // Extract unique provinces if (key === 'province' && !this.itemExists('province', value) && value !== '') { @@ -95,13 +99,20 @@ class CroppingCalendar extends CsvToFireStore { // Extract unique municipalities if (key === 'municipality' && value !== '') { - const combo = `${row.prov.trim()}|${value}` + tempMunicipality = value.trim() + + if (this.susMunicipalities.includes(tempMunicipality)) { + tempMunicipality = MUNICIPALITIES_TO_REPLACE_WITH_PAGASA.find(x => + x.province === row.prov.trim() && x.municipality === value.trim())?.replace ?? value.trim() + } + + const combo = `${row.prov.trim()}|${tempMunicipality}` if (!this.itemExists('municipality', combo)) { this.municipalities.push({ id: this.municipalities.length + 1, province: row.prov.trim(), - name: value, + name: tempMunicipality, unique: combo }) } @@ -130,7 +141,9 @@ class CroppingCalendar extends CsvToFireStore { } if (include && ['province', 'municipality', 'crop'].includes(key)) { - obj[key] = value + obj[key] = (key === 'municipality') + ? tempMunicipality + : value } }) diff --git a/src/04_find_mismatching_municipalities/index.js b/src/04_find_mismatching_municipalities/index.js new file mode 100644 index 0000000..7e00d6e --- /dev/null +++ b/src/04_find_mismatching_municipalities/index.js @@ -0,0 +1,38 @@ +const path = require('path') +const { CsvToFireStore } = require('csv-firestore') +const { municipalitiesFromCalendar } = require('./src/calendar') +const { municipalitiesFromForecast } = require('./src/forecast') + +/** + * Creates a list of mismatching municipality names from the cropping calendar and PAGASA 10-day weather forecast excel file. + * Requires the cropping calendar CSV file and 1 of PAGASA's 10-day weather forecast EXCEL file. + */ +const main = async () => { + // Read the cropping calendar file + const { municipalities: calendarMunicipalities } = await municipalitiesFromCalendar(path.join(__dirname, '..', '02_crop_calendar', process.env.CSV_FILENAME)) + + // PAGASA municipalities + const { municipalities: forecastMunicipalities } = municipalitiesFromForecast(path.join(__dirname, process.env.EXCEL_FILENAME)) + + const handler = new CsvToFireStore() + + try { + // Find municipalities in calendarMunicipalities that are not available in forecastMunicipalities + const mismatchingNames = calendarMunicipalities.reduce((list, item) => { + if (!forecastMunicipalities.find(x => x.province === item.province && + x.municipality === item.municipality) + ) { + list.push(item) + } + + return list + }, []) + + console.log(mismatchingNames) + handler.write(mismatchingNames, path.join(__dirname, 'municipalities.csv')) + } catch (err) { + console.log(err) + } +} + +main() diff --git a/src/04_find_mismatching_municipalities/src/calendar.js b/src/04_find_mismatching_municipalities/src/calendar.js new file mode 100644 index 0000000..161f5d1 --- /dev/null +++ b/src/04_find_mismatching_municipalities/src/calendar.js @@ -0,0 +1,20 @@ +const CroppingCalendar = require('../../02_crop_calendar/cropping_calendar') + +/** + * Extracts the municipalities with province list from the cropping calendar + * @param {String} filepath - Full file path to a CSV file + * @returns {Object[]} List of municipalities + */ +module.exports.municipalitiesFromCalendar = async (filepath) => { + try { + // Read the cropping calendar file + const cropCalendarData = new CroppingCalendar(filepath) + + await cropCalendarData.readCSV() + const municipalities = cropCalendarData.municipalities.map(x => ({ id: x.id, province: x.province, municipality: x.name })) + + return { municipalities } + } catch (err) { + throw new Error(err.message) + } +} diff --git a/src/04_find_mismatching_municipalities/src/forecast.js b/src/04_find_mismatching_municipalities/src/forecast.js new file mode 100644 index 0000000..28ae70a --- /dev/null +++ b/src/04_find_mismatching_municipalities/src/forecast.js @@ -0,0 +1,26 @@ +const XLSXWrapper = require('../../lib/xlsxwrapper') + +/** + * Extracts the municipalities with province list from a 10-day weather forecast excel file + * @param {String} filepath - Full file path to an EXCEL file + * @returns {Object[]} List of municipalities + */ +module.exports.municipalitiesFromForecast = (filepath) => { + // Read the excel file + const tendayForecastFile = new XLSXWrapper(filepath) + const tendayForecastData = tendayForecastFile.getDataSheet(0) + const bicolProvinces = ['Albay', 'Camarines Norte', 'Camarines Sur', 'Catanduanes', 'Masbate', 'Sorsogon'] + + const municipalities = tendayForecastData + .filter(x => (x.__EMPTY !== undefined && bicolProvinces.find(y => x.__EMPTY.toString().includes(y)))) + .map((x, id) => { + const province = bicolProvinces.find(y => x.__EMPTY.toString().includes(y)) + return { + id, + province, + municipality: x.__EMPTY.toString().split(`(${province})`)[0].trim() + } + }) + + return { municipalities } +}