From 9d9b53c2c056590528c7667a8bed356c3c594765 Mon Sep 17 00:00:00 2001 From: ciatph Date: Wed, 11 Jan 2023 04:34:11 +0800 Subject: [PATCH 1/4] chore: List municipalities from the cropping calendar --- package.json | 1 + .../index.js | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 src/04_find_mismatching_municipalities/index.js diff --git a/package.json b/package.json index 0326ac2..3e76ba7 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "process:recommendations": "node src/01_recommendations", "process:calendar": "node src/02_crop_calendar", "process:details": "node src/03_recommendations_detail", + "find:mismatching": "node src/04_find_mismatching_municipalities", "lint": "eslint src", "lint:fix": "eslint src --fix" }, diff --git a/src/04_find_mismatching_municipalities/index.js b/src/04_find_mismatching_municipalities/index.js new file mode 100644 index 0000000..3fbfb62 --- /dev/null +++ b/src/04_find_mismatching_municipalities/index.js @@ -0,0 +1,45 @@ +const path = require('path') +const CroppingCalendar = require('../02_crop_calendar/cropping_calendar') + +/** + * Creates a list of mismatching municipality names from the cropping calendar and PAGASA 10-day weather forecast excel file. + * Requires the cropping calendar CSV file and 1 of PAGASA's 10-day weather forecast EXCEL file. + */ +const main = async () => { + const handler = new CroppingCalendar(path.resolve(__dirname, '..', '02_crop_calendar', process.env.CSV_FILENAME)) + const write = true + + // Cropping Calendar-specific tables and firestore collection names + const newTables = { + provinces: 'n_provinces', + municipalities: 'n_municipalities', + crops: 'n_crops', + crop_stages: 'n_crop_stages' + } + + try { + console.log('Reading CSV...') + await handler.readCSV() + + if (write) { + // Write cropping calendar data to CSV files + console.log('\nWriting data to CSV...') + handler.write(handler.data(), path.resolve(__dirname, 'data.csv')) + + for (const collection in newTables) { + handler.write( + (collection === 'municipalities') + ? handler[collection].map(x => ({ id: x.id, province: x.province, name: x.name })) + : handler[collection], + path.resolve(__dirname, `${newTables[collection]}.csv` + )) + + console.log(`${collection}: ${handler[collection].length}`) + } + } + } catch (err) { + console.log(err) + } +} + +main() \ No newline at end of file From 5293dbebdbb917b1366d24ca0e825791591a4045 Mon Sep 17 00:00:00 2001 From: ciatph Date: Wed, 11 Jan 2023 05:56:08 +0800 Subject: [PATCH 2/4] chore: List municipalities that are not present in the 10-day weather forecast list of municipalities --- .../index.js | 79 ++++++++----------- .../src/calendar.js | 20 +++++ .../src/forecast.js | 26 ++++++ 3 files changed, 80 insertions(+), 45 deletions(-) create mode 100644 src/04_find_mismatching_municipalities/src/calendar.js create mode 100644 src/04_find_mismatching_municipalities/src/forecast.js diff --git a/src/04_find_mismatching_municipalities/index.js b/src/04_find_mismatching_municipalities/index.js index 3fbfb62..05149f0 100644 --- a/src/04_find_mismatching_municipalities/index.js +++ b/src/04_find_mismatching_municipalities/index.js @@ -1,45 +1,34 @@ -const path = require('path') -const CroppingCalendar = require('../02_crop_calendar/cropping_calendar') - -/** - * Creates a list of mismatching municipality names from the cropping calendar and PAGASA 10-day weather forecast excel file. - * Requires the cropping calendar CSV file and 1 of PAGASA's 10-day weather forecast EXCEL file. - */ -const main = async () => { - const handler = new CroppingCalendar(path.resolve(__dirname, '..', '02_crop_calendar', process.env.CSV_FILENAME)) - const write = true - - // Cropping Calendar-specific tables and firestore collection names - const newTables = { - provinces: 'n_provinces', - municipalities: 'n_municipalities', - crops: 'n_crops', - crop_stages: 'n_crop_stages' - } - - try { - console.log('Reading CSV...') - await handler.readCSV() - - if (write) { - // Write cropping calendar data to CSV files - console.log('\nWriting data to CSV...') - handler.write(handler.data(), path.resolve(__dirname, 'data.csv')) - - for (const collection in newTables) { - handler.write( - (collection === 'municipalities') - ? handler[collection].map(x => ({ id: x.id, province: x.province, name: x.name })) - : handler[collection], - path.resolve(__dirname, `${newTables[collection]}.csv` - )) - - console.log(`${collection}: ${handler[collection].length}`) - } - } - } catch (err) { - console.log(err) - } -} - -main() \ No newline at end of file +const path = require('path') +const { municipalitiesFromCalendar } = require('./src/calendar') +const { municipalitiesFromForecast } = require('./src/forecast') + +/** + * Creates a list of mismatching municipality names from the cropping calendar and PAGASA 10-day weather forecast excel file. + * Requires the cropping calendar CSV file and 1 of PAGASA's 10-day weather forecast EXCEL file. + */ +const main = async () => { + // Read the cropping calendar file + const { municipalities: calendarMunicipalities } = await municipalitiesFromCalendar(path.join(__dirname, '..', '02_crop_calendar', process.env.CSV_FILENAME)) + + // PAGASA municipalities + const { municipalities: forecastMunicipalities } = municipalitiesFromForecast(path.join(__dirname, process.env.EXCEL_FILENAME)) + + try { + // Find municipalities in calendarMunicipalities that are not available in forecastMunicipalities + const mismatchingNames = calendarMunicipalities.reduce((list, item) => { + if (!forecastMunicipalities.find(x => x.province === item.province && + x.municipality === item.municipality) + ) { + list.push(item) + } + + return list + }, []) + + console.log(mismatchingNames) + } catch (err) { + console.log(err) + } +} + +main() diff --git a/src/04_find_mismatching_municipalities/src/calendar.js b/src/04_find_mismatching_municipalities/src/calendar.js new file mode 100644 index 0000000..161f5d1 --- /dev/null +++ b/src/04_find_mismatching_municipalities/src/calendar.js @@ -0,0 +1,20 @@ +const CroppingCalendar = require('../../02_crop_calendar/cropping_calendar') + +/** + * Extracts the municipalities with province list from the cropping calendar + * @param {String} filepath - Full file path to a CSV file + * @returns {Object[]} List of municipalities + */ +module.exports.municipalitiesFromCalendar = async (filepath) => { + try { + // Read the cropping calendar file + const cropCalendarData = new CroppingCalendar(filepath) + + await cropCalendarData.readCSV() + const municipalities = cropCalendarData.municipalities.map(x => ({ id: x.id, province: x.province, municipality: x.name })) + + return { municipalities } + } catch (err) { + throw new Error(err.message) + } +} diff --git a/src/04_find_mismatching_municipalities/src/forecast.js b/src/04_find_mismatching_municipalities/src/forecast.js new file mode 100644 index 0000000..28ae70a --- /dev/null +++ b/src/04_find_mismatching_municipalities/src/forecast.js @@ -0,0 +1,26 @@ +const XLSXWrapper = require('../../lib/xlsxwrapper') + +/** + * Extracts the municipalities with province list from a 10-day weather forecast excel file + * @param {String} filepath - Full file path to an EXCEL file + * @returns {Object[]} List of municipalities + */ +module.exports.municipalitiesFromForecast = (filepath) => { + // Read the excel file + const tendayForecastFile = new XLSXWrapper(filepath) + const tendayForecastData = tendayForecastFile.getDataSheet(0) + const bicolProvinces = ['Albay', 'Camarines Norte', 'Camarines Sur', 'Catanduanes', 'Masbate', 'Sorsogon'] + + const municipalities = tendayForecastData + .filter(x => (x.__EMPTY !== undefined && bicolProvinces.find(y => x.__EMPTY.toString().includes(y)))) + .map((x, id) => { + const province = bicolProvinces.find(y => x.__EMPTY.toString().includes(y)) + return { + id, + province, + municipality: x.__EMPTY.toString().split(`(${province})`)[0].trim() + } + }) + + return { municipalities } +} From f00c2543dc7b7fa56fc734d9e23661fbe3d4c795 Mon Sep 17 00:00:00 2001 From: ciatph Date: Wed, 11 Jan 2023 06:03:41 +0800 Subject: [PATCH 3/4] chore: Write mismatching municipality names to a CSV file --- README.md | 4 ++++ src/04_find_mismatching_municipalities/index.js | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/README.md b/README.md index a6e6535..1870cd3 100644 --- a/README.md +++ b/README.md @@ -131,5 +131,9 @@ Normalize and upload the new cropping calendar data. Normalize and upload the new municipalities and other details attached to the crop recommendations. +### `npm run find:mismatching` + +Writes a list of municipalities from the cropping calendar that have mismatching names from the 10-day weather forecast excel file into a `municipalities.csv` CSV file. + @ciatph
20221205 diff --git a/src/04_find_mismatching_municipalities/index.js b/src/04_find_mismatching_municipalities/index.js index 05149f0..7e00d6e 100644 --- a/src/04_find_mismatching_municipalities/index.js +++ b/src/04_find_mismatching_municipalities/index.js @@ -1,4 +1,5 @@ const path = require('path') +const { CsvToFireStore } = require('csv-firestore') const { municipalitiesFromCalendar } = require('./src/calendar') const { municipalitiesFromForecast } = require('./src/forecast') @@ -13,6 +14,8 @@ const main = async () => { // PAGASA municipalities const { municipalities: forecastMunicipalities } = municipalitiesFromForecast(path.join(__dirname, process.env.EXCEL_FILENAME)) + const handler = new CsvToFireStore() + try { // Find municipalities in calendarMunicipalities that are not available in forecastMunicipalities const mismatchingNames = calendarMunicipalities.reduce((list, item) => { @@ -26,6 +29,7 @@ const main = async () => { }, []) console.log(mismatchingNames) + handler.write(mismatchingNames, path.join(__dirname, 'municipalities.csv')) } catch (err) { console.log(err) } From 65772566a5951040082842852dcb7ee4ddc94d61 Mon Sep 17 00:00:00 2001 From: ciatph Date: Wed, 11 Jan 2023 06:50:09 +0800 Subject: [PATCH 4/4] chore: Use the 10-day weather forecast municipality names on the cropping calendar municipalities --- src/02_crop_calendar/constants.js | 15 ++++++++++++++- src/02_crop_calendar/cropping_calendar.js | 21 +++++++++++++++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/02_crop_calendar/constants.js b/src/02_crop_calendar/constants.js index 1ce24ce..1e2257a 100644 --- a/src/02_crop_calendar/constants.js +++ b/src/02_crop_calendar/constants.js @@ -5,6 +5,19 @@ const CROP_STAGE_LABELS = { lprep: 'Preparation Stage' } +// Uncommon municipality names detected from /04_find_mismatching_municipalities +// Replace the following with municipality names from the 10-day weather forecast file +const MUNICIPALITIES_TO_REPLACE_WITH_PAGASA = [ + { province: 'Albay', municipality: 'Sto.Domingo', replace: 'Santo Domingo' }, + { province: 'Albay', municipality: 'Pioduran', replace: 'Pio Duran' }, + { province: 'Camarines Norte', municipality: 'Sta. Elena', replace: 'Santa Elena' }, + { province: 'Camarines Norte', municipality: 'SL Ruiz', replace: 'San Lorenzo Ruiz' }, + { province: 'Camarines Sur', municipality: 'Sangay', replace: 'Sagnay' }, + { province: 'Sorsogon', municipality: 'Pto. Diaz', replace: 'Prieto Diaz' }, + { province: 'Sorsogon', municipality: 'Sta. Magdalena', replace: 'Santa Magdalena' } +] + module.exports = { - CROP_STAGE_LABELS + CROP_STAGE_LABELS, + MUNICIPALITIES_TO_REPLACE_WITH_PAGASA } diff --git a/src/02_crop_calendar/cropping_calendar.js b/src/02_crop_calendar/cropping_calendar.js index 5df64e4..b403fc9 100644 --- a/src/02_crop_calendar/cropping_calendar.js +++ b/src/02_crop_calendar/cropping_calendar.js @@ -1,5 +1,5 @@ const { CsvToFireStore } = require('csv-firestore') -const { CROP_STAGE_LABELS } = require('./constants') +const { CROP_STAGE_LABELS, MUNICIPALITIES_TO_REPLACE_WITH_PAGASA } = require('./constants') class CroppingCalendar extends CsvToFireStore { constructor (csvFilePath) { @@ -18,6 +18,9 @@ class CroppingCalendar extends CsvToFireStore { this.crop_stages = [] this.count = 0 + + // Municipalities that have different naming convention from the 10-day weather forecast files + this.susMunicipalities = MUNICIPALITIES_TO_REPLACE_WITH_PAGASA.map(item => item.municipality) } /** @@ -84,6 +87,7 @@ class CroppingCalendar extends CsvToFireStore { } const value = row[item].trim() + let tempMunicipality = null // Extract unique provinces if (key === 'province' && !this.itemExists('province', value) && value !== '') { @@ -95,13 +99,20 @@ class CroppingCalendar extends CsvToFireStore { // Extract unique municipalities if (key === 'municipality' && value !== '') { - const combo = `${row.prov.trim()}|${value}` + tempMunicipality = value.trim() + + if (this.susMunicipalities.includes(tempMunicipality)) { + tempMunicipality = MUNICIPALITIES_TO_REPLACE_WITH_PAGASA.find(x => + x.province === row.prov.trim() && x.municipality === value.trim())?.replace ?? value.trim() + } + + const combo = `${row.prov.trim()}|${tempMunicipality}` if (!this.itemExists('municipality', combo)) { this.municipalities.push({ id: this.municipalities.length + 1, province: row.prov.trim(), - name: value, + name: tempMunicipality, unique: combo }) } @@ -130,7 +141,9 @@ class CroppingCalendar extends CsvToFireStore { } if (include && ['province', 'municipality', 'crop'].includes(key)) { - obj[key] = value + obj[key] = (key === 'municipality') + ? tempMunicipality + : value } })