From 516bf065624674394b356e93eaba41124932e063 Mon Sep 17 00:00:00 2001 From: ciatph Date: Sat, 31 Dec 2022 23:44:59 +0800 Subject: [PATCH 1/9] chore: Normalize the seasonal weather forecast text --- src/01_recommendations/src/classes/seasonaltab.js | 8 ++++++++ src/01_recommendations/src/extract.js | 14 ++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/01_recommendations/src/classes/seasonaltab.js b/src/01_recommendations/src/classes/seasonaltab.js index cc33ae7..c5e96f4 100644 --- a/src/01_recommendations/src/classes/seasonaltab.js +++ b/src/01_recommendations/src/classes/seasonaltab.js @@ -18,6 +18,14 @@ class SeasonalTab extends ExcelTabDefinition { __EMPTY_2: this.NORMAL_COLUMN_NAMES.PRACTICE, __EMPTY_3: this.NORMAL_COLUMN_NAMES.PRACTICE_TAGALOG } + + /** Normalized weather forecast text */ + this.NORMAL_FORECAST_NAMES = { + 'WAY BELOW NORMAL': 'Way Below Normal', + 'BELOW NORMAL': 'Below Normal', + 'NEAR NORMAL': 'Near Normal', + 'ABOVE NORMAL': 'Above Normal' + } } } diff --git a/src/01_recommendations/src/extract.js b/src/01_recommendations/src/extract.js index bb812be..c301d98 100644 --- a/src/01_recommendations/src/extract.js +++ b/src/01_recommendations/src/extract.js @@ -1,4 +1,5 @@ const XLSXWrapper = require('../../lib/xlsxwrapper') +const { RECOMMEDATIONS_TYPE } = require('./constants') /** * Extract normalized recommendations data and other metadata from an excel sheet tab @@ -28,8 +29,17 @@ module.exports.extractExcelData = (ExcelTab, excelFilePath) => { const obj = {} for (const key in ExcelTab.EXCEL_COLUMN_NAMES) { - obj[ExcelTab.EXCEL_COLUMN_NAMES[key]] = item[key] || '' - obj[ExcelTab.EXCEL_COLUMN_NAMES[key]] = obj[ExcelTab.EXCEL_COLUMN_NAMES[key]].trim() + let value = item[key] || '' + value = value.trim() + + // Normalize the forecast text + if (ExcelTab.type === RECOMMEDATIONS_TYPE.SEASONAL && + ExcelTab.EXCEL_COLUMN_NAMES[key] === ExcelTab.NORMAL_COLUMN_NAMES.FORECAST + ) { + value = ExcelTab.NORMAL_FORECAST_NAMES[value] + } + + obj[ExcelTab.EXCEL_COLUMN_NAMES[key]] = value } list.push(obj) From 54536cca302c5f356ba36519d5e603cb6cd59f9e Mon Sep 17 00:00:00 2001 From: ciatph Date: Sun, 1 Jan 2023 00:53:53 +0800 Subject: [PATCH 2/9] chore: Normalize the special recommendations farm operations - expand vertically --- src/01_recommendations/src/extract.js | 35 +++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/01_recommendations/src/extract.js b/src/01_recommendations/src/extract.js index c301d98..31d4f58 100644 --- a/src/01_recommendations/src/extract.js +++ b/src/01_recommendations/src/extract.js @@ -32,11 +32,14 @@ module.exports.extractExcelData = (ExcelTab, excelFilePath) => { let value = item[key] || '' value = value.trim() - // Normalize the forecast text - if (ExcelTab.type === RECOMMEDATIONS_TYPE.SEASONAL && - ExcelTab.EXCEL_COLUMN_NAMES[key] === ExcelTab.NORMAL_COLUMN_NAMES.FORECAST - ) { - value = ExcelTab.NORMAL_FORECAST_NAMES[value] + switch (ExcelTab.type) { + case RECOMMEDATIONS_TYPE.SEASONAL: + // Normalize the seasonal forecast text + if (ExcelTab.EXCEL_COLUMN_NAMES[key] === ExcelTab.NORMAL_COLUMN_NAMES.FORECAST) { + value = ExcelTab.NORMAL_FORECAST_NAMES[value] + } + break + default: break } obj[ExcelTab.EXCEL_COLUMN_NAMES[key]] = value @@ -48,6 +51,28 @@ module.exports.extractExcelData = (ExcelTab, excelFilePath) => { return list }, []) + // Normalize the special recommendations' farm operations + if (ExcelTab.type === RECOMMEDATIONS_TYPE.SPECIAL) { + recommendations.data = recommendations.data.reduce((list, item, index) => { + const farmoperation = item[ExcelTab.NORMAL_COLUMN_NAMES.FARM_OPERATION] + + // Expand the merged farm operations vertically + if (farmoperation.includes('/') && farmoperation !== 'Planting/Transplanting') { + const farmoperations = farmoperation.split('/').map(operations => operations.trim()) + + farmoperations.forEach((operation) => { + const temp = { ...item } + temp[ExcelTab.NORMAL_COLUMN_NAMES.FARM_OPERATION] = operation + list.push(temp) + }) + } else { + list.push(item) + } + + return list + }, []) + } + // List unique crop stages const cropstages = recommendations.data.map(x => x[ExcelTab.NORMAL_COLUMN_NAMES.CROP_STAGE]) .filter((x, i, a) => a.indexOf(x) === i) From 23660b7a4a3ca45a3c87d120137a6129218d43c7 Mon Sep 17 00:00:00 2001 From: ciatph Date: Sun, 1 Jan 2023 00:57:59 +0800 Subject: [PATCH 3/9] chore: Upload each recommendatiom to a document --- src/01_recommendations/index.js | 13 +++++++++++++ src/lib/uploadtofirestore/index.js | 6 ++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/01_recommendations/index.js b/src/01_recommendations/index.js index fede17b..7c89142 100644 --- a/src/01_recommendations/index.js +++ b/src/01_recommendations/index.js @@ -1,5 +1,6 @@ require('dotenv').config() const path = require('path') +const { CsvToFireStore } = require('csv-firestore') const SeasonalTab = require('./src/classes/seasonaltab') const TendayTab = require('./src/classes/tendaytab') const SpecialTab = require('./src/classes/specialtab') @@ -16,6 +17,9 @@ const main = async () => { // Excel file path const filePath = path.join(__dirname, process.env.EXCEL_FILENAME) + // Firestore documents upload handler + const handler = new CsvToFireStore() + // Excel tabs column names definitions const excelTabs = [ new SeasonalTab(), @@ -57,7 +61,16 @@ const main = async () => { if (upload) { try { data.forEach((item, index) => { + // Simple merged JSON data query.push(uploadToFirestore('n_list_crop_recommendations', item.recommendations.type, item.recommendations)) + + // Upload each recommendation row to a Document + // Path: /n_list_crop_recommendations_{type} + query.push(handler.firestoreUpload( + `n_list_crop_recommendations_${item.recommendations.type}`, + true, + item.recommendations.data + )) }) // Upload data to Firestore diff --git a/src/lib/uploadtofirestore/index.js b/src/lib/uploadtofirestore/index.js index 216f147..fb81b8d 100644 --- a/src/lib/uploadtofirestore/index.js +++ b/src/lib/uploadtofirestore/index.js @@ -9,11 +9,13 @@ const firestore = new FirestoreData() * - {Object[]} data - crop recommendations mapped to crop stages, farm operations and other keys * - {String} type - type of recommendations. One of RECOMMEDATIONS_TYPE. * - {String} description - Brief text description describing the nature of data + * @param {Object} metadata - Key-value pairs description and other information about the data * @returns {Timestamp} Firestore timestamp of successful data upload */ -module.exports.uploadToFirestore = async (collectionName, docName, jsonData) => { +module.exports.uploadToFirestore = async (collectionName, docName, jsonData, metadata) => { // CSV and Firestore handler - jsonData.date_created = firestore.admin.firestore.Timestamp.now() + jsonData.metadata = metadata ?? {} + jsonData.metadata.date_created = firestore.admin.firestore.Timestamp.now() try { // Upload data to Firestore From 441a60feb393e21857ebf883d43108b945927cdb Mon Sep 17 00:00:00 2001 From: ciatph Date: Sun, 1 Jan 2023 02:00:07 +0800 Subject: [PATCH 4/9] chore: Upload the new cropping calendar to a new collection --- .../src/classes/seasonaltab.js | 2 +- src/02_crop_calendar/index.js | 23 ++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/01_recommendations/src/classes/seasonaltab.js b/src/01_recommendations/src/classes/seasonaltab.js index c5e96f4..05c3de3 100644 --- a/src/01_recommendations/src/classes/seasonaltab.js +++ b/src/01_recommendations/src/classes/seasonaltab.js @@ -23,7 +23,7 @@ class SeasonalTab extends ExcelTabDefinition { this.NORMAL_FORECAST_NAMES = { 'WAY BELOW NORMAL': 'Way Below Normal', 'BELOW NORMAL': 'Below Normal', - 'NEAR NORMAL': 'Near Normal', + 'NEAR NORMAL': 'Near normal', 'ABOVE NORMAL': 'Above Normal' } } diff --git a/src/02_crop_calendar/index.js b/src/02_crop_calendar/index.js index c2c3b0e..36a0f02 100644 --- a/src/02_crop_calendar/index.js +++ b/src/02_crop_calendar/index.js @@ -41,7 +41,7 @@ const main = async () => { return { ...group } }, {}) - console.log('\nUploading data to firestore...') + console.log('\nUploading list data to firestore...') const query = [] // Upload full collections @@ -63,11 +63,28 @@ const main = async () => { logs += `${province}: ${data[province].length} items\n` // Upload query - query.push(uploadToFirestore('n_cropping_calendar_merged', province, { data: data[province] })) + query.push(uploadToFirestore('n_cropping_calendar_lite', province, { data: data[province] })) } console.log(logs) - console.log('Uploading data to Firestore...') + console.log('Uploading calendar data to Firestore...') + + // Upload list data as documents + console.log('Uploading lists in a single Firestore document...') + + query.push(uploadToFirestore('constant_data', 'provinces', { + data: handler.provinces.reduce((list, province, index) => { + const temp = { id: province.id, label: province.name } + + temp.municipalities = handler.municipalities.filter((municipality) => municipality.province === province.name) + .sort((a, b) => a.name > b.name ? 1 : (a.name < b.name) ? -1 : 0) + .map((municipality, id) => ({ id, label: municipality.name })) + + list.push(temp) + return list + }, []) + })) + await Promise.all(query) console.log('Upload success!') } From 08f87c7c5be03c44227f764aedcef95029c796eb Mon Sep 17 00:00:00 2001 From: ciatph Date: Fri, 6 Jan 2023 11:56:26 +0800 Subject: [PATCH 5/9] chore: Include province in the row crop calendar object --- src/02_crop_calendar/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/02_crop_calendar/index.js b/src/02_crop_calendar/index.js index 36a0f02..39afb1f 100644 --- a/src/02_crop_calendar/index.js +++ b/src/02_crop_calendar/index.js @@ -30,7 +30,8 @@ const main = async () => { group[province] = [] } - const obj = {} + const obj = { province } + for (const key in row) { if (!['id', 'province'].includes(key)) { obj[key] = row[key].trim() From ee44d9c34d198b90216dd6d41fb9ba097e19cafd Mon Sep 17 00:00:00 2001 From: ciatph Date: Fri, 6 Jan 2023 11:56:41 +0800 Subject: [PATCH 6/9] fix: Typo error --- src/01_recommendations/src/constants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/01_recommendations/src/constants.js b/src/01_recommendations/src/constants.js index 8fbbd12..e315a9e 100644 --- a/src/01_recommendations/src/constants.js +++ b/src/01_recommendations/src/constants.js @@ -1,7 +1,7 @@ const RECOMMEDATIONS_TYPE = { SEASONAL: 'seasonal', TENDAY: 'tenday', - SPECIAL: 'sepcial' + SPECIAL: 'special' } module.exports = { From d721003a3cea57af651858fa4ee45c03405f28e3 Mon Sep 17 00:00:00 2001 From: ciatph Date: Tue, 10 Jan 2023 20:07:58 +0800 Subject: [PATCH 7/9] chore: Use codes for the crop stage and rainfall forecast values --- src/01_recommendations/src/classes/excelsheetdef.js | 9 +++++++++ src/01_recommendations/src/classes/seasonaltab.js | 8 ++++++++ src/01_recommendations/src/extract.js | 7 ++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/01_recommendations/src/classes/excelsheetdef.js b/src/01_recommendations/src/classes/excelsheetdef.js index b944fba..e5d0029 100644 --- a/src/01_recommendations/src/classes/excelsheetdef.js +++ b/src/01_recommendations/src/classes/excelsheetdef.js @@ -1,3 +1,5 @@ +const { CROP_STAGE_LABELS } = require('../../../02_crop_calendar/constants') + class ExcelTabDefinition { /** Excel sheet tab number */ excelTabNumber = -1 @@ -21,6 +23,13 @@ class ExcelTabDefinition { PRACTICE: 'practice', PRACTICE_TAGALOG: 'practice_tagalog' } + + NORMAL_CROPSTAGE_CODES = { + 'Newly Planted': Object.keys(CROP_STAGE_LABELS).find(key => CROP_STAGE_LABELS[key] === 'Newly Planted'), + 'Vegetative/Reproductive': Object.keys(CROP_STAGE_LABELS).find(key => CROP_STAGE_LABELS[key] === 'Vegetative/Reproductive'), + Maturing: Object.keys(CROP_STAGE_LABELS).find(key => CROP_STAGE_LABELS[key] === 'Maturing'), + 'Preparation Stage': Object.keys(CROP_STAGE_LABELS).find(key => CROP_STAGE_LABELS[key] === 'Preparation Stage') + } } module.exports = ExcelTabDefinition diff --git a/src/01_recommendations/src/classes/seasonaltab.js b/src/01_recommendations/src/classes/seasonaltab.js index 05c3de3..148bedb 100644 --- a/src/01_recommendations/src/classes/seasonaltab.js +++ b/src/01_recommendations/src/classes/seasonaltab.js @@ -26,6 +26,14 @@ class SeasonalTab extends ExcelTabDefinition { 'NEAR NORMAL': 'Near normal', 'ABOVE NORMAL': 'Above Normal' } + + /** Normalized weather forecast codes */ + this.NORMAL_FORECAST_CODES = { + 'WAY BELOW NORMAL': 'wb_normal', + 'BELOW NORMAL': 'b_normal', + 'NEAR NORMAL': 'near_normal', + 'ABOVE NORMAL': 'above_normal' + } } } diff --git a/src/01_recommendations/src/extract.js b/src/01_recommendations/src/extract.js index 31d4f58..4315b69 100644 --- a/src/01_recommendations/src/extract.js +++ b/src/01_recommendations/src/extract.js @@ -36,12 +36,17 @@ module.exports.extractExcelData = (ExcelTab, excelFilePath) => { case RECOMMEDATIONS_TYPE.SEASONAL: // Normalize the seasonal forecast text if (ExcelTab.EXCEL_COLUMN_NAMES[key] === ExcelTab.NORMAL_COLUMN_NAMES.FORECAST) { - value = ExcelTab.NORMAL_FORECAST_NAMES[value] + value = ExcelTab.NORMAL_FORECAST_CODES[value] } break default: break } + // Normalize the crop stage name - use crop calendar codes + if (ExcelTab.EXCEL_COLUMN_NAMES[key] === ExcelTab.NORMAL_COLUMN_NAMES.CROP_STAGE) { + value = ExcelTab.NORMAL_CROPSTAGE_CODES[value] + } + obj[ExcelTab.EXCEL_COLUMN_NAMES[key]] = value } From b8122f59b048f1d05ab09078a0a8685627c55b00 Mon Sep 17 00:00:00 2001 From: ciatph Date: Tue, 10 Jan 2023 20:08:49 +0800 Subject: [PATCH 8/9] chore: Insert tags between
  • tags --- src/01_recommendations/src/extract.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/01_recommendations/src/extract.js b/src/01_recommendations/src/extract.js index 4315b69..15d7fc7 100644 --- a/src/01_recommendations/src/extract.js +++ b/src/01_recommendations/src/extract.js @@ -42,6 +42,19 @@ module.exports.extractExcelData = (ExcelTab, excelFilePath) => { default: break } + // Insert in
  • + switch (ExcelTab.EXCEL_COLUMN_NAMES[key]) { + case ExcelTab.NORMAL_COLUMN_NAMES.PRACTICE: + case ExcelTab.NORMAL_COLUMN_NAMES.PRACTICE_TAGALOG: + case ExcelTab.NORMAL_COLUMN_NAMES.IMPACT: + case ExcelTab.NORMAL_COLUMN_NAMES.IMPACT_TAGALOG: + value = value.replace(/
  • /g, '
  • ') + value = value.replace(/<\/li>/g, '
    • ') + break + default: + break + } + // Normalize the crop stage name - use crop calendar codes if (ExcelTab.EXCEL_COLUMN_NAMES[key] === ExcelTab.NORMAL_COLUMN_NAMES.CROP_STAGE) { value = ExcelTab.NORMAL_CROPSTAGE_CODES[value] From 429bb60535ad7d23c04abe7cc10b7feef345cdd7 Mon Sep 17 00:00:00 2001 From: ciatph Date: Tue, 10 Jan 2023 20:14:13 +0800 Subject: [PATCH 9/9] chore: Update comments --- src/02_crop_calendar/index.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/02_crop_calendar/index.js b/src/02_crop_calendar/index.js index 39afb1f..188b459 100644 --- a/src/02_crop_calendar/index.js +++ b/src/02_crop_calendar/index.js @@ -3,7 +3,7 @@ const path = require('path') const CroppingCalendar = require('./cropping_calendar') const { uploadToFirestore } = require('../lib/uploadtofirestore') -// Path: /n_cropping_calendar_merged/{province}.data[] +// Path: /n_cropping_calendar_lite/{province}.data[] const main = async () => { const handler = new CroppingCalendar(path.resolve(__dirname, process.env.CSV_FILENAME)) const upload = false @@ -11,8 +11,8 @@ const main = async () => { // Cropping Calendar-specific tables and firestore collection names const newTables = { - provinces: 'n_provinces', - municipalities: 'n_municipalities', + // provinces: 'n_provinces', + // municipalities: 'n_municipalities', crops: 'n_crops', crop_stages: 'n_crop_stages' } @@ -73,6 +73,7 @@ const main = async () => { // Upload list data as documents console.log('Uploading lists in a single Firestore document...') + // Upload the list of provinces and respective municipalities for use as constant, static values query.push(uploadToFirestore('constant_data', 'provinces', { data: handler.provinces.reduce((list, province, index) => { const temp = { id: province.id, label: province.name }