diff --git a/src/01_recommendations/index.js b/src/01_recommendations/index.js index fede17b..7c89142 100644 --- a/src/01_recommendations/index.js +++ b/src/01_recommendations/index.js @@ -1,5 +1,6 @@ require('dotenv').config() const path = require('path') +const { CsvToFireStore } = require('csv-firestore') const SeasonalTab = require('./src/classes/seasonaltab') const TendayTab = require('./src/classes/tendaytab') const SpecialTab = require('./src/classes/specialtab') @@ -16,6 +17,9 @@ const main = async () => { // Excel file path const filePath = path.join(__dirname, process.env.EXCEL_FILENAME) + // Firestore documents upload handler + const handler = new CsvToFireStore() + // Excel tabs column names definitions const excelTabs = [ new SeasonalTab(), @@ -57,7 +61,16 @@ const main = async () => { if (upload) { try { data.forEach((item, index) => { + // Simple merged JSON data query.push(uploadToFirestore('n_list_crop_recommendations', item.recommendations.type, item.recommendations)) + + // Upload each recommendation row to a Document + // Path: /n_list_crop_recommendations_{type} + query.push(handler.firestoreUpload( + `n_list_crop_recommendations_${item.recommendations.type}`, + true, + item.recommendations.data + )) }) // Upload data to Firestore diff --git a/src/01_recommendations/src/classes/excelsheetdef.js b/src/01_recommendations/src/classes/excelsheetdef.js index b944fba..e5d0029 100644 --- a/src/01_recommendations/src/classes/excelsheetdef.js +++ b/src/01_recommendations/src/classes/excelsheetdef.js @@ -1,3 +1,5 @@ +const { CROP_STAGE_LABELS } = require('../../../02_crop_calendar/constants') + class ExcelTabDefinition { /** Excel sheet tab number */ excelTabNumber = -1 @@ -21,6 +23,13 @@ class ExcelTabDefinition { PRACTICE: 'practice', PRACTICE_TAGALOG: 'practice_tagalog' } + + NORMAL_CROPSTAGE_CODES = { + 'Newly Planted': Object.keys(CROP_STAGE_LABELS).find(key => CROP_STAGE_LABELS[key] === 'Newly Planted'), + 'Vegetative/Reproductive': Object.keys(CROP_STAGE_LABELS).find(key => CROP_STAGE_LABELS[key] === 'Vegetative/Reproductive'), + Maturing: Object.keys(CROP_STAGE_LABELS).find(key => CROP_STAGE_LABELS[key] === 'Maturing'), + 'Preparation Stage': Object.keys(CROP_STAGE_LABELS).find(key => CROP_STAGE_LABELS[key] === 'Preparation Stage') + } } module.exports = ExcelTabDefinition diff --git a/src/01_recommendations/src/classes/seasonaltab.js b/src/01_recommendations/src/classes/seasonaltab.js index cc33ae7..148bedb 100644 --- a/src/01_recommendations/src/classes/seasonaltab.js +++ b/src/01_recommendations/src/classes/seasonaltab.js @@ -18,6 +18,22 @@ class SeasonalTab extends ExcelTabDefinition { __EMPTY_2: this.NORMAL_COLUMN_NAMES.PRACTICE, __EMPTY_3: this.NORMAL_COLUMN_NAMES.PRACTICE_TAGALOG } + + /** Normalized weather forecast text */ + this.NORMAL_FORECAST_NAMES = { + 'WAY BELOW NORMAL': 'Way Below Normal', + 'BELOW NORMAL': 'Below Normal', + 'NEAR NORMAL': 'Near normal', + 'ABOVE NORMAL': 'Above Normal' + } + + /** Normalized weather forecast codes */ + this.NORMAL_FORECAST_CODES = { + 'WAY BELOW NORMAL': 'wb_normal', + 'BELOW NORMAL': 'b_normal', + 'NEAR NORMAL': 'near_normal', + 'ABOVE NORMAL': 'above_normal' + } } } diff --git a/src/01_recommendations/src/constants.js b/src/01_recommendations/src/constants.js index 8fbbd12..e315a9e 100644 --- a/src/01_recommendations/src/constants.js +++ b/src/01_recommendations/src/constants.js @@ -1,7 +1,7 @@ const RECOMMEDATIONS_TYPE = { SEASONAL: 'seasonal', TENDAY: 'tenday', - SPECIAL: 'sepcial' + SPECIAL: 'special' } module.exports = { diff --git a/src/01_recommendations/src/extract.js b/src/01_recommendations/src/extract.js index bb812be..15d7fc7 100644 --- a/src/01_recommendations/src/extract.js +++ b/src/01_recommendations/src/extract.js @@ -1,4 +1,5 @@ const XLSXWrapper = require('../../lib/xlsxwrapper') +const { RECOMMEDATIONS_TYPE } = require('./constants') /** * Extract normalized recommendations data and other metadata from an excel sheet tab @@ -28,8 +29,38 @@ module.exports.extractExcelData = (ExcelTab, excelFilePath) => { const obj = {} for (const key in ExcelTab.EXCEL_COLUMN_NAMES) { - obj[ExcelTab.EXCEL_COLUMN_NAMES[key]] = item[key] || '' - obj[ExcelTab.EXCEL_COLUMN_NAMES[key]] = obj[ExcelTab.EXCEL_COLUMN_NAMES[key]].trim() + let value = item[key] || '' + value = value.trim() + + switch (ExcelTab.type) { + case RECOMMEDATIONS_TYPE.SEASONAL: + // Normalize the seasonal forecast text + if (ExcelTab.EXCEL_COLUMN_NAMES[key] === ExcelTab.NORMAL_COLUMN_NAMES.FORECAST) { + value = ExcelTab.NORMAL_FORECAST_CODES[value] + } + break + default: break + } + + // Insert in
  • + switch (ExcelTab.EXCEL_COLUMN_NAMES[key]) { + case ExcelTab.NORMAL_COLUMN_NAMES.PRACTICE: + case ExcelTab.NORMAL_COLUMN_NAMES.PRACTICE_TAGALOG: + case ExcelTab.NORMAL_COLUMN_NAMES.IMPACT: + case ExcelTab.NORMAL_COLUMN_NAMES.IMPACT_TAGALOG: + value = value.replace(/
  • /g, '
  • ') + value = value.replace(/<\/li>/g, '
    • ') + break + default: + break + } + + // Normalize the crop stage name - use crop calendar codes + if (ExcelTab.EXCEL_COLUMN_NAMES[key] === ExcelTab.NORMAL_COLUMN_NAMES.CROP_STAGE) { + value = ExcelTab.NORMAL_CROPSTAGE_CODES[value] + } + + obj[ExcelTab.EXCEL_COLUMN_NAMES[key]] = value } list.push(obj) @@ -38,6 +69,28 @@ module.exports.extractExcelData = (ExcelTab, excelFilePath) => { return list }, []) + // Normalize the special recommendations' farm operations + if (ExcelTab.type === RECOMMEDATIONS_TYPE.SPECIAL) { + recommendations.data = recommendations.data.reduce((list, item, index) => { + const farmoperation = item[ExcelTab.NORMAL_COLUMN_NAMES.FARM_OPERATION] + + // Expand the merged farm operations vertically + if (farmoperation.includes('/') && farmoperation !== 'Planting/Transplanting') { + const farmoperations = farmoperation.split('/').map(operations => operations.trim()) + + farmoperations.forEach((operation) => { + const temp = { ...item } + temp[ExcelTab.NORMAL_COLUMN_NAMES.FARM_OPERATION] = operation + list.push(temp) + }) + } else { + list.push(item) + } + + return list + }, []) + } + // List unique crop stages const cropstages = recommendations.data.map(x => x[ExcelTab.NORMAL_COLUMN_NAMES.CROP_STAGE]) .filter((x, i, a) => a.indexOf(x) === i) diff --git a/src/02_crop_calendar/index.js b/src/02_crop_calendar/index.js index c2c3b0e..188b459 100644 --- a/src/02_crop_calendar/index.js +++ b/src/02_crop_calendar/index.js @@ -3,7 +3,7 @@ const path = require('path') const CroppingCalendar = require('./cropping_calendar') const { uploadToFirestore } = require('../lib/uploadtofirestore') -// Path: /n_cropping_calendar_merged/{province}.data[] +// Path: /n_cropping_calendar_lite/{province}.data[] const main = async () => { const handler = new CroppingCalendar(path.resolve(__dirname, process.env.CSV_FILENAME)) const upload = false @@ -11,8 +11,8 @@ const main = async () => { // Cropping Calendar-specific tables and firestore collection names const newTables = { - provinces: 'n_provinces', - municipalities: 'n_municipalities', + // provinces: 'n_provinces', + // municipalities: 'n_municipalities', crops: 'n_crops', crop_stages: 'n_crop_stages' } @@ -30,7 +30,8 @@ const main = async () => { group[province] = [] } - const obj = {} + const obj = { province } + for (const key in row) { if (!['id', 'province'].includes(key)) { obj[key] = row[key].trim() @@ -41,7 +42,7 @@ const main = async () => { return { ...group } }, {}) - console.log('\nUploading data to firestore...') + console.log('\nUploading list data to firestore...') const query = [] // Upload full collections @@ -63,11 +64,29 @@ const main = async () => { logs += `${province}: ${data[province].length} items\n` // Upload query - query.push(uploadToFirestore('n_cropping_calendar_merged', province, { data: data[province] })) + query.push(uploadToFirestore('n_cropping_calendar_lite', province, { data: data[province] })) } console.log(logs) - console.log('Uploading data to Firestore...') + console.log('Uploading calendar data to Firestore...') + + // Upload list data as documents + console.log('Uploading lists in a single Firestore document...') + + // Upload the list of provinces and respective municipalities for use as constant, static values + query.push(uploadToFirestore('constant_data', 'provinces', { + data: handler.provinces.reduce((list, province, index) => { + const temp = { id: province.id, label: province.name } + + temp.municipalities = handler.municipalities.filter((municipality) => municipality.province === province.name) + .sort((a, b) => a.name > b.name ? 1 : (a.name < b.name) ? -1 : 0) + .map((municipality, id) => ({ id, label: municipality.name })) + + list.push(temp) + return list + }, []) + })) + await Promise.all(query) console.log('Upload success!') } diff --git a/src/lib/uploadtofirestore/index.js b/src/lib/uploadtofirestore/index.js index 216f147..fb81b8d 100644 --- a/src/lib/uploadtofirestore/index.js +++ b/src/lib/uploadtofirestore/index.js @@ -9,11 +9,13 @@ const firestore = new FirestoreData() * - {Object[]} data - crop recommendations mapped to crop stages, farm operations and other keys * - {String} type - type of recommendations. One of RECOMMEDATIONS_TYPE. * - {String} description - Brief text description describing the nature of data + * @param {Object} metadata - Key-value pairs description and other information about the data * @returns {Timestamp} Firestore timestamp of successful data upload */ -module.exports.uploadToFirestore = async (collectionName, docName, jsonData) => { +module.exports.uploadToFirestore = async (collectionName, docName, jsonData, metadata) => { // CSV and Firestore handler - jsonData.date_created = firestore.admin.firestore.Timestamp.now() + jsonData.metadata = metadata ?? {} + jsonData.metadata.date_created = firestore.admin.firestore.Timestamp.now() try { // Upload data to Firestore