From 8e2390121bf222712e99d00f12c3cf6ddbf4880a Mon Sep 17 00:00:00 2001 From: Irit <35077819+iritush@users.noreply.github.com> Date: Wed, 22 Jan 2020 14:59:39 -0500 Subject: [PATCH 01/10] =?UTF-8?q?-Update=20goal=20fields=20to=20the=20fiel?= =?UTF-8?q?d=20names=20provided=20in=20the=20new=20goal=20csv=20file=20-se?= =?UTF-8?q?parate=20between=20parsing=20and=20saving=20goal=20csv=20and=20?= =?UTF-8?q?client=20csv=20-switch=20to=20use=20collection=20name=20?= =?UTF-8?q?=E2=80=98testusers=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- functions/index.js | 142 +++++++++++++++++++++++++-------------- functions/models/goal.js | 50 ++++++++------ functions/models/user.js | 2 +- 3 files changed, 120 insertions(+), 74 deletions(-) diff --git a/functions/index.js b/functions/index.js index c490ae8..cc881df 100644 --- a/functions/index.js +++ b/functions/index.js @@ -9,14 +9,14 @@ const {isValidEmail} = require('./models/data-validators'); const {db} = require('./models/user'); exports.helloWorld = functions.https.onRequest((request, response) => { - response.send("Hello from Firebase!"); + response.send("Hello from Firebase!"); console.log("Attempting to access environment configuration data") //Must be configured on the firebase side using the firebase cli: https://firebase.google.com/docs/functions/config-env try { console.log(`user: ${functions.config().credentials.user} password: ${functions.config().credentials.password}`) } catch (error) { console.log(`Got the following error when trying to access credentials: ${{error}}`) - } + } }); /*This firebase function is for testing purposes to be able to use a file saved locally as input. @@ -34,7 +34,8 @@ exports.pullDataFromLocalCSVFileTEST = functions.https.onRequest((request, respo console.log('Found ' + zipEntries.length + ' entry in the zip file'); fileContent += csvzip.readAsText(zipEntries[0]); } - parseCSVAndSaveToFireStore (fileContent); + // parseClientCSVAndSaveToFireStore (fileContent); + parseGoalCSVAndSaveToFireStore (fileContent); response.send('done'); }) @@ -75,19 +76,23 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { // // Name Year Month Day Hour // res[1] res[2] res[3] res[4] res[5] - fRegEx = /(gm_clients_served_)(\d\d\d\d)-(\d?\d)-(\d?\d)-?(\d?\d)?/; + fRegEx_gm_clients = /(gm_clients_served_)(\d\d\d\d)-(\d?\d)-(\d?\d)-?(\d?\d)?/; + fRegEx_gm_goals = /(gm_goals_details_)(\d\d\d\d)-(\d?\d)-(\d?\d)-?(\d?\d)?/; // Applies the regex pattern to our filename and stores result in fnParts (file name parts) - fnParts = fRegEx.exec(fileName); + // fnParts = fRegEx_gm_clients.exec(fileName); + fnParts = fRegEx_gm_goals.exec(fileName); // we'll construct a new object to keep track of each files important details // and more importantly, to make it easy to sort/search + if(fnParts) { let newFile = { name: fileName, year: parseInt(fnParts[2]), // year month: parseInt(fnParts[3]), // month day: parseInt(fnParts[4]), // day hour: parseInt(fnParts[5]) || 0 // hour - first file of day has no hour in name so use 0 instead - }; + }; fileNames.push(newFile) // toss new object into array + } } // sort() is an instance of "fast-sort" // So to fine the newest file... sort by the year, then month, then day, then hour in a descending fashion. @@ -147,7 +152,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { sftpConnectionToCvoeo.end(); // Finally send the response string along with the official A-OK code (200) console.log('Parsing file content'); - parseCSVAndSaveToFireStore (fileContent); + parseClientCSVAndSaveToFireStore (fileContent); response.send(outString, 200); return true; }) @@ -170,7 +175,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { If user does not exist, create a new user document under the 'users' collection + create new goal TODO: add more error handling to this function */ - function parseCSVAndSaveToFireStore(fileContent) { + function parseClientCSVAndSaveToFireStore(fileContent) { //*** Known issue: When parsing a csv file with multiple lines that have goal data, saving to firestore is not working properly */ // TODO: Ideally data validation will be handles in the user class but add any validations that are needed here Papa.parse(fileContent, { @@ -186,7 +191,6 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { } else { let user = new User(results.data[i]['System Name ID']); - let goal = new Goal(user.uid); for (let key in results.data[i]) { if(results.data[i][key] != "") { switch (key) { @@ -203,55 +207,18 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { break; } } - if(results.data[i]['GOAL ID']) { - if (results.data[i][key] != "") { - switch (key) { - case 'GOAL ID': - goal.goaluid = results.data[i][key]; - break; - case 'GOAL TYPE': - goal.goalType = results.data[i][key]; - break; - case 'GOAL DUE': - goal.goalDueDate = results.data[i][key]; - break; - case 'GOAL NOTES': - goal.goalNotes = results.data[i][key]; - break; - case 'GOAL COMPLETE': - goal.isGoalComplete = results.data[i][key]; - break; - } - } - } - } + } - let usersCollection = db.collection('users'); + let usersCollection = db.collection('testusers'); usersCollection.where('uid', '==', user.uid).get() .then(userSnapshot => { if (userSnapshot.empty) { console.log("Did not find a matching document with uid " + user.uid); - user.createNewUserInFirestore(); - if (goal.goaluid) { - goal.createNewGoalInFirestore(); + user.createNewUserInFirestore(); } - } else { - console.log("Found a matching document for uid " + user.uid); - user.updateExistingUserInFirestore(); - if (goal.goaluid) { - usersCollection.doc(user.uid).collection('goals').where('goaluid', '==', goal.goaluid).get() - .then(goalSnapshot => { - if (goalSnapshot.empty) { - console.log("Did not find a matching document with goal id " + goal.goaluid + " for user " + goal.useruid); - goal.createNewGoalInFirestore(); - } - else { - console.log("Found a matching document for goal id " + goal.goaluid + " under document for user " + goal.useruid); - goal.updateExistingGoalInFirestore(); - } - }) - } + console.log("Found a matching document for user uid " + user.uid); + user.updateExistingUserInFirestore(); } }) .catch(err => { @@ -262,3 +229,76 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { } }) } + + function parseGoalCSVAndSaveToFireStore(fileContent) { + //*** Known issue: When parsing a csv file with multiple lines that have goal data, saving to firestore is not working properly */ + // TODO: Ideally data validation will be handles in the user class but add any validations that are needed here + Papa.parse(fileContent, { + //papaparse (https://www.papaparse.com)returns 'results' which has an array 'data'. + // Each entry in 'data' is an object, a set of key/values that match the header at the head of the csv file. + header: true, + skipEmptyLines: true, + complete: function(results) { + console.log("Found "+ results.data.length + " lines in file content\n"); + for (let i = 0;i { + if (userSnapshot.empty) { + console.log("Did not find a matching document with uid " + user.uid); + user.createNewUserInFirestore(); + goal.createNewGoalInFirestore(); + } + else { + console.log("Found a matching document for user uid " + user.uid); + usersCollection.doc(user.uid).collection('goals').where('goaluid', '==', goal.goaluid).get() + .then(goalSnapshot => { + if (goalSnapshot.empty) { + console.log("Did not find a matching document with goal id " + goal.goaluid + " for user " + goal.useruid); + goal.createNewGoalInFirestore(); + } + else { + console.log("Found a matching document for goal id " + goal.goaluid + " under document for user " + goal.useruid); + goal.updateExistingGoalInFirestore(); + } + }) + } + }) + .catch(err => { + console.log('Error getting documents', err); + }); + } + } + } + }) + } \ No newline at end of file diff --git a/functions/models/goal.js b/functions/models/goal.js index c91e6b6..0c5ab09 100644 --- a/functions/models/goal.js +++ b/functions/models/goal.js @@ -1,5 +1,5 @@ const {db} = require('./user'); -let usersCollection = db.collection('users'); +let usersCollection = db.collection('testusers'); let userDoc; class Goal { //TODO: add data validation to all properties @@ -11,10 +11,11 @@ class Goal { } this.goaluid = ''; this.useruid = useruid;//unique id of the user which this goal corresponds to - this.goalType = ''; - this.goalDueDate = ''; - this.goalNotes = ''; - this.isGoalComplete = ''; + this.goalCategory = ''; + this.goalDate = ''; + this.goalProgress = ''; + this.isGoalComplete = false; + this.goalNextSteps = ''; this.created = Date.now(); userDoc = usersCollection.doc(this.useruid); } @@ -23,42 +24,47 @@ class Goal { console.log ( "User uid: " + this.useruid + "\n" + "Goal uid: " + this.goaluid + "\n" + - "Goal type: " + this.goalType + "\n" + - "Goal due date: " + this.goalDueDate + "\n" + - "Goal notes: " + this.goalNotes + "\n" + - "Goal Complete?: " + this.isGoalComplete + "\n") + "Goal category: " + this.goalCategory + "\n" + + "Goal date: " + this.goalDate + "\n" + + "Goal progress: " + this.goalProgress + "\n" + + "Goal complete?: " + this.isGoalComplete + "\n" + + "Goal next steps: " + this.goalNextSteps + "\n") } createNewGoalInFirestore() { - console.log("Creating a new goal for user " + this.useruid + " with the following data:\n"); + console.log("Creating a new goal " + this.goaluid + "for user " + this.useruid + " with the following data:\n"); this.printAllFieldsToConsole(); userDoc.collection('goals').doc(this.goaluid).set({ created: this.created, goaluid: this.goaluid, useruid: this.useruid, - goalType: this.goalType, - goalDue: this.goalDueDate, - goalNotes: this.goalNotes, - isGoalComplete: this.isGoalComplete + goalCategory: this.goalCategory, + goalDue: this.goalDate, + goalProgress: this.goalProgress, + isGoalComplete: this.isGoalComplete, + goalNextSteps: this.goalNextSteps }); } - + //TODO inspect value of the field 'progress' and update 'isGoalComplete' to True if progress = 100% updateExistingGoalInFirestore () { let goalDoc = userDoc.collection('goals').doc(this.goaluid); - console.log("Updating goal id " + this.goaluid + " with the following:\n"); + console.log("Updating goal " + this.goaluid + "for user " + this.useruid + " with the following data:\n"); this.printAllFieldsToConsole(); - if (this.goalType) { - goalDoc.update({goalType: this.goalType}); + if (this.goalCategory) { + goalDoc.update({goalCategory: this.goalCategory}); } - if (this.goalDueDate) { - goalDoc.update({goalDue: this.goalDueDate}); + if (this.goalDate) { + goalDoc.update({goalDue: this.goalDate}); } - if (this.goalNotes) { - goalDoc.update({goalNotes: this.goalNotes}); + if (this.goalProgress) { + goalDoc.update({goalProgress: this.goalProgress}); } if (this.isGoalComplete) { goalDoc.update({isGoalComplete: this.isGoalComplete}); } + if (this.goalNextSteps) { + goalDoc.update({goalNextSteps: this.goalNextSteps}); + } } } module.exports = Goal; \ No newline at end of file diff --git a/functions/models/user.js b/functions/models/user.js index ee14475..6570370 100644 --- a/functions/models/user.js +++ b/functions/models/user.js @@ -2,7 +2,7 @@ const admin = require('firebase-admin'); // TODO: add consts for field names in the db admin.initializeApp(); const db = admin.firestore(); -let usersCollection = db.collection('users'); +let usersCollection = db.collection('testusers'); class User { //TODO: add data validation to all properties constructor(uid) { From 879113007fe798eb6df32e10f1339c2bb5e370fd Mon Sep 17 00:00:00 2001 From: Irit <35077819+iritush@users.noreply.github.com> Date: Wed, 22 Jan 2020 19:56:22 -0500 Subject: [PATCH 02/10] Fix the issue where goals were updated under the wrong user doc. --- functions/models/goal.js | 12 ++++++------ functions/models/user.js | 10 ++++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/functions/models/goal.js b/functions/models/goal.js index 0c5ab09..3aa8eab 100644 --- a/functions/models/goal.js +++ b/functions/models/goal.js @@ -1,6 +1,5 @@ const {db} = require('./user'); let usersCollection = db.collection('testusers'); -let userDoc; class Goal { //TODO: add data validation to all properties constructor(useruid) { @@ -17,7 +16,6 @@ class Goal { this.isGoalComplete = false; this.goalNextSteps = ''; this.created = Date.now(); - userDoc = usersCollection.doc(this.useruid); } printAllFieldsToConsole() { @@ -27,14 +25,15 @@ class Goal { "Goal category: " + this.goalCategory + "\n" + "Goal date: " + this.goalDate + "\n" + "Goal progress: " + this.goalProgress + "\n" + - "Goal complete?: " + this.isGoalComplete + "\n" + + "Goal omplete?: " + this.isGoalComplete + "\n" + "Goal next steps: " + this.goalNextSteps + "\n") } createNewGoalInFirestore() { - console.log("Creating a new goal " + this.goaluid + "for user " + this.useruid + " with the following data:\n"); + console.log("Creating a new goal " + this.goaluid + " for user " + this.useruid + " with the following data:\n"); this.printAllFieldsToConsole(); - userDoc.collection('goals').doc(this.goaluid).set({ + let currentUserDoc = usersCollection.doc(this.useruid); + currentUserDoc.collection('goals').doc(this.goaluid).set({ created: this.created, goaluid: this.goaluid, useruid: this.useruid, @@ -47,9 +46,10 @@ class Goal { } //TODO inspect value of the field 'progress' and update 'isGoalComplete' to True if progress = 100% updateExistingGoalInFirestore () { - let goalDoc = userDoc.collection('goals').doc(this.goaluid); console.log("Updating goal " + this.goaluid + "for user " + this.useruid + " with the following data:\n"); this.printAllFieldsToConsole(); + let currentUserDoc = usersCollection.doc(this.uid); + let goalDoc = currentUserDoc.collection('goals').doc(this.goaluid); if (this.goalCategory) { goalDoc.update({goalCategory: this.goalCategory}); } diff --git a/functions/models/user.js b/functions/models/user.js index 6570370..1eddf7b 100644 --- a/functions/models/user.js +++ b/functions/models/user.js @@ -27,7 +27,8 @@ class User { createNewUserInFirestore() { console.log("Creating a new document with uid " + this.uid + " with the following data:\n"); this.printAllFieldsToConsole(); - usersCollection.doc(this.uid).set({ + let currentUserDoc = usersCollection.doc(this.uid); + currentUserDoc.set({ created: this.dateCreated, uid: this.uid, displayName: this.firstName, @@ -39,14 +40,15 @@ class User { updateExistingUserInFirestore () { console.log("Updating uid " + this.uid + " with the following:\n"); this.printAllFieldsToConsole(); + let currentUserDoc = usersCollection.doc(this.uid); if (this.firstName) { - usersCollection.doc(this.uid).update({displayName: this.firstName}); + currentUserDoc.update({displayName: this.firstName}); } if (this.lastName) { - usersCollection.doc(this.uid).update({lastName: this.lastName}); + currentUserDoc.update({lastName: this.lastName}); } if (this.email) { - usersCollection.doc(this.uid).update({email: this.email}); + currentUserDoc.update({email: this.email}); } } } From 129a61341fe943ea898e762b9ea4ceef6ae56220 Mon Sep 17 00:00:00 2001 From: Irit <35077819+iritush@users.noreply.github.com> Date: Thu, 23 Jan 2020 11:25:08 -0500 Subject: [PATCH 03/10] change goal due field to goal date field + fix uid dield name --- functions/models/goal.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/models/goal.js b/functions/models/goal.js index 3aa8eab..4b35c8d 100644 --- a/functions/models/goal.js +++ b/functions/models/goal.js @@ -38,7 +38,7 @@ class Goal { goaluid: this.goaluid, useruid: this.useruid, goalCategory: this.goalCategory, - goalDue: this.goalDate, + goalDate: this.goalDate, goalProgress: this.goalProgress, isGoalComplete: this.isGoalComplete, goalNextSteps: this.goalNextSteps @@ -48,13 +48,13 @@ class Goal { updateExistingGoalInFirestore () { console.log("Updating goal " + this.goaluid + "for user " + this.useruid + " with the following data:\n"); this.printAllFieldsToConsole(); - let currentUserDoc = usersCollection.doc(this.uid); + let currentUserDoc = usersCollection.doc(this.useruid); let goalDoc = currentUserDoc.collection('goals').doc(this.goaluid); if (this.goalCategory) { goalDoc.update({goalCategory: this.goalCategory}); } if (this.goalDate) { - goalDoc.update({goalDue: this.goalDate}); + goalDoc.update({goalDate: this.goalDate}); } if (this.goalProgress) { goalDoc.update({goalProgress: this.goalProgress}); From c05d36159d459731be431bfde700ae60b9ed5208 Mon Sep 17 00:00:00 2001 From: Irit <35077819+iritush@users.noreply.github.com> Date: Thu, 23 Jan 2020 11:28:53 -0500 Subject: [PATCH 04/10] add argument to be passed to indicate which csv file to parse (goal or client) --- functions/index.js | 58 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/functions/index.js b/functions/index.js index cc881df..4c1705f 100644 --- a/functions/index.js +++ b/functions/index.js @@ -26,16 +26,27 @@ exports.helloWorld = functions.https.onRequest((request, response) => { exports.pullDataFromLocalCSVFileTEST = functions.https.onRequest((request, response) => { let fileContent=""; let pathToFile=""; - pathToFile = request.body.pathToFile + let clientOrGoalCSV = ""; + pathToFile = request.body.pathToFile; + clientOrGoalCSV = request.body.clientOrGoalCSV; console.log('Extracting data from the following file: ' + JSON.stringify(pathToFile)); let csvzip = new AdmZip(pathToFile); let zipEntries = csvzip.getEntries(); if (zipEntries.length > 0) { console.log('Found ' + zipEntries.length + ' entry in the zip file'); fileContent += csvzip.readAsText(zipEntries[0]); -} - // parseClientCSVAndSaveToFireStore (fileContent); - parseGoalCSVAndSaveToFireStore (fileContent); + } + switch (clientOrGoalCSV) { + case 'client': + parseClientCSVAndSaveToFireStore (fileContent); + break; + case 'goal': + parseGoalCSVAndSaveToFireStore (fileContent); + break; + default: + throw ("Must provide argument "); + + } response.send('done'); }) @@ -43,9 +54,14 @@ exports.pullDataFromLocalCSVFileTEST = functions.https.onRequest((request, respo gets the most recent zipped CSV file and extracts the content. // TODO: look into tracking the name of the last parsed file and pulling all new files that were added to the server since then + Must pass in an argument stating which csv file to parse (client/goal): + curl -X POST -H "Content-Type:application/json" -d '{" + clientOrGoalCSV:""}' */ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { // TODO: add more error handlng to this function + let clientOrGoalCSV = ""; + clientOrGoalCSV = request.body.clientOrGoalCSV; const sftpConnectionToCvoeo = new Client(); let outString = ""; let fileContent = ""; @@ -76,11 +92,21 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { // // Name Year Month Day Hour // res[1] res[2] res[3] res[4] res[5] - fRegEx_gm_clients = /(gm_clients_served_)(\d\d\d\d)-(\d?\d)-(\d?\d)-?(\d?\d)?/; - fRegEx_gm_goals = /(gm_goals_details_)(\d\d\d\d)-(\d?\d)-(\d?\d)-?(\d?\d)?/; + // Applies the regex pattern to our filename and stores result in fnParts (file name parts) - // fnParts = fRegEx_gm_clients.exec(fileName); - fnParts = fRegEx_gm_goals.exec(fileName); + switch (clientOrGoalCSV) { + case 'client': + fRegEx_gm_clients = /(gm_clients_served_)(\d\d\d\d)-(\d?\d)-(\d?\d)-?(\d?\d)?/; + fnParts = fRegEx_gm_clients.exec(fileName); + break; + case 'goal': + fRegEx_gm_goals = /(gm_goals_details_)(\d\d\d\d)-(\d?\d)-(\d?\d)-?(\d?\d)?/; + fnParts = fRegEx_gm_goals.exec(fileName); + break; + default: + throw ("Must provide argument "); + } + // we'll construct a new object to keep track of each files important details // and more importantly, to make it easy to sort/search if(fnParts) { @@ -152,7 +178,16 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { sftpConnectionToCvoeo.end(); // Finally send the response string along with the official A-OK code (200) console.log('Parsing file content'); - parseClientCSVAndSaveToFireStore (fileContent); + switch (clientOrGoalCSV) { + case 'client': + parseClientCSVAndSaveToFireStore (fileContent); + break; + case 'goal': + parseGoalCSVAndSaveToFireStore (fileContent); + break; + default: + throw ("Must provide argument "); + } response.send(outString, 200); return true; }) @@ -176,7 +211,6 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { TODO: add more error handling to this function */ function parseClientCSVAndSaveToFireStore(fileContent) { - //*** Known issue: When parsing a csv file with multiple lines that have goal data, saving to firestore is not working properly */ // TODO: Ideally data validation will be handles in the user class but add any validations that are needed here Papa.parse(fileContent, { //papaparse (https://www.papaparse.com)returns 'results' which has an array 'data'. @@ -241,12 +275,10 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { complete: function(results) { console.log("Found "+ results.data.length + " lines in file content\n"); for (let i = 0;i { usersCollection.where('uid', '==', user.uid).get() .then(userSnapshot => { if (userSnapshot.empty) { - console.log("Did not find a matching document with uid " + user.uid); + console.log("Did not find a matching document with uid " + user.uid); user.createNewUserInFirestore(); goal.createNewGoalInFirestore(); } From 4f53c74e76fe68b9b5629bed6e567b82afb49f14 Mon Sep 17 00:00:00 2001 From: iritush <35077819+iritush@users.noreply.github.com> Date: Mon, 3 Feb 2020 11:59:01 -0500 Subject: [PATCH 05/10] Update functions/index.js suggestions from Micah Co-Authored-By: Micah Mutrux --- functions/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/functions/index.js b/functions/index.js index 4c1705f..6f6117c 100644 --- a/functions/index.js +++ b/functions/index.js @@ -20,6 +20,7 @@ exports.helloWorld = functions.https.onRequest((request, response) => { }); /*This firebase function is for testing purposes to be able to use a file saved locally as input. + * @param request.body.clientOrGoalCSV {string} Either 'client' or 'goal', to correspond with the submitted csv To run this function, have a firebase server set locally then run the following command: curl -X POST -H "Content-Type:application/json" -d '{"pathToFile":""}' */ @@ -333,4 +334,4 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { } } }) - } \ No newline at end of file + } From cb5cf0dfaeccf36b04d8fc95b5ccf06e32dd6ee4 Mon Sep 17 00:00:00 2001 From: iritush <35077819+iritush@users.noreply.github.com> Date: Mon, 3 Feb 2020 11:59:16 -0500 Subject: [PATCH 06/10] Update functions/index.js suggestions from Micah Co-Authored-By: Micah Mutrux --- functions/index.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/functions/index.js b/functions/index.js index 6f6117c..7b935c0 100644 --- a/functions/index.js +++ b/functions/index.js @@ -45,7 +45,10 @@ exports.pullDataFromLocalCSVFileTEST = functions.https.onRequest((request, respo parseGoalCSVAndSaveToFireStore (fileContent); break; default: - throw ("Must provide argument "); + response + .type('application/json') + .status(409) + .send({status:409, message: "Missing required param clientOrGoalCSV of either 'client ' or 'goal'" }); } response.send('done'); From 8bbc490b79ee9c7d2a2885017ee93484e5f0f39a Mon Sep 17 00:00:00 2001 From: Irit <35077819+iritush@users.noreply.github.com> Date: Tue, 4 Feb 2020 18:42:53 -0500 Subject: [PATCH 07/10] config for firebase emulator --- firebase.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/firebase.json b/firebase.json index 0059b49..304dde5 100644 --- a/firebase.json +++ b/firebase.json @@ -11,5 +11,16 @@ "npm --prefix \"$RESOURCE_DIR\" run lint" ], "source": "functions" + }, + "emulators": { + "functions": { + "port": 5001 + }, + "firestore": { + "port": 8080 + }, + "database": { + "port": 9000 + } } } From 4186883c30d5fa17dea3a3252dad9ea0e8c49d34 Mon Sep 17 00:00:00 2001 From: Irit <35077819+iritush@users.noreply.github.com> Date: Fri, 7 Feb 2020 14:49:54 -0500 Subject: [PATCH 08/10] added a check for required arguments on request and return a 409 reponse if these are missing --- functions/index.js | 360 +++++++++++++++++++++++++-------------------- 1 file changed, 199 insertions(+), 161 deletions(-) diff --git a/functions/index.js b/functions/index.js index 7b935c0..e270674 100644 --- a/functions/index.js +++ b/functions/index.js @@ -30,28 +30,50 @@ exports.pullDataFromLocalCSVFileTEST = functions.https.onRequest((request, respo let clientOrGoalCSV = ""; pathToFile = request.body.pathToFile; clientOrGoalCSV = request.body.clientOrGoalCSV; - console.log('Extracting data from the following file: ' + JSON.stringify(pathToFile)); - let csvzip = new AdmZip(pathToFile); - let zipEntries = csvzip.getEntries(); - if (zipEntries.length > 0) { - console.log('Found ' + zipEntries.length + ' entry in the zip file'); - fileContent += csvzip.readAsText(zipEntries[0]); - } - switch (clientOrGoalCSV) { - case 'client': - parseClientCSVAndSaveToFireStore (fileContent); - break; - case 'goal': - parseGoalCSVAndSaveToFireStore (fileContent); - break; - default: - response + if ((!(clientOrGoalCSV == 'client' || clientOrGoalCSV =='goal')) || (!(pathToFile))) { + response .type('application/json') .status(409) - .send({status:409, message: "Missing required param clientOrGoalCSV of either 'client ' or 'goal'" }); - - } - response.send('done'); + .send({status:409, message: "Missing required params: clientOrGoalCSV must be 'client' or 'goal';pathToFile must have the path to the csv file being used" }); + } + else { + try { + console.log('Extracting data from the following file: ' + JSON.stringify(pathToFile)); + let csvzip = new AdmZip(pathToFile); + let zipEntries = csvzip.getEntries(); + if (zipEntries.length > 0) { + console.log('Found ' + zipEntries.length + ' entry in the zip file'); + fileContent += csvzip.readAsText(zipEntries[0]); + } + switch (clientOrGoalCSV) { + case 'client': + parseClientCSVAndSaveToFireStore (fileContent); + response + .type('application/json') + .status(200) + .send({status:200, message: "Completed parsing CSV" }); + break; + case 'goal': + parseGoalCSVAndSaveToFireStore (fileContent); + response + .type('application/json') + .status(200) + .send({status:200, message: "Completed parsing CSV" }); + break; + default: + response + .type('application/json') + .status(500) + .send({status:500, message: "Unexpected Failure" }); + } + } + catch (err) { + response + .type('application/json') + .status(500) + .send({status:500, message: err }); + } + } }) /* This firebase function connects to the client's sftp server, @@ -66,143 +88,159 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { // TODO: add more error handlng to this function let clientOrGoalCSV = ""; clientOrGoalCSV = request.body.clientOrGoalCSV; - const sftpConnectionToCvoeo = new Client(); - let outString = ""; - let fileContent = ""; - const directoryName = '/dropbox/'; - //Connect to cvoeo sftp server using environment configuration. These have to be configured and deployed to firebase using the firebase cli. - //https://firebase.google.com/docs/functions/config-env - console.log('Establishing a connection with the sftp server') - sftpConnectionToCvoeo.connect({ - host: `${functions.config().cvoeosftp.host}`, - username: `${functions.config().cvoeosftp.username}`, - password: `${functions.config().cvoeosftp.password}` - }) - .then( - () => { - console.log('Getting the list of files in \'' + directoryName + '\' directory'); - return sftpConnectionToCvoeo.list(directoryName); - }) - .then( - (fileList) => { - let fileNames = []; // create array to dump file names into and to sort later - for (zipFileIdx in fileList) { - let fileName = fileList[zipFileIdx].name; // actual name of file - // Do a regex match using capturing parens to break up the items we want to pull out. - // Results from regex will be in array of items matched to the capturing parentheses - // - // If regex match results were in an array named "res"... - // res[0] contains the entire input string - // - // Name Year Month Day Hour - // res[1] res[2] res[3] res[4] res[5] - - // Applies the regex pattern to our filename and stores result in fnParts (file name parts) - switch (clientOrGoalCSV) { - case 'client': - fRegEx_gm_clients = /(gm_clients_served_)(\d\d\d\d)-(\d?\d)-(\d?\d)-?(\d?\d)?/; - fnParts = fRegEx_gm_clients.exec(fileName); - break; - case 'goal': - fRegEx_gm_goals = /(gm_goals_details_)(\d\d\d\d)-(\d?\d)-(\d?\d)-?(\d?\d)?/; - fnParts = fRegEx_gm_goals.exec(fileName); - break; - default: - throw ("Must provide argument "); - } - - // we'll construct a new object to keep track of each files important details - // and more importantly, to make it easy to sort/search - if(fnParts) { - let newFile = { - name: fileName, - year: parseInt(fnParts[2]), // year - month: parseInt(fnParts[3]), // month - day: parseInt(fnParts[4]), // day - hour: parseInt(fnParts[5]) || 0 // hour - first file of day has no hour in name so use 0 instead - }; - fileNames.push(newFile) // toss new object into array - } - } - // sort() is an instance of "fast-sort" - // So to fine the newest file... sort by the year, then month, then day, then hour in a descending fashion. - // The sort mutates the fileNames array... changes it directly. - // In the end, the newest file on the server would be at fileNames[0] in our array. - sort(fileNames).desc( - [ - 'year', - 'month', - 'day', - 'hour' - ] - ); - console.log('Most recent file on server: ' + fileNames[0].name); - return fileNames[0].name; + if (!(clientOrGoalCSV == 'client' || clientOrGoalCSV =='goal')) { + response + .type('application/json') + .status(409) + .send({status:409, message: "Missing required param: clientOrGoalCSV must be 'client' or 'goal'" }); + } + else { + const sftpConnectionToCvoeo = new Client(); + let outString = ""; + let fileContent = ""; + const directoryName = '/dropbox/'; + //Connect to cvoeo sftp server using environment configuration. These have to be configured and deployed to firebase using the firebase cli. + //https://firebase.google.com/docs/functions/config-env + console.log('Establishing a connection with the sftp server') + sftpConnectionToCvoeo.connect({ + host: `${functions.config().cvoeosftp.host}`, + username: `${functions.config().cvoeosftp.username}`, + password: `${functions.config().cvoeosftp.password}` }) - .then( - (newestFileName) => { - // Names are like this 'gm_clients_served_2019-07-08-8.zip' - // Request this specific ZIP file - console.log('Getting ' + newestFileName + ' from server'); - let readableSFTP = sftpConnectionToCvoeo.get(directoryName + newestFileName); - // Tell the server log about it... - console.log('readableSFTP: ' + JSON.stringify(readableSFTP)); - // Returning the variable here passes it back out to be caught - // in the next '.then' clause - return readableSFTP; - }) - .then( - // We can call the incoming data anything. Chunk is fairly - // common with working with Node's in-memory streams/buffers. - // Chunk also refers to the true hero in The Goonies. - (chunk) => { - // Tell the server log what's going on - // console.log('chunk: ' + JSON.stringify(chunk)); - // Collect output for future response - //outString += chunk; // Display ZIP file as binary output... looks ugly and is useless. - // Create a new unzipper using the Chunk as input... - let csvzip = new AdmZip(chunk); - // Figure out how many files are in the Chunk-zip - // Presumably always 1, but it could be any number. - let zipEntries = csvzip.getEntries(); - // Again, collect output for future response... - outString += "Zip Entries: " + JSON.stringify(zipEntries) + "\n"; - // Assuming that there is at least 1 entry in the Zip... - // that is at least a single file inside the Zip... - if (zipEntries.length > 0) { - // HERE IS WHERE WE COULD PUT CODE OR A FUNCTION CALL - // TO LOAD THE CSV INTO THE FIREBASE DATABASE - // - // Right now we just read the first file in the Zip, whatever it is, - // but we are assuming it is probably a CSV file, as text. - // We append the CSV content to our forthcoming response output. - console.log('Found ' + zipEntries.length + ' entry in the zip file'); - fileContent += csvzip.readAsText(zipEntries[0]); - } - sftpConnectionToCvoeo.end(); - // Finally send the response string along with the official A-OK code (200) - console.log('Parsing file content'); - switch (clientOrGoalCSV) { - case 'client': - parseClientCSVAndSaveToFireStore (fileContent); - break; - case 'goal': - parseGoalCSVAndSaveToFireStore (fileContent); - break; - default: - throw ("Must provide argument "); - } - response.send(outString, 200); - return true; + .then( + () => { + console.log('Getting the list of files in \'' + directoryName + '\' directory'); + return sftpConnectionToCvoeo.list(directoryName); }) - // Error handler ... which just spits out the error message. - .catch( - (err) => { - outString = 'ERROR: ' + err + "\n"; - console.log (outString); - // Note we use a code of 500 here instead of 200 as above. - response.send(outString, 500); - }); + .then( + (fileList) => { + let fileNames = []; // create array to dump file names into and to sort later + for (zipFileIdx in fileList) { + let fileName = fileList[zipFileIdx].name; // actual name of file + // Do a regex match using capturing parens to break up the items we want to pull out. + // Results from regex will be in array of items matched to the capturing parentheses + // + // If regex match results were in an array named "res"... + // res[0] contains the entire input string + // + // Name Year Month Day Hour + // res[1] res[2] res[3] res[4] res[5] + + // Applies the regex pattern to our filename and stores result in fnParts (file name parts) + switch (clientOrGoalCSV) { + case 'client': + fRegEx_gm_clients = /(gm_clients_served_)(\d\d\d\d)-(\d?\d)-(\d?\d)-?(\d?\d)?/; + fnParts = fRegEx_gm_clients.exec(fileName); + break; + case 'goal': + fRegEx_gm_goals = /(gm_goals_details_)(\d\d\d\d)-(\d?\d)-(\d?\d)-?(\d?\d)?/; + fnParts = fRegEx_gm_goals.exec(fileName); + break; + default: + response + .type('application/json') + .status(500) + .send({status:500, message: "Unexpected Failure" }); + } + // we'll construct a new object to keep track of each files important details + // and more importantly, to make it easy to sort/search + if(fnParts) { + let newFile = { + name: fileName, + year: parseInt(fnParts[2]), // year + month: parseInt(fnParts[3]), // month + day: parseInt(fnParts[4]), // day + hour: parseInt(fnParts[5]) || 0 // hour - first file of day has no hour in name so use 0 instead + }; + fileNames.push(newFile) // toss new object into array + } + } + // sort() is an instance of "fast-sort" + // So to fine the newest file... sort by the year, then month, then day, then hour in a descending fashion. + // The sort mutates the fileNames array... changes it directly. + // In the end, the newest file on the server would be at fileNames[0] in our array. + sort(fileNames).desc( + [ + 'year', + 'month', + 'day', + 'hour' + ] + ); + console.log('Most recent file on server: ' + fileNames[0].name); + return fileNames[0].name; + }) + .then( + (newestFileName) => { + // Names are like this 'gm_clients_served_2019-07-08-8.zip' + // Request this specific ZIP file + console.log('Getting ' + newestFileName + ' from server'); + let readableSFTP = sftpConnectionToCvoeo.get(directoryName + newestFileName); + // Returning the variable here passes it back out to be caught + // in the next '.then' clause + return readableSFTP; + }) + .then( + // We can call the incoming data anything. Chunk is fairly + // common with working with Node's in-memory streams/buffers. + // Chunk also refers to the true hero in The Goonies. + (chunk) => { + // Create a new unzipper using the Chunk as input... + let csvzip = new AdmZip(chunk); + // Figure out how many files are in the Chunk-zip + // Presumably always 1, but it could be any number. + let zipEntries = csvzip.getEntries(); + // Again, collect output for future response... + outString += "Zip Entries: " + JSON.stringify(zipEntries) + "\n"; + // Assuming that there is at least 1 entry in the Zip... + // that is at least a single file inside the Zip... + if (zipEntries.length > 0) { + // HERE IS WHERE WE COULD PUT CODE OR A FUNCTION CALL + // TO LOAD THE CSV INTO THE FIREBASE DATABASE + // + // Right now we just read the first file in the Zip, whatever it is, + // but we are assuming it is probably a CSV file, as text. + // We append the CSV content to our forthcoming response output. + console.log('Found ' + zipEntries.length + ' entry in the zip file'); + fileContent += csvzip.readAsText(zipEntries[0]); + } + sftpConnectionToCvoeo.end(); + console.log('Parsing file content'); + switch (clientOrGoalCSV) { + case 'client': + parseClientCSVAndSaveToFireStore (fileContent); + response + .type('application/json') + .status(200) + .send({status:200, message: "Completed parsing CSV" }); + break; + case 'goal': + parseGoalCSVAndSaveToFireStore (fileContent); + response + .type('application/json') + .status(200) + .send({status:200, message: "Completed parsing CSV" }); + break; + default: + console.log ("Unexpected Failure"); + response + .type('application/json') + .status(500) + .send({status:500, message: "Unexpected Failure" }); + } + }) + // Error handler ... which just spits out the error message. + .catch( + (err) => { + outString = 'ERROR: ' + err + "\n"; + console.log (outString); + // Note we use a code of 500 here instead of 200 as above. + response + .type('application/json') + .status(500) + .send({status:500, message: err }); + }); + } }); /*This function parses the content provided and saves it to the firestore db: @@ -215,7 +253,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { TODO: add more error handling to this function */ function parseClientCSVAndSaveToFireStore(fileContent) { - // TODO: Ideally data validation will be handles in the user class but add any validations that are needed here + // TODO: Ideally data validation will be handled in the user/goal class but add any validations that are needed here Papa.parse(fileContent, { //papaparse (https://www.papaparse.com)returns 'results' which has an array 'data'. // Each entry in 'data' is an object, a set of key/values that match the header at the head of the csv file. @@ -224,7 +262,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { complete: function(results) { console.log("Found "+ results.data.length + " lines in file content\n"); for (let i = 0;i { } }) .catch(err => { - console.log('Error getting documents', err); - }); + throw (err); + }); } } } @@ -311,7 +349,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { usersCollection.where('uid', '==', user.uid).get() .then(userSnapshot => { if (userSnapshot.empty) { - console.log("Did not find a matching document with uid " + user.uid); + console.log("Did not find a matching document with uid " + user.uid); user.createNewUserInFirestore(); goal.createNewGoalInFirestore(); } @@ -331,7 +369,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { } }) .catch(err => { - console.log('Error getting documents', err); + throw (err); }); } } From f74a102f902cc9c88213f820380103d471feb83e Mon Sep 17 00:00:00 2001 From: Irit <35077819+iritush@users.noreply.github.com> Date: Fri, 7 Feb 2020 14:51:38 -0500 Subject: [PATCH 09/10] remove some console prints --- functions/models/goal.js | 4 ---- functions/models/user.js | 4 ---- 2 files changed, 8 deletions(-) diff --git a/functions/models/goal.js b/functions/models/goal.js index 4b35c8d..2760aed 100644 --- a/functions/models/goal.js +++ b/functions/models/goal.js @@ -30,8 +30,6 @@ class Goal { } createNewGoalInFirestore() { - console.log("Creating a new goal " + this.goaluid + " for user " + this.useruid + " with the following data:\n"); - this.printAllFieldsToConsole(); let currentUserDoc = usersCollection.doc(this.useruid); currentUserDoc.collection('goals').doc(this.goaluid).set({ created: this.created, @@ -46,8 +44,6 @@ class Goal { } //TODO inspect value of the field 'progress' and update 'isGoalComplete' to True if progress = 100% updateExistingGoalInFirestore () { - console.log("Updating goal " + this.goaluid + "for user " + this.useruid + " with the following data:\n"); - this.printAllFieldsToConsole(); let currentUserDoc = usersCollection.doc(this.useruid); let goalDoc = currentUserDoc.collection('goals').doc(this.goaluid); if (this.goalCategory) { diff --git a/functions/models/user.js b/functions/models/user.js index 1eddf7b..46fd95f 100644 --- a/functions/models/user.js +++ b/functions/models/user.js @@ -25,8 +25,6 @@ class User { "email: " + this.email + "\n") } createNewUserInFirestore() { - console.log("Creating a new document with uid " + this.uid + " with the following data:\n"); - this.printAllFieldsToConsole(); let currentUserDoc = usersCollection.doc(this.uid); currentUserDoc.set({ created: this.dateCreated, @@ -38,8 +36,6 @@ class User { } updateExistingUserInFirestore () { - console.log("Updating uid " + this.uid + " with the following:\n"); - this.printAllFieldsToConsole(); let currentUserDoc = usersCollection.doc(this.uid); if (this.firstName) { currentUserDoc.update({displayName: this.firstName}); From ed348db38923adf44c3d6b0bd4402ce1cf15f199 Mon Sep 17 00:00:00 2001 From: Irit <35077819+iritush@users.noreply.github.com> Date: Fri, 7 Feb 2020 16:45:42 -0500 Subject: [PATCH 10/10] removed some extra spaces --- functions/index.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/functions/index.js b/functions/index.js index e270674..547a04c 100644 --- a/functions/index.js +++ b/functions/index.js @@ -34,7 +34,7 @@ exports.pullDataFromLocalCSVFileTEST = functions.https.onRequest((request, respo response .type('application/json') .status(409) - .send({status:409, message: "Missing required params: clientOrGoalCSV must be 'client' or 'goal';pathToFile must have the path to the csv file being used" }); + .send({status:409, message: "Missing required params: clientOrGoalCSV must be 'client' or 'goal';pathToFile must have the path to the csv file being used"}); } else { try { @@ -51,27 +51,27 @@ exports.pullDataFromLocalCSVFileTEST = functions.https.onRequest((request, respo response .type('application/json') .status(200) - .send({status:200, message: "Completed parsing CSV" }); + .send({status:200, message: "Completed parsing CSV"}); break; case 'goal': parseGoalCSVAndSaveToFireStore (fileContent); response .type('application/json') .status(200) - .send({status:200, message: "Completed parsing CSV" }); + .send({status:200, message: "Completed parsing CSV"}); break; default: response .type('application/json') .status(500) - .send({status:500, message: "Unexpected Failure" }); + .send({status:500, message: "Unexpected Failure"}); } } catch (err) { response .type('application/json') .status(500) - .send({status:500, message: err }); + .send({status:500, message: err}); } } }) @@ -92,7 +92,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { response .type('application/json') .status(409) - .send({status:409, message: "Missing required param: clientOrGoalCSV must be 'client' or 'goal'" }); + .send({status:409, message: "Missing required param: clientOrGoalCSV must be 'client' or 'goal'"}); } else { const sftpConnectionToCvoeo = new Client(); @@ -140,7 +140,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { response .type('application/json') .status(500) - .send({status:500, message: "Unexpected Failure" }); + .send({status:500, message: "Unexpected Failure"}); } // we'll construct a new object to keep track of each files important details // and more importantly, to make it easy to sort/search @@ -212,21 +212,21 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { response .type('application/json') .status(200) - .send({status:200, message: "Completed parsing CSV" }); + .send({status:200, message: "Completed parsing CSV"}); break; case 'goal': parseGoalCSVAndSaveToFireStore (fileContent); response .type('application/json') .status(200) - .send({status:200, message: "Completed parsing CSV" }); + .send({status:200, message: "Completed parsing CSV"}); break; default: console.log ("Unexpected Failure"); response .type('application/json') .status(500) - .send({status:500, message: "Unexpected Failure" }); + .send({status:500, message: "Unexpected Failure"}); } }) // Error handler ... which just spits out the error message. @@ -238,7 +238,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => { response .type('application/json') .status(500) - .send({status:500, message: err }); + .send({status:500, message: err}); }); } });