Skip to content

Commit

Permalink
Merge pull request #169 from codeforbtv/write-data-to-firestore
Browse files Browse the repository at this point in the history
Write data to firestore
  • Loading branch information
iritush authored Dec 16, 2019
2 parents 366fad8 + 63de666 commit 12373cc
Show file tree
Hide file tree
Showing 4 changed files with 260 additions and 33 deletions.
170 changes: 137 additions & 33 deletions functions/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ const Client = require('ssh2-sftp-client');
const AdmZip = require('adm-zip');
const sort = require('fast-sort');
const Papa = require('papaparse');
const User = require('./models/user');
const Goal = require('./models/goal');
const {isValidEmail} = require('./models/data-validators');
const {db} = require('./models/user');

exports.helloWorld = functions.https.onRequest((request, response) => {
response.send("Hello from Firebase!");
Expand All @@ -15,10 +19,35 @@ exports.helloWorld = functions.https.onRequest((request, response) => {
}
});

/*This firebase function is for testing purposes to be able to use a file saved locally as input.
To run this function, have a firebase server set locally then run the following command:
curl -X POST <local path to firebase fuunction> -H "Content-Type:application/json" -d '{"pathToFile":"<path to local file>"}'
*/
exports.pullDataFromLocalCSVFileTEST = functions.https.onRequest((request, response) => {
let fileContent="";
let pathToFile="";
pathToFile = request.body.pathToFile
console.log('Extracting data from the following file: ' + JSON.stringify(pathToFile));
let csvzip = new AdmZip(pathToFile);
let zipEntries = csvzip.getEntries();
if (zipEntries.length > 0) {
console.log('Found ' + zipEntries.length + ' entry in the zip file');
fileContent += csvzip.readAsText(zipEntries[0]);
}
parseCSVAndSaveToFireStore (fileContent);
response.send('done');
})

/* This firebase function connects to the client's sftp server,
gets the most recent zipped CSV file and extracts the content.
// TODO: look into tracking the name of the last parsed file and pulling all new files that were
added to the server since then
*/
exports.pullDataFromSftp= functions.https.onRequest((request, response) => {
// TODO: add more error handlng to this function
const sftpConnectionToCvoeo = new Client();
var outString = "";
var fileContent = "";
let outString = "";
let fileContent = "";
const directoryName = '/dropbox/';
//Connect to cvoeo sftp server using environment configuration. These have to be configured and deployed to firebase using the firebase cli.
//https://firebase.google.com/docs/functions/config-env
Expand All @@ -35,7 +64,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => {
})
.then(
(fileList) => {
var fileNames = []; // create array to dump file names into and to sort later
let fileNames = []; // create array to dump file names into and to sort later
for (zipFileIdx in fileList) {
let fileName = fileList[zipFileIdx].name; // actual name of file
// Do a regex match using capturing parens to break up the items we want to pull out.
Expand Down Expand Up @@ -80,7 +109,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => {
// Names are like this 'gm_clients_served_2019-07-08-8.zip'
// Request this specific ZIP file
console.log('Getting ' + newestFileName + ' from server');
var readableSFTP = sftpConnectionToCvoeo.get(directoryName + newestFileName);
let readableSFTP = sftpConnectionToCvoeo.get(directoryName + newestFileName);
// Tell the server log about it...
console.log('readableSFTP: ' + JSON.stringify(readableSFTP));
// Returning the variable here passes it back out to be caught
Expand All @@ -97,10 +126,10 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => {
// Collect output for future response
//outString += chunk; // Display ZIP file as binary output... looks ugly and is useless.
// Create a new unzipper using the Chunk as input...
var csvzip = new AdmZip(chunk);
let csvzip = new AdmZip(chunk);
// Figure out how many files are in the Chunk-zip
// Presumably always 1, but it could be any number.
var zipEntries = csvzip.getEntries();
let zipEntries = csvzip.getEntries();
// Again, collect output for future response...
outString += "Zip Entries: " + JSON.stringify(zipEntries) + "\n";
// Assuming that there is at least 1 entry in the Zip...
Expand All @@ -118,7 +147,7 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => {
sftpConnectionToCvoeo.end();
// Finally send the response string along with the official A-OK code (200)
console.log('Parsing file content');
parseCSVFromServer (fileContent);
parseCSVAndSaveToFireStore (fileContent);
response.send(outString, 200);
return true;
})
Expand All @@ -132,29 +161,104 @@ exports.pullDataFromSftp= functions.https.onRequest((request, response) => {
});
});

function parseCSVFromServer(fileContent) {
//papaparse (https://www.papaparse.com)returns 'results' which has an array 'data'.
// Each entry in 'data' is an object, a set of key/values that match the header at the head of the csv file.
Papa.parse(fileContent, {
header: true,
skipEmptyLines: true,
complete: function(results) {
console.log("Found "+ results.data.length + " lines in file content\n");
//printing all the key values in the csv file to console ** for now **
// Next step is to write this information to the firebase db.
for (var i = 0;i<results.data.length ;i++) {
console.log("Entry number", i, ":");
console.log("---------------");
for (var key in results.data[i]) {
if(results.data[i][key] != "") {
console.log("key " + key + " has value " + results.data[i][key]);
}
else {
console.log("key " + key + " has no value ");
}
}
console.log("**************************************\n");
}
}
});
}
/*This function parses the content provided and saves it to the firestore db:
-Each user should have a firestore document in the "users" firestore collection
-The 'System Name ID' field in the CSV file is used as the user unique ID
TODO: confirm with cvoeo that the 'System Name ID' field is a reliable unique id to use
-The function checks if the user already exists in the db:
If user exists, update db with non empty fields + update/create new doc for goal if there is a goal
If user does not exist, create a new user document under the 'users' collection + create new goal
TODO: add more error handling to this function
*/
function parseCSVAndSaveToFireStore(fileContent) {
//*** Known issue: When parsing a csv file with multiple lines that have goal data, saving to firestore is not working properly */
// TODO: Ideally data validation will be handles in the user class but add any validations that are needed here
Papa.parse(fileContent, {
//papaparse (https://www.papaparse.com)returns 'results' which has an array 'data'.
// Each entry in 'data' is an object, a set of key/values that match the header at the head of the csv file.
header: true,
skipEmptyLines: true,
complete: function(results) {
console.log("Found "+ results.data.length + " lines in file content\n");
for (let i = 0;i<results.data.length ;i++) {
if(!results.data[i]['System Name ID']) {
console.log ("Missing 'System Name ID' field in file. This field is mandatory for creating and updating data in db");
}
else {
let user = new User(results.data[i]['System Name ID']);
let goal = new Goal(user.uid);
for (let key in results.data[i]) {
if(results.data[i][key] != "") {
switch (key) {
case 'First Name':
user.firstName = results.data[i][key];
break;
case 'Last Name':
user.lastName = results.data[i][key];
break;
case 'Email Address':
user.email = isValidEmail(results.data[i][key])
? results.data[i][key].trim().toLowerCase()
: null;
break;
}
}
if(results.data[i]['GOAL ID']) {
if (results.data[i][key] != "") {
switch (key) {
case 'GOAL ID':
goal.goaluid = results.data[i][key];
break;
case 'GOAL TYPE':
goal.goalType = results.data[i][key];
break;
case 'GOAL DUE':
goal.goalDueDate = results.data[i][key];
break;
case 'GOAL NOTES':
goal.goalNotes = results.data[i][key];
break;
case 'GOAL COMPLETE':
goal.isGoalComplete = results.data[i][key];
break;
}
}
}
}

let usersCollection = db.collection('users');
usersCollection.where('uid', '==', user.uid).get()
.then(userSnapshot => {
if (userSnapshot.empty) {
console.log("Did not find a matching document with uid " + user.uid);
user.createNewUserInFirestore();
if (goal.goaluid) {
goal.createNewGoalInFirestore();
}
}
else {
console.log("Found a matching document for uid " + user.uid);
user.updateExistingUserInFirestore();
if (goal.goaluid) {
usersCollection.doc(user.uid).collection('goals').where('goaluid', '==', goal.goaluid).get()
.then(goalSnapshot => {
if (goalSnapshot.empty) {
console.log("Did not find a matching document with goal id " + goal.goaluid + " for user " + goal.useruid);
goal.createNewGoalInFirestore();
}
else {
console.log("Found a matching document for goal id " + goal.goaluid + " under document for user " + goal.useruid);
goal.updateExistingGoalInFirestore();
}
})
}
}
})
.catch(err => {
console.log('Error getting documents', err);
});
}
}
}
})
}
5 changes: 5 additions & 0 deletions functions/models/data-validators.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
// TODO look into using an existing library instead: https://blog.mailtrap.io/react-native-email-validation
const emailRegex = /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;

const isValidEmail = (address) => emailRegex.test(address);
module.exports.isValidEmail = isValidEmail;
64 changes: 64 additions & 0 deletions functions/models/goal.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
const {db} = require('./user');
let usersCollection = db.collection('users');
let userDoc;
class Goal {
//TODO: add data validation to all properties
constructor(useruid) {
if (!useruid) {
console.log("Must provide a user uid when creating a new goal");
return;
// Add better error handling
}
this.goaluid = '';
this.useruid = useruid;//unique id of the user which this goal corresponds to
this.goalType = '';
this.goalDueDate = '';
this.goalNotes = '';
this.isGoalComplete = '';
this.created = Date.now();
userDoc = usersCollection.doc(this.useruid);
}

printAllFieldsToConsole() {
console.log (
"User uid: " + this.useruid + "\n" +
"Goal uid: " + this.goaluid + "\n" +
"Goal type: " + this.goalType + "\n" +
"Goal due date: " + this.goalDueDate + "\n" +
"Goal notes: " + this.goalNotes + "\n" +
"Goal Complete?: " + this.isGoalComplete + "\n")
}

createNewGoalInFirestore() {
console.log("Creating a new goal for user " + this.useruid + " with the following data:\n");
this.printAllFieldsToConsole();
userDoc.collection('goals').doc(this.goaluid).set({
created: this.created,
goaluid: this.goaluid,
useruid: this.useruid,
goalType: this.goalType,
goalDue: this.goalDueDate,
goalNotes: this.goalNotes,
isGoalComplete: this.isGoalComplete
});
}

updateExistingGoalInFirestore () {
let goalDoc = userDoc.collection('goals').doc(this.goaluid);
console.log("Updating goal id " + this.goaluid + " with the following:\n");
this.printAllFieldsToConsole();
if (this.goalType) {
goalDoc.update({goalType: this.goalType});
}
if (this.goalDueDate) {
goalDoc.update({goalDue: this.goalDueDate});
}
if (this.goalNotes) {
goalDoc.update({goalNotes: this.goalNotes});
}
if (this.isGoalComplete) {
goalDoc.update({isGoalComplete: this.isGoalComplete});
}
}
}
module.exports = Goal;
54 changes: 54 additions & 0 deletions functions/models/user.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
const admin = require('firebase-admin');
// TODO: add consts for field names in the db
admin.initializeApp();
const db = admin.firestore();
let usersCollection = db.collection('users');
class User {
//TODO: add data validation to all properties
constructor(uid) {
if (!uid) {
console.log("Must provide a user uid when creating a new user");
return;
// TODO: Add better error handling here
}
this.uid = uid; // this corresponds to Outcome Tracker's "System Name ID"
this.email = '';
this.firstName = '';
this.lastName = '';
this.dateCreated = Date.now();//TODO: switch this to a more readable date format
}
printAllFieldsToConsole() {
console.log (
"uid: " + this.uid + "\n" +
"First name: " + this.firstName + "\n" +
"Last name: " + this.lastName + "\n" +
"email: " + this.email + "\n")
}
createNewUserInFirestore() {
console.log("Creating a new document with uid " + this.uid + " with the following data:\n");
this.printAllFieldsToConsole();
usersCollection.doc(this.uid).set({
created: this.dateCreated,
uid: this.uid,
displayName: this.firstName,
lastName: this.lastName,
email: this.email
});
}

updateExistingUserInFirestore () {
console.log("Updating uid " + this.uid + " with the following:\n");
this.printAllFieldsToConsole();
if (this.firstName) {
usersCollection.doc(this.uid).update({displayName: this.firstName});
}
if (this.lastName) {
usersCollection.doc(this.uid).update({lastName: this.lastName});
}
if (this.email) {
usersCollection.doc(this.uid).update({email: this.email});
}
}
}
module.exports = User;
module.exports.db = db;

0 comments on commit 12373cc

Please sign in to comment.