From 40cac84e1ef339448988814e9ff6abc17c13c26e Mon Sep 17 00:00:00 2001 From: DanDai Date: Wed, 25 Oct 2017 11:59:41 -0700 Subject: [PATCH 1/3] Create server and database directories, create index.js --- README.md | 4 ++-- server/index.js | 0 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 server/index.js diff --git a/README.md b/README.md index 8ba3826..4306a45 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Project Name +#analytics -The project description +Service responsible for analyzing user behavior and correlating with given lists ## Roadmap diff --git a/server/index.js b/server/index.js new file mode 100644 index 0000000..e69de29 From f3766149f315736e2f66a594a1760d3270f351db Mon Sep 17 00:00:00 2001 From: DanDai Date: Wed, 25 Oct 2017 12:01:01 -0700 Subject: [PATCH 2/3] Enter boilerplate code for index.js --- server/index.js | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/server/index.js b/server/index.js index e69de29..acaab31 100644 --- a/server/index.js +++ b/server/index.js @@ -0,0 +1,37 @@ +const express = require('express'); +const app = express(); +const server = require('http').Server(app); +const bodyParser = require('body-parser'); +const request = require('request'); +const rp = require('request-promise-native'); +const port = process.env.PORT || 2424; +const restaurantList = require('./database/restaurantdb.js'); +const appServerDB = require('./database/mysql.js'); +const handleQuery = require('./controller/queryHandler.js'); + +// import entire SDK +//var AWS = require('aws-sdk'); +//AWS.config.loadFromPath('./config.json'); + +app.use(bodyParser.json()); +app.use(bodyParser.urlencoded({extended: true})); + + + +app.get('/', (req, res) => { + res.status(200); + res.send('Serving up webpage'); +}); + + + + + + + + + + +server.listen(port, () => { + console.log(`(>^.^)> Server now listening on ${port}!`); +}); \ No newline at end of file From e3b85bd41df3c3c7960bc3d0e726c522cb637cc3 Mon Sep 17 00:00:00 2001 From: DanDai Date: Sat, 4 Nov 2017 15:22:52 -0700 Subject: [PATCH 3/3] Pull down messages from SQS, process and then send to customer profiline --- .gitignore | 1 + Dockerfile | 7 ++++ notes.txt | 16 ++++++++++ package.json | 42 ++++++++++++++++++++++++ server/database/querydb.js | 34 ++++++++++++++++++++ server/helpers/DBHelpers.js | 64 +++++++++++++++++++++++++++++++++++++ server/helpers/SQSHelper.js | 31 ++++++++++++++++++ server/index.js | 63 ++++++++++++++++++------------------ 8 files changed, 226 insertions(+), 32 deletions(-) create mode 100644 Dockerfile create mode 100644 notes.txt create mode 100644 package.json create mode 100644 server/database/querydb.js create mode 100644 server/helpers/DBHelpers.js create mode 100644 server/helpers/SQSHelper.js diff --git a/.gitignore b/.gitignore index 66428dc..5bc1ab7 100755 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ coverage # Compiled Dirs (http://nodejs.org/api/addons.html) build/ dist/ +server/config # Dependency directorys # Deployed apps should consider commenting these lines out: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f7455ab --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM node:boron +WORKDIR /server +# Install app dependencies +COPY package.json . +RUN npm install +COPY . . +CMD [ "npm", "start" ] \ No newline at end of file diff --git a/notes.txt b/notes.txt new file mode 100644 index 0000000..ce223a8 --- /dev/null +++ b/notes.txt @@ -0,0 +1,16 @@ +order of events for analytics service: + +1) user query is submitted (from app-server) with user/queryID/listID +2) events are collected and matched to the initial query + +parsed events (that have been matched with a query) are sent up to elasticsearch DB for analysis/visualization + +notes: analytics service should be built modular enough for new queries to be made on the existing and new datasets put together to upload to elasticsearch + + +RESERVATION r-03fe84e46141e651d 478994730514 +INSTANCE i-003ddec692665e4ea ami-05355a6c ip-172-31-19-234.ec2.internal pending app-server 0 m1.large 2017-11-04T18:26:54+0000 us-east-1c aki-88aa75e1 monitoring-disabled 172.31.19.234 vpc-e0f34598 subnet-08269943 ebs paravirtual xen 7915ab10-a096-4852-90df-0570e45317f2 sg-21a99753 default true +NIC eni-6e7113de subnet-08269943 vpc-e0f34598 478994730514 in-use 172.31.19.234 ip-172-31-19-234.ec2.internal true +NICATTACHMENT eni-attach-22f458e5 0 attaching 2017-11-04T11:26:54-0700 true +GROUP sg-21a99753 yelpalytics-mongoDB +PRIVATEIPADDRESS 172.31.19.234 ip-172-31-19-234.ec2.internal \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..4f26ad2 --- /dev/null +++ b/package.json @@ -0,0 +1,42 @@ +{ + "name": "analytics", + "version": "1.0.0", + "description": "Analytics service for Yelp-Alytics recommendation engine simulator", + "main": "server/index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start-dev": "NODE_ENV=development node server/index.js", + "start": "NODE_ENV=production node server/index.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/git-reset-hard/analytics.git" + }, + "author": "Daniel Dai", + "license": "ISC", + "bugs": { + "url": "https://github.com/git-reset-hard/analytics/issues" + }, + "homepage": "https://github.com/git-reset-hard/analytics#readme", + "dependencies": { + "assert": "^1.4.1", + "aws-sdk": "^2.140.0", + "body-parser": "^1.18.2", + "chai": "^4.1.2", + "elasticsearch": "^13.3.1", + "express": "^4.16.2", + "faker": "^4.1.0", + "hash-generator": "^0.1.0", + "http": "0.0.0", + "mocha": "^4.0.1", + "mongoose": "^4.12.4", + "mysql": "^2.15.0", + "mysql2": "^1.4.2", + "request": "^2.83.0", + "request-promise-native": "^1.0.5", + "sequelize": "^4.17.0", + "shortid": "^2.2.8", + "sqs-consumer": "^3.8.0", + "winston": "^3.0.0-rc1" + } +} diff --git a/server/database/querydb.js b/server/database/querydb.js new file mode 100644 index 0000000..56f94a5 --- /dev/null +++ b/server/database/querydb.js @@ -0,0 +1,34 @@ +const config = require('../config/env.json')[process.env.NODE_ENV || 'development']; +const mongoose = require('mongoose'); +mongoose.Promise = require('bluebird'); +var db = mongoose.connect(config.mongoConnect, { + useMongoClient: true +}); + + +var querySchema = mongoose.Schema({ + queryID: String, + userId: String, + list: Array, + searchTerm: String, + location: String, + date: Date, + isPersonalized: Boolean +}); + +var Query = mongoose.model('Query', querySchema); + +var listSchema = mongoose.Schema({ + queryID: String, + listID: String, + list: Array, + isPersonalized: Boolean +}); + +var List = mongoose.model('List', listSchema); + + +module.exports = { + Query, + List +}; \ No newline at end of file diff --git a/server/helpers/DBHelpers.js b/server/helpers/DBHelpers.js new file mode 100644 index 0000000..3c16809 --- /dev/null +++ b/server/helpers/DBHelpers.js @@ -0,0 +1,64 @@ +const mongodb = require('../database/querydb.js'); +const SQSHelp = require('./SQSHelper.js'); + +const listInsert = function(query, done) { + let listObj = { + queryID: query.queryID, + listID: query.id, + list: [], + isPersonalized: query.isPersonalized + }; + + for (let i = 1; i <= 10; i++) { + listObj.list.push(query['restaurantID_' + i]); + } + mongodb.List.create(listObj) + .then((result) => { + return mongodb.Query.findOne({queryID: query.queryID}); + }) + .then((query) => { + if (query) { + query.list = listObj.list; + //send query to SQS + SQSHelp.sendMessage(query); + return mongodb.Query.findOneAndUpdate({queryID: query.queryID}, query); + } + }) + .then((result) => { + //console.log('query successfully updated with list', result); + done(); + }) + .catch((err) => { + console.log('error with insertion'. err); + done(); + }); +}; + +const queryInsert = function(query, done) { + query['queryID'] = query.id; + + mongodb.List.findOne({listID: query.servedList}) + .then((list) => { + if (list) { + query.list = list.list; + //send query to SQS + SQSHelp.sendMessage(query); + + } + return mongodb.Query.create(query); + }) + .then((result) => { + //console.log('query inserted into DB', result); + done(); + }) + .catch((err) => { + console.log('error with insertion'. err); + done(); + }); +}; + + +module.exports = { + listInsert, + queryInsert +}; \ No newline at end of file diff --git a/server/helpers/SQSHelper.js b/server/helpers/SQSHelper.js new file mode 100644 index 0000000..a1ef89a --- /dev/null +++ b/server/helpers/SQSHelper.js @@ -0,0 +1,31 @@ +// Load the AWS SDK for Node.js +var AWS = require('aws-sdk'); +// Load credentials and set the region from the JSON file +AWS.config.loadFromPath('./server/config/config.json'); + +// Create an SQS service object +var sqs = new AWS.SQS({apiVersion: '2012-11-05'}); +var queueURL = 'https://sqs.us-west-1.amazonaws.com/478994730514/analyticsToCustomer'; + + +const sendMessage = function(messageObj) { + + let querySQS = { + DelaySeconds: 10, + MessageBody: JSON.stringify(messageObj), + QueueUrl: queueURL + }; + + sqs.sendMessage(querySQS, function(err, data) { + if (err) { + console.log('Error"', err); + } else { + console.log('SQS Send Success', data.MessageId); + } + }); +}; + + +module.exports = { + sendMessage +}; \ No newline at end of file diff --git a/server/index.js b/server/index.js index acaab31..d7a1f50 100644 --- a/server/index.js +++ b/server/index.js @@ -1,37 +1,36 @@ -const express = require('express'); -const app = express(); -const server = require('http').Server(app); -const bodyParser = require('body-parser'); -const request = require('request'); -const rp = require('request-promise-native'); -const port = process.env.PORT || 2424; -const restaurantList = require('./database/restaurantdb.js'); -const appServerDB = require('./database/mysql.js'); -const handleQuery = require('./controller/queryHandler.js'); - -// import entire SDK -//var AWS = require('aws-sdk'); -//AWS.config.loadFromPath('./config.json'); - -app.use(bodyParser.json()); -app.use(bodyParser.urlencoded({extended: true})); - - - -app.get('/', (req, res) => { - res.status(200); - res.send('Serving up webpage'); +const DBHelpers = require('./helpers/DBHelpers.js'); +const Consumer = require('sqs-consumer'); +// Load the AWS SDK for Node.js +var AWS = require('aws-sdk'); +// Load credentials and set the region from the JSON file +AWS.config.loadFromPath('./server/config/config.json'); + + + +const app = Consumer.create({ + queueUrl: 'https://sqs.us-west-1.amazonaws.com/478994730514/app-serverToAnalytics', + handleMessage: (message, done) => { + message = JSON.parse(message.Body); + if (message.type === 'list') { + DBHelpers.listInsert(message, done); + } + if (message.type === 'query') { + DBHelpers.queryInsert(message, done); + } + }, + sqs: new AWS.SQS() }); + +app.on('error', (err) => { + console.log(err.message); +}); + +app.start(); +console.log('listening for messages...'); +//pull down check-in/reviews from Customer Profiling +//pull down clicks from Customer Profiling - - - - - - -server.listen(port, () => { - console.log(`(>^.^)> Server now listening on ${port}!`); -}); \ No newline at end of file +//match clicks with lists \ No newline at end of file