From e690973d07c9457e87ea67c0827cafd7bb3ac4d7 Mon Sep 17 00:00:00 2001 From: Jonathan Hooper Date: Tue, 14 Feb 2017 11:42:24 -0600 Subject: [PATCH] Add module to write results to a postgres database This commit adds a `write-to-db-module`. This module takes the result of fetching and processing a report, and adds the data to a postgres database. The plan is for the reporter to insert this data in such a way that another application can come in behind it and serve the data via an API. To enable this feature, the reporter is run with the `--write-to-database` option. The reporter needs the database to be configured in the config under the `postgres` prop. Ref #194 --- .travis.yml | 6 + index.js | 16 +- package.json | 4 +- src/config.js | 7 + src/write-results-to-db.js | 91 ++ test/fixtures/results.js | 1383 ++++++++++++++++++++++++++++++ test/support/database.js | 22 + test/write-results-to-db.test.js | 174 ++++ 8 files changed, 1699 insertions(+), 4 deletions(-) create mode 100644 src/write-results-to-db.js create mode 100644 test/fixtures/results.js create mode 100644 test/support/database.js create mode 100644 test/write-results-to-db.test.js diff --git a/.travis.yml b/.travis.yml index 105d9be0..79cc66f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,12 @@ node_js: sudo: false os: - linux +services: + - postgresql +addons: + postgresql: "9.4" +before_script: +- "createdb travis_ci_test" deploy: edge: true provider: cloudfoundry diff --git a/index.js b/index.js index 81b5a90f..9516ad94 100644 --- a/index.js +++ b/index.js @@ -6,6 +6,8 @@ var Analytics = require("./src/analytics"), csv = require("fast-csv"), zlib = require('zlib'); +const writeResultsToDatabase = require("./src/write-results-to-db") + // AWS credentials are looked for in env vars or in ~/.aws/config. // AWS bucket and path need to be set in env vars mentioned in config.js. @@ -76,9 +78,16 @@ var run = function(options) { // JSON else { // some reports can be slimmed down for direct rendering - if (options.slim && report.slim) delete data.data; - - writeReport(name, JSON.stringify(data, null, 2), ".json", done); + if (options.slim && report.slim) { + delete data.data; + writeReport(name, JSON.stringify(data, null, 2), ".json", done); + } else if (options["write-to-database"]) { + writeResultsToDatabase(data, { realtime: report.realtime }).then(() => { + writeReport(name, JSON.stringify(data, null, 2), ".json", done); + }).catch(err => done(err)) + } else { + writeReport(name, JSON.stringify(data, null, 2), ".json", done); + } } }); }; @@ -112,6 +121,7 @@ var run = function(options) { } if (options.debug) console.log("All done."); + process.exit(0); }); }; diff --git a/package.json b/package.json index d6b21b29..9a44eba5 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,8 @@ "proxyquire": "^1.7.11" }, "optionalDependencies": { - "newrelic": "^1.36.1" + "knex": "^0.12.6", + "newrelic": "^1.36.1", + "pg": "^6.1.2" } } diff --git a/src/config.js b/src/config.js index 5a867ee2..836753a3 100644 --- a/src/config.js +++ b/src/config.js @@ -32,6 +32,13 @@ module.exports = { hostname: process.env.ANALYTICS_HOSTNAME || "" }, + postgres: { + host : process.env.POSTGRES_HOST, + user : process.env.POSTGRES_USER, + password : process.env.POSTGRES_PASSWORD, + database : process.env.POSTGRES_DATABASE || "analytics-reporter", + }, + static: { path: '../analytics.usa.gov/' } diff --git a/src/write-results-to-db.js b/src/write-results-to-db.js new file mode 100644 index 00000000..b034f21e --- /dev/null +++ b/src/write-results-to-db.js @@ -0,0 +1,91 @@ +const knex = require("knex") +const config = require("./config") + +const db = knex({ client: "pg", connection: config.postgres }) + +const writeResultsToDatabase = (results, { realtime } = {}) => { + if (realtime) { + return _writeRealtimeResults(results) + } else if (results.query.dimensions.match(/ga:date/)) { + return _writeRegularResults(results) + } else { + return Promise.resolve() + } +} + +const _dataForDataPoint = (dataPoint, { realtime } = {}) => { + const data = Object.assign({}, dataPoint) + let dateTime + if (realtime) { + dateTime = new Date() + } else { + dateTime = _dateTimeForDataPoint(dataPoint) + } + delete data.date + delete data.hour + + return { + date_time: dateTime, + data: data, + } +} + +const _dateTimeForDataPoint = (dataPoint) => { + let dateString = dataPoint.date + if (dataPoint.hour) { + dateString = `${dateString}T${dataPoint.hour}:00:00` + } + if (!isNaN(Date.parse(dateString))) { + return new Date(dateString) + } +} + +const _rowForDataPoint = ({ results, dataPoint, realtime }) => { + const row = _dataForDataPoint(dataPoint, { realtime }) + row.report_name = results.name + row.report_agency = results.agency + return row +} + +const _rowWasAlreadyInserted = (row) => { + const query = Object.assign({}, row) + Object.keys(query).forEach(key => { + if (query[key] === undefined) { + delete query[key] + } + }) + return db("analytics_data").where(query).count().then(result => { + const count = parseInt(result[0].count) + return count > 0 + }) +} + +const _writeRealtimeResults = (results) => { + const rows = results.data.map(dataPoint => { + return _rowForDataPoint({ results, dataPoint, realtime: true }) + }) + return db("analytics_data").insert(rows) +} + +const _writeRegularResults = (results) => { + const rows = results.data.map(dataPoint => { + return _rowForDataPoint({ results, dataPoint }) + }) + + const rowPromises = rows.map(row => { + return _rowWasAlreadyInserted(row).then(inserted => { + if (!inserted) { + return row + } + }) + }) + + return Promise.all(rowPromises).then(rows => { + rows = rows.filter(row => { + return row !== undefined && row.date_time !== undefined + }) + return db("analytics_data").insert(rows) + }) +} + +module.exports = writeResultsToDatabase diff --git a/test/fixtures/results.js b/test/fixtures/results.js new file mode 100644 index 00000000..fb6fbb3a --- /dev/null +++ b/test/fixtures/results.js @@ -0,0 +1,1383 @@ +module.exports = { + "name": "devices", + "query": { + "start-date": "90daysAgo", + "end-date": "yesterday", + "dimensions": "ga:date,ga:deviceCategory", + "metrics": [ + "ga:sessions" + ], + "sort": [ + "ga:date" + ], + "start-index": 1, + "max-results": 10000, + "samplingLevel": "HIGHER_PRECISION" + }, + "meta": { + "name": "Devices", + "description": "90 days of desktop/mobile/tablet visits for all sites." + }, + "data": [ + { + "date": "2016-11-17", + "device": "desktop", + "visits": "17944716" + }, + { + "date": "2016-11-17", + "device": "mobile", + "visits": "8927140" + }, + { + "date": "2016-11-17", + "device": "tablet", + "visits": "1493615" + }, + { + "date": "2016-11-18", + "device": "desktop", + "visits": "15266887" + }, + { + "date": "2016-11-18", + "device": "mobile", + "visits": "8188620" + }, + { + "date": "2016-11-18", + "device": "tablet", + "visits": "1359252" + }, + { + "date": "2016-11-19", + "device": "desktop", + "visits": "7486523" + }, + { + "date": "2016-11-19", + "device": "mobile", + "visits": "6802302" + }, + { + "date": "2016-11-19", + "device": "tablet", + "visits": "1244910" + }, + { + "date": "2016-11-20", + "device": "desktop", + "visits": "8095419" + }, + { + "date": "2016-11-20", + "device": "mobile", + "visits": "6355972" + }, + { + "date": "2016-11-20", + "device": "tablet", + "visits": "1301498" + }, + { + "date": "2016-11-21", + "device": "desktop", + "visits": "18290260" + }, + { + "date": "2016-11-21", + "device": "mobile", + "visits": "8660823" + }, + { + "date": "2016-11-21", + "device": "tablet", + "visits": "1478005" + }, + { + "date": "2016-11-22", + "device": "desktop", + "visits": "16994015" + }, + { + "date": "2016-11-22", + "device": "mobile", + "visits": "8599485" + }, + { + "date": "2016-11-22", + "device": "tablet", + "visits": "1413091" + }, + { + "date": "2016-11-23", + "device": "desktop", + "visits": "13510470" + }, + { + "date": "2016-11-23", + "device": "mobile", + "visits": "8133319" + }, + { + "date": "2016-11-23", + "device": "tablet", + "visits": "1279496" + }, + { + "date": "2016-11-24", + "device": "desktop", + "visits": "6234988" + }, + { + "date": "2016-11-24", + "device": "mobile", + "visits": "5953655" + }, + { + "date": "2016-11-24", + "device": "tablet", + "visits": "1022314" + }, + { + "date": "2016-11-25", + "device": "desktop", + "visits": "8768054" + }, + { + "date": "2016-11-25", + "device": "mobile", + "visits": "7241617" + }, + { + "date": "2016-11-25", + "device": "tablet", + "visits": "1212316" + }, + { + "date": "2016-11-26", + "device": "desktop", + "visits": "6981808" + }, + { + "date": "2016-11-26", + "device": "mobile", + "visits": "6722048" + }, + { + "date": "2016-11-26", + "device": "tablet", + "visits": "1190519" + }, + { + "date": "2016-11-27", + "device": "desktop", + "visits": "8225314" + }, + { + "date": "2016-11-27", + "device": "mobile", + "visits": "6672403" + }, + { + "date": "2016-11-27", + "device": "tablet", + "visits": "1302649" + }, + { + "date": "2016-11-28", + "device": "desktop", + "visits": "19526901" + }, + { + "date": "2016-11-28", + "device": "mobile", + "visits": "9300099" + }, + { + "date": "2016-11-28", + "device": "tablet", + "visits": "1547016" + }, + { + "date": "2016-11-29", + "device": "desktop", + "visits": "19881628" + }, + { + "date": "2016-11-29", + "device": "mobile", + "visits": "9665025" + }, + { + "date": "2016-11-29", + "device": "tablet", + "visits": "1579273" + }, + { + "date": "2016-11-30", + "device": "desktop", + "visits": "19573065" + }, + { + "date": "2016-11-30", + "device": "mobile", + "visits": "10083858" + }, + { + "date": "2016-11-30", + "device": "tablet", + "visits": "1601741" + }, + { + "date": "2016-12-01", + "device": "desktop", + "visits": "18611610" + }, + { + "date": "2016-12-01", + "device": "mobile", + "visits": "10212056" + }, + { + "date": "2016-12-01", + "device": "tablet", + "visits": "1564647" + }, + { + "date": "2016-12-02", + "device": "desktop", + "visits": "16303740" + }, + { + "date": "2016-12-02", + "device": "mobile", + "visits": "9595214" + }, + { + "date": "2016-12-02", + "device": "tablet", + "visits": "1452885" + }, + { + "date": "2016-12-03", + "device": "desktop", + "visits": "8145522" + }, + { + "date": "2016-12-03", + "device": "mobile", + "visits": "8038915" + }, + { + "date": "2016-12-03", + "device": "tablet", + "visits": "1328963" + }, + { + "date": "2016-12-04", + "device": "desktop", + "visits": "8753097" + }, + { + "date": "2016-12-04", + "device": "mobile", + "visits": "7206951" + }, + { + "date": "2016-12-04", + "device": "tablet", + "visits": "1365981" + }, + { + "date": "2016-12-05", + "device": "desktop", + "visits": "20527426" + }, + { + "date": "2016-12-05", + "device": "mobile", + "visits": "10433381" + }, + { + "date": "2016-12-05", + "device": "tablet", + "visits": "1670167" + }, + { + "date": "2016-12-06", + "device": "desktop", + "visits": "19967407" + }, + { + "date": "2016-12-06", + "device": "mobile", + "visits": "10023434" + }, + { + "date": "2016-12-06", + "device": "tablet", + "visits": "1657519" + }, + { + "date": "2016-12-07", + "device": "desktop", + "visits": "19532055" + }, + { + "date": "2016-12-07", + "device": "mobile", + "visits": "10063789" + }, + { + "date": "2016-12-07", + "device": "tablet", + "visits": "1646568" + }, + { + "date": "2016-12-08", + "device": "desktop", + "visits": "19218012" + }, + { + "date": "2016-12-08", + "device": "mobile", + "visits": "10323528" + }, + { + "date": "2016-12-08", + "device": "tablet", + "visits": "1714556" + }, + { + "date": "2016-12-09", + "device": "desktop", + "visits": "16651672" + }, + { + "date": "2016-12-09", + "device": "mobile", + "visits": "9478158" + }, + { + "date": "2016-12-09", + "device": "tablet", + "visits": "1564344" + }, + { + "date": "2016-12-10", + "device": "desktop", + "visits": "8394504" + }, + { + "date": "2016-12-10", + "device": "mobile", + "visits": "8008296" + }, + { + "date": "2016-12-10", + "device": "tablet", + "visits": "1438817" + }, + { + "date": "2016-12-11", + "device": "desktop", + "visits": "8769674" + }, + { + "date": "2016-12-11", + "device": "mobile", + "visits": "7318707" + }, + { + "date": "2016-12-11", + "device": "tablet", + "visits": "1471781" + }, + { + "date": "2016-12-12", + "device": "desktop", + "visits": "20124799" + }, + { + "date": "2016-12-12", + "device": "mobile", + "visits": "10002557" + }, + { + "date": "2016-12-12", + "device": "tablet", + "visits": "1677637" + }, + { + "date": "2016-12-13", + "device": "desktop", + "visits": "19692582" + }, + { + "date": "2016-12-13", + "device": "mobile", + "visits": "9946246" + }, + { + "date": "2016-12-13", + "device": "tablet", + "visits": "1664839" + }, + { + "date": "2016-12-14", + "device": "desktop", + "visits": "19450673" + }, + { + "date": "2016-12-14", + "device": "mobile", + "visits": "10324397" + }, + { + "date": "2016-12-14", + "device": "tablet", + "visits": "1713116" + }, + { + "date": "2016-12-15", + "device": "desktop", + "visits": "19047361" + }, + { + "date": "2016-12-15", + "device": "mobile", + "visits": "10346150" + }, + { + "date": "2016-12-15", + "device": "tablet", + "visits": "1728800" + }, + { + "date": "2016-12-16", + "device": "desktop", + "visits": "16873358" + }, + { + "date": "2016-12-16", + "device": "mobile", + "visits": "9932215" + }, + { + "date": "2016-12-16", + "device": "tablet", + "visits": "1663874" + }, + { + "date": "2016-12-17", + "device": "desktop", + "visits": "8866860" + }, + { + "date": "2016-12-17", + "device": "mobile", + "visits": "8772502" + }, + { + "date": "2016-12-17", + "device": "tablet", + "visits": "1627369" + }, + { + "date": "2016-12-18", + "device": "desktop", + "visits": "8105408" + }, + { + "date": "2016-12-18", + "device": "mobile", + "visits": "7414904" + }, + { + "date": "2016-12-18", + "device": "tablet", + "visits": "1469536" + }, + { + "date": "2016-12-19", + "device": "desktop", + "visits": "19220918" + }, + { + "date": "2016-12-19", + "device": "mobile", + "visits": "10438620" + }, + { + "date": "2016-12-19", + "device": "tablet", + "visits": "1677447" + }, + { + "date": "2016-12-20", + "device": "desktop", + "visits": "18241079" + }, + { + "date": "2016-12-20", + "device": "mobile", + "visits": "10558487" + }, + { + "date": "2016-12-20", + "device": "tablet", + "visits": "1618781" + }, + { + "date": "2016-12-21", + "device": "desktop", + "visits": "17147953" + }, + { + "date": "2016-12-21", + "device": "mobile", + "visits": "10422959" + }, + { + "date": "2016-12-21", + "device": "tablet", + "visits": "1563992" + }, + { + "date": "2016-12-22", + "device": "desktop", + "visits": "15503945" + }, + { + "date": "2016-12-22", + "device": "mobile", + "visits": "10305992" + }, + { + "date": "2016-12-22", + "device": "tablet", + "visits": "1529405" + }, + { + "date": "2016-12-23", + "device": "desktop", + "visits": "11361437" + }, + { + "date": "2016-12-23", + "device": "mobile", + "visits": "9521278" + }, + { + "date": "2016-12-23", + "device": "tablet", + "visits": "1446075" + }, + { + "date": "2016-12-24", + "device": "desktop", + "visits": "5600182" + }, + { + "date": "2016-12-24", + "device": "mobile", + "visits": "7144987" + }, + { + "date": "2016-12-24", + "device": "tablet", + "visits": "1190168" + }, + { + "date": "2016-12-25", + "device": "desktop", + "visits": "4408666" + }, + { + "date": "2016-12-25", + "device": "mobile", + "visits": "5531137" + }, + { + "date": "2016-12-25", + "device": "tablet", + "visits": "1026063" + }, + { + "date": "2016-12-26", + "device": "desktop", + "visits": "7825098" + }, + { + "date": "2016-12-26", + "device": "mobile", + "visits": "7232890" + }, + { + "date": "2016-12-26", + "device": "tablet", + "visits": "1355893" + }, + { + "date": "2016-12-27", + "device": "desktop", + "visits": "13935273" + }, + { + "date": "2016-12-27", + "device": "mobile", + "visits": "8975892" + }, + { + "date": "2016-12-27", + "device": "tablet", + "visits": "1445369" + }, + { + "date": "2016-12-28", + "device": "desktop", + "visits": "14480665" + }, + { + "date": "2016-12-28", + "device": "mobile", + "visits": "9244411" + }, + { + "date": "2016-12-28", + "device": "tablet", + "visits": "1495648" + }, + { + "date": "2016-12-29", + "device": "desktop", + "visits": "14178667" + }, + { + "date": "2016-12-29", + "device": "mobile", + "visits": "9223986" + }, + { + "date": "2016-12-29", + "device": "tablet", + "visits": "1501026" + }, + { + "date": "2016-12-30", + "device": "desktop", + "visits": "11547674" + }, + { + "date": "2016-12-30", + "device": "mobile", + "visits": "8372061" + }, + { + "date": "2016-12-30", + "device": "tablet", + "visits": "1373276" + }, + { + "date": "2016-12-31", + "device": "desktop", + "visits": "6126765" + }, + { + "date": "2016-12-31", + "device": "mobile", + "visits": "6393735" + }, + { + "date": "2016-12-31", + "device": "tablet", + "visits": "1188851" + }, + { + "date": "2017-01-01", + "device": "desktop", + "visits": "5717572" + }, + { + "date": "2017-01-01", + "device": "mobile", + "visits": "6002253" + }, + { + "date": "2017-01-01", + "device": "tablet", + "visits": "1219702" + }, + { + "date": "2017-01-02", + "device": "desktop", + "visits": "10414034" + }, + { + "date": "2017-01-02", + "device": "mobile", + "visits": "8280913" + }, + { + "date": "2017-01-02", + "device": "tablet", + "visits": "1572182" + }, + { + "date": "2017-01-03", + "device": "desktop", + "visits": "19074040" + }, + { + "date": "2017-01-03", + "device": "mobile", + "visits": "10002388" + }, + { + "date": "2017-01-03", + "device": "tablet", + "visits": "1634073" + }, + { + "date": "2017-01-04", + "device": "desktop", + "visits": "19474263" + }, + { + "date": "2017-01-04", + "device": "mobile", + "visits": "10263370" + }, + { + "date": "2017-01-04", + "device": "tablet", + "visits": "1707684" + }, + { + "date": "2017-01-05", + "device": "desktop", + "visits": "19466017" + }, + { + "date": "2017-01-05", + "device": "mobile", + "visits": "10736442" + }, + { + "date": "2017-01-05", + "device": "tablet", + "visits": "1762507" + }, + { + "date": "2017-01-06", + "device": "desktop", + "visits": "17268777" + }, + { + "date": "2017-01-06", + "device": "mobile", + "visits": "10204089" + }, + { + "date": "2017-01-06", + "device": "tablet", + "visits": "1700304" + }, + { + "date": "2017-01-07", + "device": "desktop", + "visits": "8771825" + }, + { + "date": "2017-01-07", + "device": "mobile", + "visits": "8622569" + }, + { + "date": "2017-01-07", + "device": "tablet", + "visits": "1657525" + }, + { + "date": "2017-01-08", + "device": "desktop", + "visits": "8468167" + }, + { + "date": "2017-01-08", + "device": "mobile", + "visits": "7523797" + }, + { + "date": "2017-01-08", + "device": "tablet", + "visits": "1573548" + }, + { + "date": "2017-01-09", + "device": "desktop", + "visits": "19946515" + }, + { + "date": "2017-01-09", + "device": "mobile", + "visits": "10112103" + }, + { + "date": "2017-01-09", + "device": "tablet", + "visits": "1724557" + }, + { + "date": "2017-01-10", + "device": "desktop", + "visits": "20321640" + }, + { + "date": "2017-01-10", + "device": "mobile", + "visits": "10515776" + }, + { + "date": "2017-01-10", + "device": "tablet", + "visits": "1795632" + }, + { + "date": "2017-01-11", + "device": "desktop", + "visits": "19671577" + }, + { + "date": "2017-01-11", + "device": "mobile", + "visits": "10465313" + }, + { + "date": "2017-01-11", + "device": "tablet", + "visits": "1732368" + }, + { + "date": "2017-01-12", + "device": "desktop", + "visits": "19589937" + }, + { + "date": "2017-01-12", + "device": "mobile", + "visits": "10277052" + }, + { + "date": "2017-01-12", + "device": "tablet", + "visits": "1703584" + }, + { + "date": "2017-01-13", + "device": "desktop", + "visits": "17146743" + }, + { + "date": "2017-01-13", + "device": "mobile", + "visits": "9619211" + }, + { + "date": "2017-01-13", + "device": "tablet", + "visits": "1585216" + }, + { + "date": "2017-01-14", + "device": "desktop", + "visits": "8330783" + }, + { + "date": "2017-01-14", + "device": "mobile", + "visits": "8038168" + }, + { + "date": "2017-01-14", + "device": "tablet", + "visits": "1474055" + }, + { + "date": "2017-01-15", + "device": "desktop", + "visits": "7940108" + }, + { + "date": "2017-01-15", + "device": "mobile", + "visits": "7377663" + }, + { + "date": "2017-01-15", + "device": "tablet", + "visits": "1420365" + }, + { + "date": "2017-01-16", + "device": "desktop", + "visits": "14829426" + }, + { + "date": "2017-01-16", + "device": "mobile", + "visits": "9257283" + }, + { + "date": "2017-01-16", + "device": "tablet", + "visits": "1558470" + }, + { + "date": "2017-01-17", + "device": "desktop", + "visits": "21076771" + }, + { + "date": "2017-01-17", + "device": "mobile", + "visits": "11441390" + }, + { + "date": "2017-01-17", + "device": "tablet", + "visits": "1742698" + }, + { + "date": "2017-01-18", + "device": "desktop", + "visits": "20446130" + }, + { + "date": "2017-01-18", + "device": "mobile", + "visits": "10970693" + }, + { + "date": "2017-01-18", + "device": "tablet", + "visits": "1717717" + }, + { + "date": "2017-01-19", + "device": "desktop", + "visits": "20157052" + }, + { + "date": "2017-01-19", + "device": "mobile", + "visits": "11228989" + }, + { + "date": "2017-01-19", + "device": "tablet", + "visits": "1726224" + }, + { + "date": "2017-01-20", + "device": "desktop", + "visits": "19344217" + }, + { + "date": "2017-01-20", + "device": "mobile", + "visits": "12884804" + }, + { + "date": "2017-01-20", + "device": "tablet", + "visits": "1873116" + }, + { + "date": "2017-01-21", + "device": "desktop", + "visits": "9950647" + }, + { + "date": "2017-01-21", + "device": "mobile", + "visits": "10568161" + }, + { + "date": "2017-01-21", + "device": "tablet", + "visits": "1783297" + }, + { + "date": "2017-01-22", + "device": "desktop", + "visits": "10151644" + }, + { + "date": "2017-01-22", + "device": "mobile", + "visits": "9316374" + }, + { + "date": "2017-01-22", + "device": "tablet", + "visits": "1822457" + }, + { + "date": "2017-01-23", + "device": "desktop", + "visits": "23257771" + }, + { + "date": "2017-01-23", + "device": "mobile", + "visits": "12281874" + }, + { + "date": "2017-01-23", + "device": "tablet", + "visits": "1957768" + }, + { + "date": "2017-01-24", + "device": "desktop", + "visits": "21802654" + }, + { + "date": "2017-01-24", + "device": "mobile", + "visits": "11787571" + }, + { + "date": "2017-01-24", + "device": "tablet", + "visits": "1840512" + }, + { + "date": "2017-01-25", + "device": "desktop", + "visits": "21217961" + }, + { + "date": "2017-01-25", + "device": "mobile", + "visits": "12259488" + }, + { + "date": "2017-01-25", + "device": "tablet", + "visits": "1824556" + }, + { + "date": "2017-01-26", + "device": "desktop", + "visits": "20151178" + }, + { + "date": "2017-01-26", + "device": "mobile", + "visits": "11692776" + }, + { + "date": "2017-01-26", + "device": "tablet", + "visits": "1720242" + }, + { + "date": "2017-01-27", + "device": "desktop", + "visits": "17657726" + }, + { + "date": "2017-01-27", + "device": "mobile", + "visits": "10761667" + }, + { + "date": "2017-01-27", + "device": "tablet", + "visits": "1574402" + }, + { + "date": "2017-01-28", + "device": "desktop", + "visits": "9175780" + }, + { + "date": "2017-01-28", + "device": "mobile", + "visits": "9316210" + }, + { + "date": "2017-01-28", + "device": "tablet", + "visits": "1486173" + }, + { + "date": "2017-01-29", + "device": "desktop", + "visits": "9761406" + }, + { + "date": "2017-01-29", + "device": "mobile", + "visits": "9702597" + }, + { + "date": "2017-01-29", + "device": "tablet", + "visits": "1606222" + }, + { + "date": "2017-01-30", + "device": "desktop", + "visits": "22638067" + }, + { + "date": "2017-01-30", + "device": "mobile", + "visits": "12653369" + }, + { + "date": "2017-01-30", + "device": "tablet", + "visits": "1858651" + }, + { + "date": "2017-01-31", + "device": "desktop", + "visits": "22251428" + }, + { + "date": "2017-01-31", + "device": "mobile", + "visits": "12268125" + }, + { + "date": "2017-01-31", + "device": "tablet", + "visits": "1819209" + }, + { + "date": "2017-02-01", + "device": "desktop", + "visits": "21087290" + }, + { + "date": "2017-02-01", + "device": "mobile", + "visits": "12257163" + }, + { + "date": "2017-02-01", + "device": "tablet", + "visits": "1791769" + }, + { + "date": "2017-02-02", + "device": "desktop", + "visits": "20524207" + }, + { + "date": "2017-02-02", + "device": "mobile", + "visits": "12114547" + }, + { + "date": "2017-02-02", + "device": "tablet", + "visits": "1757504" + }, + { + "date": "2017-02-03", + "device": "desktop", + "visits": "17997793" + }, + { + "date": "2017-02-03", + "device": "mobile", + "visits": "11483512" + }, + { + "date": "2017-02-03", + "device": "tablet", + "visits": "1646621" + }, + { + "date": "2017-02-04", + "device": "desktop", + "visits": "9313172" + }, + { + "date": "2017-02-04", + "device": "mobile", + "visits": "9544262" + }, + { + "date": "2017-02-04", + "device": "tablet", + "visits": "1503310" + }, + { + "date": "2017-02-05", + "device": "desktop", + "visits": "8833525" + }, + { + "date": "2017-02-05", + "device": "mobile", + "visits": "8273273" + }, + { + "date": "2017-02-05", + "device": "tablet", + "visits": "1436846" + }, + { + "date": "2017-02-06", + "device": "desktop", + "visits": "21775734" + }, + { + "date": "2017-02-06", + "device": "mobile", + "visits": "12223955" + }, + { + "date": "2017-02-06", + "device": "tablet", + "visits": "1821893" + }, + { + "date": "2017-02-07", + "device": "desktop", + "visits": "22100599" + }, + { + "date": "2017-02-07", + "device": "mobile", + "visits": "12625240" + }, + { + "date": "2017-02-07", + "device": "tablet", + "visits": "1899859" + }, + { + "date": "2017-02-08", + "device": "desktop", + "visits": "22031758" + }, + { + "date": "2017-02-08", + "device": "mobile", + "visits": "13262193" + }, + { + "date": "2017-02-08", + "device": "tablet", + "visits": "1931228" + }, + { + "date": "2017-02-09", + "device": "desktop", + "visits": "20575032" + }, + { + "date": "2017-02-09", + "device": "mobile", + "visits": "12979335" + }, + { + "date": "2017-02-09", + "device": "tablet", + "visits": "1921387" + }, + { + "date": "2017-02-10", + "device": "desktop", + "visits": "17711813" + }, + { + "date": "2017-02-10", + "device": "mobile", + "visits": "11965905" + }, + { + "date": "2017-02-10", + "device": "tablet", + "visits": "1675788" + }, + { + "date": "2017-02-11", + "device": "desktop", + "visits": "9097741" + }, + { + "date": "2017-02-11", + "device": "mobile", + "visits": "10059393" + }, + { + "date": "2017-02-11", + "device": "tablet", + "visits": "1542236" + }, + { + "date": "2017-02-12", + "device": "desktop", + "visits": "9652936" + }, + { + "date": "2017-02-12", + "device": "mobile", + "visits": "9133410" + }, + { + "date": "2017-02-12", + "device": "tablet", + "visits": "1592009" + }, + { + "date": "2017-02-13", + "device": "desktop", + "visits": "20780584" + }, + { + "date": "2017-02-13", + "device": "mobile", + "visits": "12435261" + }, + { + "date": "2017-02-13", + "device": "tablet", + "visits": "1753516" + }, + { + "date": "2017-02-14", + "device": "desktop", + "visits": "19207139" + }, + { + "date": "2017-02-14", + "device": "mobile", + "visits": "11879814" + }, + { + "date": "2017-02-14", + "device": "tablet", + "visits": "1642179" + } + ], + "totals": { + "visits": 2380289500, + "devices": { + "desktop": 1369555309, + "mobile": 868783942, + "tablet": 141950249 + } + }, + "taken_at": "2017-02-15T15:44:53.044Z" +} + diff --git a/test/support/database.js b/test/support/database.js new file mode 100644 index 00000000..cbe9d03d --- /dev/null +++ b/test/support/database.js @@ -0,0 +1,22 @@ +const knex = require("knex") + +const connection = { + host: "localhost", + database: process.env.TRAVIS ? "travis_ci_test" : "analytics_reporter_test", +} + +const resetSchema = () => { + const db = knex({ client: "pg", connection }) + return db.schema.dropTableIfExists("analytics_data").then(() => { + return db.schema.createTable("analytics_data", (table) => { + table.increments("id") + table.string("report_name") + table.string("report_agency") + table.dateTime("date_time") + table.jsonb("data") + table.timestamps(true, true) + }) + }) +} + +module.exports = { connection, resetSchema } diff --git a/test/write-results-to-db.test.js b/test/write-results-to-db.test.js new file mode 100644 index 00000000..34043578 --- /dev/null +++ b/test/write-results-to-db.test.js @@ -0,0 +1,174 @@ +const expect = require("chai").expect +const knex = require("knex") +const proxyquire = require("proxyquire") +const database = require("./support/database") +const resultsFixture = require("./fixtures/results") + +proxyquire.noCallThru() + +const writeResultsToDatabase = proxyquire("../src/write-results-to-db", { + "./config": { postgres: database.connection }, +}) + +const databaseClient = knex({ client: "pg", connection: database.connection }) + +describe(".writeResultsToDatabase(results)", () => { + let results + + beforeEach(done => { + results = Object.assign({}, resultsFixture) + + database.resetSchema().then(() => { + done() + }).catch(done) + }) + + context("when the report is not realtime", () => { + it("should insert a record for each results.data element", done => { + results.name = "report-name" + results.data = [ + { + date: "2017-02-11", + name: "abc", + }, + { + date: "2017-02-12", + name: "def", + }, + ] + + writeResultsToDatabase(results).then(() => { + return databaseClient.select().table("analytics_data") + }).then(rows => { + expect(rows).to.have.length(2) + rows.forEach((row, index) => { + const data = results.data[index] + expect(row.report_name).to.equal("report-name") + expect(row.date_time.getTime()).to.equal((new Date(data.date)).getTime()) + expect(row.data.name).to.equal(data.name) + expect(row.data.date).to.be.undefined + }) + done() + }).catch(done) + }) + + it("should use the ga:hour dimension in the date if it is present", done => { + results.data = [{ + date: "2017-02-15", + hour: "12", + }] + const date = new Date("2017-02-15T12:00:00") + + writeResultsToDatabase(results).then(() => { + return databaseClient.select().table("analytics_data") + }).then(rows => { + const row = rows[0] + expect(row.date_time.getTime()).to.equal(date.getTime()) + done() + }).catch(done) + }) + + it("should ignore reports that don't have a ga:date dimension", done => { + results.query = { dimensions: "ga:something,ga:somethingElse" } + + writeResultsToDatabase(results).then(() => { + return databaseClient.select().table("analytics_data") + }).then(rows => { + expect(rows).to.have.length(0) + done() + }).catch(done) + }) + + it("should ignore data points that have already been inserted", done => { + firstResults = Object.assign({}, results) + secondResults = Object.assign({}, results) + + firstResults.data = [ + { + date: "2017-02-11", + visits: "123", + browser: "Chrome", + }, + { + date: "2017-02-11", + visits: "456", + browser: "Safari" + }, + ] + secondResults.data = [ + { + date: "2017-02-11", + visits: "456", + browser: "Safari", + }, + { + date: "2017-02-11", + visits: "789", + browser: "Internet Explorer" + }, + ] + + writeResultsToDatabase(firstResults).then(() => { + return writeResultsToDatabase(secondResults) + }).then(() => { + return databaseClient.select().table("analytics_data") + }).then(rows => { + expect(rows).to.have.length(3) + done() + }).catch(done) + }) + + it("should not not insert a record if the date is invalid", done => { + results.data = [ + { + date: "(other)", + visits: "123", + }, + { + date: "2017-02-16", + visits: "456", + }, + ] + + writeResultsToDatabase(results).then(() => { + return databaseClient.select().table("analytics_data") + }).then(rows => { + expect(rows).to.have.length(1) + expect(rows[0].data.visits).to.equal("456") + done() + }).catch(done) + }) + }) + + context("when the report is realtime", () => { + it("should insert a record for each results.data element with the current date", done => { + const currentDateTime = new Date() + + results.name = "report-name" + results.data = [ + { + "city": "Washington DC", + "active_visitors": 123, + }, + { + "city": "Baton Rouge", + "active_visitors": 456, + } + ] + + writeResultsToDatabase(results, { realtime: true }).then(() => { + return databaseClient.select().table("analytics_data") + }).then(rows => { + expect(rows).to.have.length(2) + rows.forEach((row, index) => { + const data = results.data[index] + expect(row.report_name).to.equal("report-name") + expect(row.date_time - currentDateTime).to.be.below(1000) + expect(row.data.city).to.equal(data.city) + expect(row.data.active_visitors).to.equal(data.active_visitors) + }) + done() + }).catch(done) + }) + }) +})