-
Notifications
You must be signed in to change notification settings - Fork 153
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add module to write results to a postgres database
This commit adds a `write-to-db-module`. This module takes the result of fetching and processing a report, and adds the data to a postgres database. The plan is for the reporter to insert this data in such a way that another application can come in behind it and serve the data via an API. To enable this feature, the reporter is run with the `--write-to-database` option. The reporter needs the database to be configured in the config under the `postgres` prop. Ref #194
- Loading branch information
Showing
8 changed files
with
1,699 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,91 @@ | ||
const knex = require("knex") | ||
const config = require("./config") | ||
|
||
const db = knex({ client: "pg", connection: config.postgres }) | ||
|
||
const writeResultsToDatabase = (results, { realtime } = {}) => { | ||
if (realtime) { | ||
return _writeRealtimeResults(results) | ||
} else if (results.query.dimensions.match(/ga:date/)) { | ||
return _writeRegularResults(results) | ||
} else { | ||
return Promise.resolve() | ||
} | ||
} | ||
|
||
const _dataForDataPoint = (dataPoint, { realtime } = {}) => { | ||
const data = Object.assign({}, dataPoint) | ||
let dateTime | ||
if (realtime) { | ||
dateTime = new Date() | ||
} else { | ||
dateTime = _dateTimeForDataPoint(dataPoint) | ||
} | ||
delete data.date | ||
delete data.hour | ||
|
||
return { | ||
date_time: dateTime, | ||
data: data, | ||
} | ||
} | ||
|
||
const _dateTimeForDataPoint = (dataPoint) => { | ||
let dateString = dataPoint.date | ||
if (dataPoint.hour) { | ||
dateString = `${dateString}T${dataPoint.hour}:00:00` | ||
} | ||
if (!isNaN(Date.parse(dateString))) { | ||
return new Date(dateString) | ||
} | ||
} | ||
|
||
const _rowForDataPoint = ({ results, dataPoint, realtime }) => { | ||
const row = _dataForDataPoint(dataPoint, { realtime }) | ||
row.report_name = results.name | ||
row.report_agency = results.agency | ||
return row | ||
} | ||
|
||
const _rowWasAlreadyInserted = (row) => { | ||
const query = Object.assign({}, row) | ||
Object.keys(query).forEach(key => { | ||
if (query[key] === undefined) { | ||
delete query[key] | ||
} | ||
}) | ||
return db("analytics_data").where(query).count().then(result => { | ||
const count = parseInt(result[0].count) | ||
return count > 0 | ||
}) | ||
} | ||
|
||
const _writeRealtimeResults = (results) => { | ||
const rows = results.data.map(dataPoint => { | ||
return _rowForDataPoint({ results, dataPoint, realtime: true }) | ||
}) | ||
return db("analytics_data").insert(rows) | ||
} | ||
|
||
const _writeRegularResults = (results) => { | ||
const rows = results.data.map(dataPoint => { | ||
return _rowForDataPoint({ results, dataPoint }) | ||
}) | ||
|
||
const rowPromises = rows.map(row => { | ||
return _rowWasAlreadyInserted(row).then(inserted => { | ||
if (!inserted) { | ||
return row | ||
} | ||
}) | ||
}) | ||
|
||
return Promise.all(rowPromises).then(rows => { | ||
rows = rows.filter(row => { | ||
return row !== undefined && row.date_time !== undefined | ||
}) | ||
return db("analytics_data").insert(rows) | ||
}) | ||
} | ||
|
||
module.exports = writeResultsToDatabase |
Oops, something went wrong.