Skip to content
This repository has been archived by the owner on Oct 27, 2022. It is now read-only.

Commit

Permalink
Add app-side gzip compression. Start from first recorded History and …
Browse files Browse the repository at this point in the history 
…add changed items.
  • Loading branch information
@chirale
chirale committed Jan 14, 2020
1 parent 2f6ffad commit 7431ebf
Show file tree
Hide file tree
Showing 3 changed files with 103 additions and 12 deletions.
4 changes: 4 additions & 0 deletions app.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ if (argv['nosentry']) {
}
const util = require('util');
// external dependencies ///////////////////////////////////////////////////////
const compression = require('compression');
const express = require('express');
const apicache = require('apicache').options({ debug: false }).middleware;
const morgan = require('morgan');
Expand All @@ -24,6 +25,9 @@ const port = parseInt(argv.port ? argv.port : "8080");
const db = require(util.format('./db/connector/%s', localconfig.database.engine));
const models = require('./db/models');

// compress all responses @see https://www.npmjs.com/package/compression#examples
app.use(compression());

//NEXT TWO LINES FOR READ BODY FROM POST
app.use(morgan('common'));

Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"dependencies": {
"apicache": "^1.4.0",
"commander": "^2.20.0",
"compression": "^1.7.4",
"cron": "^1.7.2",
"deep-diff": "^1.0.2",
"express": "^4.16.4",
Expand Down
110 changes: 98 additions & 12 deletions urls.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,9 +231,12 @@ module.exports = function(app, apicache) {
* @return {GeoJSON string} send JSON of past results merged with direct
* Wikidata query results, inverse order (direct query before, past after)
*/
app.get('/api/timedata', apicache('30 minutes'), async function (req, res) {
// apicache('30 minutes'),
app.get('/api/timedata', async function (req, res) {
// all results from all timeshot and real-time, merged together
let sparqlResultsArray = [];
let sparqlResultsFirstShotArray = [];
let sparqlResultsChangedDuplicatesArray = [];
// let now = parseInt(Math.round(new Date().getTime() / 1000));
let now = new Date().getTime();
console.log(now);
Expand Down Expand Up @@ -265,11 +268,65 @@ module.exports = function(app, apicache) {
// offset: ???,
limit: localconfig.historyTimelineLimit
}).then(async hists => {
/**
* Get only the times an element isn't changed.
* It's used to avoid duplicates pin when element change and to display
* correct counters.
* @param {Object} el Object from JSON, to get Wikidata id
* @param {[type]} changedTimes [description]
* @param {[type]} allTimes [description]
* @return {Array} array of unchanged times
*/
function getOnlyUnchangedTimes (el, changedTimes, allTimes) {
let unchangedTimes = [];
// previously stored changed times
let ct = changedTimes[el.properties.wikidata];
if (typeof ct !== 'undefined') {
// get difference between all stored times from history
// and changed times
for (at of allTimes) {
if (ct.indexOf(at) === -1) {
unchangedTimes.push(at);
}
else {
// do not collect anymore unchangedTimes if is changed
// new result will be kept from now on
break;
}
}
console.log(el.properties.wikidata);
console.log(unchangedTimes);
return unchangedTimes;
}
else {
// this pin is never changed in History
return allTimes;
}
}

function getOnlyNextTimes(el, changedTimes, allTimes) {
let ct = changedTimes[el.properties.wikidata];
let allTimeStartInd = allTimes.indexOf(el.properties.times[0]);
// TODO: exclude elements changed another time later! duplicates in this case!
return allTimes.slice(allTimeStartInd + 1);
}

// store all timeshot date from all Histories for this map
let allTimes = [];
if (hists) {
let changedTimes = {};

// copy first valid timeshot History
let n;
for (n = 0; n < hists.length; n++) {
if (!hists[n].error) {
sparqlResultsFirstShotArray = JSON.parse(hists[0].json).data;
break;
}
}
// get only differences between sparqlResultsFirstShotArray and other Histories
if (hists.slice(n+1)) {
// Past timeshots //////////////////////////////////////////////
for (histInd in hists) {
for (histInd in hists.slice(n+1)) {
let hist = hists[histInd];
if (DEBUG) {
console.log('mapId', hist.mapId, 'published', hist.map.published, 'histInd (array)', histInd); // DEBUG
Expand All @@ -287,32 +344,61 @@ module.exports = function(app, apicache) {
if (elk.hasOwnProperty('postProcess')) {
elk.properties.times = [thisHistoryTime];
sparqlResultsArray.push(elk);
// save a map with wikidata id and changed times for later use
if (typeof changedTimes[elk.properties.wikidata] === 'undefined') {
changedTimes[elk.properties.wikidata] = [];
}
changedTimes[elk.properties.wikidata].push(thisHistoryTime);
}
// add when different from previous record
// add to global time array when time is different from previous record
if (!allTimes.length || thisHistoryTime !== allTimes[allTimes.length - 1]) {
// run only on first element of histories
allTimes.push(thisHistoryTime);
}
}
// isFirstFoundHistory = false;
}
}

}
// now as last time in array
allTimes.push(now);
// first results: assign times only if are unchanged on all timeshots
for (fstel of sparqlResultsFirstShotArray) {
// el.properties.time = now;
fstel.properties.times = getOnlyUnchangedTimes(fstel, changedTimes, allTimes);
}
// create an array with old items changed at least one time, to be displayed as circle

// TODO:
for (htel of sparqlResultsArray) {
let newObj = JSON.parse(JSON.stringify(htel));
// display as a circle, removing postProcess
delete newObj.postProcess;
// get times after the change and create a new record with it
newObj.properties.times = getOnlyNextTimes(htel, changedTimes, allTimes);
console.log(newObj);
sparqlResultsChangedDuplicatesArray.push(newObj);
}
// sparqlResultsChangedDuplicatesArray = []; // XXX DEBUG
// console.log(sparqlResultsChangedDuplicatesArray);
//
//
//
// Real-time data ////////////////////////////////////////////////
// Reuse previous (cached) query from previous screen
let sparqlResultsRealtimeArray = await data.getJSONfromInternalUrl(req.query.q);
// let sparqlResultsRealtimeArray = await data.getJSONfromInternalUrl(req.query.q);
// apply now for current Query from Wikidata
// flag real time results to be populated with the very current time
// using client-side javascript (see enrichFeatures on mapdata.js)
for (el of sparqlResultsRealtimeArray) {
// el.properties.time = now;
el.properties.times = allTimes;
el.properties.current = true;
}
res.send(sparqlResultsArray.concat(sparqlResultsRealtimeArray));
// for (rtel of sparqlResultsRealtimeArray) {
// // el.properties.time = now;
// rtel.properties.times = getOnlyUnchangedTimes(rtel, changedTimes, allTimes);
// rtel.properties.current = true;
// }
////// res.send(sparqlResultsArray.concat(sparqlResultsRealtimeArray));

// output: First results [circle], all results of changed elements first time changed [pin], changed in the past [circle]
res.send(sparqlResultsFirstShotArray.concat(sparqlResultsArray).concat(sparqlResultsChangedDuplicatesArray));
});
});

Expand Down

0 comments on commit 7431ebf

Please sign in to comment.