Reduce running time (#49)

* speed 
* code cleaning

bin/run.js

@@ -1,8 +1,8 @@
 const {roadFlow}  = require('../src/index.js')
 
 const inputFiles = {
-  "OS": './input/OSMM_HIGHWAYS_June18.gpkg',
-  "OSM": './input/UK_OSM.pbf'
+  "OS": './input/London_OS.gpkg',
+  "OSM": './input/London_OSM.pbf'
 };
 
 const outputFiles = {

package.json

@@ -1,5 +1,5 @@
 {
-  "name": "roads",
+  "name": "roadFlow",
   "version": "1.0.0",
   "description": "The direction of traffic for oneway roads can change, leading to out-of-date online data. The current process for detecting these changes in direction is a manual approach, including trawling through newspaper extracts and through word of mouth from local authorities to see which roads have changed. This results in a slow, time-consuming update process where the majority of surveyors will be sent to random locations in order to determine if the directions are still valid, most of which will already be correct. There is a clear need for a better suited detection system, to avoid the waste of time in sending surveyors to locations where the directions of roads are likely already correct.",
   "main": "index.js",
@@ -13,7 +13,7 @@
     "coveralls": "nyc report --reporter=text-lcov | coveralls"
   },
   "author": "Geovation Hub",
-  "license": "ISC",
+  "license": "MIT",
   "dependencies": {
     "@turf/line-overlap": "^6.0.2",
     "@turf/turf": "^5.1.6",

src/comparator/README.md

@@ -1,2 +1,5 @@
-* `overlap.js` contains `distToler = 0.004` and `overlapPercentage = 0.5`, which are distance between links to be overlap and the overlap percentage between overlap segment and road links (0.5 = 50%).
-* `note-generator.js` contains `tolerance = 170` which is tolerance of the angle in degree and is exported to be used by other modules.
+The code in this folder is to compare between OS and OSM datasets and find mismatches in oneway road links. The script used to read two input files, then compare road name of each road in first set against all links in second set. If a name match found then compare coordinates if there is an overlap between two links. call function to calculate direction of both links and check if in same or opposite direction. After finishing all links write output to output files.
+The first file executed is `comparator.js`, then calls `compareOSroadWithOSM.js` which calls `checker.js` and `direction.js`. `comparator.js` calls `print.js` for printing information to console, `progress.js` called in time intervals to calculate progress of the execution. `io.js` is used to read and write input and output files.
+
+* `checker.js` contains two parametric values `distToler = 0.004` and `overlapPercent = 0.5` inside function `isOverlapping`, which are distance between links to be overlap and the overlap percentage between overlap segment and road links (0.5 = 50%).
+* `direction.js` contains `tolerance = 10` which is tolerance of the angle in degree and is exported to be used by other modules.

src/comparator/checker.js

@@ -14,7 +14,7 @@ exports.compareNames  = (nameOne, nameTwo) => {  //function compares two road na
 }
 
 exports.isOverlapping = (geometryOne, geometryTwo) => { //function comapres overlap between 2 road links
-  //distToler is error tolerance in km, overlapPercent is min acceptance ratio between overlap and road link
+  //distToler is tolerance in km, overlapPercent is min acceptance ratio between overlap and road link
   const distToler = 0.004, overlapPercent = 0.5;
   //calculate the overlap sections between OS and OSM road links
   const overlap = turf.lineOverlap(geometryOne, geometryTwo, {tolerance: distToler});
@@ -30,13 +30,11 @@ exports.isOverlapping = (geometryOne, geometryTwo) => { //function comapres over
 }
 
 exports.inRange = (roadOne, roadTwo, range = 1) => { //function compares if 2 road links are close together
-    if ( _.has(roadOne , "geometry.coordinates[0]") && _.has(roadTwo , "geometry.coordinates[0]")) {
-      const pointOne = roadOne.geometry.coordinates[0]; //get 1st point in 1st road
-      const pointTwo = roadTwo.geometry.coordinates[0]; //get 1st point in 2nd road
-      const distance = turf.distance(pointOne, pointTwo); //calculate distance
-      if (distance < range) { //if in range (in km) return true
-        return true;
-      }
-    }
+  const pointOne = roadOne.geometry.coordinates[0]; //get 1st point in 1st road
+  const pointTwo = roadTwo.geometry.coordinates[0]; //get 1st point in 2nd road
+  const distance = turf.distance(pointOne, pointTwo); //calculate distance
+  if (distance < range) { //if in range (in km) return true
+    return true;
+  }
   return false;
 }

src/comparator/compareOSroadWithOSM.js

@@ -1,50 +1,56 @@
+//module has 2 function: compareOSroadWithOSM that loop through OSM links,
+//prepare output data in arrays and returns number of matches found in OSM for OS link
+//linkComparisions function compares two links and return true if direction mismatch found
+
 const {inRange} = require('./checker.js');
 const {compareNames} = require('./checker.js');
 const {isOverlapping} = require('./checker.js');
-const {calculateAngle} = require('./direction.js')
+const {calculateAngle} = require('./direction.js');
 const {isMismatch} = require('./direction.js');
 
-let matchesCounter;
+let matchesCounter; //counter for howmany matches found in OSM for each OS link
 
-conditions = (roadOS, roadOSM) => {
+//========== compare links, when match check if opposite direction ==========
+linkComparisions = (roadOS, roadOSM) => {
   const osName = roadOS.properties.name.slice(3, (roadOS.properties.name.length - 1));
   if ( compareNames(osName, roadOSM.properties.name) ) { //comapre if names matches
     if ( isOverlapping(roadOS.geometry, roadOSM.geometry) ) { //check if links overlap
       matchesCounter ++; //increment links' match counter
       let angleOS = calculateAngle(roadOS.geometry.coordinates); //find OS angle
-      angleOS = roadOS.properties.direction ? angleOS : (angleOS + 180) % 360; //if opposite direction rotate 180
-      let angleOSM = calculateAngle(roadOSM.geometry.coordinates); //find OSM angle
+      angleOS = roadOS.properties.direction ? angleOS : (angleOS + 180) % 360; //if link direction is opposite rotate 180
+      const angleOSM = calculateAngle(roadOSM.geometry.coordinates); //find OSM angle
       return isMismatch(angleOS, angleOSM); //return true if mismatch occure
     }
   }
   return false;
 }
-exports.conditions = conditions;
+exports.linkComparisions = linkComparisions; //export inner function for unit testing
 
+//========== loops through OSM data ==========
 exports.compareOSroadWithOSM = (roadOS, dataOSM, outputData) => {
   matchesCounter = 0; //reset counter for number of matches
 
   dataOSM.features
-    .filter(roadOSM => inRange(roadOS, roadOSM))
-    .forEach(roadOSM => {
-      if (conditions(roadOS, roadOSM)) {
-        let data = {
+    .filter(roadOSM => inRange(roadOS, roadOSM))  //filter features that are in range
+    .forEach(roadOSM => {  //loop through filter features in range
+      if (linkComparisions(roadOS, roadOSM)) { //check return value of call to inner function that compare links
+        let data = { //create object for output file
           "roadName": roadOSM.properties.name,
           "OSId": (roadOS.properties.id).toString(),
           "OSMId": (roadOSM.properties.id).toString(),
         };
-        outputData.info.push(data);
-        outputData.OS.push(roadOS);
-        outputData.OSM.push(roadOSM);
+        outputData.info.push(data);  //push object to info array
+        outputData.OS.push(roadOS);  //push OS link to OS array
+        outputData.OSM.push(roadOSM);  //push OSM link to OSM array
       }
     });
 
-  if (matchesCounter === 0) {
+  if (matchesCounter === 0) { //if number of matches ZERO set to 'noMatch'
     matchesCounter = 'noMatch';
-  } else if (matchesCounter === 1) {
+  } else if (matchesCounter === 1) { //if number of matches ONE set to 'oneMatch'
     matchesCounter = 'oneMatch';
-  } else {
+  } else { //else set to 'multiMatch'
     matchesCounter = 'multiMatch';
   }
-  return matchesCounter;
+  return matchesCounter; //return string
 }

src/comparator/direction.js

@@ -1,40 +1,43 @@
 //module has 2 function calculateAngle (calculates angle from coordinates)
 //and isMismatch (checks if angles has opposite directions)
 
+
 const turf = require('@turf/turf');
 
-const tolerance = 170; //tolerance of road vector angles in Degree.
-exports.tolerance = tolerance;
+const tolerance = 10; //deviation from 180 degree.
+exports.tolerance = tolerance; //export tolerance
 
+//========== calculates angle from coordinates ==========
 exports.calculateAngle = (coordinates) => {
   let index = coordinates.length - 1; //index points to last element in array
-  let angle;
   //calculate difference in x and y coordinates between 1st and last points of link
-  let xDifference = coordinates[index][0] - coordinates[0][0];
-  let yDifference = coordinates[index][1] - coordinates[0][1];
+  const xDifference = coordinates[index][0] - coordinates[0][0];
+  const yDifference = coordinates[index][1] - coordinates[0][1];
   //calculate angle of link using INVERSEtan()
   let tanTheta = Math.atan( Math.abs(yDifference / xDifference) );
   //switch to check in which quadrant the angle is so execute the right formula
   switch(true){
     case (xDifference >= 0 && yDifference >= 0):  //angle in 1st quadrant
-      angle = tanTheta
+      tanTheta = tanTheta;
       break;
     case (xDifference < 0 && yDifference > 0):  //angle in 2nd quadrant
-      angle = (Math.PI - tanTheta);
+      tanTheta = (Math.PI - tanTheta);
       break;
     case (xDifference <= 0 && yDifference <= 0):  //angle in 3rd quadrant
-      angle = (tanTheta - Math.PI);
+      tanTheta = (tanTheta - Math.PI);
       break;
     case (xDifference > 0 && yDifference < 0):  //angle in 4th quadrant
-      angle = (-tanTheta);
+      tanTheta = (-tanTheta);
   }
-  return ((angle * 180 / Math.PI) + 360 ) % 360;
+  //convert radian to degree, add 360 to convert -ve to +ve, and %360 to range 0 to 360
+  return ((tanTheta * 180 / Math.PI) + 360 ) % 360;
 }
 
+//========== compare if two angles are in opposite direction ==========
 exports.isMismatch = (angleOne, angleTwo) => {
     //compare angle differnce falls in range to be considered opposite direction
-    if ( (Math.abs(angleOne - angleTwo) >= tolerance )
-      && (Math.abs(angleOne - angleTwo) <= (360 - tolerance)  ) ) {
+    if ( (Math.abs(angleOne - angleTwo) >= (180 - tolerance) )
+      && (Math.abs(angleOne - angleTwo) <= (180 + tolerance)  ) ) {
       return true;
     }
   return false;

src/comparator/io.js

@@ -1,27 +1,28 @@
 //module for reading and writing files to disk
+
 const fs = require('fs');
 
-  //========== read files and return array of data in file ==========
-  exports.read = (files) => {
-    if (!files.OS || !files.OSM) { //check file names are given
-      throw 'ERROR! One or both file names are missing'; //throw error
-    }
-    let dataOne = fs.readFileSync(files.OS); //read in OS file first
-    let dataTwo = fs.readFileSync(files.OSM); //read in OSM file second
-    dataOne = JSON.parse(dataOne.toString()); //parse OS data
-    dataTwo = JSON.parse(dataTwo.toString()); //parse OSM data
-    return [dataOne, dataTwo]; //return array of data from both files
+//========== read files and return array of data in file ==========
+exports.read = (files) => {
+  if (!files.OS || !files.OSM) { //check file names are given
+    throw 'ERROR! One or both file names are missing'; //throw error
   }
+  let dataOne = fs.readFileSync(files.OS); //read in OS file first
+  let dataTwo = fs.readFileSync(files.OSM); //read in OSM file second
+  dataOne = JSON.parse(dataOne.toString()); //parse OS data
+  dataTwo = JSON.parse(dataTwo.toString()); //parse OSM data
+  return [dataOne, dataTwo]; //return array of data from both files
+}
 
-  //========== write data to output files ==========
-  exports.write = (file = '', data = {}) => {
-    if (!file || JSON.stringify(data) == '{}') { //check file name given & data not empty
-      throw 'ERROR! Either file name or data is missing'; //throw error
-    }
-    let output = { //add "type" and "feature" keys to make data readible by GIS programmes
-                  "type": "FeatureCollection",
-                  "features": data
-                  };
-    fs.writeFileSync(file, JSON.stringify(output, null, 2)); //write data to file
-    return true; //return true
+//========== write data to output files ==========
+exports.write = (file = '', data = {}) => {
+  if (!file || JSON.stringify(data) == '{}') { //check file name given & data not empty
+    throw 'ERROR! Either file name or data is missing'; //throw error
   }
+  const output = { //add "type" and "feature" keys to make data readible by GIS programmes
+    "type": "FeatureCollection",
+    "features": data
+  };
+  fs.writeFileSync(file, JSON.stringify(output, null, 2)); //write data to file
+  return true; //return true
+}

src/comparator/print.js

@@ -1,32 +1,37 @@
+//module to print information through out the programme
+
 const tm = require('../time.js');
 
 exports.message = (text) => {
-    console.info(text);
+  console.info(text);
 }
 
+//========== print info at start of programe ==========
 exports.header = (lengthOS, lengthOSM) => {
   console.info('\n\t\t*****\t comparator Script started at ' +
             new Date().toLocaleTimeString() + ' \t*****\n');
   console.info('Nubmer of roads OS= ' + lengthOS + ', and OSM= ' + lengthOSM);
 }
 
+//========== print info about link matche at end of programe ==========
 exports.report = (counter) => {
   console.info('Number of OS links with NONE match in OSM: ' + counter.noMatch);
   console.info('Number of OS links with ONE  match in OSM: ' + counter.oneMatch);
   console.info('Number of OS links with MULTImatch in OSM: ' + counter.multiMatch);
   console.info('Number of road links without a Name in OS: ' + counter.noName);
 }
 
+//========== print progress info at set intervals ==========
 exports.progress = (obj) => {
-  if (obj.toPrint) {
+  if (obj.toPrint) { //check if toPrint is true
     console.info('Time passed: ' + tm.format(obj.timePassed));
     console.info('Estimate Time Left: ' + tm.format(obj.estimateTimeLeft));
     console.info('Progress: ' + obj.progressPercent.toFixed(2) + '%');
   }
 }
 
+//========== print info at end of programe ==========
 exports.footer = (time) => {
-  let duration = new Date() - time;
   console.info('\t***************************************\n');
   console.info('\t\tTotal time taken: \t' + tm.format(new Date() - time) + '\n');
 }

src/comparator/progress.js

@@ -1,26 +1,26 @@
 //module to calculate how many roads processed and estimate time left
 
 const print = require('./print.js');
-const _ = require('lodash');
 
+const intervalPrintTime = 10000; //time intervals to print progress in millisecond
 let startDate;
 let lastPrintDate;
 
 init();
 
-//-----------/
+//========== initiate function ==========
 function init() {
   startDate = new Date();
   lastPrintDate = new Date();
 }
 
 exports.init = init;
+exports.intervalPrintTime = intervalPrintTime;
 
 exports.calculateProgress = (counters) => {
-  let calcObj = {};
-
+  let calcObj = {}; //object to hold progress information
   // print only once evere 3000 mills
-  if ( (new Date() - lastPrintDate) > 3000 ) {
+  if ( (new Date() - lastPrintDate) > intervalPrintTime ) {
     lastPrintDate = new Date();
     let timePassed = lastPrintDate - startDate;
     let roadRatio = (counters.totalRoadsOS / counters.processedOS);

src/process-features/extra-brackets.js

@@ -1,19 +1,20 @@
 /*
-module to delete extra brackets in arrays. mainly used for GIS coordinates array
+module to delete extra brackets in arrays. used to convert multiLineString to lineString
 array output format [[X1, Y1], [X2, Y2], [X3, Y3], .....[Xn, Yn]]
 */
 
-exports.delete = (input, output = []) => {
+exports.delete = (input) => {
+  let output = [];
   if (!Array.isArray(input)) { //if input not an array return empty array []
-    return [];
+    return output;
   }
   //loop through array elements
   for (let temp of input) {
     //check if first element is not array (no extra brucket) push to output array
     if (!Array.isArray(temp[0])) {
       output.push(temp);
-    } else { //else element array call same function again (recursion)
-      output = this.delete(temp, output);
+    } else { //else element is array, cocatenate to output array
+      output = output.concat(temp);
     }
   }
   return output; //return output array

src/process-features/feature-extractor.js

@@ -4,22 +4,21 @@ calls module to process coordinates array
 returns output GIS data with processed coordinates array
 */
 
-const brackets = require('./extra-brackets')
+const deleteBrackets = require('./extra-brackets').delete
 
 //this is the input file to be fixed
 exports.filter = (features) => {
   for (let i = 0; i < features.length; i++) {
+    let geometry = features[i].geometry;
     // check if type is MultiLineString convert it to LineString
-    if (features[i].geometry.type == "MultiLineString"){
-      features[i].geometry.type = "LineString";
+    if (geometry.type == "MultiLineString"){
+      geometry.type = "LineString";
     }
 
-    let coordinates = brackets.delete(features[i].geometry.coordinates);
+    let coordinates = deleteBrackets(geometry.coordinates);
+    geometry.coordinates = coordinates;
     //check if coordinates are empty delete entry
-    if (coordinates.length > 0) {
-      //delete extra brackets in coordinates array and save them
-      features[i].geometry.coordinates = coordinates;
-    } else {
+    if (coordinates.length === 0) {
       //delete element i form features
       features.splice(i, 1);
     }

test/findAngle.js → test/calculateAngle.js

@@ -2,7 +2,7 @@ const assert = require('assert');
 const {expect} = require('chai');
 const {calculateAngle} = require('../src/comparator/direction.js');
 
-describe('findAngle.js finds the dirction of the road based on angle', () => {
+describe('calculateAngle.js finds the dirction of the road based on angle', () => {
   it('Test when angle in 1st quadrant', () => {
     const input = [[6, 3], [9, 6]];
     const expected = 45;