Merge pull request #163 from ideal-postcodes/NOV_2017_ONSPD

Nov 2017 ONSPD Update

.circleci/config.yml

@@ -1,22 +1,67 @@
 version: 2
 jobs:
-   build:
+   node6:
      docker:
        - image: circleci/node:6.10
          environment:
            - PGHOST=localhost
            - PGUSER=postgres
-       - image: mdillon/postgis
+       - image: circleci/postgres:9.6-alpine-postgis
 
      steps:
+       - run:
+          name: Wait for db
+          command: dockerize -wait tcp://localhost:5432 -timeout 2m
        - checkout
        - run: sudo apt-get update
        - run: sudo apt install postgresql-client
+       - run: POSTGRES_USER=postgres npm run setup_test_db
+       - restore_cache:
+           keys:
+             - npm-deps-{{ checksum "package.json" }}
+       - run: npm install
+       - save_cache:
+           key: npm-deps-{{ checksum "package.json" }}
+           paths:
+             - node_modules/
+       - run: npm run test-ci
+       - run: npm run coverage
+
+
+   node8:
+     docker:
+       - image: circleci/node:8.8
+         environment:
+           - PGHOST=localhost
+           - PGUSER=postgres
+       - image: circleci/postgres:9.6-alpine-postgis
+
+     steps:
        - run:
           name: Wait for db
-          command: dockerize -wait tcp://localhost:5432 -timeout 1m
+          command: dockerize -wait tcp://localhost:5432 -timeout 2m
+       - checkout
+       - run: sudo apt-get update
+       - run: sudo apt install postgresql-client
        - run: POSTGRES_USER=postgres npm run setup_test_db
+       - restore_cache:
+           keys:
+             - npm-deps-{{ checksum "package.json" }}
        - run: npm install
-       - run: npm test
+       - save_cache:
+           key: npm-deps-{{ checksum "package.json" }}
+           paths:
+             - node_modules/
+       - run: npm run test-ci
+       - run: npm run coverage
+
+workflows:
+  version: 2
+  node6_and_node8:
+    jobs:
+      - node6
+      - node8
+
+
 
 

.gitignore

@@ -10,3 +10,4 @@ config/deploy/
 process.json
 .nyc_output/
 log/
+npm-debug.log

CHANGELOG.md

@@ -2,6 +2,17 @@
 
 Any changes, including backwards incompatible changes will be listed here
 
+## 7.0.0 (8/12/2017)
+
+- *Breaking Change* Place schema has been amended with additional columns to support better text search for place names. When upgrading, `places` will need to be rebuilt
+- Added Dockerfile & Dockerhub Repository (thanks to @jamescun and @billinghamj)
+- Fix: Added missing filterable attributes for `?filter=`
+- Updated dependencies
+- Updated NUTS, Wards, Parishses GSS codes
+- Added optional rate limiting on bulk lookup endpoints
+- Updated ONSPD dataset to Nov 2017
+- Updated OS Names dataset to Oct 2017
+
 ## 6.1.1 (28/9/2017)
 
 - Updated dependencies

README.md

@@ -1,4 +1,4 @@
-# Postcodes.io [![Build Status](https://travis-ci.org/ideal-postcodes/postcodes.io.png)](https://travis-ci.org/ideal-postcodes/postcodes.io) [![Coverage Status](https://coveralls.io/repos/github/ideal-postcodes/postcodes.io/badge.svg?branch=code_coverage_btn)](https://coveralls.io/github/ideal-postcodes/postcodes.io?branch=code_coverage_btn)
+# Postcodes.io [![Build Status](https://travis-ci.org/ideal-postcodes/postcodes.io.png)](https://travis-ci.org/ideal-postcodes/postcodes.io) [![Coverage Status](https://coveralls.io/repos/github/goingdotin/postcodes.io/badge.svg?branch=master)](https://coveralls.io/github/goingdotin/postcodes.io?branch=master)
 
 Query for UK postcodes and geolocations over HTTP. Postcodes.io uses the Office for National Statistics Postcode Directory.
 

app/controllers/postcodes_controller.js

@@ -96,6 +96,9 @@ exports.bulk = (request, response, next) => {
 	}
 };
 
+const MAX_GEOLOCATIONS = defaults.bulkGeocode.geolocations.MAX;
+const GEO_ASYNC_LIMIT = defaults.bulkGeocode.geolocations.ASYNC_LIMIT || MAX_GEOLOCATIONS;
+
 function bulkGeocode (request, response, next) {
 	const geolocations = request.body.geolocations;
 
@@ -115,9 +118,19 @@ function bulkGeocode (request, response, next) {
 		};
 		return next();
 	}
+
+	const lookupGeolocation = (location, callback) => {
+		Postcode.nearestPostcodes(location, (error, postcodes) => {
+			if (error || !postcodes) {
+				return callback(null, { query: location, result: null });
+			}
+			callback(null, {
+				query: sanitizeQuery(location),
+				result: postcodes.map(postcode => Postcode.toJson(postcode))
+			});
+		});
+	};
 
-	const result = [];
-	const execution = [];
 	const whitelist = ["limit", "longitude", "latitude", "radius", "widesearch"];
 	const sanitizeQuery = query => {
 		const result = {};
@@ -128,40 +141,20 @@ function bulkGeocode (request, response, next) {
 		}
 		return result;
 	};
-
-	geolocations.forEach(location => {
-		execution.push(callback => {
-			const params = location;
-
-			Postcode.nearestPostcodes(params, (error, postcodes) => {
-				if (error || !postcodes) {
-					result.push({
-						query: location,
-						result: null
-					});
-				} else {
-					result.push({
-						query: sanitizeQuery(location),
-						result: postcodes.map(postcode => Postcode.toJson(postcode))
-					});
-				}
-				callback();
-			});
-
-		});
-	});
-
-	const onComplete = () => {
+
+	async.mapLimit(geolocations, GEO_ASYNC_LIMIT, lookupGeolocation, (error, data) => {
+		if (error) return next(error);
 		response.jsonApiResponse = {
 			status: 200,
-			result: result
+			result: data
 		};
 		return next();
-	};
-
-	async.parallel(execution, onComplete);
+	});
 }
 
+const MAX_POSTCODES = defaults.bulkLookups.postcodes.MAX;
+const BULK_ASYNC_LIMIT = defaults.bulkLookups.postcodes.ASYNC_LIMIT || MAX_POSTCODES;
+
 function bulkLookupPostcodes (request, response, next) {
 	const postcodes = request.body.postcodes;
 
@@ -172,43 +165,32 @@ function bulkLookupPostcodes (request, response, next) {
 		};
 		return next();
 	}
-
-	const MAX_POSTCODES = defaults.bulkLookups.postcodes.MAX;
-
+
 	if (postcodes.length > MAX_POSTCODES) {
 		response.jsonApiResponse = {
 			status: 400,
 			error: `Too many postcodes submitted. Up to ${MAX_POSTCODES} postcodes can be bulk requested at a time`
 		};
 		return next();
 	}
-
-	const result = [];
-	const execution = [];
-
-	postcodes.forEach(postcode => {
-		execution.push(callback => {
-			Postcode.find(postcode, (error, postcodeInfo) => {
-				if (error || !postcodeInfo) {
-					result.push({
-						query: postcode,
-						result: null
-					});
-				} else {
-					result.push({
-						query: postcode,
-						result: Postcode.toJson(postcodeInfo)
-					});
-				}
-				callback();
+
+	const lookupPostcode = (postcode, callback) => {
+		Postcode.find(postcode, (error, postcodeInfo) => {
+			if (error || !postcodeInfo) { 
+				return callback(null, { query: postcode, result: null });
+			}
+			callback(null, { 
+				query: postcode,
+				result: Postcode.toJson(postcodeInfo)
 			});
 		});
-	});
-
-	async.parallel(execution, () => {
+	};
+
+	async.mapLimit(postcodes, BULK_ASYNC_LIMIT, lookupPostcode, (error, data) => {
+		if (error) return next(error);
 		response.jsonApiResponse = {
 			status: 200,
-			result: result
+			result: data
 		};
 		return next();
 	});

app/models/index.js

@@ -129,7 +129,7 @@ Base.prototype._csvSeed = function (options, callback) {
 	const transform = options.transform || (row => row);
 	const query = `COPY ${this.relation} (${columns}) FROM STDIN DELIMITER ',' CSV`;
 
-	async.eachSeries(filepath, (filepath, cb) => {
+	async.eachLimit(filepath, 5, (filepath, cb) => {
 		pool.connect((error, client, done) => {
 			const pgStream = client.query(copyFrom(query))
 				.on("end", () => {