Merge pull request #440 from ideal-postcodes/onspd_updates

ONSPD updates

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 Any changes, including backwards incompatible changes will be listed here
 
+## 11.0.0 (3/1/2020)
+
+- *Breaking Change* CCG Short Code column added to ccgs table
+- CCG Short Code returned in `codes` object
+- Update ONSPD to November 2019
+- Update OS Open Names to October 2019
+- Updated GSS codes for midding NUTS
+- Updated GSS codes for midding wards
+- New docker-compose setup for test, dev and s3 pg_dump testing
+
 ## 10.2.1 (2/9/2019)
 
 - Return specific error message if postcode not in SPD but in ONSPD. Thanks to [@mashedkeyboard](https://github.com/mashedkeyboard)
@@ -100,7 +110,7 @@ Minor updates and fixes. Many related to docker improvements
 ## 9.0.0 (8/6/2018)
 
 - *Breaking Change* Updated `postcode` and `terminated_postcode` models for the new schema in ONSPD CSV file. This means `>9.0.0` will not be able to import ONSPD CSV files produced before May 2018. `pg_dump` imports will not be affected
-- Added package.lock file 
+- Added package.lock file
 - Updated wards, districts, nuts, ccgs GSS Codes
 - Updated ONSPD dataset to May 2018
 - Updated OS Names dataset to April 2018
@@ -207,22 +217,22 @@ For the exact changes, you can may inspect the most recent commit applied to the
 - Minimum advised required Postgresql version of 9.5 (due to backwards incompatible pg_dump)
 - Updated latest pg_dump for Feb 2017 ONSPD, January 2017 OS Places data
 
-## 3.0.3 
+## 3.0.3
 - Updated latest pg_dump for Nov 2016 ONSPD
 - Updated NUTS GSS codes
 - Added support for Node.js 6.9
 
-## 3.0.2 
+## 3.0.2
 - Updated latest pg_dump for Aug 2016 ONSPD
 
-## 3.0.1 
+## 3.0.1
 - Added some missing GSS codes. Fix import script for Aug 2016 ONSPD
 
-## 3.0.0 
+## 3.0.0
 - Drop support for node.js 0.10 and 0.12
 
-## 2.0.1 
+## 2.0.1
 - Expanded accept headers and HTTP methods in CORS preflight requests
 
-## 2.0.0 
+## 2.0.0
 - Updated dataset to February 2016

Makefile

@@ -51,6 +51,18 @@ test-down:
 test-shell:
 	docker-compose -f docker/test/docker-compose.yml exec api /bin/bash
 
+## -- Live Test Methods --
+
+## Build local pg and api images using pg_dump from S3
+.PHONY: live-up
+live-up:
+	docker-compose -f docker/live-test/docker-compose.yml up -d --build
+
+## Shut down live test services
+.PHONY: live-down
+live-down:
+	docker-compose -f docker/live-test/docker-compose.yml down -v
+
 ## -- Misc --
 
 ## Update repository against origin/master

app/models/base.js

@@ -1,15 +1,15 @@
 "use strict";
 
 const fs = require("fs");
-const pg = require("pg");
-const copyFrom = require("pg-copy-streams").from;
+const { Pool } = require("pg");
+const { from } = require("pg-copy-streams");
 const async = require("async");
 const csv = require("csv");
 const defaults = require("../../config/config")();
 const config = defaults.postgres;
 
 // Instantiate postgres client pool
-const pool = pg.Pool(config);
+const pool = new Pool(config);
 
 // All models inherit from base
 // Requires schema and relation name
@@ -134,7 +134,7 @@ Base.prototype._csvSeed = function(options, callback) {
     (filepath, cb) => {
       pool.connect((error, client, done) => {
         const pgStream = client
-          .query(copyFrom(query))
+          .query(from(query))
           .on("end", () => {
             done();
             return cb();

app/models/ccg.js

@@ -3,13 +3,41 @@
 const TABLE_NAME = "ccgs";
 
 const { inherits } = require("util");
+const path = require("path");
+const fs = require("fs");
+const async = require("async");
 const AttributeBase = require("./attribute_base.js");
 
 function Model() {
-	AttributeBase.call(this, TABLE_NAME);
+  const schemaAddition = {
+    "ccg19cdh": "VARCHAR(32) NULL UNIQUE",
+  };
+
+	AttributeBase.call(this, TABLE_NAME, schemaAddition);
 }
 
 inherits(Model, AttributeBase);
 
+// Override Seed Data Method
+Model.prototype.seedData = function (callback) {
+  const self = this;
+  const dataPath = path.join(__dirname, "../../data/");
+  const dataObject = JSON.parse(fs.readFileSync(path.join(dataPath, self.relation + ".json")));
+  const insertQueue = [];
+
+  for (let code in dataObject) {
+    insertQueue.push([code, dataObject[code].name, dataObject[code].ccg19cdh]);
+  }
+
+  async.parallel(insertQueue.map(elem => {
+    return function (callback) {
+      const query = `
+				INSERT INTO ${self.relation} (code, name, ccg19cdh) VALUES ($1, $2, $3)
+			`;
+      self._query(query, elem, callback);
+    };
+  }), callback);
+};
+
 module.exports = new Model();