Skip to content

Commit

Permalink
Merge pull request #229 from ideal-postcodes/10.0.0
Browse files Browse the repository at this point in the history
10.0.0
  • Loading branch information
cblanc committed Dec 10, 2018
2 parents a164aa6 + bec41fa commit 550e4ec
Show file tree
Hide file tree
Showing 57 changed files with 3,626 additions and 1,812 deletions.
135 changes: 49 additions & 86 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -1,91 +1,54 @@
version: 2
---
default_steps: &default_steps
steps:
- run:
command: "dockerize -wait tcp://localhost:5432 -timeout 2m"
name: "Wait for db"
- checkout
- run: "sudo apt-get update"
- run: "sudo apt install postgresql-client"
- run: "POSTGRES_USER=postgres npm run setup_test_db"
- restore_cache:
keys:
- "nodemodules-deps-{{ checksum \"package-lock.json\" }}"
- run: "npm install"
- save_cache:
key: "nodemodules-deps-{{ checksum \"package-lock.json\" }}"
paths:
- node_modules/
- run: "npm run test"
- run: "npm run coverage"
jobs:
node6:
docker:
- image: circleci/node:6.10
environment:
- PGHOST=localhost
- PGUSER=postgres
- image: circleci/postgres:9.6-alpine-postgis
steps:
- run:
name: Wait for db
command: dockerize -wait tcp://localhost:5432 -timeout 2m
- checkout
- run: sudo apt-get update
- run: sudo apt install postgresql-client
- run: POSTGRES_USER=postgres npm run setup_test_db
- restore_cache:
keys:
- nodemodules-deps-{{ checksum "package-lock.json" }}
- nodemodules-deps-
- run: npm install
- save_cache:
key: nodemodules-deps-{{ checksum "package-lock.json" }}
paths:
- node_modules/
- run: npm run test
- run: npm run coverage


node8:
docker:
- image: circleci/node:8.8
environment:
- PGHOST=localhost
- PGUSER=postgres
- image: circleci/postgres:9.6-alpine-postgis
steps:
- run:
name: Wait for db
command: dockerize -wait tcp://localhost:5432 -timeout 2m
- checkout
- run: sudo apt-get update
- run: sudo apt install postgresql-client
- run: POSTGRES_USER=postgres npm run setup_test_db
- restore_cache:
keys:
- nodemodules-deps-{{ checksum "package-lock.json" }}
- nodemodules-deps-
- run: npm install
- save_cache:
key: nodemodules-deps-{{ checksum "package-lock.json" }}
paths:
- node_modules/
- run: npm run test
- run: npm run coverage
node10:
docker:
- image: circleci/node:10
environment:
- PGHOST=localhost
- PGUSER=postgres
- image: circleci/postgres:9.6-alpine-postgis
steps:
- run:
name: Wait for db
command: dockerize -wait tcp://localhost:5432 -timeout 2m
- checkout
- run: sudo apt-get update
- run: sudo apt install postgresql-client
- run: POSTGRES_USER=postgres npm run setup_test_db
- restore_cache:
keys:
- nodemodules-deps-{{ checksum "package-lock.json" }}
- nodemodules-deps-
- run: npm install
- save_cache:
key: nodemodules-deps-{{ checksum "package-lock.json" }}
paths:
- node_modules/
- run: npm run test
- run: npm run coverage

node10:
<<: *default_steps
docker:
- environment:
- PGHOST=localhost
- PGUSER=postgres
image: "circleci/node:10"
- image: "circleci/postgres:10-alpine-postgis"
node11:
<<: *default_steps
docker:
- environment:
- PGHOST=localhost
- PGUSER=postgres
image: "circleci/node:11"
- image: "circleci/postgres:10-alpine-postgis"
node8:
<<: *default_steps
docker:
- environment:
- PGHOST=localhost
- PGUSER=postgres
image: "circleci/node:8"
- image: "circleci/postgres:10-alpine-postgis"
version: 2
workflows:
version: 2
node6_node8_node10:
supported_nodes:
jobs:
- node6
- node8
- node10

- node11
version: 2

4 changes: 4 additions & 0 deletions .env
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
POSTGRES_DB=postcodesiodb
POSTGRES_PASSWORD=secret
POSTGRES_USER=postcodesio
POSTGRES_HOST=db
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,4 @@ process.json
log/
npm-debug.log
*.swp
*.vim
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,18 @@

Any changes, including backwards incompatible changes will be listed here

## 10.0.0 (29/11/2018)

- *Breaking Change* Install dependencies upgraded. Going forward, only the following minimum versions will be tested:
- Node.js 8
- PostgreSQL 10
- Postcode responses now include County Electoral Districts (ceds)
- Updated documentation (outcodes, installation)
- Updated dependencies
- Updated GSS codes for missing wards
- Updated ONSPD dataset to November 2018
- Updated OS Names dataset to October 2018

## 9.0.3 (26/10/2018)

Minor updates and fixes. Many related to docker improvements
Expand Down
8 changes: 5 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM node:9
FROM node:10-alpine

WORKDIR /usr/src/app

Expand All @@ -10,8 +10,10 @@ HEALTHCHECK --interval=5s CMD node healthcheck.js

COPY package.json .

RUN npm install --only=production --no-package-lock && \
npm cache clean --force
RUN apk --no-cache add --virtual build-dependencies build-base gcc python && \
npm install --only=production --no-package-lock && \
npm cache clean --force && \
apk del build-dependencies

COPY . .

Expand Down
10 changes: 4 additions & 6 deletions Dockerfile.pg
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
FROM mdillon/postgis:9.6
FROM mdillon/postgis:10-alpine

# Copy the pointer to the latest pg_dump
COPY latest /tmp/latest

# Download and decompress latest pg_dump into image to be loaded on container startup
RUN apt-get update && \
apt-get install wget -y && \
wget -nv -O - $(cat /tmp/latest) > /docker-entrypoint-initdb.d/_postcodesiodata.sql.gz
RUN mkdir -p /docker-entrypoint-initdb.d/ && \
apk add --no-cache curl && \
curl -o /docker-entrypoint-initdb.d/_postcodesiodata.sql.gz $(cat /tmp/latest)

96 changes: 10 additions & 86 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,77 +10,23 @@ Documentation can be found at [postcodes.io](http://postcodes.io)

Please make sure you have the following available:

- Node.js v6.x [(Instructions)](http://nodejs.org/)
- Postgres (9.5 or greater)
- Node.js v8.9 [(Instructions)](http://nodejs.org/)
- PostgreSQL (10 or greater)
- PostGIS extension [(Instructions)](http://postgis.net/install)

We also have an end-to-end guide to install Postcodes.io on a fresh virtual machine [in the wiki](https://github.com/ideal-postcodes/postcodes.io/wiki/Installing-Postcodes.io).

## Install & Download Data & Run

**Download and install**

```bash
$ git clone https://github.com/ideal-postcodes/postcodes.io.git

$ cd postcodes.io/ && npm install
```

**Configure Postgres and Seed Database**

Postcodes.io is packaged with a script to setup and download the ONS Postcode Directory. To run this, navigate into the repository directory and run:
## Run with Docker

```
$ npm run setup
docker-compose up
```

This script will prompt you for Postgres superuser credentials. This privilege is required to create a new user, database, extensions and then to load the data. For the security conscious, you can find out how this works by [reading our installation notes](http://postcodes.io/docs#Install-notes) and [the script itself](/bin/setup.sh). Other install methods are available but require a bit of extra work.

You can change the Postgres username/password yourself but you will need to update 'config/config.js' with the relevant credentials.
## Self Hosting

The import process takes around 10 minutes to complete.
Instructions for installing and hosting postcodes.io yourself can be found at [postcodes.io/about](https://postcodes.io/about#Install-notes). Methods include:

**Run it**

```
node server.js // Default environment is development
```

## Docker

This repository builds both the application and database services as docker containers.

### Application

The Postcodes.io application is packaged as a Docker container identified on the Docker Hub as `idealpostcodes/postcodes.io`.

Example usage:

```
docker run -d -p 8000 idealpostcodes/postcodes.io
```

Configuration available via Environment Variables:

- `POSTGRES_USER`
- `POSTGRES_PASSWORD`
- `POSTGRES_DATABASE`
- `POSTGRES_HOST`
- `POSTGRES_PORT`
- `PORT` (HTTP Port to run on)
- `MAPBOX_PUBLIC_KEY`

### Database

`idealpostcodes/postcodes.io.db` points to a [PostgreSQL & PostGIS](https://hub.docker.com/r/mdillon/postgis/) base image and preloads the latest `pg_dump` in the build process. Upon starting a container for the first time, PostGIS will be enabled and the `pg_dump` will take around a minute to restore.

Example usage:

```
docker run -p 5432 -e POSTGRES_USER=postcodesio -e POSTGRES_DB=postcodesiodb -e POSTGRES_PASSWORD=password idealpostcodes/postcodes.io.db
```

As this contains the official postgres base image, the available configuration environment variables are documented here: [https://hub.docker.com/_/postgres/](https://hub.docker.com/_/postgres/)
- [Install as Docker containers](https://postcodes.io/about#docker-install)
- [Install on host with Node.js and Postgresql](https://postcodes.io/about#install-requirements)
- [Download and import the raw dataset to Postgresql](https://postcodes.io/about#import-from-pgdump)

## Testing

Expand All @@ -92,29 +38,7 @@ $ npm test

## External Libraries

1. [Java Library](https://github.com/spdeepak/postcodes-io-java) by [Deepak Sunanda Prabhakar](https://github.com/spdeepak)

2. PHP Libraries
* [postcodes-io-bundle](https://github.com/boxuk/postcodes-io-bundle) by the people at [Box UK](https://www.boxuk.com/). Read the [Blog post](https://www.boxuk.com/insight/tech-posts/geocoding-postcodes-symfony2)<br/>
* [postcodes-io laravel](https://github.com/adityamenon/postcodes-io-laravel) package by [Aditya Menon](http://adityamenon.co)<br/>
* [Postcodes Laravel 5+](https://github.com/codescheme/postcodes) package by [Codescheme](https://github.com/codescheme)<br/>
* [PHP Class for Postcodes.io](https://github.com/hart1994/Postcodes-IO-PHP) by [Ryan](https://github.com/hart1994/)<br/>

3. [Ruby Library](https://github.com/jamesruston/postcodes_io) by [James Ruston](https://github.com/jamesruston)

4. [Node.JS Library](https://github.com/cuvva/postcodesio-client-node) by [billinghamj](https://github.com/billinghamj)

5. [Python Library](https://github.com/previousdeveloper/PythonPostcodesWrapper) by [Gokhan Karadas](https://github.com/previousdeveloper)

6. [C# Library](https://github.com/markembling/MarkEmbling.PostcodesIO) by [Mark Embling](https://github.com/markembling)

7. [R Library](https://github.com/erzk/PostcodesioR) by [Eryk Walczak](http://walczak.org). Read the [blog post](http://walczak.org/2016/07/postcode-and-geolocation-api-for-the-uk/).

8. [Hack Library](https://github.com/Matt-Barber/HackPostcodes) HackPostcodes by [Matt Barber](https://recursiveiterator.wordpress.com/)

9. [Wolfram Language Library](https://github.com/arnoudbuzing/postcode) Postcode by [Arnoud Buzing](https://github.com/arnoudbuzing)

10. [Google Sheets Addon](https://chrome.google.com/webstore/detail/uk-postcode-geocoder/bjkecdilmiedfkihpgfhfikchkghliia?utm_source=permalink) by [Ed Patrick](http://edwebdeveloper.com/)
A list of external libraries can be found on the [about page](https://postcodes.io/about)

## License

Expand Down
52 changes: 52 additions & 0 deletions app/lib/setup.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
"use strict";

const { series } = require("async");
const models = require("../models/index.js");
const { Outcode } = models;

// List of support tables generated from data/ directory
const SUPPORT_TABLES = [
"Ccg",
"Ced",
"Constituency",
"County",
"District",
"Nuts",
"Parish",
"Ward",
].map(name => models[name]);

/**
* Generates support tables in DB
*
* @returns {Promise}
*/
const setupSupportTables = () => {
return new Promise((resolve, reject) => {
series(SUPPORT_TABLES.map(m => m._setupTable.bind(m)), (error, result) => {
if (error) return reject(error);
return resolve(result);
});
});
};

/**
* Generates outcodes from existing postcodes relation
*
* @returns {Promise}
*/
const setupOutcodeTable = () => {
return new Promise((resolve, reject) => {
Outcode._setupTable((error, result) => {
if (error) return reject(error);
return resolve(result);
});
});
};

module.exports = {
setupSupportTables,
setupOutcodeTable,
SUPPORT_TABLES,
};

2 changes: 1 addition & 1 deletion app/models/attribute_base.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ const fs = require("fs");
const util = require("util");
const path = require("path");
const async = require("async");
const Base = require("./index").Base;
const Base = require("./base").Base;

const requiredAttributes = {
"code": "VARCHAR(32) NOT NULL UNIQUE",
Expand Down
Loading

0 comments on commit 550e4ec

Please sign in to comment.