Merge pull request #140 from nhsuk/feature/replace-postcodes-io

Use in-house postcode search

CHANGELOG.md

@@ -1,11 +1,13 @@
 0.20.0 / TBC
 ===================
 - Update npm dependencies
-- Upgrade to `node:10.14.0-alpine` for latest security and bug patches
-- Use Azure Search rather than Elasticsearch
+- Upgrade to `node:10.15.0-alpine` for latest security and bug patches
+- Use Azure Search rather than Elasticsearch for service search
+- Use in-house, Azure Search backed postcode search rather postcodes.io
 - Enable debugger for development
 - Calculate distance between the search origin and each result items
 - Preserve original formatting for service information and opening times
+- Be more polite with error messaging
 
 0.19.0 / 2018-11-20
 ===================

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:10.14.0-alpine
+FROM node:10.15.0-alpine
 
 RUN apk add --no-cache python=2.7.15-r1 git-perl bash make gcc g++
 RUN rm /bin/sh && ln -s /bin/bash /bin/sh

README.md

@@ -57,7 +57,7 @@ environment.
 | `LOG_LEVEL`                        | Numeric [log level](https://github.com/trentm/node-bunyan#levels)                                                                                          | Depends on `NODE_ENV`     | No         |
 | `NODE_ENV`                         | Node environment                                                                                                                                           | development               | Yes        |
 | `PORT`                             | Server port                                                                                                                                                | 3000                      | Yes        |
-| `SEARCH_API_HOST`                  | Host name for the [NHS Developer API](https://developer.api.nhs.uk/)                                                                                       | nhsapiint.azure-api.net   | Yes        |
+| `SEARCH_API_HOST`                  | Host name for the [NHS Developer API](https://developer.api.nhs.uk/)                                                                                       | api.nhs.uk                | Yes        |
 | `SEARCH_API_KEY`                   | `subscription-key` for the [NHS Developer API](https://developer.api.nhs.uk/)                                                                              |                           | Yes        |
 | `SEARCH_API_VERSION`               | Version of the [NHS Developer API](https://developer.api.nhs.uk/)                                                                                          | 1                         | Yes        |
 | `WEBTRENDS_ANALYTICS_TRACKING_ID`  | [Webtrends](https://www.webtrends.com/) tracking id                                                                                                        |                           | No         |

app/lib/displayUtils/messages.js

@@ -1,17 +1,12 @@
 function mandatorySelectionMessage() {
-  return 'You must choose one of the options.';
+  return 'Please choose one of the options.';
 }
 
 function emptyPostcodeMessage() {
-  return 'You must enter a postcode.';
-}
-
-function invalidPostcodeMessage(location) {
-  return `We can't find the postcode '${location.toUpperCase()}'. Check the postcode is correct and try again.`;
+  return 'Please enter a postcode.';
 }
 
 module.exports = {
   emptyPostcodeMessage,
-  invalidPostcodeMessage,
   mandatorySelectionMessage,
 };

app/lib/prometheus/counters.js

@@ -4,6 +4,6 @@ module.exports = {
   applicationStarts: new promClient.Counter({ help: 'The number of times the application has been started', name: 'app_starts' }),
   emptySearchLocationErrors: new promClient.Counter({ help: 'The number of empty search location errors', name: 'empty_search_location_errors' }),
   errorPageViews: new promClient.Counter({ help: 'The number of error page views', name: 'error_page_views' }),
-  outOfEnglandLocationWarnings: new promClient.Counter({ help: 'The number of out of England location warnings', name: 'out_of_england_location_warnings' }),
+  postcodeNotFoundWarnings: new promClient.Counter({ help: 'The number of postcodes that have not been found', name: 'postcode_not_found_warnings' }),
   validationLocationErrors: new promClient.Counter({ help: 'The number of location validation errors', name: 'validation_location_errors' }),
 };

app/lib/prometheus/histograms.js

@@ -3,7 +3,7 @@ const buckets = require('../constants').promHistogramBuckets;
 const promQueryLabelName = require('../constants').promQueryLabelName;
 
 module.exports = {
-  postcodesIORequest: new promClient.Histogram({ buckets, help: 'Duration histogram of postcodes.io request', name: 'postcodes_io_request_duration' }),
+  postcodeSearch: new promClient.Histogram({ buckets, help: 'Duration histogram of postcode search', name: 'postcode_search_duration' }),
   searchGetServices: new promClient.Histogram({
     buckets, help: 'Duration histogram of Search request to get Services', labelNames: [promQueryLabelName], name: 'search_get_services',
   }),

app/lib/search/postcodeSearchQueryBuilder.js

@@ -0,0 +1,9 @@
+function build(location) {
+  return {
+    search: location,
+    select: 'Latitude,Longitude,Type,Name1',
+    top: 1,
+  };
+}
+
+module.exports = build;

app/lib/search/request.js

@@ -1,15 +1,16 @@
 const rp = require('request-promise-native');
 
-const search = require('../../../config/config').search;
 const headers = require('./headers');
+const search = require('../../../config/config').search;
 
-async function request(query) {
-  return rp({
+async function request(query, path) {
+  const response = await rp({
     body: JSON.stringify(query),
     headers,
     method: 'POST',
-    url: `https://${search.host}/service-search/search?api-version=${search.version}`,
+    url: `https://${search.host}/service-search/${path}?api-version=${search.version}`,
   });
+  return JSON.parse(response);
 }
 
 module.exports = request;

app/middleware/getServices.js

@@ -1,13 +1,13 @@
 const VError = require('verror').VError;
 
-const searchRequest = require('../lib/search/request');
-const maxNumberOfResults = require('../../config/config').search.maxNumberOfResults;
-const searchServicesHistogram = require('../lib/prometheus/histograms').searchGetServices;
-const promQueryLabelName = require('../lib/constants').promQueryLabelName;
 const log = require('../lib/logger');
 const mapResults = require('../lib/mappers/mapResults');
-const queryBuilder = require('../lib/search/queryBuilder');
+const maxNumberOfResults = require('../../config/config').search.maxNumberOfResults;
+const promQueryLabelName = require('../lib/constants').promQueryLabelName;
+const queryBuilder = require('../lib/search/serviceSearchQueryBuilder');
 const queryMapper = require('../lib/utils/queryMapper');
+const searchRequest = require('../lib/search/request');
+const searchServicesHistogram = require('../lib/prometheus/histograms').searchGetServices;
 
 function handleError(error, next) {
   const errMsg = 'Error making request to Search API';
@@ -17,8 +17,7 @@ function handleError(error, next) {
   next(newError);
 }
 
-function processResults(response, searchOrigin, logResults) {
-  const results = JSON.parse(response);
+function processResults(results, searchOrigin, logResults) {
   const resultsCount = results['@odata.count'];
   logResults(resultsCount);
   return [mapResults(results, searchOrigin), resultsCount];
@@ -46,9 +45,9 @@ async function getServices(req, res, next) {
   };
 
   try {
-    const response = await searchRequest(query);
+    const results = await searchRequest(query, 'search');
     [res.locals.services, res.locals.resultsCount] = processResults(
-      response, searchOrigin, logResults
+      results, searchOrigin, logResults
     );
     next();
   } catch (error) {

app/middleware/postcodeSearch.js

@@ -0,0 +1,60 @@
+const errorCounter = require('../lib/prometheus/counters').errorPageViews;
+const log = require('../lib/logger');
+const postcodeNotFoundWarnings = require('../lib/prometheus/counters').postcodeNotFoundWarnings;
+const postcodeSearchHistogram = require('../lib/prometheus/histograms').postcodeSearch;
+const queryBuilder = require('../lib/search/postcodeSearchQueryBuilder');
+const renderer = require('./renderer');
+const searchRequest = require('../lib/search/request');
+
+function isOutcode(postcodeDetails) {
+  return postcodeDetails.Type === 'PostcodeOutCode';
+}
+
+function postcodeDetailsMapper(postcodeDetails) {
+  return {
+    isOutcode: isOutcode(postcodeDetails),
+    location: {
+      lat: postcodeDetails.Latitude,
+      lon: postcodeDetails.Longitude,
+    },
+  };
+}
+
+async function performPostcodeSearch(location) {
+  const query = queryBuilder(location);
+  const path = 'postcodesandplaces/search';
+  const response = await searchRequest(query, path);
+  log.debug({ postcodeSearchResponse: { response } }, 'Postcode Search response');
+  return response;
+}
+
+async function postcodeSearch(req, res, next) {
+  const location = res.locals.location;
+
+  log.debug({ location }, 'Postcode search text');
+  const endTimer = postcodeSearchHistogram.startTimer();
+  if (location) {
+    try {
+      const postcodeDetails = await performPostcodeSearch(location);
+
+      if (postcodeDetails && postcodeDetails.value && postcodeDetails.value.length) {
+        res.locals.postcodeLocationDetails = postcodeDetailsMapper(postcodeDetails.value[0]);
+        next();
+      } else {
+        postcodeNotFoundWarnings.inc(1);
+        renderer.postcodeNotFound(req, res);
+      }
+    } catch (error) {
+      log.debug({ location }, 'Error in postcode Search');
+      errorCounter.inc(1);
+      next(error);
+    } finally {
+      endTimer();
+    }
+  } else {
+    log.debug('No postcode');
+    next();
+  }
+}
+
+module.exports = postcodeSearch;

app/middleware/renderer.js

@@ -54,15 +54,9 @@ function emptyPostcode(req, res) {
   location(req, res);
 }
 
-function invalidPostcode(req, res, loc) {
-  log.debug({ location: loc }, 'Location failed validation');
-  res.locals.errorMessage = messages.invalidPostcodeMessage(loc);
-  location(req, res);
-}
-
-function outsideOfEngland(req, res) {
-  log.debug({ location: res.locals.location }, 'Outside of England');
-  res.locals.outsideOfEnglandPostcodeFlag = true;
+function postcodeNotFound(req, res) {
+  log.debug({ location: res.locals.location }, 'Postcode not found');
+  res.locals.postcodeNotFound = true;
   location(req, res);
 }
 
@@ -71,9 +65,8 @@ module.exports = {
   chlamydiaOnlineRedirect,
   choose,
   emptyPostcode,
-  invalidPostcode,
   location,
-  outsideOfEngland,
+  postcodeNotFound,
   recommend,
   results,
   startPage,

app/views/includes/postcodeNotFoundMessage.nunjucks

@@ -0,0 +1,3 @@
+We can't find the postcode '{{ location }}'. Please check the postcode is correct and try again.
+<br>
+Or get help to find a chlamydia test in <a href="{{ siteRoot }}/#scotland"><span class='sr-only'>find a chlamydia test in </span>Scotland</a>, <a href="{{ siteRoot }}/#wales"><span class='sr-only'>find a chlamydia test in </span>Wales</a> or <a href="{{ siteRoot }}/#northern"><span class='sr-only'>find a chlamydia test in </span>Northern Ireland</a>.

app/views/location.nunjucks

@@ -10,11 +10,11 @@
 {% endblock %}
 
 {% block content %}
-{% if errorMessage or outsideOfEnglandPostcodeFlag %}
+{% if errorMessage or postcodeNotFound %}
 <div class="error-summary">
   <h2 class="local-header--error error-summary-heading">
-    {% if (outsideOfEnglandPostcodeFlag) %}
-      {% include 'includes/outOfEnglandMessage.nunjucks' %}
+    {% if (postcodeNotFound) %}
+      {% include 'includes/postcodeNotFoundMessage.nunjucks' %}
     {% elif errorMessage %}
       {{ errorMessage }}
     {% endif %}
@@ -25,7 +25,7 @@
 <form method="get" class="form" action="{{ siteRoot }}/results">
   <div class="reading-width">
     <div class="form-item-wrapper">
-      <div class="form-group {% if errorMessage %} error {% endif %}">
+      <div class="form-group {% if errorMessage or postcodeNotFound %} error {% endif %}">
         <label class="form-label-bold" for="location">Enter a postcode in England</label>
           <p class="form-group">
             <input autocomplete="off" type="text" class="form-control" name="location" id="location" value=""{% if errorMessage %} aria-describedby="error-message" aria-invalid="true"{% endif %}>

config/config.js

@@ -17,7 +17,7 @@ module.exports = {
   },
   search: {
     apiKey: process.env.SEARCH_API_KEY,
-    host: process.env.SEARCH_API_HOST || 'nhsapiint.azure-api.net',
+    host: process.env.SEARCH_API_HOST || 'api.nhs.uk',
     maxNumberOfResults: 30,
     version: process.env.SEARCH_API_VERSION || '1',
   },

config/routes.js

@@ -2,8 +2,7 @@ const router = require('express').Router();
 
 const getServices = require('../app/middleware/getServices');
 const locationValidator = require('../app/middleware/locationValidator');
-const notInEnglandHandler = require('../app/middleware/notInEnglandHandler');
-const postcodeLookup = require('../app/middleware/postcodeLookup');
+const postcodeSearch = require('../app/middleware/postcodeSearch');
 const prerender = require('../app/middleware/prerender');
 const renderer = require('../app/middleware/renderer');
 const selectionValidatorAge = require('../app/middleware/selectionValidatorAge');
@@ -51,8 +50,7 @@ router.get(
 router.get(
   '/results',
   locationValidator,
-  postcodeLookup,
-  notInEnglandHandler,
+  postcodeSearch,
   getServices,
   prerender.results,
   renderer.results