Merge branch 'develop' into greenkeeper/request-2.87.0

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
+package-lock.json
 
 # Config
 config.development.json
@@ -23,4 +24,4 @@ persist
 public/**/*
 
 workspace/_exif/*
-workspace/_tmp/*
+workspace/_tmp/*

.npmignore

@@ -0,0 +1,8 @@
+coverage/
+domains/
+images/
+public/
+test/
+workspace/_tmp/
+log/
+cache/

.snyk

@@ -0,0 +1,10 @@
+# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
+version: v1.12.0
+ignore: {}
+# patches apply the minimum changes required to fix a vulnerability
+patch:
+  'npm:lodash:20180130':
+    - '@dadi/logger > aws-kinesis-writable > lodash':
+        patched: '2018-07-04T02:27:36.332Z'
+    - '@dadi/boot > cli-table2 > lodash':
+        patched: '2018-07-04T02:27:36.332Z'

CHANGELOG.md

@@ -4,6 +4,50 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [3.1.0] - 2018-07-04
+
+### Changed
+
+* [#394](https://github.com/dadi/cdn/pull/394): performance improvements.
+
+## [3.0.5] - 2018-06-27
+
+### Changed
+
+* work queue added, ensures if multiple requests are made for the same resource _before_ the first one has finished processing, remaining requests wait for the result of the first one _instead of requesting a new computation each time_. When the processing for the first request finishes, all waiting requests are served and the request is removed from the work queue.
+
+## [3.0.4] - 2018-06-22
+
+### Fixed
+
+* [#388](https://github.com/dadi/cdn/pull/388): fix issue where not all chunks from a remote HTTP call were passed to the calling function.
+
+## [3.0.3] - 2018-06-06
+
+### Changed
+
+* [#372](https://github.com/dadi/cdn/pull/372): add proper support for range requests, enabling seekable content such as audio and video to be served correctly.
+
+## [3.0.2] - 2018-06-04
+
+### Changed
+
+* changed `.npmignore` to correctly exclude any files and directories used for development purposes only.
+
+## [3.0.1] - 2018-06-04
+
+### Changed
+
+* [#369](https://github.com/dadi/cdn/pull/369): fix issue where `devicePixelRatio` was ignored when a resize style other than `crop` was used; change default resize style to `aspectfill` when `width`, `height` and `gravity` are supplied.
+
+## [3.0.0] - 2018-05-21
+
+Full public release of Release Candidate 4.
+
+### Breaking changes
+
+* The file upload function in CDN was recently removed. We came to the conclusion that CDN should remain a delivery tool - how a user gets their assets to their storage location for CDN to serve should be up to them. There are a great number of ways to do that, including via DADI API (see https://docs.dadi.tech/api/latest).
+
 ## [3.0.0-RC4] - 2018-05-16
 
 ### Added

README.md

@@ -1,7 +1,7 @@
-<img src="https://dadi.tech/assets/products/dadi-cdn-full.png" alt="DADI CDN" height="65"/>
+<img src="https://dadi.cloud/assets/products/dadi-cdn-full.png" alt="DADI CDN" height="65"/>
 
 [![npm (scoped)](https://img.shields.io/npm/v/@dadi/cdn.svg?maxAge=10800&style=flat-square)](https://www.npmjs.com/package/@dadi/cdn)
-[![coverage](https://img.shields.io/badge/coverage-88%25-yellow.svg?style=flat)](https://github.com/dadi/cdn)
+[![Coverage Status](https://coveralls.io/repos/github/dadi/cdn/badge.svg?branch=develop)](https://coveralls.io/github/dadi/cdn?branch=develop)
 [![Build Status](https://travis-ci.org/dadi/cdn.svg?branch=master)](https://travis-ci.org/dadi/cdn)
 [![JavaScript Style Guide](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat-square)](http://standardjs.com/)
 [![Greenkeeper badge](https://badges.greenkeeper.io/dadi/cdn.svg)](https://greenkeeper.io/)
@@ -32,7 +32,7 @@ CDN is part of DADI, a suite of components covering the full development stack,
 
 ### Install dependencies
 
-Ensure you have the required dependencies installed. See the first sections in the CDN [installation](https://docs.dadi.tech/cdn) documentation.
+Ensure you have the required dependencies installed. See the first sections in the CDN [installation](https://docs.dadi.cloud/cdn) documentation.
 
 ### Install CDN
 
@@ -68,7 +68,7 @@ With the default configuration, our CDN server is available at http://localhost:
 
 CDN requires a configuration file specific to the application environment. For example in the production environment it will look for a file named `config.production.json`.
 
-When CDN was installed, a development configuration file was created for you in a `config` folder at your application root. Full configuration documentation can be found at https://docs.dadi.tech/cdn.
+When CDN was installed, a development configuration file was created for you in a `config` folder at your application root. Full configuration documentation can be found at https://docs.dadi.cloud/cdn.
 
 
 ### Run CDN as a service
@@ -84,7 +84,7 @@ $ forever-service install -s index.js -e NODE_ENV=production cdn --start
 
 ### Configuring an image source
 
-Before you can serve assets or images you need to tell CDN where your files are located. Currently, CDN can serve your files from three types of source: [Amazon S3](https://docs.dadi.tech/cdn/#amazon-s3), [a remote server](https://docs.dadi.tech/cdn/#remote-server), and the [the local filesystem](https://docs.dadi.tech/cdn/#local-filesystem). We'll start using the local filesystem, but see the [full documentation](https://docs.dadi.tech/cdn/#defining-sources) for details on using the other source types.
+Before you can serve assets or images you need to tell CDN where your files are located. Currently, CDN can serve your files from three types of source: [Amazon S3](https://docs.dadi.cloud/cdn/#amazon-s3), [a remote server](https://docs.dadi.cloud/cdn/#remote-server), and the [the local filesystem](https://docs.dadi.cloud/cdn/#local-filesystem). We'll start using the local filesystem, but see the [full documentation](https://docs.dadi.cloud/cdn/#defining-sources) for details on using the other source types.
 
 The sample configuration file defines a local filesystem source. The `path` property is set to use an directory called `images` at the root of your application. CDN will look for your files at the location defined in this `path` property every time it handles a request.
 
@@ -123,14 +123,14 @@ drwxr-xr-x  4 root  wheel      136 13 Mar 13:01 ..
 http://127.0.0.1:8001/92875.jpg
 
 ## Links
-* [CDN Documentation](https://docs.dadi.tech/cdn)
+* [CDN Documentation](https://docs.dadi.cloud/cdn)
 
 ## Licence
 
 DADI is a data centric development and delivery stack, built specifically in support of the principles of API first and COPE.
 
 Copyright notice<br />
-(C) 2018 DADI+ Limited <support@dadi.tech><br />
+(C) 2018 DADI+ Limited <support@dadi.cloud><br />
 All rights reserved
 
 This product is part of DADI.<br />

config.js

@@ -277,7 +277,8 @@ const schema = {
       path: {
         doc: 'The remote host to request assets from, for example http://media.example.com',
         format: String,
-        default: ''
+        default: '',
+        allowDomainOverride: true
       }
     }
   },

dadi/lib/cache/index.js

@@ -133,10 +133,12 @@ Cache.prototype.isEnabled = function () {
  * @param {String} key   cache key
  * @param {[type]} value
  */
-Cache.prototype.set = function (key, value) {
+Cache.prototype.set = function (key, value, options) {
   if (!this.isEnabled()) return Promise.resolve(null)
 
-  return cache.set(key, value)
+  let encryptedKey = this.getNormalisedKey(key)
+
+  return cache.set(encryptedKey, value, options)
 }
 
 let instance

dadi/lib/controller/index.js

@@ -1,10 +1,11 @@
 const cloudfront = require('cloudfront')
-const concat = require('concat-stream')
+const compressible = require('compressible')
 const etag = require('etag')
 const fs = require('fs')
-const lengthStream = require('length-stream')
 const mime = require('mime')
 const path = require('path')
+const seek = require('./seek')
+const sha1 = require('sha1')
 const urlParser = require('url')
 const zlib = require('zlib')
 
@@ -14,13 +15,18 @@ const logger = require('@dadi/logger')
 const HandlerFactory = require(path.join(__dirname, '/../handlers/factory'))
 const RecipeController = require(path.join(__dirname, '/recipe'))
 const RouteController = require(path.join(__dirname, '/route'))
+const WorkQueue = require('./../workQueue')
 const workspace = require(path.join(__dirname, '/../models/workspace'))
 
 logger.init(config.get('logging'), config.get('logging.aws'), config.get('env'))
 
+let workQueue = new WorkQueue()
+
 const Controller = function (router) {
   router.use(logger.requestLogger)
 
+  router.use(seek)
+
   router.get('/robots.txt', (req, res) => {
     const robotsFile = config.get('robots')
 
@@ -37,68 +43,83 @@ const Controller = function (router) {
   })
 
   router.get(/(.+)/, (req, res) => {
-    const factory = new HandlerFactory(workspace.get())
+    let factory = new HandlerFactory(workspace.get())
+    let queueKey = sha1(req.__domain + req.url)
 
-    factory.create(req).then(handler => {
-      return handler.get().then(stream => {
-        this.addContentTypeHeader(res, handler)
-        this.addCacheControlHeader(res, handler, req.__domain)
-        this.addLastModifiedHeader(res, handler)
+    return workQueue.run(queueKey, () => {
+      return factory.create(req).then(handler => {
+        return handler.get().then(data => {
+          return { handler, data }
+        }).catch(err => {
+          err.__handler = handler
 
-        if (handler.storageHandler && handler.storageHandler.notFound) {
-          res.statusCode = config.get('notFound.statusCode', req.__domain) || 404
-        }
+          return Promise.reject(err)
+        })
+      })
+    }).then(({handler, data}) => {
+      this.addContentTypeHeader(res, handler)
+      this.addCacheControlHeader(res, handler, req.__domain)
+      this.addLastModifiedHeader(res, handler)
 
-        if (handler.storageHandler && handler.storageHandler.cleanUp) {
-          handler.storageHandler.cleanUp()
-        }
+      if (handler.storageHandler && handler.storageHandler.notFound) {
+        res.statusCode = config.get('notFound.statusCode', req.__domain) || 404
+      }
 
-        let contentLength = 0
+      if (handler.storageHandler && handler.storageHandler.cleanUp) {
+        handler.storageHandler.cleanUp()
+      }
 
-        // receive the concatenated buffer and send the response
-        // unless the etag hasn't changed, then send 304 and end the response
-        function sendBuffer (buffer) {
-          res.setHeader('Content-Length', contentLength)
-          res.setHeader('ETag', etag(buffer))
+      let etagResult = etag(data)
+      let contentLength = Buffer.isBuffer(data)
+        ? data.byteLength
+        : data.length
 
-          if (req.headers['if-none-match'] === etag(buffer) && handler.getContentType() !== 'application/json') {
-            res.statusCode = 304
-            res.end()
-          } else {
-            let cacheHeader = (handler.getHeader && handler.getHeader('x-cache')) ||
-              (handler.isCached ? 'HIT' : 'MISS')
+      res.setHeader('Content-Length', contentLength)
+      res.setHeader('ETag', etagResult)
 
-            res.setHeader('X-Cache', cacheHeader)
-            res.end(buffer)
-          }
-        }
+      let shouldCompress =
+        req.headers['accept-encoding'] === 'gzip' &&
+        compressible(handler.getContentType())
 
-        function lengthListener (length) {
-          contentLength = length
-        }
+      if (
+        shouldCompress &&
+        config.get('headers.useGzipCompression', req.__domain) &&
+        handler.getContentType() !== 'application/json'
+      ) {
+        res.setHeader('Content-Encoding', 'gzip')
 
-        let concatStream = concat(sendBuffer)
+        data = new Promise((resolve, reject) => {
+          zlib.gzip(data, (err, compressedData) => {
+            if (err) return reject(err)
 
-        if (
-          config.get('headers.useGzipCompression', req.__domain) &&
-          handler.getContentType() !== 'application/json'
-        ) {
-          res.setHeader('Content-Encoding', 'gzip')
+            resolve(compressedData)
+          })
+        })
+      }
 
-          let gzipStream = stream.pipe(zlib.createGzip())
-          gzipStream = gzipStream.pipe(lengthStream(lengthListener))
-          gzipStream.pipe(concatStream)
+      return Promise.resolve(data).then(data => {
+        if (req.headers.range) {
+          res.sendSeekable(data)
+        } else if (req.headers['if-none-match'] === etagResult && handler.getContentType() !== 'application/json') {
+          res.statusCode = 304
+          res.end()
         } else {
-          stream.pipe(lengthStream(lengthListener)).pipe(concatStream)
-        }
-      }).catch(err => {
-        logger.error({err: err})
+          let cacheHeader = (handler.getHeader && handler.getHeader('x-cache')) ||
+            (handler.isCached ? 'HIT' : 'MISS')
 
-        res.setHeader('X-Cache', handler.isCached ? 'HIT' : 'MISS')
-
-        help.sendBackJSON(err.statusCode || 400, err, res)
+          res.setHeader('X-Cache', cacheHeader)
+          res.end(data)
+        }
       })
     }).catch(err => {
+      logger.error({err: err})
+
+      if (err.__handler) {
+        res.setHeader('X-Cache', err.__handler.isCached ? 'HIT' : 'MISS')
+
+        delete err.__handler
+      }
+
       help.sendBackJSON(err.statusCode || 400, err, res)
     })
   })