From d11f0cd8f247f914cd0fe4fddcd214a03138930d Mon Sep 17 00:00:00 2001 From: Joshua Holbrook Date: Sun, 2 Sep 2018 18:22:11 -0400 Subject: [PATCH 1/2] Revert "Revert "Add support for brotli encoding"" --- README.md | 11 ++ lib/ecstatic.js | 62 +++++-- lib/ecstatic/defaults.json | 1 + lib/ecstatic/opts.js | 6 + test/compression.js | 187 ++++++++++++++++++++++ test/public/brotli/fake_ecstatic | 1 + test/public/brotli/fake_ecstatic.br | Bin 0 -> 11 bytes test/public/brotli/index.html | 1 + test/public/brotli/index.html.br | 3 + test/public/brotli/not_actually_brotli.br | 1 + test/public/brotli/real_ecstatic | 1 + test/public/brotli/real_ecstatic.br | 2 + 12 files changed, 264 insertions(+), 12 deletions(-) create mode 100644 test/compression.js create mode 100644 test/public/brotli/fake_ecstatic create mode 100644 test/public/brotli/fake_ecstatic.br create mode 100644 test/public/brotli/index.html create mode 100644 test/public/brotli/index.html.br create mode 100644 test/public/brotli/not_actually_brotli.br create mode 100644 test/public/brotli/real_ecstatic create mode 100644 test/public/brotli/real_ecstatic.br diff --git a/README.md b/README.md index a250245..e82247f 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ const opts = { cache: 'max-age=3600', cors: false, gzip: true, + brotli: false, defaultExt: 'html', handleError: true, serverHeader: true, @@ -209,6 +210,16 @@ that the behavior is appropriate. If `./public/some-file.js.gz` is not valid gzip, this will fall back to `./public/some-file.js`. You can turn this off with `opts.gzip === false`. +### `opts.brotli` +### `--brotli` + +Serve `./public/some-file.js.br` in place of `./public/some-file.js` when the +[brotli encoded](https://github.com/google/brotli) version exists and ecstatic +determines that the behavior is appropriate. If the request does not contain +`br` in the HTTP `accept-encoding` header, ecstatic will instead attempt to +serve a gzipped version (if `opts.gzip` is `true`), or fall back to +`./public.some-file.js`. Defaults to **false**. + ### `opts.serverHeader` ### `--no-server-header` diff --git a/lib/ecstatic.js b/lib/ecstatic.js index cf73e24..7827408 100644 --- a/lib/ecstatic.js +++ b/lib/ecstatic.js @@ -32,7 +32,7 @@ function decodePathname(pathname) { // Check to see if we should try to compress a file with gzip. -function shouldCompress(req) { +function shouldCompressGzip(req) { const headers = req.headers; return headers && headers['accept-encoding'] && @@ -42,6 +42,16 @@ function shouldCompress(req) { ; } +function shouldCompressBrotli(req) { + const headers = req.headers; + + return headers && headers['accept-encoding'] && + headers['accept-encoding'] + .split(',') + .some(el => ['*', 'br'].indexOf(el.trim()) !== -1) + ; +} + function hasGzipId12(gzipped, cb) { const stream = fs.createReadStream(gzipped, { start: 0, end: 1 }); let buffer = Buffer(''); @@ -166,7 +176,8 @@ module.exports = function createMiddleware(_dir, _options) { const parsed = url.parse(req.url); let pathname = null; let file = null; - let gzipped = null; + let gzippedFile = null; + let brotliFile = null; // Strip any null bytes from the url // This was at one point necessary because of an old bug in url.parse @@ -198,7 +209,9 @@ module.exports = function createMiddleware(_dir, _options) { path.relative(path.join('/', baseDir), pathname) ) ); - gzipped = `${file}.gz`; + // determine compressed forms if they were to exist + gzippedFile = `${file}.gz`; + brotliFile = `${file}.br`; if (serverHeader !== false) { // Set common headers. @@ -229,7 +242,7 @@ module.exports = function createMiddleware(_dir, _options) { function serve(stat) { // Do a MIME lookup, fall back to octet-stream and handle gzip - // special case. + // and brotli special case. const defaultType = opts.contentType || 'application/octet-stream'; let contentType = mime.lookup(file, defaultType); let charSet; @@ -238,7 +251,6 @@ module.exports = function createMiddleware(_dir, _options) { const etag = generateEtag(stat, weakEtags); let cacheControl = cache; let stream = null; - if (contentType) { charSet = mime.charsets.lookup(contentType, 'utf-8'); if (charSet) { @@ -246,11 +258,14 @@ module.exports = function createMiddleware(_dir, _options) { } } - if (file === gzipped) { // is .gz picked up + if (file === gzippedFile) { // is .gz picked up res.setHeader('Content-Encoding', 'gzip'); - // strip gz ending and lookup mime type contentType = mime.lookup(path.basename(file, '.gz'), defaultType); + } else if (file === brotliFile) { // is .br picked up + res.setHeader('Content-Encoding', 'br'); + // strip br ending and lookup mime type + contentType = mime.lookup(path.basename(file, '.br'), defaultType); } if (typeof cacheControl === 'function') { @@ -401,13 +416,13 @@ module.exports = function createMiddleware(_dir, _options) { }); } - // Look for a gzipped file if this is turned on - if (opts.gzip && shouldCompress(req)) { - fs.stat(gzipped, (err, stat) => { + // serve gzip file if exists and is valid + function tryServeWithGzip() { + fs.stat(gzippedFile, (err, stat) => { if (!err && stat.isFile()) { - hasGzipId12(gzipped, (gzipErr, isGzip) => { + hasGzipId12(gzippedFile, (gzipErr, isGzip) => { if (!gzipErr && isGzip) { - file = gzipped; + file = gzippedFile; serve(stat); } else { statFile(); @@ -417,6 +432,29 @@ module.exports = function createMiddleware(_dir, _options) { statFile(); } }); + } + + // serve brotli file if exists, otherwise try gzip + function tryServeWithBrotli(shouldTryGzip) { + fs.stat(brotliFile, (err, stat) => { + if (!err && stat.isFile()) { + file = brotliFile; + serve(stat); + } else if (shouldTryGzip) { + tryServeWithGzip(); + } else { + statFile(); + } + }); + } + + const shouldTryBrotli = opts.brotli && shouldCompressBrotli(req); + const shouldTryGzip = opts.gzip && shouldCompressGzip(req); + // always try brotli first, next try gzip, finally serve without compression + if (shouldTryBrotli) { + tryServeWithBrotli(shouldTryGzip); + } else if (shouldTryGzip) { + tryServeWithGzip(); } else { statFile(); } diff --git a/lib/ecstatic/defaults.json b/lib/ecstatic/defaults.json index df222be..d890da8 100644 --- a/lib/ecstatic/defaults.json +++ b/lib/ecstatic/defaults.json @@ -8,6 +8,7 @@ "cache": "max-age=3600", "cors": false, "gzip": true, + "brotli": false, "defaultExt": ".html", "handleError": true, "serverHeader": true, diff --git a/lib/ecstatic/opts.js b/lib/ecstatic/opts.js index 4e6a8d4..d4b170d 100644 --- a/lib/ecstatic/opts.js +++ b/lib/ecstatic/opts.js @@ -14,6 +14,7 @@ module.exports = (opts) => { let si = defaults.si; let cache = defaults.cache; let gzip = defaults.gzip; + let brotli = defaults.brotli; let defaultExt = defaults.defaultExt; let handleError = defaults.handleError; const headers = {}; @@ -105,6 +106,10 @@ module.exports = (opts) => { gzip = opts.gzip; } + if (typeof opts.brotli !== 'undefined' && opts.brotli !== null) { + brotli = opts.brotli; + } + aliases.handleError.some((k) => { if (isDeclared(k)) { handleError = opts[k]; @@ -195,6 +200,7 @@ module.exports = (opts) => { defaultExt, baseDir: (opts && opts.baseDir) || '/', gzip, + brotli, handleError, headers, serverHeader, diff --git a/test/compression.js b/test/compression.js new file mode 100644 index 0000000..279512b --- /dev/null +++ b/test/compression.js @@ -0,0 +1,187 @@ +'use strict'; + +const test = require('tap').test; +const ecstatic = require('../lib/ecstatic'); +const http = require('http'); +const request = require('request'); + +const root = `${__dirname}/public`; + +test('serves brotli-encoded file when available', (t) => { + t.plan(3); + + const server = http.createServer(ecstatic({ + root, + brotli: true, + autoIndex: true + })); + + server.listen(() => { + const port = server.address().port; + const options = { + uri: `http://localhost:${port}/brotli`, + headers: { + 'accept-encoding': 'gzip, deflate, br' + } + }; + + request.get(options, (err, res) => { + t.ifError(err); + t.equal(res.statusCode, 200); + t.equal(res.headers['content-encoding'], 'br'); + }); + }); + t.once('end', () => { + server.close(); + }); +}); + +test('serves gzip-encoded file when brotli not available', (t) => { + t.plan(3); + + const server = http.createServer(ecstatic({ + root, + brotli: true, + gzip: true, + autoIndex: true + })); + + server.listen(() => { + const port = server.address().port; + const options = { + uri: `http://localhost:${port}/gzip`, + headers: { + 'accept-encoding': 'gzip, deflate, br' + } + }; + + request.get(options, (err, res) => { + t.ifError(err); + t.equal(res.statusCode, 200); + t.equal(res.headers['content-encoding'], 'gzip'); + }); + }); + t.once('end', () => { + server.close(); + }); +}); + +test('serves gzip-encoded file when brotli not accepted', (t) => { + t.plan(3); + + const server = http.createServer(ecstatic({ + root, + brotli: true, + gzip: true, + autoIndex: true + })); + + server.listen(() => { + const port = server.address().port; + const options = { + uri: `http://localhost:${port}/brotli`, + headers: { + 'accept-encoding': 'gzip, deflate' + } + }; + + request.get(options, (err, res) => { + t.ifError(err); + t.equal(res.statusCode, 200); + t.equal(res.headers['content-encoding'], 'gzip'); + }); + }); + t.once('end', () => { + server.close(); + }); +}); + +test('serves gzip-encoded file when brotli not enabled', (t) => { + t.plan(3); + + const server = http.createServer(ecstatic({ + root, + brotli: false, + gzip: true, + autoIndex: true + })); + + server.listen(() => { + const port = server.address().port; + const options = { + uri: `http://localhost:${port}/brotli`, + headers: { + 'accept-encoding': 'gzip, deflate, br' + } + }; + + request.get(options, (err, res) => { + t.ifError(err); + t.equal(res.statusCode, 200); + t.equal(res.headers['content-encoding'], 'gzip'); + }); + }); + t.once('end', () => { + server.close(); + }); +}); + +test('serves unencoded file when compression not accepted', (t) => { + t.plan(3); + + const server = http.createServer(ecstatic({ + root, + brotli: true, + gzip: true, + autoIndex: true + })); + + server.listen(() => { + const port = server.address().port; + const options = { + uri: `http://localhost:${port}/brotli`, + headers: { + 'accept-encoding': '' + } + }; + + request.get(options, (err, res) => { + t.ifError(err); + t.equal(res.statusCode, 200); + t.equal(res.headers['content-encoding'], undefined); + }); + }); + t.once('end', () => { + server.close(); + }); +}); + +test('serves unencoded file when compression not enabled', (t) => { + t.plan(3); + + const server = http.createServer(ecstatic({ + root, + brotli: false, + gzip: false, + autoIndex: true + })); + + server.listen(() => { + const port = server.address().port; + const options = { + uri: `http://localhost:${port}/brotli`, + headers: { + 'accept-encoding': 'gzip, deflate, br' + } + }; + + request.get(options, (err, res) => { + t.ifError(err); + t.equal(res.statusCode, 200); + t.equal(res.headers['content-encoding'], undefined); + }); + }); + t.once('end', () => { + server.close(); + }); +}); diff --git a/test/public/brotli/fake_ecstatic b/test/public/brotli/fake_ecstatic new file mode 100644 index 0000000..f99e840 --- /dev/null +++ b/test/public/brotli/fake_ecstatic @@ -0,0 +1 @@ +ecstatic \ No newline at end of file diff --git a/test/public/brotli/fake_ecstatic.br b/test/public/brotli/fake_ecstatic.br new file mode 100644 index 0000000000000000000000000000000000000000..2914d81823ec424e3bcaa6f7ebf668a8d948de1b GIT binary patch literal 11 Scmb1UXJA-z>{QdF6R!Xi#04Dy literal 0 HcmV?d00001 diff --git a/test/public/brotli/index.html b/test/public/brotli/index.html new file mode 100644 index 0000000..1b52eea --- /dev/null +++ b/test/public/brotli/index.html @@ -0,0 +1 @@ +brotli, but I'm not compressed!!! diff --git a/test/public/brotli/index.html.br b/test/public/brotli/index.html.br new file mode 100644 index 0000000..2567ab9 --- /dev/null +++ b/test/public/brotli/index.html.br @@ -0,0 +1,3 @@ + +€brotli, compressed!! + \ No newline at end of file diff --git a/test/public/brotli/not_actually_brotli.br b/test/public/brotli/not_actually_brotli.br new file mode 100644 index 0000000..cad943c --- /dev/null +++ b/test/public/brotli/not_actually_brotli.br @@ -0,0 +1 @@ +You've been duped! This is not compressed! diff --git a/test/public/brotli/real_ecstatic b/test/public/brotli/real_ecstatic new file mode 100644 index 0000000..0389ee1 --- /dev/null +++ b/test/public/brotli/real_ecstatic @@ -0,0 +1 @@ +ecstatic diff --git a/test/public/brotli/real_ecstatic.br b/test/public/brotli/real_ecstatic.br new file mode 100644 index 0000000..750f429 --- /dev/null +++ b/test/public/brotli/real_ecstatic.br @@ -0,0 +1,2 @@ + €ecstatic + \ No newline at end of file From b877d5298d67a30b684e3152c7d934c2dac18022 Mon Sep 17 00:00:00 2001 From: Jade Thornton Date: Sun, 2 Sep 2018 17:37:22 -0500 Subject: [PATCH 2/2] add gzip compression test files --- .gitignore | 1 - test/public/brotli/index.html.gz | Bin 0 -> 50 bytes 2 files changed, 1 deletion(-) create mode 100644 test/public/brotli/index.html.gz diff --git a/.gitignore b/.gitignore index 9169638..a37059e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ coverage *.dat *.out *.pid -*.gz pids logs diff --git a/test/public/brotli/index.html.gz b/test/public/brotli/index.html.gz new file mode 100644 index 0000000000000000000000000000000000000000..f7a6fde738ec589a6e5b3dfa11ae270d648ee2ca GIT binary patch literal 50 zcmb2|=HQsvRT9m>oSB!BTA`OwlAFWuIYi$()Wgg7Y_P}rQ#u+RzJX7e7