diff --git a/README.md b/README.md index 0ffbbc1f..6366b349 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ This will install `http-server` globally so that it may be run from the command |`--log-ip` |Enable logging of the client's IP address |`false` | |`-P` or `--proxy` |Proxies all requests which can't be resolved locally to the given url. e.g.: -P http://someurl.com | | |`--proxy-options` |Pass proxy [options](https://github.com/http-party/node-http-proxy#options) using nested dotted objects. e.g.: --proxy-options.secure false | +|`--proxy-cache` |Enable disk caching of proxy responses to specified folder | | |`--username` |Username for basic authentication | | |`--password` |Password for basic authentication | | |`-S`, `--tls` or `--ssl` |Enable secure request serving with TLS/SSL (HTTPS)|`false`| diff --git a/bin/http-server b/bin/http-server index 7c597fa8..2e491c7c 100755 --- a/bin/http-server +++ b/bin/http-server @@ -10,6 +10,7 @@ var chalk = require('chalk'), fs = require('fs'), url = require('url'); +const {getPort} = require("../lib/core/get-port.js"); var argv = require('minimist')(process.argv.slice(2), { alias: { tls: 'ssl' @@ -46,6 +47,7 @@ if (argv.h || argv.help) { '', ' -P --proxy Fallback proxy if the request cannot be resolved. e.g.: http://someurl.com', ' --proxy-options Pass options to proxy using nested dotted objects. e.g.: --proxy-options.secure false', + ' --proxy-cache Enable disk caching of proxy responses to specified folder', '', ' --username Username for basic authentication [none]', ' Can also be specified with the env variable NODE_HTTP_SERVER_USERNAME', @@ -71,6 +73,7 @@ var port = argv.p || argv.port || parseInt(process.env.PORT, 10), sslPassphrase = process.env.NODE_HTTP_SERVER_SSL_PASSPHRASE, proxy = argv.P || argv.proxy, proxyOptions = argv['proxy-options'], + proxyCache = argv['proxy-cache'], utc = argv.U || argv.utc, version = argv.v || argv.version, logger; @@ -126,11 +129,7 @@ if (version) { } if (!port) { - portfinder.basePort = 8080; - portfinder.getPort(function (err, port) { - if (err) { throw err; } - listen(port); - }); + getPort(8080).then(listen) } else { listen(port); @@ -150,6 +149,7 @@ function listen(port) { logFn: logger.request, proxy: proxy, proxyOptions: proxyOptions, + proxyCache: proxyCache, showDotfiles: argv.dotfiles, mimetypes: argv.mimetypes, username: argv.username || process.env.NODE_HTTP_SERVER_USERNAME, @@ -240,6 +240,7 @@ function listen(port) { else { logger.info('Unhandled requests will be served from: ' + proxy); } + logger.info('And will cached to: ' + proxyCache); } logger.info('Hit CTRL-C to stop the server'); diff --git a/doc/http-server.1 b/doc/http-server.1 index 8e2796e6..426ab619 100644 --- a/doc/http-server.1 +++ b/doc/http-server.1 @@ -89,6 +89,10 @@ Fallback proxy if the request cannot be resolved. .BI \-\-proxy\-options Pass proxy options using nested dotted objects. +.TP +.BI \-\-proxy\-cache +Enable disk caching of proxy responses to specified folder. + .TP .BI \-\-username " " \fIUSERNAME\fR Username for basic authentication. diff --git a/lib/core/get-port.js b/lib/core/get-port.js new file mode 100644 index 00000000..0ac9f62e --- /dev/null +++ b/lib/core/get-port.js @@ -0,0 +1,14 @@ +const portfinder = require('portfinder'); + +exports.getPort = function (basePort) { + return new Promise((resolve, reject) => { + portfinder.basePort = basePort || 8080; + portfinder.getPort(function (err, port) { + if (err) { + reject(err); + } else { + resolve(port); + } + }); + }); +} \ No newline at end of file diff --git a/lib/http-server.js b/lib/http-server.js index dfe4c474..f47ff58d 100644 --- a/lib/http-server.js +++ b/lib/http-server.js @@ -1,6 +1,8 @@ 'use strict'; var fs = require('fs'), + path = require('path'), + zlib = require('zlib'), union = require('union'), httpServerCore = require('./core'), auth = require('basic-auth'), @@ -142,6 +144,50 @@ function HttpServer(options) { if (typeof options.proxy === 'string') { var proxyOptions = options.proxyOptions || {}; var proxy = httpProxy.createProxyServer(proxyOptions); + + if (typeof options.proxyCache === 'string') { + var proxyCache = options.proxyCache; + var proxyCacheLogFn = options.proxyCacheLogFn || function () {}; + proxy.on('proxyRes', async function (proxyRes, req, res) { + var localFile = path.isAbsolute(proxyCache) ? proxyCache : path.join(process.cwd(), proxyCache, req.url.split('?')[0]); + var localDir = path.dirname(localFile); + if (proxyRes.statusCode !== 200) { + proxyCacheLogFn({ + status: proxyRes.statusCode, + message: proxyRes.statusMessage + }, proxyRes, req, res, localFile); + return; + } + var contentEncoding = proxyRes.headers['content-encoding']; + if (!fs.existsSync(localDir)) { + fs.mkdirSync(localDir, { recursive: true }); + } + + await new Promise((resolve, reject) => { + var stream = fs.createWriteStream(localFile); + + if (contentEncoding === 'gzip' || contentEncoding === 'deflate') { + proxyRes.pipe(zlib.createGunzip()).pipe(stream); + } else if (contentEncoding === 'br') { + proxyRes.pipe(zlib.createBrotliDecompress()).pipe(stream); + } else { + proxyRes.pipe(stream); + } + + stream.on('finish', () => { + proxyCacheLogFn(null, proxyRes, req, res, localFile); + resolve(); + }); + stream.on('error', (err) => { + proxyCacheLogFn({ + status: proxyRes.statusCode, + message: err.message + }, proxyRes, req, res, localFile); + reject(err); + }); + }); + }); + } before.push(function (req, res) { proxy.web(req, res, { target: options.proxy, diff --git a/test/proxy-cache.test.js b/test/proxy-cache.test.js new file mode 100644 index 00000000..5f9b9827 --- /dev/null +++ b/test/proxy-cache.test.js @@ -0,0 +1,210 @@ +const test = require('tap').test +const promisify = require('util').promisify +const httpServer = require('../lib/http-server') +const request = require('request') +const fs = require("fs"); +const path = require("path"); +const {getPort} = require("../lib/core/get-port.js"); +const requestAsync = promisify(request) +const fsReadFile = promisify(fs.readFile) + +// Prevent errors from being swallowed +process.on('uncaughtException', console.error) + +test('cache response when configured', (t) => { + t.plan(4); + new Promise((resolve) => { + const remoteServerRoot = path.join(__dirname, 'fixtures', 'root'); + const serverRoot = path.join(__dirname, 'cache'); + const cachedFilePath = path.join(serverRoot, 'file'); + const remoteFilePath = path.join(remoteServerRoot, 'file'); + + const remoteServer = httpServer.createServer({ + root: remoteServerRoot, + }); + + getPort().then(remotePort => { + remoteServer.listen(remotePort, async () => { + try { + const serverWithCache = httpServer.createServer({ + root: serverRoot, + proxy: `http://localhost:${remotePort}`, + proxyCache: './', + }); + + const serverPort = await getPort(); + + await new Promise((resolve) => { + serverWithCache.listen(serverPort, async () => { + try { + await requestAsync(`http://localhost:${serverPort}/file`).then(async (res) => { + t.ok(res) + t.equal(res.statusCode, 200); + + const cachedFile = await fsReadFile(cachedFilePath, 'utf8'); + const remoteFile = await fsReadFile(remoteFilePath, 'utf8'); + + t.equal(res.body.trim(), cachedFile.trim(), 'cached file content matches'); + t.equal(cachedFile.trim(), remoteFile.trim(), 'cached file content matches remote file content'); + }) + } catch (err) { + t.fail(err.toString()) + } finally { + fs.rmSync(cachedFilePath); + serverWithCache.close(); + resolve(); + } + }); + }); + } catch (err) { + t.fail(err.toString()) + } finally { + remoteServer.close(); + resolve(); + } + }); + }) + + }) + .then(() => t.end()) + .catch(err => { + t.fail(err.toString()) + t.end() + }) +}); + +test('decompress gzipped response before caching', (t) => { + new Promise(resolve => { + const remoteServerRoot = path.join(__dirname, 'public', 'gzip'); + const serverRoot = path.join(__dirname, 'cache'); + + const remoteFilePath = path.join(remoteServerRoot, 'real_ecstatic'); + const cachedFilePath = path.join(serverRoot, 'real_ecstatic'); + + + const remoteServer = httpServer.createServer({ + root: remoteServerRoot, + gzip: true, + }); + + getPort().then(remotePort => { + remoteServer.listen(remotePort, async () => { + try { + const serverWithCache = httpServer.createServer({ + root: serverRoot, + proxy: `http://localhost:${remotePort}`, + proxyCache: './', + }); + + const serverPort = await getPort(); + await new Promise((resolve) => { + serverWithCache.listen(serverPort, async () => { + try { + await requestAsync({ + uri: `http://localhost:${serverPort}/real_ecstatic`, + headers: { + 'accept-encoding': 'gzip, deflate, br' + } + }).then(async (res) => { + t.ok(res) + t.equal(res.statusCode, 200, 'response is 200'); + t.equal(res.headers['content-encoding'], 'gzip', 'response is gzipped'); + + const cachedFile = await fsReadFile(cachedFilePath, 'utf8'); + const remoteFile = await fsReadFile(remoteFilePath, 'utf8'); + + t.equal(cachedFile.trim(), remoteFile.trim(), 'cached file content matches remote file content'); + }) + } catch (err) { + t.fail(err.toString()) + } finally { + fs.rmSync(cachedFilePath); + serverWithCache.close(); + resolve(); + } + }); + }); + } catch (err) { + t.fail(err.toString()) + } finally { + remoteServer.close(); + resolve(); + } + + }); + }) + }) + .then(() => t.end()) + .catch(err => { + t.fail(err.toString()) + t.end() + }) +}); + +test('decompress brotli response before caching', (t) => { + new Promise(resolve => { + const remoteServerRoot = path.join(__dirname, 'public', 'brotli'); + const serverRoot = path.join(__dirname, 'cache'); + + const remoteFilePath = path.join(remoteServerRoot, 'real_ecstatic'); + const cachedFilePath = path.join(serverRoot, 'real_ecstatic'); + + + const remoteServer = httpServer.createServer({ + root: remoteServerRoot, + brotli: true, + }); + + getPort().then(remotePort => { + remoteServer.listen(remotePort, async () => { + try { + const serverWithCache = httpServer.createServer({ + root: serverRoot, + proxy: `http://localhost:${remotePort}`, + proxyCache: './', + }); + + const serverPort = await getPort(); + await new Promise((resolve) => { + serverWithCache.listen(serverPort, async () => { + try { + await requestAsync({ + uri: `http://localhost:${serverPort}/real_ecstatic`, + headers: { + 'accept-encoding': 'gzip, deflate, br' + } + }).then(async (res) => { + t.ok(res) + t.equal(res.statusCode, 200, 'response is 200'); + t.equal(res.headers['content-encoding'], 'br', 'response is brotli'); + + const cachedFile = await fsReadFile(cachedFilePath, 'utf8'); + const remoteFile = await fsReadFile(remoteFilePath, 'utf8'); + + t.equal(cachedFile.trim(), remoteFile.trim(), 'cached file content matches remote file content'); + }) + } catch (err) { + t.fail(err.toString()) + } finally { + fs.rmSync(cachedFilePath); + serverWithCache.close(); + resolve(); + } + }); + }); + } catch (err) { + t.fail(err.toString()) + } finally { + remoteServer.close(); + resolve(); + } + + }); + }) + }) + .then(() => t.end()) + .catch(err => { + t.fail(err.toString()) + t.end() + }) +});