From 3594db7e5eed9cf382c433a5eb09a258fd6fcbe8 Mon Sep 17 00:00:00 2001 From: happy wang Date: Tue, 17 Oct 2023 15:19:45 +0800 Subject: [PATCH 01/12] Implement a new proxyCache configuration options in HttpServer --- lib/http-server.js | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/lib/http-server.js b/lib/http-server.js index dfe4c474..2d1e1e56 100644 --- a/lib/http-server.js +++ b/lib/http-server.js @@ -1,6 +1,8 @@ 'use strict'; var fs = require('fs'), + path = require('path'), + zlib = require('zlib'), union = require('union'), httpServerCore = require('./core'), auth = require('basic-auth'), @@ -142,6 +144,33 @@ function HttpServer(options) { if (typeof options.proxy === 'string') { var proxyOptions = options.proxyOptions || {}; var proxy = httpProxy.createProxyServer(proxyOptions); + var localRoot = this.root; + + if (typeof options.proxyCache === 'string') { + var proxyCache = options.proxyCache; + proxy.on('proxyRes', async function (proxyRes, req, res) { + var localFile = path.join(localRoot, proxyCache, req.url); + var localDir = path.dirname(localFile); + var contentEncoding = proxyRes.headers['content-encoding']; + if (!fs.existsSync(localDir)) { + fs.mkdirSync(localDir, { recursive: true }); + } + + await new Promise((resolve, reject) => { + var stream = fs.createWriteStream(localFile); + + if (contentEncoding === 'gzip' || contentEncoding === 'deflate') { + proxyRes.pipe(zlib.createGunzip()).pipe(stream); + } else if (contentEncoding === 'br') { + proxyRes.pipe(zlib.createBrotliDecompress()).pipe(stream); + } else { + proxyRes.pipe(stream); + } + + stream.on('finish', resolve); + }); + }); + } before.push(function (req, res) { proxy.web(req, res, { target: options.proxy, From f60091ce5f4549f034820d8ab2ca7dd171612ad2 Mon Sep 17 00:00:00 2001 From: happy wang Date: Tue, 17 Oct 2023 15:21:36 +0800 Subject: [PATCH 02/12] Create a promised-based version of the getPort() function --- bin/http-server | 7 ++----- lib/core/get-port.js | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 lib/core/get-port.js diff --git a/bin/http-server b/bin/http-server index 7c597fa8..7751b44c 100755 --- a/bin/http-server +++ b/bin/http-server @@ -10,6 +10,7 @@ var chalk = require('chalk'), fs = require('fs'), url = require('url'); +const {getPort} = require("../lib/core/get-port.js"); var argv = require('minimist')(process.argv.slice(2), { alias: { tls: 'ssl' @@ -126,11 +127,7 @@ if (version) { } if (!port) { - portfinder.basePort = 8080; - portfinder.getPort(function (err, port) { - if (err) { throw err; } - listen(port); - }); + getPort(8080).then(listen) } else { listen(port); diff --git a/lib/core/get-port.js b/lib/core/get-port.js new file mode 100644 index 00000000..0ac9f62e --- /dev/null +++ b/lib/core/get-port.js @@ -0,0 +1,14 @@ +const portfinder = require('portfinder'); + +exports.getPort = function (basePort) { + return new Promise((resolve, reject) => { + portfinder.basePort = basePort || 8080; + portfinder.getPort(function (err, port) { + if (err) { + reject(err); + } else { + resolve(port); + } + }); + }); +} \ No newline at end of file From 9ebc64e66ef94920992e17e2ae79481c61568ecc Mon Sep 17 00:00:00 2001 From: happy wang Date: Tue, 17 Oct 2023 15:25:33 +0800 Subject: [PATCH 03/12] Add a --proxy-cache option for the http-server command --- bin/http-server | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/http-server b/bin/http-server index 7751b44c..2e491c7c 100755 --- a/bin/http-server +++ b/bin/http-server @@ -47,6 +47,7 @@ if (argv.h || argv.help) { '', ' -P --proxy Fallback proxy if the request cannot be resolved. e.g.: http://someurl.com', ' --proxy-options Pass options to proxy using nested dotted objects. e.g.: --proxy-options.secure false', + ' --proxy-cache Enable disk caching of proxy responses to specified folder', '', ' --username Username for basic authentication [none]', ' Can also be specified with the env variable NODE_HTTP_SERVER_USERNAME', @@ -72,6 +73,7 @@ var port = argv.p || argv.port || parseInt(process.env.PORT, 10), sslPassphrase = process.env.NODE_HTTP_SERVER_SSL_PASSPHRASE, proxy = argv.P || argv.proxy, proxyOptions = argv['proxy-options'], + proxyCache = argv['proxy-cache'], utc = argv.U || argv.utc, version = argv.v || argv.version, logger; @@ -147,6 +149,7 @@ function listen(port) { logFn: logger.request, proxy: proxy, proxyOptions: proxyOptions, + proxyCache: proxyCache, showDotfiles: argv.dotfiles, mimetypes: argv.mimetypes, username: argv.username || process.env.NODE_HTTP_SERVER_USERNAME, @@ -237,6 +240,7 @@ function listen(port) { else { logger.info('Unhandled requests will be served from: ' + proxy); } + logger.info('And will cached to: ' + proxyCache); } logger.info('Hit CTRL-C to stop the server'); From a49fe29ece9d3f1bf5afc50e22748cd0a9aa9a95 Mon Sep 17 00:00:00 2001 From: happy wang Date: Tue, 17 Oct 2023 15:30:23 +0800 Subject: [PATCH 04/12] Add unit tests covering the proxyCache feature --- test/proxy-cache.test.js | 210 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 test/proxy-cache.test.js diff --git a/test/proxy-cache.test.js b/test/proxy-cache.test.js new file mode 100644 index 00000000..5f9b9827 --- /dev/null +++ b/test/proxy-cache.test.js @@ -0,0 +1,210 @@ +const test = require('tap').test +const promisify = require('util').promisify +const httpServer = require('../lib/http-server') +const request = require('request') +const fs = require("fs"); +const path = require("path"); +const {getPort} = require("../lib/core/get-port.js"); +const requestAsync = promisify(request) +const fsReadFile = promisify(fs.readFile) + +// Prevent errors from being swallowed +process.on('uncaughtException', console.error) + +test('cache response when configured', (t) => { + t.plan(4); + new Promise((resolve) => { + const remoteServerRoot = path.join(__dirname, 'fixtures', 'root'); + const serverRoot = path.join(__dirname, 'cache'); + const cachedFilePath = path.join(serverRoot, 'file'); + const remoteFilePath = path.join(remoteServerRoot, 'file'); + + const remoteServer = httpServer.createServer({ + root: remoteServerRoot, + }); + + getPort().then(remotePort => { + remoteServer.listen(remotePort, async () => { + try { + const serverWithCache = httpServer.createServer({ + root: serverRoot, + proxy: `http://localhost:${remotePort}`, + proxyCache: './', + }); + + const serverPort = await getPort(); + + await new Promise((resolve) => { + serverWithCache.listen(serverPort, async () => { + try { + await requestAsync(`http://localhost:${serverPort}/file`).then(async (res) => { + t.ok(res) + t.equal(res.statusCode, 200); + + const cachedFile = await fsReadFile(cachedFilePath, 'utf8'); + const remoteFile = await fsReadFile(remoteFilePath, 'utf8'); + + t.equal(res.body.trim(), cachedFile.trim(), 'cached file content matches'); + t.equal(cachedFile.trim(), remoteFile.trim(), 'cached file content matches remote file content'); + }) + } catch (err) { + t.fail(err.toString()) + } finally { + fs.rmSync(cachedFilePath); + serverWithCache.close(); + resolve(); + } + }); + }); + } catch (err) { + t.fail(err.toString()) + } finally { + remoteServer.close(); + resolve(); + } + }); + }) + + }) + .then(() => t.end()) + .catch(err => { + t.fail(err.toString()) + t.end() + }) +}); + +test('decompress gzipped response before caching', (t) => { + new Promise(resolve => { + const remoteServerRoot = path.join(__dirname, 'public', 'gzip'); + const serverRoot = path.join(__dirname, 'cache'); + + const remoteFilePath = path.join(remoteServerRoot, 'real_ecstatic'); + const cachedFilePath = path.join(serverRoot, 'real_ecstatic'); + + + const remoteServer = httpServer.createServer({ + root: remoteServerRoot, + gzip: true, + }); + + getPort().then(remotePort => { + remoteServer.listen(remotePort, async () => { + try { + const serverWithCache = httpServer.createServer({ + root: serverRoot, + proxy: `http://localhost:${remotePort}`, + proxyCache: './', + }); + + const serverPort = await getPort(); + await new Promise((resolve) => { + serverWithCache.listen(serverPort, async () => { + try { + await requestAsync({ + uri: `http://localhost:${serverPort}/real_ecstatic`, + headers: { + 'accept-encoding': 'gzip, deflate, br' + } + }).then(async (res) => { + t.ok(res) + t.equal(res.statusCode, 200, 'response is 200'); + t.equal(res.headers['content-encoding'], 'gzip', 'response is gzipped'); + + const cachedFile = await fsReadFile(cachedFilePath, 'utf8'); + const remoteFile = await fsReadFile(remoteFilePath, 'utf8'); + + t.equal(cachedFile.trim(), remoteFile.trim(), 'cached file content matches remote file content'); + }) + } catch (err) { + t.fail(err.toString()) + } finally { + fs.rmSync(cachedFilePath); + serverWithCache.close(); + resolve(); + } + }); + }); + } catch (err) { + t.fail(err.toString()) + } finally { + remoteServer.close(); + resolve(); + } + + }); + }) + }) + .then(() => t.end()) + .catch(err => { + t.fail(err.toString()) + t.end() + }) +}); + +test('decompress brotli response before caching', (t) => { + new Promise(resolve => { + const remoteServerRoot = path.join(__dirname, 'public', 'brotli'); + const serverRoot = path.join(__dirname, 'cache'); + + const remoteFilePath = path.join(remoteServerRoot, 'real_ecstatic'); + const cachedFilePath = path.join(serverRoot, 'real_ecstatic'); + + + const remoteServer = httpServer.createServer({ + root: remoteServerRoot, + brotli: true, + }); + + getPort().then(remotePort => { + remoteServer.listen(remotePort, async () => { + try { + const serverWithCache = httpServer.createServer({ + root: serverRoot, + proxy: `http://localhost:${remotePort}`, + proxyCache: './', + }); + + const serverPort = await getPort(); + await new Promise((resolve) => { + serverWithCache.listen(serverPort, async () => { + try { + await requestAsync({ + uri: `http://localhost:${serverPort}/real_ecstatic`, + headers: { + 'accept-encoding': 'gzip, deflate, br' + } + }).then(async (res) => { + t.ok(res) + t.equal(res.statusCode, 200, 'response is 200'); + t.equal(res.headers['content-encoding'], 'br', 'response is brotli'); + + const cachedFile = await fsReadFile(cachedFilePath, 'utf8'); + const remoteFile = await fsReadFile(remoteFilePath, 'utf8'); + + t.equal(cachedFile.trim(), remoteFile.trim(), 'cached file content matches remote file content'); + }) + } catch (err) { + t.fail(err.toString()) + } finally { + fs.rmSync(cachedFilePath); + serverWithCache.close(); + resolve(); + } + }); + }); + } catch (err) { + t.fail(err.toString()) + } finally { + remoteServer.close(); + resolve(); + } + + }); + }) + }) + .then(() => t.end()) + .catch(err => { + t.fail(err.toString()) + t.end() + }) +}); From 684e305f51e735fd38a91aee7c5fa588a4416c8d Mon Sep 17 00:00:00 2001 From: happy wang Date: Tue, 17 Oct 2023 15:42:53 +0800 Subject: [PATCH 05/12] Add description for --proxy-cache option in README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0ffbbc1f..6366b349 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ This will install `http-server` globally so that it may be run from the command |`--log-ip` |Enable logging of the client's IP address |`false` | |`-P` or `--proxy` |Proxies all requests which can't be resolved locally to the given url. e.g.: -P http://someurl.com | | |`--proxy-options` |Pass proxy [options](https://github.com/http-party/node-http-proxy#options) using nested dotted objects. e.g.: --proxy-options.secure false | +|`--proxy-cache` |Enable disk caching of proxy responses to specified folder | | |`--username` |Username for basic authentication | | |`--password` |Password for basic authentication | | |`-S`, `--tls` or `--ssl` |Enable secure request serving with TLS/SSL (HTTPS)|`false`| From 8250e55bacc9d6cccf645ea03c3c7d78da222422 Mon Sep 17 00:00:00 2001 From: happy wang Date: Tue, 17 Oct 2023 15:53:34 +0800 Subject: [PATCH 06/12] Add description for --proxy-cache option in doc/http-server.1 --- doc/http-server.1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/http-server.1 b/doc/http-server.1 index 8e2796e6..426ab619 100644 --- a/doc/http-server.1 +++ b/doc/http-server.1 @@ -89,6 +89,10 @@ Fallback proxy if the request cannot be resolved. .BI \-\-proxy\-options Pass proxy options using nested dotted objects. +.TP +.BI \-\-proxy\-cache +Enable disk caching of proxy responses to specified folder. + .TP .BI \-\-username " " \fIUSERNAME\fR Username for basic authentication. From 77898f5091430e23c783e88dbfc19d39845cdd3b Mon Sep 17 00:00:00 2001 From: happy wang Date: Wed, 18 Oct 2023 15:04:47 +0800 Subject: [PATCH 07/12] Refactor the handling logic for the proxyCache --- lib/http-server.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/http-server.js b/lib/http-server.js index 2d1e1e56..5cba80ec 100644 --- a/lib/http-server.js +++ b/lib/http-server.js @@ -144,12 +144,11 @@ function HttpServer(options) { if (typeof options.proxy === 'string') { var proxyOptions = options.proxyOptions || {}; var proxy = httpProxy.createProxyServer(proxyOptions); - var localRoot = this.root; if (typeof options.proxyCache === 'string') { var proxyCache = options.proxyCache; proxy.on('proxyRes', async function (proxyRes, req, res) { - var localFile = path.join(localRoot, proxyCache, req.url); + var localFile = path.isAbsolute(proxyCache) ? proxyCache : path.join(process.cwd(), proxyCache, req.url); var localDir = path.dirname(localFile); var contentEncoding = proxyRes.headers['content-encoding']; if (!fs.existsSync(localDir)) { From 5b4e154b300029851ce77567934a8b11bdd9a543 Mon Sep 17 00:00:00 2001 From: happy wang Date: Wed, 18 Oct 2023 15:06:49 +0800 Subject: [PATCH 08/12] Add options.proxyCacheLogFn to output logs when resources are proxied and cached --- lib/http-server.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/http-server.js b/lib/http-server.js index 5cba80ec..6426595c 100644 --- a/lib/http-server.js +++ b/lib/http-server.js @@ -147,6 +147,7 @@ function HttpServer(options) { if (typeof options.proxyCache === 'string') { var proxyCache = options.proxyCache; + var proxyCacheLogFn = options.proxyCacheLogFn || function () {}; proxy.on('proxyRes', async function (proxyRes, req, res) { var localFile = path.isAbsolute(proxyCache) ? proxyCache : path.join(process.cwd(), proxyCache, req.url); var localDir = path.dirname(localFile); @@ -166,7 +167,10 @@ function HttpServer(options) { proxyRes.pipe(stream); } - stream.on('finish', resolve); + stream.on('finish', () => { + proxyCacheLogFn(proxyRes, req, res, localFile); + resolve(); + }); }); }); } From 8b650bc5d6649acd4c79ce1e2a5c98727f1d1012 Mon Sep 17 00:00:00 2001 From: happy wang Date: Wed, 18 Oct 2023 15:11:36 +0800 Subject: [PATCH 09/12] Expand logging to cover the cache failures --- lib/http-server.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/http-server.js b/lib/http-server.js index 6426595c..0f118e0a 100644 --- a/lib/http-server.js +++ b/lib/http-server.js @@ -168,9 +168,13 @@ function HttpServer(options) { } stream.on('finish', () => { - proxyCacheLogFn(proxyRes, req, res, localFile); + proxyCacheLogFn(null, proxyRes, req, res, localFile); resolve(); }); + stream.on('error', (err) => { + proxyCacheLogFn(err, proxyRes, req, res, localFile); + reject(err); + }); }); }); } From 4809525989e7c5f84fe730537b67ae7449cf0c42 Mon Sep 17 00:00:00 2001 From: happy wang Date: Wed, 22 Nov 2023 19:42:13 +0800 Subject: [PATCH 10/12] Correct the way to generate err --- lib/http-server.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/http-server.js b/lib/http-server.js index 0f118e0a..7c054e9b 100644 --- a/lib/http-server.js +++ b/lib/http-server.js @@ -172,7 +172,10 @@ function HttpServer(options) { resolve(); }); stream.on('error', (err) => { - proxyCacheLogFn(err, proxyRes, req, res, localFile); + proxyCacheLogFn({ + status: proxyRes.statusCode, + message: err.message + }, proxyRes, req, res, localFile); reject(err); }); }); From 06f580bf5f2f08b79e6bd557006edadaab1450cd Mon Sep 17 00:00:00 2001 From: happy wang Date: Wed, 22 Nov 2023 19:42:41 +0800 Subject: [PATCH 11/12] Not cache when proxy response is not 200 --- lib/http-server.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/http-server.js b/lib/http-server.js index 7c054e9b..fbadbaed 100644 --- a/lib/http-server.js +++ b/lib/http-server.js @@ -151,6 +151,13 @@ function HttpServer(options) { proxy.on('proxyRes', async function (proxyRes, req, res) { var localFile = path.isAbsolute(proxyCache) ? proxyCache : path.join(process.cwd(), proxyCache, req.url); var localDir = path.dirname(localFile); + if (proxyRes.statusCode !== 200) { + proxyCacheLogFn({ + status: proxyRes.statusCode, + message: proxyRes.statusMessage + }, proxyRes, req, res, localFile); + return; + } var contentEncoding = proxyRes.headers['content-encoding']; if (!fs.existsSync(localDir)) { fs.mkdirSync(localDir, { recursive: true }); From ebe7773014a3f91fbdab28d7bd70076f4aa599c4 Mon Sep 17 00:00:00 2001 From: happy wang Date: Thu, 23 Nov 2023 15:47:44 +0800 Subject: [PATCH 12/12] Prevent save query to filename --- lib/http-server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/http-server.js b/lib/http-server.js index fbadbaed..f47ff58d 100644 --- a/lib/http-server.js +++ b/lib/http-server.js @@ -149,7 +149,7 @@ function HttpServer(options) { var proxyCache = options.proxyCache; var proxyCacheLogFn = options.proxyCacheLogFn || function () {}; proxy.on('proxyRes', async function (proxyRes, req, res) { - var localFile = path.isAbsolute(proxyCache) ? proxyCache : path.join(process.cwd(), proxyCache, req.url); + var localFile = path.isAbsolute(proxyCache) ? proxyCache : path.join(process.cwd(), proxyCache, req.url.split('?')[0]); var localDir = path.dirname(localFile); if (proxyRes.statusCode !== 200) { proxyCacheLogFn({