From c786ee2ea19d2fcef078a30cecb70d69036a4803 Mon Sep 17 00:00:00 2001 From: Vojta Jina Date: Sat, 30 Nov 2013 20:08:40 -0800 Subject: [PATCH] feat(web-server): cache preprocessed files Now Karma does not write preprocessed files into disk (as the web server would read them) and store that in memory. Especially when running with multiple browsers (each browser has to fetch the file), this saves FS access (1 write per preprocessed file; x reads per preprocessed file, where x is number of captured browsers). This is in trade for slightly higher memory usage. I however tried AngularJS with this change (forcing all the files to be processed/cached) and the memory consumption was not significantly higher. We can optimize this to clean up the cache, especially if there is only a single browser captured --- lib/middleware/common.js | 19 ++++++++++-- lib/middleware/source-files.js | 4 +-- lib/preprocessor.js | 11 ++----- test/unit/middleware/source-files.spec.coffee | 28 +++++++++++++++++ test/unit/preprocessor.spec.coffee | 31 ++----------------- 5 files changed, 53 insertions(+), 40 deletions(-) diff --git a/lib/middleware/common.js b/lib/middleware/common.js index b7d3fa2c7..6376dface 100644 --- a/lib/middleware/common.js +++ b/lib/middleware/common.js @@ -26,11 +26,26 @@ var serve404 = function(response, path) { var createServeFile = function(fs, directory) { - return function(filepath, response, transform) { + return function(filepath, response, transform, content) { + var responseData; + if (directory) { filepath = directory + filepath; } + // serve from cache + if (content) { + response.setHeader('Content-Type', mime.lookup(filepath, 'text/plain')); + + // call custom transform fn to transform the data + responseData = transform && transform(content) || content; + + response.writeHead(200); + + log.debug('serving (cached): ' + filepath); + return response.end(responseData); + } + return fs.readFile(filepath, function(error, data) { if (error) { return serve404(response, filepath); @@ -39,7 +54,7 @@ var createServeFile = function(fs, directory) { response.setHeader('Content-Type', mime.lookup(filepath, 'text/plain')); // call custom transform fn to transform the data - var responseData = transform && transform(data.toString()) || data; + responseData = transform && transform(data.toString()) || data; response.writeHead(200); diff --git a/lib/middleware/source-files.js b/lib/middleware/source-files.js index 551ce7722..aa10b40f8 100644 --- a/lib/middleware/source-files.js +++ b/lib/middleware/source-files.js @@ -37,7 +37,7 @@ var createSourceFilesMiddleware = function(filesPromise, serveFile, var file = findByPath(files.served, requestedFilePath); if (file) { - serveFile(file.contentPath, response, function() { + serveFile(file.contentPath || file.path, response, function() { if (/\?\d+/.test(request.url)) { // files with timestamps - cache one year, rely on timestamps common.setHeavyCacheHeaders(response); @@ -45,7 +45,7 @@ var createSourceFilesMiddleware = function(filesPromise, serveFile, // without timestamps - no cache (debug) common.setNoCacheHeaders(response); } - }); + }, file.content); } else { next(); } diff --git a/lib/preprocessor.js b/lib/preprocessor.js index 88f3c67cd..3743ad170 100644 --- a/lib/preprocessor.js +++ b/lib/preprocessor.js @@ -1,13 +1,9 @@ -var path = require('path'); var fs = require('graceful-fs'); var crypto = require('crypto'); var mm = require('minimatch'); var log = require('./logger').create('preprocess'); -// TODO(vojta): extract get/create temp dir somewhere else (use the same for launchers etc) -var TMP = process.env.TMPDIR || process.env.TMP || process.env.TEMP || '/tmp'; - var sha1 = function(data) { var hash = crypto.createHash('sha1'); hash.update(data); @@ -23,10 +19,9 @@ var createPreprocessor = function(config, basePath, injector) { var preprocessors = []; var nextPreprocessor = function(content) { if (!preprocessors.length) { - file.contentPath = path.normalize(TMP + '/' + sha1(file.path) + path.extname(file.path)); - return fs.writeFile(file.contentPath, content, function() { - done(); - }); + file.contentPath = null; + file.content = content; + return done(); } preprocessors.shift()(content, file, nextPreprocessor); diff --git a/test/unit/middleware/source-files.spec.coffee b/test/unit/middleware/source-files.spec.coffee index 8c09c5cc1..669eb3a3c 100644 --- a/test/unit/middleware/source-files.spec.coffee +++ b/test/unit/middleware/source-files.spec.coffee @@ -12,6 +12,7 @@ describe 'middleware.source-files', -> base: path: 'a.js': mocks.fs.file(0, 'js-src-a') + 'index.html': mocks.fs.file(0, '') src: 'some.js': mocks.fs.file(0, 'js-source') 'utf8ášč': @@ -133,3 +134,30 @@ describe 'middleware.source-files', -> done() callHandlerWith '/base/some.js' + + it 'should set content-type headers', (done) -> + servedFiles [ + new File('/base/path/index.html') + ] + + response.once 'end', -> + expect(response._headers['Content-Type']).to.equal 'text/html' + done() + + callHandlerWith '/base/index.html' + + + it 'should use cached content if available', (done) -> + cachedFile = new File('/some/file.js') + cachedFile.content = 'cached-content' + + servedFiles [ + cachedFile + ] + + response.once 'end', -> + expect(nextSpy).not.to.have.been.called + expect(response).to.beServedAs 200, 'cached-content' + done() + + callHandlerWith '/absolute/some/file.js' diff --git a/test/unit/preprocessor.spec.coffee b/test/unit/preprocessor.spec.coffee index 3e0e12bbe..cb9742be7 100644 --- a/test/unit/preprocessor.spec.coffee +++ b/test/unit/preprocessor.spec.coffee @@ -11,20 +11,12 @@ describe 'preprocessor', -> mockFs = mocks.fs.create some: 'a.js': mocks.fs.file 0, 'content' - 'style.less': mocks.fs.file 0, 'whatever' - temp: {} # so that we can write preprocessed content here - mocks_ = 'graceful-fs': mockFs minimatch: require 'minimatch' - globals_ = - process: - env: TMPDIR: '/temp' - nextTick: process.nextTick - - m = mocks.loadFile __dirname + '/../../lib/preprocessor.js', mocks_, globals_ + m = mocks.loadFile __dirname + '/../../lib/preprocessor.js', mocks_ it 'should preprocess matching file', (done) -> @@ -40,7 +32,7 @@ describe 'preprocessor', -> pp file, -> expect(fakePreprocessor).to.have.been.called expect(file.path).to.equal 'path-preprocessed' - expect(mockFs.readFileSync(file.contentPath).toString()).to.equal 'new-content' + expect(file.content).to.equal 'new-content' done() @@ -80,22 +72,5 @@ describe 'preprocessor', -> expect(fakePreprocessor1).to.have.been.calledOnce expect(fakePreprocessor2).to.have.been.calledOnce expect(file.path).to.equal 'path-p1-p2' - expect(mockFs.readFileSync(file.contentPath).toString()).to.equal 'content-c1-c2' - done() - - - it 'should keep processed extension', (done) -> - fakePreprocessor = sinon.spy (content, file, done) -> - file.path = file.path.replace '.less', '.css' - done content - - injector = new di.Injector [{ - 'preprocessor:less': ['factory', -> fakePreprocessor] - }] - - pp = m.createPreprocessor {'**/*.less': ['less']}, null, injector - file = {originalPath: '/some/style.less', path: '/some/style.less'} - - pp file, -> - expect(file.contentPath).to.match /\.css$/ + expect(file.content).to.equal 'content-c1-c2' done()