Permalink
Browse files

fetch HTML pages to benchmark

fixes #612
  • Loading branch information...
1 parent 2604d6a commit d3eb8a407ea23c4e2402f0f33e7e25cef39f12cc @alexlamsl alexlamsl committed Apr 9, 2016
View
@@ -3,6 +3,7 @@
.DS_Store
/.jekyll-metadata
/_site/
+/benchmarks/*.html
/benchmarks/generated
/node_modules/
/npm-debug.log
View
@@ -7,6 +7,7 @@ var brotli = require('brotli'),
fork = require('child_process').fork,
fs = require('fs'),
http = require('http'),
+ https = require('https'),
lzma = require('lzma'),
Minimize = require('minimize'),
path = require('path'),
@@ -28,7 +29,9 @@ var progress = new Progress('[:bar] :etas :fileName', {
var table = new Table({
head: ['File', 'Before', 'After', 'Minimize', 'Will Peavy', 'htmlcompressor.com', 'Savings', 'Time'],
- colWidths: [20, 25, 25, 25, 25, 25, 20, 10]
+ colWidths: [fileNames.reduce(function(length, fileName) {
+ return Math.max(length, fileName.length);
+ }, 0) + 2, 25, 25, 25, 25, 25, 20, 10]
});
function toKb(size) {
@@ -121,9 +124,9 @@ function run(tasks, done) {
var rows = {};
run(fileNames.map(function(fileName) {
- return function(done) {
- progress.tick(0, { fileName: fileName });
- var filePath = path.join('benchmarks/', fileName + '.html');
+ var filePath = path.join('benchmarks/', fileName + '.html');
+
+ function processFile(site, done) {
var original = {
filePath: filePath,
gzFilePath: path.join('benchmarks/generated/', fileName + '.html.gz'),
@@ -199,7 +202,7 @@ run(fileNames.map(function(fileName) {
function testHTMLMinifier(done) {
var info = infos.minifier;
info.startTime = Date.now();
- var args = [filePath, '-c', 'sample-cli-config-file.conf', '--minify-urls', urls[fileName], '-o', info.filePath];
+ var args = [filePath, '-c', 'sample-cli-config-file.conf', '--minify-urls', site, '-o', info.filePath];
fork('./cli', args).on('exit', function() {
readSizes(info, done);
});
@@ -328,6 +331,35 @@ run(fileNames.map(function(fileName) {
progress.tick({ fileName: '' });
done();
});
+ }
+
+ function get(site, callback) {
+ var options = url.parse(site);
+ var protocol = options.protocol === 'https:' ? https : http;
+ protocol.get(options, function(res) {
+ var status = res.statusCode;
+ if (200 <= status && status < 300) {
+ if (res.headers['content-encoding'] === 'gzip') {
+ res = res.pipe(zlib.createGunzip());
+ }
+ res.pipe(fs.createWriteStream(filePath)).on('finish', function() {
+ callback(site);
+ });
+ }
+ else if (300 <= status && status < 400 && res.headers.location) {
+ get(url.resolve(site, res.headers.location), callback);
+ }
+ else {
+ throw new Error('HTTP error ' + status + '\n' + site);
+ }
+ });
+ }
+
+ return function(done) {
+ progress.tick(0, { fileName: fileName });
+ get(urls[fileName], function(site) {
+ processFile(site, done);
+ });
};
}), function() {
fileNames.forEach(function(fileName) {
View
Oops, something went wrong.
View
Oops, something went wrong.
Oops, something went wrong.
Oops, something went wrong.

0 comments on commit d3eb8a4

Please sign in to comment.