Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
23 changed files
with
6,133 additions
and
709 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
#!/usr/bin/env node | ||
|
||
// see lib/main.js for info | ||
|
||
const ghAuth = require('./lib/ghauth') | ||
, main = require('./lib/main') | ||
, stats = require('./lib/stats') | ||
|
||
const config = require('./config') | ||
|
||
config.verbose = process.argv.indexOf('-v') > -1 | ||
config.quiet = process.argv.indexOf('-q') > -1 | ||
|
||
if (process.argv.indexOf('-s') > -1) | ||
return stats(config) | ||
|
||
ghAuth(config, main.bind(null, config)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
module.exports = { | ||
inFile : 'data.js' | ||
, outFile : 'data-min.js' | ||
// for now; need to tweak this later | ||
, githubLastPushCutoffMonths : 48 | ||
, sizeExceptions : [ | ||
'Backbone' | ||
, 'Zepto' | ||
] | ||
, minifyExceptions : [ | ||
'cm.js' | ||
] | ||
, errorCountThreshold : 10 | ||
, maxGzipped : 5 * 1024 | ||
// these 3 are basic sanity cecks, minimums that we would expect the contents | ||
// to be, otherwise something might be wrong | ||
, minRaw : 300 | ||
, minMinified : 150 | ||
, minGzipped : 100 | ||
// this is to compare raw with minified, if it's too close then we may have | ||
// been given the minified source in the first place | ||
, minifierTollerance : 50 | ||
// give them this much wriggle room with the reported 'size' vs actual gzipped size | ||
, expectedTolleranceFraction : 0.5 | ||
} |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,73 +1,70 @@ | ||
var uuid = require('node-uuid') | ||
, fs = require('fs') | ||
, async = require('async') | ||
, xml = require('./xml') | ||
const uuid = require('node-uuid') | ||
, fs = require('fs') | ||
, memo = require('async-memo') | ||
, moment = require('moment') | ||
, xml = require('./xml') | ||
|
||
require('date-utils') | ||
|
||
var file = 'atom.xml' | ||
, footRegex = /(\n<\/feed>\n)/ | ||
, dateStr = new Date().addMinutes(new Date().getTimezoneOffset()).toFormat('YYYY-MM-DD\'T\'HH24:MI:SSZ') | ||
, entryTemplate = | ||
'\t<entry>\n' | ||
+ '\t\t<title>{xmlEncodedName}</title>\n' | ||
+ '\t\t<link href="{url}" />\n' | ||
+ '\t\t<id>urn:uuid:{atomUuid}</id>\n' | ||
+ '\t\t<updated>{atomDate}</updated>\n' | ||
+ '\t\t<summary>{xmlEncodedDescription}</summary>\n' | ||
+ '\t</entry>\n\n' | ||
+ '$1' | ||
, contents | ||
|
||
// memoize so we only read once, but keep contents global so we can edit it | ||
, fileContents = async.memoize( | ||
const file = 'atom.xml' | ||
, footRegex = /(\n<\/feed>\n?)/ | ||
, dateStr = moment.utc().format('YYYY-MM-DD\'T\'HH:mm:ssZ') | ||
, entryTemplate = | ||
'\t<entry>\n' | ||
+ '\t\t<title>{xmlEncodedName}</title>\n' | ||
+ '\t\t<link href="{url}" />\n' | ||
+ '\t\t<id>urn:uuid:{atomUuid}</id>\n' | ||
+ '\t\t<updated>{atomDate}</updated>\n' | ||
+ '\t\t<summary>{xmlEncodedDescription}</summary>\n' | ||
+ '\t</entry>\n\n' | ||
+ '$1' | ||
// memoize so we only read once, but keep contents global so we can edit it | ||
, fileContents = memo( | ||
function (callback) { | ||
fs.readFile(file, 'utf-8', function (err, _contents) { | ||
contents = _contents | ||
callback(err) | ||
}) | ||
} | ||
, function () { return '_' } | ||
) | ||
}) | ||
|
||
, escapeRegExp = function (string) { | ||
return string.replace(/[\-\[\]\{\}\(\)*+?.\\\^$|,#\s]/g, function (match) { return '\\' + match }) | ||
} | ||
var contents | ||
|
||
, exists = function (contents, lib) { | ||
return new RegExp(escapeRegExp('<title>' + lib.xmlEncodedName + '</title>')).test(contents) | ||
} | ||
function escapeRegExp (string) { | ||
return string.replace(/[\-\[\]\{\}\(\)*+?.\\\^$|,#\s]/g, function (match) { return '\\' + match }) | ||
} | ||
|
||
, t = function (s, d){ | ||
for (var p in d) | ||
s = s.replace(new RegExp('{' + p + '}', 'g'), d[p]) | ||
return s | ||
} | ||
function exists (contents, lib) { | ||
return new RegExp(escapeRegExp('<title>' + lib.xmlEncodedName + '</title>')).test(contents) | ||
} | ||
|
||
, processLibrary = function (lib, callback) { | ||
fileContents(function (err) { | ||
if (err) | ||
throw err | ||
function t (s, d) { | ||
for (var p in d) | ||
s = s.replace(new RegExp('{' + p + '}', 'g'), d[p]) | ||
return s | ||
} | ||
|
||
lib.xmlEncodedName = xml.encodeXmlEntities(lib.name) | ||
if (exists(contents, lib)) | ||
return callback() | ||
function processLibrary (lib, callback) { | ||
fileContents(function (err) { | ||
if (err) | ||
throw err | ||
|
||
lib.atomUuid = uuid.v1() | ||
lib.atomDate = dateStr | ||
lib.xmlEncodedDescription = xml.encodeXmlEntities(lib.description) | ||
contents = contents.replace(footRegex, t(entryTemplate, lib)) | ||
;delete lib.atomUuid | ||
;delete lib.atomDate | ||
callback() | ||
}) | ||
} | ||
lib.xmlEncodedName = xml.encodeXmlEntities(lib.name) | ||
if (exists(contents, lib)) | ||
return callback() | ||
|
||
, write = function (callback) { | ||
fs.writeFile(file, contents, 'utf-8', callback) | ||
} | ||
lib.atomUuid = uuid.v1() | ||
lib.atomDate = dateStr | ||
lib.xmlEncodedDescription = xml.encodeXmlEntities(lib.description) | ||
contents = contents.replace(footRegex, t(entryTemplate, lib)) | ||
;delete lib.atomUuid | ||
;delete lib.atomDate | ||
;delete lib.xmlEncodedName | ||
;delete lib.xmlEncodedDescription | ||
callback() | ||
}) | ||
} | ||
|
||
module.exports = { | ||
processLibrary: processLibrary | ||
, write: write | ||
function write (callback) { | ||
fs.writeFile(file, contents, 'utf-8', callback) | ||
} | ||
|
||
module.exports.processLibrary = processLibrary | ||
module.exports.write = write |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,47 +1,39 @@ | ||
var zip = require('zip') | ||
, request = require('request') | ||
var zip = require('zip') | ||
, hyperquest = require('hyperquest') | ||
, bl = require('bl') | ||
|
||
, requestPool = { maxSockets: 20 } | ||
function unzip (zipEntry, buffer) { | ||
var reader = zip.Reader(buffer) | ||
, data = null | ||
|
||
, unzip = function (zipEntry, buffer) { | ||
var reader = zip.Reader(buffer) | ||
, data = null | ||
// iterate through zipfile entries looking for the right name | ||
reader.forEach(function(entry) { | ||
if (zipEntry === entry._header.file_name) { | ||
data = entry._stream.toString(); | ||
} | ||
}) | ||
return data | ||
// iterate through zipfile entries looking for the right name | ||
reader.forEach(function(entry) { | ||
if (zipEntry === entry._header.file_name) { | ||
data = entry._stream.toString(); | ||
} | ||
}) | ||
return data | ||
} | ||
|
||
, fetch = function (sourceUrl, zipEntry, callback) { | ||
var options = { | ||
url: sourceUrl | ||
, encoding: zipEntry ? null : 'utf8' | ||
, timeout: 1 * 60 * 1000 | ||
, pool: requestPool | ||
, maxRedirects: 2 | ||
, headers: {"User-Agent" : "microjs.com-build-script"} | ||
} | ||
, handle = function (err, response, body) { | ||
if (err) | ||
return callback(err) | ||
function fetch (sourceUrl, zipEntry, callback) { | ||
hyperquest(sourceUrl, { headers: { 'User-Agent': 'microjs.com-build-script' } }) | ||
.pipe(bl(function (err, content) { | ||
if (err) | ||
return callback(err) | ||
|
||
if (zipEntry) { | ||
try { | ||
body = unzip(zipEntry, body) | ||
} catch (e) { | ||
callback('unzip error' + e) | ||
} | ||
if (!body) | ||
return callback('could not find entry "' + zipEntry + '" inside zip file') | ||
} | ||
if (zipEntry) { | ||
try { | ||
content = unzip(zipEntry, content) | ||
} catch (e) { | ||
return callback('unzip error' + e) | ||
} | ||
|
||
callback(null, body) | ||
} | ||
if (!content) | ||
return callback('could not find entry ' + zipEntry + ' inside zip file') | ||
} | ||
|
||
request(options, handle) | ||
} | ||
callback(null, content.toString()) | ||
})) | ||
} | ||
|
||
module.exports = fetch |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,21 +1,21 @@ | ||
var read = require('read') | ||
, colors = require('colors') | ||
const read = require('read') | ||
, colors = require('colors') | ||
|
||
, auth = function (config, callback) { | ||
console.log('To collect GitHub stats we need you to authenticate so we can exceed API rate-limiting.'.bold.green) | ||
console.log('Leave the username field blank to skip stats collection.'.bold.green) | ||
console.log() | ||
function auth (config, callback) { | ||
console.log('To collect GitHub stats we need you to authenticate so we can exceed API rate-limiting.'.bold.green) | ||
console.log('Leave the username field blank to skip stats collection.'.bold.green) | ||
console.log() | ||
|
||
read({ prompt: 'GitHub username:'.bold }, function (err, data) { | ||
if (err) return callback(err) | ||
if (data === '') return callback() | ||
config.ghUser = data | ||
read({ prompt: 'GitHub password:'.bold, silent: true, replace: '\u2714' }, function (err, data) { | ||
if (err) return callback(err) | ||
config.ghPass = data | ||
callback() | ||
}) | ||
}) | ||
} | ||
read({ prompt: 'GitHub username:'.bold }, function (err, data) { | ||
if (err) return callback(err) | ||
if (data === '') return callback() | ||
config.ghUser = data | ||
read({ prompt: 'GitHub password:'.bold, silent: true, replace: '\u2714' }, function (err, data) { | ||
if (err) return callback(err) | ||
config.ghPass = data | ||
callback() | ||
}) | ||
}) | ||
} | ||
|
||
module.exports = auth | ||
module.exports = auth |
Oops, something went wrong.