Permalink
Browse files

rewrote builder

  • Loading branch information...
1 parent e023d00 commit 2b4b9f5015e901ab540f3ca7140fce61c19dddc5 @rvagg rvagg committed Aug 18, 2013
Showing with 6,133 additions and 709 deletions.
  1. +1 −1 README.md
  2. +0 −24 build
  3. +17 −0 build.js
  4. +25 −0 config.js
  5. +5,437 −1 data-min.js
  6. +55 −58 lib/atom.js
  7. +30 −38 lib/fetcher.js
  8. +18 −18 lib/ghauth.js
  9. +0 −69 lib/ghstats.js
  10. +17 −24 lib/input.js
  11. +70 −63 lib/logger.js
  12. +160 −142 lib/main.js
  13. +9 −13 lib/output.js
  14. +68 −0 lib/process-github.js
  15. +84 −0 lib/process-source.js
  16. +28 −0 lib/process-tweets.js
  17. +0 −91 lib/processor.js
  18. +7 −5 lib/sizer.js
  19. +52 −60 lib/stats.js
  20. +0 −31 lib/twitterstats.js
  21. +24 −44 lib/validator.js
  22. +9 −9 lib/xml.js
  23. +22 −18 package.json
View
@@ -37,7 +37,7 @@ file with a name that's unlikely to change (no version numbers).
(1) It's ok to target a specific platform, like node.js, or WebKit.
-(2) To check the minified and compressed size of your library from the command line, either use `./build -v` (see below) or run:
+(2) To check the minified and compressed size of your library from the command line, either use `./build.js -v` (see below) or run:
$ uglifyjs yourlib.js | gzip -9f | wc -c
View
24 build
@@ -1,24 +0,0 @@
-#!/usr/bin/env node
-
-// see lib/main.js for info
-
-var config = {
- inFile: 'data.js'
- , outFile: 'data-min.js'
- , githubLastPushCutoffMonths: 48 // for now—need to tweak this later
- , sizeExceptions: [
- 'Backbone'
- , 'Zepto'
- ]
- , minifyExceptions: [ 'cm.js' ]
- , errorCountThreshold: 10
- , verbose: process.argv.indexOf('-v') != -1
- , quiet: process.argv.indexOf('-q') != -1
- }
- , ghAuth = require('./lib/ghauth')
- , main = require('./lib/main')
- , stats = require('./lib/stats')
-
-ghAuth(config, function () {
- (process.argv.indexOf('-s') != -1 ? stats : main).create(config).start()
-})
View
@@ -0,0 +1,17 @@
+#!/usr/bin/env node
+
+// see lib/main.js for info
+
+const ghAuth = require('./lib/ghauth')
+ , main = require('./lib/main')
+ , stats = require('./lib/stats')
+
+const config = require('./config')
+
+config.verbose = process.argv.indexOf('-v') > -1
+config.quiet = process.argv.indexOf('-q') > -1
+
+if (process.argv.indexOf('-s') > -1)
+ return stats(config)
+
+ghAuth(config, main.bind(null, config))
View
@@ -0,0 +1,25 @@
+module.exports = {
+ inFile : 'data.js'
+ , outFile : 'data-min.js'
+ // for now; need to tweak this later
+ , githubLastPushCutoffMonths : 48
+ , sizeExceptions : [
+ 'Backbone'
+ , 'Zepto'
+ ]
+ , minifyExceptions : [
+ 'cm.js'
+ ]
+ , errorCountThreshold : 10
+ , maxGzipped : 5 * 1024
+ // these 3 are basic sanity cecks, minimums that we would expect the contents
+ // to be, otherwise something might be wrong
+ , minRaw : 300
+ , minMinified : 150
+ , minGzipped : 100
+ // this is to compare raw with minified, if it's too close then we may have
+ // been given the minified source in the first place
+ , minifierTollerance : 50
+ // give them this much wriggle room with the reported 'size' vs actual gzipped size
+ , expectedTolleranceFraction : 0.5
+}
View
5,438 data-min.js

Large diffs are not rendered by default.

Oops, something went wrong.
View
@@ -1,73 +1,70 @@
-var uuid = require('node-uuid')
- , fs = require('fs')
- , async = require('async')
- , xml = require('./xml')
+const uuid = require('node-uuid')
+ , fs = require('fs')
+ , memo = require('async-memo')
+ , moment = require('moment')
+ , xml = require('./xml')
-require('date-utils')
-
-var file = 'atom.xml'
- , footRegex = /(\n<\/feed>\n)/
- , dateStr = new Date().addMinutes(new Date().getTimezoneOffset()).toFormat('YYYY-MM-DD\'T\'HH24:MI:SSZ')
- , entryTemplate =
- '\t<entry>\n'
- + '\t\t<title>{xmlEncodedName}</title>\n'
- + '\t\t<link href="{url}" />\n'
- + '\t\t<id>urn:uuid:{atomUuid}</id>\n'
- + '\t\t<updated>{atomDate}</updated>\n'
- + '\t\t<summary>{xmlEncodedDescription}</summary>\n'
- + '\t</entry>\n\n'
- + '$1'
- , contents
-
- // memoize so we only read once, but keep contents global so we can edit it
- , fileContents = async.memoize(
+const file = 'atom.xml'
+ , footRegex = /(\n<\/feed>\n?)/
+ , dateStr = moment.utc().format('YYYY-MM-DD\'T\'HH:mm:ssZ')
+ , entryTemplate =
+ '\t<entry>\n'
+ + '\t\t<title>{xmlEncodedName}</title>\n'
+ + '\t\t<link href="{url}" />\n'
+ + '\t\t<id>urn:uuid:{atomUuid}</id>\n'
+ + '\t\t<updated>{atomDate}</updated>\n'
+ + '\t\t<summary>{xmlEncodedDescription}</summary>\n'
+ + '\t</entry>\n\n'
+ + '$1'
+ // memoize so we only read once, but keep contents global so we can edit it
+ , fileContents = memo(
function (callback) {
fs.readFile(file, 'utf-8', function (err, _contents) {
contents = _contents
callback(err)
})
- }
- , function () { return '_' }
- )
+ })
- , escapeRegExp = function (string) {
- return string.replace(/[\-\[\]\{\}\(\)*+?.\\\^$|,#\s]/g, function (match) { return '\\' + match })
- }
+var contents
- , exists = function (contents, lib) {
- return new RegExp(escapeRegExp('<title>' + lib.xmlEncodedName + '</title>')).test(contents)
- }
+function escapeRegExp (string) {
+ return string.replace(/[\-\[\]\{\}\(\)*+?.\\\^$|,#\s]/g, function (match) { return '\\' + match })
+}
- , t = function (s, d){
- for (var p in d)
- s = s.replace(new RegExp('{' + p + '}', 'g'), d[p])
- return s
- }
+function exists (contents, lib) {
+ return new RegExp(escapeRegExp('<title>' + lib.xmlEncodedName + '</title>')).test(contents)
+}
- , processLibrary = function (lib, callback) {
- fileContents(function (err) {
- if (err)
- throw err
+function t (s, d) {
+ for (var p in d)
+ s = s.replace(new RegExp('{' + p + '}', 'g'), d[p])
+ return s
+}
- lib.xmlEncodedName = xml.encodeXmlEntities(lib.name)
- if (exists(contents, lib))
- return callback()
+function processLibrary (lib, callback) {
+ fileContents(function (err) {
+ if (err)
+ throw err
- lib.atomUuid = uuid.v1()
- lib.atomDate = dateStr
- lib.xmlEncodedDescription = xml.encodeXmlEntities(lib.description)
- contents = contents.replace(footRegex, t(entryTemplate, lib))
- ;delete lib.atomUuid
- ;delete lib.atomDate
- callback()
- })
- }
+ lib.xmlEncodedName = xml.encodeXmlEntities(lib.name)
+ if (exists(contents, lib))
+ return callback()
- , write = function (callback) {
- fs.writeFile(file, contents, 'utf-8', callback)
- }
+ lib.atomUuid = uuid.v1()
+ lib.atomDate = dateStr
+ lib.xmlEncodedDescription = xml.encodeXmlEntities(lib.description)
+ contents = contents.replace(footRegex, t(entryTemplate, lib))
+ ;delete lib.atomUuid
+ ;delete lib.atomDate
+ ;delete lib.xmlEncodedName
+ ;delete lib.xmlEncodedDescription
+ callback()
+ })
+}
-module.exports = {
- processLibrary: processLibrary
- , write: write
+function write (callback) {
+ fs.writeFile(file, contents, 'utf-8', callback)
}
+
+module.exports.processLibrary = processLibrary
+module.exports.write = write
View
@@ -1,47 +1,39 @@
-var zip = require('zip')
- , request = require('request')
+var zip = require('zip')
+ , hyperquest = require('hyperquest')
+ , bl = require('bl')
- , requestPool = { maxSockets: 20 }
+function unzip (zipEntry, buffer) {
+ var reader = zip.Reader(buffer)
+ , data = null
- , unzip = function (zipEntry, buffer) {
- var reader = zip.Reader(buffer)
- , data = null
- // iterate through zipfile entries looking for the right name
- reader.forEach(function(entry) {
- if (zipEntry === entry._header.file_name) {
- data = entry._stream.toString();
- }
- })
- return data
+ // iterate through zipfile entries looking for the right name
+ reader.forEach(function(entry) {
+ if (zipEntry === entry._header.file_name) {
+ data = entry._stream.toString();
}
+ })
+ return data
+}
- , fetch = function (sourceUrl, zipEntry, callback) {
- var options = {
- url: sourceUrl
- , encoding: zipEntry ? null : 'utf8'
- , timeout: 1 * 60 * 1000
- , pool: requestPool
- , maxRedirects: 2
- , headers: {"User-Agent" : "microjs.com-build-script"}
- }
- , handle = function (err, response, body) {
- if (err)
- return callback(err)
+function fetch (sourceUrl, zipEntry, callback) {
+ hyperquest(sourceUrl, { headers: { 'User-Agent': 'microjs.com-build-script' } })
+ .pipe(bl(function (err, content) {
+ if (err)
+ return callback(err)
- if (zipEntry) {
- try {
- body = unzip(zipEntry, body)
- } catch (e) {
- callback('unzip error' + e)
- }
- if (!body)
- return callback('could not find entry "' + zipEntry + '" inside zip file')
- }
+ if (zipEntry) {
+ try {
+ content = unzip(zipEntry, content)
+ } catch (e) {
+ return callback('unzip error' + e)
+ }
- callback(null, body)
- }
+ if (!content)
+ return callback('could not find entry ' + zipEntry + ' inside zip file')
+ }
- request(options, handle)
- }
+ callback(null, content.toString())
+ }))
+}
module.exports = fetch
View
@@ -1,21 +1,21 @@
-var read = require('read')
- , colors = require('colors')
+const read = require('read')
+ , colors = require('colors')
- , auth = function (config, callback) {
- console.log('To collect GitHub stats we need you to authenticate so we can exceed API rate-limiting.'.bold.green)
- console.log('Leave the username field blank to skip stats collection.'.bold.green)
- console.log()
+function auth (config, callback) {
+ console.log('To collect GitHub stats we need you to authenticate so we can exceed API rate-limiting.'.bold.green)
+ console.log('Leave the username field blank to skip stats collection.'.bold.green)
+ console.log()
- read({ prompt: 'GitHub username:'.bold }, function (err, data) {
- if (err) return callback(err)
- if (data === '') return callback()
- config.ghUser = data
- read({ prompt: 'GitHub password:'.bold, silent: true, replace: '\u2714' }, function (err, data) {
- if (err) return callback(err)
- config.ghPass = data
- callback()
- })
- })
- }
+ read({ prompt: 'GitHub username:'.bold }, function (err, data) {
+ if (err) return callback(err)
+ if (data === '') return callback()
+ config.ghUser = data
+ read({ prompt: 'GitHub password:'.bold, silent: true, replace: '\u2714' }, function (err, data) {
+ if (err) return callback(err)
+ config.ghPass = data
+ callback()
+ })
+ })
+}
-module.exports = auth
+module.exports = auth
Oops, something went wrong.

0 comments on commit 2b4b9f5

Please sign in to comment.