rewrote builder

README.md

@@ -37,7 +37,7 @@ file with a name that's unlikely to change (no version numbers).
 
 (1) It's ok to target a specific platform, like node.js, or WebKit.
 
-(2) To check the minified and compressed size of your library from the command line, either use `./build -v` (see below) or run:
+(2) To check the minified and compressed size of your library from the command line, either use `./build.js -v` (see below) or run:
 
     $ uglifyjs yourlib.js | gzip -9f | wc -c
 

build.js

@@ -0,0 +1,17 @@
+#!/usr/bin/env node
+
+// see lib/main.js for info
+
+const ghAuth   = require('./lib/ghauth')
+    , main     = require('./lib/main')
+    , stats    = require('./lib/stats')
+
+const config   = require('./config')
+
+config.verbose = process.argv.indexOf('-v') > -1
+config.quiet   = process.argv.indexOf('-q') > -1
+
+if (process.argv.indexOf('-s') > -1)
+  return stats(config)
+
+ghAuth(config, main.bind(null, config))

config.js

@@ -0,0 +1,25 @@
+module.exports = {
+    inFile                     : 'data.js'
+  , outFile                    : 'data-min.js'
+    // for now; need to tweak this later
+  , githubLastPushCutoffMonths : 48
+  , sizeExceptions             : [
+        'Backbone'
+      , 'Zepto'
+    ]
+  , minifyExceptions           : [
+        'cm.js'
+    ]
+  , errorCountThreshold        : 10
+  , maxGzipped                 : 5 * 1024
+    // these 3 are basic sanity cecks, minimums that we would expect the contents
+    // to be, otherwise something might be wrong
+  , minRaw                     : 300
+  , minMinified                : 150
+  , minGzipped                 : 100
+    // this is to compare raw with minified, if it's too close then we may have
+    // been given the minified source in the first place
+  , minifierTollerance         : 50
+    // give them this much wriggle room with the reported 'size' vs actual gzipped size
+  , expectedTolleranceFraction : 0.5
+}

lib/atom.js

@@ -1,73 +1,70 @@
-var uuid = require('node-uuid')
-  , fs = require('fs')
-  , async = require('async')
-  , xml = require('./xml')
+const uuid          = require('node-uuid')
+    , fs            = require('fs')
+    , memo          = require('async-memo')
+    , moment        = require('moment')
+    , xml           = require('./xml')
 
-require('date-utils')
-
-var file = 'atom.xml'
-  , footRegex = /(\n<\/feed>\n)/
-  , dateStr = new Date().addMinutes(new Date().getTimezoneOffset()).toFormat('YYYY-MM-DD\'T\'HH24:MI:SSZ')
-  , entryTemplate =
-        '\t<entry>\n'
-      + '\t\t<title>{xmlEncodedName}</title>\n'
-      + '\t\t<link href="{url}" />\n'
-      + '\t\t<id>urn:uuid:{atomUuid}</id>\n'
-      + '\t\t<updated>{atomDate}</updated>\n'
-      + '\t\t<summary>{xmlEncodedDescription}</summary>\n'
-      + '\t</entry>\n\n'
-      + '$1'
-  , contents
-
-    // memoize so we only read once, but keep contents global so we can edit it
-  , fileContents = async.memoize(
+const file          = 'atom.xml'
+    , footRegex     = /(\n<\/feed>\n?)/
+    , dateStr       = moment.utc().format('YYYY-MM-DD\'T\'HH:mm:ssZ')
+    , entryTemplate =
+          '\t<entry>\n'
+        + '\t\t<title>{xmlEncodedName}</title>\n'
+        + '\t\t<link href="{url}" />\n'
+        + '\t\t<id>urn:uuid:{atomUuid}</id>\n'
+        + '\t\t<updated>{atomDate}</updated>\n'
+        + '\t\t<summary>{xmlEncodedDescription}</summary>\n'
+        + '\t</entry>\n\n'
+        + '$1'
+      // memoize so we only read once, but keep contents global so we can edit it
+    , fileContents = memo(
         function (callback) {
           fs.readFile(file, 'utf-8', function (err, _contents) {
             contents = _contents
             callback(err)
           })
-        }
-      , function () { return '_' }
-    )
+        })
 
-  , escapeRegExp = function (string) {
-      return string.replace(/[\-\[\]\{\}\(\)*+?.\\\^$|,#\s]/g, function (match) { return '\\' + match })
-    }
+var contents
 
-  , exists = function (contents, lib) {
-      return new RegExp(escapeRegExp('<title>' + lib.xmlEncodedName + '</title>')).test(contents)
-    }
+function escapeRegExp (string) {
+  return string.replace(/[\-\[\]\{\}\(\)*+?.\\\^$|,#\s]/g, function (match) { return '\\' + match })
+}
 
-  , t = function (s, d){
-      for (var p in d)
-        s = s.replace(new RegExp('{' + p + '}', 'g'), d[p])
-      return s
-    }
+function exists (contents, lib) {
+  return new RegExp(escapeRegExp('<title>' + lib.xmlEncodedName + '</title>')).test(contents)
+}
 
-  , processLibrary = function (lib, callback) {
-      fileContents(function (err) {
-        if (err)
-          throw err
+function t (s, d) {
+  for (var p in d)
+    s = s.replace(new RegExp('{' + p + '}', 'g'), d[p])
+  return s
+}
 
-        lib.xmlEncodedName = xml.encodeXmlEntities(lib.name)
-        if (exists(contents, lib))
-          return callback()
+function processLibrary (lib, callback) {
+  fileContents(function (err) {
+    if (err)
+      throw err
 
-        lib.atomUuid = uuid.v1()
-        lib.atomDate = dateStr
-        lib.xmlEncodedDescription = xml.encodeXmlEntities(lib.description)
-        contents = contents.replace(footRegex, t(entryTemplate, lib))
-        ;delete lib.atomUuid
-        ;delete lib.atomDate
-        callback()
-      })
-    }
+    lib.xmlEncodedName = xml.encodeXmlEntities(lib.name)
+    if (exists(contents, lib))
+      return callback()
 
-  , write = function (callback) {
-      fs.writeFile(file, contents, 'utf-8', callback)
-    }
+    lib.atomUuid = uuid.v1()
+    lib.atomDate = dateStr
+    lib.xmlEncodedDescription = xml.encodeXmlEntities(lib.description)
+    contents = contents.replace(footRegex, t(entryTemplate, lib))
+    ;delete lib.atomUuid
+    ;delete lib.atomDate
+    ;delete lib.xmlEncodedName
+    ;delete lib.xmlEncodedDescription
+    callback()
+  })
+}
 
-module.exports = {
-    processLibrary: processLibrary
-  , write: write
+function write (callback) {
+  fs.writeFile(file, contents, 'utf-8', callback)
 }
+
+module.exports.processLibrary = processLibrary
+module.exports.write          = write

lib/fetcher.js

@@ -1,47 +1,39 @@
-var zip = require('zip')
-  , request = require('request')
+var zip        = require('zip')
+  , hyperquest = require('hyperquest')
+  , bl         = require('bl')
 
-  , requestPool = { maxSockets: 20 }
+function unzip (zipEntry, buffer) {
+  var reader = zip.Reader(buffer)
+    , data   = null
 
-  , unzip = function (zipEntry, buffer) {
-      var reader = zip.Reader(buffer)
-        , data = null
-      // iterate through zipfile entries looking for the right name
-      reader.forEach(function(entry) {
-        if (zipEntry === entry._header.file_name) {
-          data = entry._stream.toString();
-        }
-      })
-      return data
+  // iterate through zipfile entries looking for the right name
+  reader.forEach(function(entry) {
+    if (zipEntry === entry._header.file_name) {
+      data = entry._stream.toString();
     }
+  })
+  return data
+}
 
-  , fetch = function (sourceUrl, zipEntry, callback) {
-      var options = {
-              url: sourceUrl
-            , encoding: zipEntry ? null : 'utf8'
-            , timeout: 1 * 60 * 1000
-            , pool: requestPool
-            , maxRedirects: 2
-            , headers: {"User-Agent" : "microjs.com-build-script"}
-          }
-        , handle = function (err, response, body) {
-            if (err)
-              return callback(err)
+function fetch (sourceUrl, zipEntry, callback) {
+  hyperquest(sourceUrl, { headers: { 'User-Agent': 'microjs.com-build-script' } })
+    .pipe(bl(function (err, content) {
+      if (err)
+        return callback(err)
 
-            if (zipEntry) {
-              try {
-                body = unzip(zipEntry, body)
-              } catch (e) {
-                callback('unzip error' + e)
-              }
-              if (!body)
-                return callback('could not find entry "' + zipEntry + '" inside zip file')
-            }
+      if (zipEntry) {
+        try {
+          content = unzip(zipEntry, content)
+        } catch (e) {
+          return callback('unzip error' + e)
+        }
 
-            callback(null, body)
-          }
+        if (!content)
+          return callback('could not find entry ' + zipEntry + ' inside zip file')
+      }
 
-      request(options, handle)
-    }
+      callback(null, content.toString())
+    }))
+}
 
 module.exports = fetch

lib/ghauth.js

@@ -1,21 +1,21 @@
-var read = require('read')
-  , colors = require('colors')
+const read   = require('read')
+    , colors = require('colors')
 
-  , auth = function (config, callback) {
-      console.log('To collect GitHub stats we need you to authenticate so we can exceed API rate-limiting.'.bold.green)
-      console.log('Leave the username field blank to skip stats collection.'.bold.green)
-      console.log()
+function auth (config, callback) {
+  console.log('To collect GitHub stats we need you to authenticate so we can exceed API rate-limiting.'.bold.green)
+  console.log('Leave the username field blank to skip stats collection.'.bold.green)
+  console.log()
 
-      read({ prompt: 'GitHub username:'.bold }, function (err, data) {
-        if (err) return callback(err)
-        if (data === '') return callback()
-        config.ghUser = data
-        read({ prompt: 'GitHub password:'.bold, silent: true, replace: '\u2714' }, function (err, data) {
-          if (err) return callback(err)
-          config.ghPass = data
-          callback()
-        })
-      })
-    }
+  read({ prompt: 'GitHub username:'.bold }, function (err, data) {
+    if (err) return callback(err)
+    if (data === '') return callback()
+    config.ghUser = data
+    read({ prompt: 'GitHub password:'.bold, silent: true, replace: '\u2714' }, function (err, data) {
+      if (err) return callback(err)
+      config.ghPass = data
+      callback()
+    })
+  })
+}
 
-module.exports = auth  
+module.exports = auth