Work on AMD

.jshintignore

@@ -1,5 +1,5 @@
 node_modules/
 test/node_modules
 test/tmp
-lib/runner/package-commonjs/resources
+lib/runner/commonjs/resources
 test/command-integration

bin/get.js

@@ -4,11 +4,10 @@ var fs = require('fs'),
     path = require('path'),
     util = require('util'),
     Minilog = require('minilog'),
-    DetectiveList = require('../lib/detective-list.js'),
-    AmdList = require('../lib/amd-list.js'),
-    amdetective = require('amdetective'),
-    amdresolve = require('amd-resolve'),
-    SortDependencies = require('../lib/sort-dependencies.js'),
+    DetectiveList = require('../lib/list/detective.js'),
+    AmdList = require('../lib/list/amd.js'),
+    loadAMDConfig = require('../lib/runner/amd/load-config.js'),
+    runner = require('../lib/runner/amd'),
     Cache = require('minitask').Cache;
 
 var opts = require('optimist')
@@ -25,6 +24,7 @@ if(!argv['include']) {
   console.log('Usage: --include <file/dir>');
   console.log('Options:');
   console.log('  --amd');
+  console.log('  --config');
   console.log('  --main <file>');
   process.exit(1);
 }
@@ -41,16 +41,6 @@ opts = {
 var main = argv.main || Array.isArray(argv.include) ? argv.include[0] : argv.include,
     basepath = path.dirname(main);
 
-function loadAMDConfig(filepath) {
-  // the config specification for RJS is painful to parse as it's not a JSON file
-  // but rather a JS file that defines as specifically named variable
-  var sandbox = {};
-  require('vm').runInNewContext(
-    fs.readFileSync(filepath).toString(), sandbox);
-
-  return sandbox.require;
-}
-
 if(argv.amd) {
   opts.amdresolve = loadAMDConfig(argv.config);
 }
@@ -68,198 +58,38 @@ var cache = Cache.instance({
 });
 cache.begin();
 
-function lookupDeps(filepath) {
-  var key = opts['cache-hash'] + '-amdependencies',
-      noCache = false; // for easy dev
-
-  return cache.file(filepath).data(key);
-}
-
-function wrapAMD(filepath, basePath) {
-  var deps = lookupDeps(filepath),
-      // the substr here will not be correct for files under folders which have been mapped unless the path length
-      // happens to be identical e.g. app and lib
-      relativeName = (path.dirname(filepath) + '/' + path.basename(filepath, path.extname(filepath))).substr(basePath.length + 1);
-
-  return fs.readFileSync(filepath).toString().replace('define(', 'define(' +
-    JSON.stringify(relativeName) + ', ' +
-    JSON.stringify(deps) + ', '
-  );
-}
-
-var template = fs.readFileSync(__dirname + '/../lib/amd-vendor-wrap.js').toString();
-
-function wrapAMDVendor(name, filepath, deps, globalName) {
-  var result = '';
-
-  if(!filepath) {
-    return '';
-  }
-
-
-  if(globalName) {
-    result += fs.readFileSync(filepath).toString();
-    result += template.replace('%name%', JSON.stringify(name)).replace('%deps%', JSON.stringify(deps)).replace('%global%', globalName);
-  } else {
-
-    if(!deps) {
-      try {
-        deps = amdetective(fs.readFileSync(filepath).toString());
-      } catch(e) {
-      }
-      console.log(deps);
-    }
-
-    // assuming: define(function (require, exports, module) {
-    // -> define('decode',['require','exports','module'],function (require, exports, module) {
-    result += fs.readFileSync(filepath).toString().replace('define(', 'define(' +
-      JSON.stringify(name) + ', ' +
-      JSON.stringify(deps || ['require','exports','module']) + ', '
-    );
-  }
-
-  return result;
-}
-
-function uniq() {
-  var last = null;
-  return function(item) {
-    // to make a set of sorted keys unique, just check that consecutive keys are different
-    var isDuplicate = (item == last);
-    last = item;
-    return !isDuplicate;
-  };
-}
-
 console.log('Reading files: ');
 (Array.isArray(argv.include) ? argv.include : [ argv.include ]).map(function(filepath) {
   console.log('  ' + filepath);
   list.add(filepath);
 });
 
 list.exec(function(err, files) {
-  cache.end();
-  var errs = list.resolveErrors();
-  console.log('done!');
+  console.log('Processing ' + files.length + ' files.');
+  runner({ files: files }, {
+      main: argv.main,
+      basepath: basepath,
+      configjs: opts.amdresolve,
+      errs: list.resolveErrors(),
+      'cache-method': opts['cache-method'],
+      'cache-path': opts['cache-path'],
+      cache: true,
+      jobs: require('os').cpus().length * 2
+    }, fs.createWriteStream(path.resolve(process.cwd(), './bundle.js')), function() {
+    cache.end();
+    console.log('compiled');
+  });
 
 /*
   console.log(errs.map(function(item) {
       return item.dep;
     }).sort());
 */
-  console.log(files.length);
 
   files = files.filter(function(e) {
     return path.basename(e.name) != 'application.js';
   });
 
-
-  var sorter = new SortDependencies();
-
-  var basePath = path.dirname(argv.config);
-
-  var configjs = loadAMDConfig(argv.config);
-
-
-  // find the paths of the vendor files
-  var vendorNames = Object.keys(configjs.paths);
-      vendorPaths = { };
-
-  var missing = errs.map(function(item) {
-    return item.dep;
-  }).sort().filter(uniq());
-
-  missing.concat(vendorNames).forEach(function(name) {
-    if(vendorPaths[name]) {
-      return;
-    }
-    var fullpath = '';
-
-    configjs.relDir = basePath;
-    configjs.baseDir = basePath;
-
-    try {
-      fullpath = amdresolve.sync(name, configjs);
-    } catch(err) { }
-
-
-    if(fullpath) {
-      fullpath = path.normalize(fullpath);
-      stat = fs.statSync(fullpath);
-      if(stat.isFile()) {
-        vendorPaths[name] = fullpath;
-      }
-    }
-  });
-
-  missing = missing.filter(function(name) {
-    return !vendorPaths[name];
-  });
-
-  // list the still not found items
-  var failed = missing.sort().filter(uniq());
-  // console.log(vendorPaths, failed);
-
-  // TODO at this point, plug in the config data
-
-  // figure out the actual 3rd party dependencies (from the list) and extract names from paths key
-  // Note: the CANONICAL names are from the paths array
-  vendorNames.forEach(function(name) {
-    sorter.add({ name: name, deps: configjs.shim && configjs.shim[name] && configjs.shim[name].deps || [] });
-  });
-
-  var result = '';
-
-  // Use the resolver to output the vendor files first
-
-  sorter.resolve('require');
-
-  console.log('Vendor');
-  while(!sorter.isEmpty()) {
-    var next = sorter.next();
-    console.log('\t' + next.name + (vendorPaths[next.name] ? ' OK' : ''));
-
-    // due to unfilled dep on /v2/config
-    if(next.name != 'triconf') {
-
-      var vendorShimEntry = configjs.shim[next.name] || {};
-
-      result += wrapAMDVendor(next.name, vendorPaths[next.name], vendorShimEntry.deps, vendorShimEntry.exports || '');
-    }
-
-    sorter.resolve(next);
-  }
-
-  // now add the other package files
-
-  files.forEach(function(file) {
-    sorter.add({ name: file.name, deps: lookupDeps(file.name) || [] });
-  });
-
-  sorter.resolve('require');
-
-  console.log('App');
-  while(!sorter.isEmpty()) {
-    var next = sorter.next();
-    console.log('\t' + next.name);
-    result += wrapAMD(next.name, basePath);
-    sorter.resolve(next);
-  }
-
-  fs.writeFileSync(path.resolve(process.cwd(), './bundle.js'), result);
-
-  /*
-  console.log(deps);
-
-  console.log(files.map(function(file) { return file.name + ' ' + lookupDeps(file.name); }));
-
-  console.log(files.length);
-
-
-  files.map();
-
-  */
-
   /*
   var inferPackages = require('../lib/list-tasks/infer-packages.js'),
       filterPackages = require('../lib/list-tasks/filter-packages.js');

index.js

@@ -1,7 +1,7 @@
 var os = require('os'),
     path = require('path'),
     List = require('minitask').list,
-    DetectiveList = require('./lib/detective-list.js'),
+    DetectiveList = require('./lib/list/detective.js'),
     packageCommonJs = require('./lib/runner/package-commonjs'),
     Capture = require('./lib/file-tasks/capture.js'),
     Minilog = require('minilog'),

lib/file-tasks/wrap-amd-vendor.js

@@ -0,0 +1,24 @@
+var fs = require('fs'),
+    template = fs.readFileSync(__dirname + '/../util/amd-vendor-wrap.js').toString();
+
+module.exports = function wrapAMDVendor(name, filepath, deps, globalName) {
+  var result = '';
+
+  if(!filepath) {
+    return '';
+  }
+
+  if(globalName) {
+    result += fs.readFileSync(filepath).toString();
+    result += template.replace('%name%', JSON.stringify(name)).replace('%deps%', JSON.stringify(deps)).replace('%global%', globalName);
+  } else {
+    // assuming: define(function (require, exports, module) {
+    // -> define('decode',['require','exports','module'],function (require, exports, module) {
+    result += fs.readFileSync(filepath).toString().replace('define(', 'define(' +
+      JSON.stringify(name) + ', ' +
+      JSON.stringify(deps || ['require','exports','module']) + ', '
+    );
+  }
+
+  return result;
+}

lib/file-tasks/wrap-amd.js

@@ -0,0 +1,13 @@
+var fs = require('fs'),
+    path = require('path');
+
+module.exports = function wrapAMD(filepath, deps, basePath) {
+  // the substr here will not be correct for files under folders which have been mapped unless the path length
+  // happens to be identical e.g. app and lib
+  var relativeName = (path.dirname(filepath) + '/' + path.basename(filepath, path.extname(filepath))).substr(basePath.length + 1);
+
+  return fs.readFileSync(filepath).toString().replace('define(', 'define(' +
+    JSON.stringify(relativeName) + ', ' +
+    JSON.stringify(deps) + ', '
+  );
+};

lib/amd-list.js → lib/list/amd.js

@@ -1,7 +1,7 @@
 var fs = require('fs'),
     path = require('path'),
-    amdetective = require('amdetective'),
     amdresolve = require('amd-resolve'),
+    amdDeps = require('../runner/amd/deps.js'),
     List = require('minitask').list,
     Cache = require('minitask').Cache,
     log = require('minilog')('parse');
@@ -16,9 +16,6 @@ function DetectiveList(opts) {
       path: opts['cache-path']
   });
 
-  var key = opts['cache-hash'] + '-amdependencies',
-      noCache = false; // for easy dev
-
   // replace the find function to use node-detective
   this.find(function(filepath, stat, onDone) {
     var self = this;
@@ -27,25 +24,11 @@ function DetectiveList(opts) {
       return onDone(null, []);
     }
 
-    var deps;
-
-    log.info('Parsing:', filepath);
+    // log.info('Parsing:', filepath);
 
-    // check the cache
-    deps = cache.file(filepath).data(key);
-    if (noCache || typeof deps === 'undefined') {
-      try {
-        deps = amdetective(fs.readFileSync(filepath).toString());
-      } catch(e) {
-        console.log('parse error: ', filepath, e);
-        cache.file(filepath).data(key, []);
-        return [];
-      }
-      // store result
-      cache.file(filepath).data(key, deps);
-    } else {
-      // console.log('using cached result', filepath, deps);
-    }
+    var deps = amdDeps(cache, filepath, function(err) {
+      console.log('parse error: ', filepath, err);
+    });
 
     // console.log(deps);
 

lib/runner/amd/deps.js

@@ -0,0 +1,23 @@
+var fs = require('fs'),
+    amdetective = require('amdetective');
+
+var key = 'amdependencies';
+
+module.exports = function(cache, filepath, onErr) {
+  // check the cache
+  deps = cache.file(filepath).data(key);
+  if (typeof deps === 'undefined') {
+    try {
+      deps = amdetective(fs.readFileSync(filepath).toString());
+    } catch(e) {
+      cache.file(filepath).data(key, []);
+      if(onErr) {
+        onErr(e);
+      }
+      return [];
+    }
+    // store result
+    cache.file(filepath).data(key, deps);
+  }
+  return deps;
+};