Permalink
Browse files

Various fixes

  • Loading branch information...
1 parent 8418d42 commit 949db3131fd4333af488f2fce6e756ea92c46c84 @mixu committed Oct 30, 2015
Showing with 208 additions and 76 deletions.
  1. +34 −37 lib/lambda.js
  2. +26 −0 lib/parse-target.js
  3. +20 −5 lib/stream/from-fs.js
  4. +4 −4 lib/stream/to-fs.js
  5. +33 −26 lib/task.js
  6. +1 −1 package.json
  7. +2 −2 test/integration.test.js
  8. +47 −0 test/lambda.test.js
  9. +40 −0 test/parse-target.test.js
  10. +1 −1 test/task.test.js
View
@@ -2,27 +2,8 @@ var url = require('url');
var Orchestrator = require('orchestrator');
var identify = require('identify-github-event');
var Task = require('./task');
-
-function parseTarget(target) {
- var result = target.match(/([^\/]+)\/([^# ]+)(.*)/);
- if (!result) {
- return { user: '', repo: '', branch: '' };
- }
- if (!result[3]) {
- return {
- user: result[1],
- repo: result[2],
- branch: 'master',
- };
- }
- var rest = result[3].match(/#?([^ ]*)( -)? (.*)/);
- return {
- user: result[1],
- repo: result[2],
- branch: rest[1] || 'master',
- name: rest[3],
- };
-}
+var parseTarget = require('./parse-target.js');
+var xtend = require('xtend');
var instances = [];
@@ -31,7 +12,6 @@ function Lambda() {
this._targetToTasks = {};
this._taskNames = {};
this._config = {};
- this._counter = 1;
// track all Lambdas to allow for the CLI to find them
instances.push(this);
}
@@ -40,7 +20,7 @@ Lambda.prototype.config = function(key, config) {
if (arguments.length === 0) {
return this._config;
} else if (arguments.length === 2) {
- this._config[key] = config;
+ this._config[key] = xtend(this._config[key] || {}, config);
} else {
this._config = key;
}
@@ -61,8 +41,18 @@ Lambda.prototype.task = function(names, deps, fn) {
(Array.isArray(names) ? names : [names]).forEach(function(str) {
var target = parseTarget(str);
var fullTarget = target.user + '/' + target.repo + '#' + target.branch;
- var name = target.name || 'task-' + self._counter++;
- self._taskNames[str] = true;
+ var name;
+ // if the task has a task name, use the full string as the task name
+ if (target.name) {
+ name = str;
+ } else {
+ var counter = 1;
+ do {
+ name = str + ' - task-' + counter++;
+ } while(self._taskNames[name]);
+ }
+
+ self._taskNames[name] = true;
function noArgs() {
return fn(new Task({
@@ -85,23 +75,22 @@ Lambda.prototype.task = function(names, deps, fn) {
// if the task wants two params, convert it into a two-param callback that
// looks like a one-item callback to Orchestrator
var runner = fn.length < 2 ? noArgs : oneArg;
- self._orchestrator.add(str, deps, runner);
+ self._orchestrator.add(name, deps, runner);
if (!self._targetToTasks[fullTarget]) {
- self._targetToTasks[fullTarget] = [ str ];
+ self._targetToTasks[fullTarget] = [ name ];
} else {
- self._targetToTasks[fullTarget].push(str);
+ self._targetToTasks[fullTarget].push(name);
}
});
};
Lambda.prototype.getTasksByEvent = function(event) {
- var self = this,
- user = event.user,
+ var user = event.user,
repo = event.repo,
branch = event.branch;
var fullTarget = event.user + '/' + event.repo + '#' + event.branch;
- return this._targetToTasks[fullTarget];
+ return this._targetToTasks[fullTarget] || [];
};
Lambda.prototype.exec = function(event, onDone) {
@@ -121,7 +110,7 @@ Lambda.prototype.exec = function(event, onDone) {
} else if (typeof event === 'object') {
var target = identify.target(event);
// find the tasks that match the event
- tasks = this.getTasksByEvent(target);
+ taskNames = this.getTasksByEvent(target);
} else {
console.log('[markdown-styles-lambda] No target event or task was specified.');
this.printKnownTasks();
@@ -165,23 +154,31 @@ Lambda.prototype.printKnownTasks = function() {
};
Lambda.prototype.snsHandler = function(events) {
+ var self = this;
if (typeof events === 'string') {
events = [events];
}
- var matchAll = events.indexOf('*') !== -1 || arguments.length === 0;
+ var matchAll = arguments.length === 0 || events.indexOf('*') !== -1 ;
return function(event, context) {
- console.log('[markdown-styles-lambda] Received event:', event);
- var eventType = lambda.identifyGithubEvent(event.Records[0].Sns.Message);
+ var unwrappedEvent = {};
+ console.log('[markdown-styles-lambda] Received event:', JSON.stringify(event, null, 2));
+ try {
+ unwrappedEvent = JSON.parse(event.Records[0].Sns.Message);
+ } catch (e) {
+ console.log('[markdown-styles-lambda] Could not parse SNS message payload as JSON.');
+ }
+ console.log('[markdown-styles-lambda] Unwrapped event:', JSON.stringify(unwrappedEvent));
+ var eventType = self.identifyGithubEvent(unwrappedEvent);
if (matchAll || events.indexOf(eventType) !== -1) {
- lambda.exec(event.Records[0].Sns.Message, context);
+ self.exec(unwrappedEvent, context);
} else {
console.log('[markdown-styles-lambda] Did nothing with ' + eventType);
context.success();
}
};
}
-Lambda.prototype.identifyGithubEvent = identify;
+Lambda.identifyGithubEvent = Lambda.prototype.identifyGithubEvent = identify;
Lambda.create = Lambda.prototype.create = function() {
return new Lambda();
View
@@ -0,0 +1,26 @@
+module.exports = function parseTarget(target) {
+ var parts = target.match(/([^\/]+)\/([^# ]+)(.*)/);
+ if (!parts) {
+ return { user: '', repo: '', branch: 'master' };
+ }
+ var result = {
+ user: parts[1],
+ repo: parts[2],
+ }
+ if (!parts[3]) {
+ result.branch = 'master';
+ return result;
+ }
+ var branch = parts[3].match(/#([^ ]+)(.*)/);
+ if (branch) {
+ result.branch = branch[1];
+ result.name = branch[2].replace(/^( -)? /, '');
+ } else {
+ result.branch = 'master';
+ result.name = parts[3].replace(/^( -)? /, '');
+ }
+ if (!result.name) {
+ delete result.name;
+ }
+ return result;
+};
View
@@ -1,15 +1,30 @@
var fs = require('fs'),
- pi = require('pipe-iterators');
+ pi = require('pipe-iterators'),
+ xtend = require('xtend');
-function read() {
+function read(opts) {
+ opts = xtend({
+ read: true,
+ buffer: true,
+ }, opts);
return pi.thru.obj(function(file, enc, onDone) {
var stat = fs.statSync(file);
if (stat.isFile()) {
- this.push({
+ var result = {
path: file,
stat: stat,
- contents: fs.readFileSync(file, 'utf8')
- });
+ contents: null,
+ }
+ if (!opts.read) {
+ this.push(result);
+ } else {
+ if (opts.buffer) {
+ result.contents = fs.readFileSync(file, 'utf8')
+ } else {
+ result.contents = fs.createReadStream(file);
+ }
+ this.push(result);
+ }
}
onDone();
});
View
@@ -14,8 +14,9 @@ function dest(output) {
file.path = output + file.path;
var writeDir = path.dirname(file.path);
- function log() {
+ function done() {
console.log('[FS] Wrote ' + originalPath + ' -> ' + file.path);
+ onDone();
}
(seen[writeDir] ? function(a, onDone) { onDone(null); } : mkdirp)(
@@ -25,11 +26,10 @@ function dest(output) {
}
seen[writeDir] = true;
if (!pi.isReadable(file.contents)) {
- fs.writeFile(file.path, file.contents, log);
+ fs.writeFile(file.path, file.contents, done);
} else {
- file.contents.pipe(fs.createWriteStream(file.path)).once('finish', log);
+ file.contents.pipe(fs.createWriteStream(file.path)).once('finish', done);
}
- onDone();
}
);
});
View
@@ -22,7 +22,11 @@ Task.prototype.config = function(key, config) {
if (arguments.length === 0) {
return this._config;
} else if (arguments.length === 2) {
- this._config[key] = config;
+ if (typeof config === 'string' || typeof config === 'boolean' || typeof config === 'number') {
+ this._config[key] = config;
+ } else {
+ this._config[key] = xtend(this._config[key] || {}, config);
+ }
} else {
this._config = key;
}
@@ -42,21 +46,21 @@ Task.prototype.github = function(glob, opts) {
opts.base += '/';
}
var readable = pi.fromAsync(function(onDone) {
- console.log('[Github API] Matching against Github Contents API with glob ' + glob);
- gglob({
- authenticate: self._config.github,
- user: self.user,
- repo: self.repo,
- branch: self.branch,
- glob: glob
- }, function(err, results, meta) {
- console.log('[Github API] Glob match ' + glob + ' completed. API limit remaining: ' + meta.limit);
- if (err) {
- console.error('[Github API] Returned error:', err);
- }
- onDone(null, results);
- });
- }).pipe(
+ console.log('[Github API] Matching against Github Contents API with glob ' + glob);
+ gglob({
+ authenticate: self._config.github,
+ user: self.user,
+ repo: self.repo,
+ branch: self.branch,
+ glob: glob
+ }, function(err, results, meta) {
+ if (err) {
+ console.error('[Github API] Returned error:', err);
+ }
+ console.log('[Github API] Glob match ' + glob + ' completed. Matched ' + results.length + ' files. API limit remaining: ' + meta.limit);
+ onDone(null, results);
+ });
+ }).pipe(
pi.thru.obj(function(file, enc, done) {
// path relative to the basepath of the glob
var path = file.path;
@@ -134,40 +138,43 @@ Task.prototype.s3 = function(target) {
var parts = url.parse(target);
var bucket = parts.host;
// S3 can write to `/foo/bar`, but only paths like `foo/bar` show up in the S3 UI and work for static site hosting
- var key = (parts.path.charAt(0) === '/' ? parts.path.substr(1) : parts.path);
+ var key = (parts.path && parts.path.charAt(0) === '/' ? parts.path.substr(1) : '');
var s3 = new AWS.S3(this._config.s3 ? this._config.s3 : {});
return pi.writable.obj(function(file, enc, onDone) {
var stream = this;
var contentType = mime.lookup(file.path) || 'text/html';
- console.log('[S3] Writing ' + file.path + ' -> s3://' + bucket + '/' + key + file.path + ' as ' + contentType);
+ var fileKey = (key + file.path).replace(/^\/+/, '');
+ console.log('[S3] Writing ' + file.path + ' -> s3://' + bucket + '/' + fileKey + ' as ' + contentType);
s3.putObject({
Bucket: bucket,
- Key: key + file.path,
+ Key: fileKey,
Body: file.contents,
ContentType: contentType
}, function(err, data) {
- console.log('[S3] Wrote ' + file.path + ' -> s3://' + bucket + '/' + key + file.path);
+ console.log('[S3] Wrote ' + file.path + ' -> s3://' + bucket + '/' + fileKey );
onDone(err);
});
});
};
-Task.prototype.fromFs = function(glob, basename) {
- var base = basename || parse.basename(glob);
- if (base.charAt(base.length - 1) !== '/') {
- base += '/';
+Task.prototype.fromFs = function(glob, opts) {
+ opts = xtend({
+ base: parse.basename(glob)
+ }, opts);
+ if (opts.base.charAt(opts.base.length - 1) !== '/') {
+ opts.base += '/';
}
return pi.pipeline([
wildglob.stream(glob),
pi.filter(function(filename) {
var stat = fs.statSync(filename);
return stat.isFile();
}),
- fromFs(),
+ fromFs(opts),
pi.mapKey('path', function(path) {
// path relative to the basepath of the glob
- var stripPath = path.substr(0, base.length) === base ? path.substr(base.length) : path;
+ var stripPath = path.substr(0, opts.base.length) === opts.base ? path.substr(opts.base.length) : path;
// ensure that the first character is a /
if (stripPath.charAt(0) !== '/') {
stripPath = '/' + stripPath;
View
@@ -4,7 +4,7 @@
"description": "",
"main": "./lib/lambda.js",
"scripts": {
- "test": "echo \"Error: no test specified\" && exit 1"
+ "test": "mocha --bail ./test/*.test.js"
},
"author": "Mikito Takada <mikito.takada@gmail.com> (http://mixu.net/)",
"license": "ISC",
@@ -26,7 +26,7 @@ describe('integration tests', function() {
'abc/bar/baz.md': '# Hello'
});
var task = new Task({});
- task.fromFs(tmpdir + '/**/*.md', tmpdir + '/abc')
+ task.fromFs(tmpdir + '/**/*.md', { base: tmpdir + '/abc'})
.pipe(pi.map(function(item) { return item.path; }))
.pipe(pi.toArray(function(results) {
assert.deepEqual(results, [ '/foo.md', '/bar/baz.md' ]);
@@ -40,7 +40,7 @@ describe('integration tests', function() {
'abc/bar/baz.md': '# Hello'
});
var task = new Task({});
- task.fromFs(tmpdir + '/**/*.md', tmpdir + '/abc/')
+ task.fromFs(tmpdir + '/**/*.md', { base: tmpdir + '/abc/' })
.pipe(pi.map(function(item) { return item.path; }))
.pipe(pi.toArray(function(results) {
assert.deepEqual(results, [ '/foo.md', '/bar/baz.md' ]);
Oops, something went wrong.

0 comments on commit 949db31

Please sign in to comment.