Permalink
Browse files

Initial code

seems to be a problem with github api
  • Loading branch information...
1 parent e0fb3de commit 0c22cf29ea0eba26501f24465e501ef0fd92ba9c @ForbesLindesay ForbesLindesay committed Mar 20, 2013
View
2 .gitignore
@@ -12,3 +12,5 @@ logs
results
npm-debug.log
+
+node_modules
View
4 README.md
@@ -1,4 +1,6 @@
bot
===
-The esdiscuss bot
+The esdiscuss bot
+
+**N.B.** this requires the change in https://github.com/ajaxorg/node-github/pull/5 or https://github.com/mikedeboer/node-github/pull/82 to run.
View
33 index.js
@@ -0,0 +1,33 @@
+var pipermail = require('pipermail');
+
+var stream = pipermail('https://mail.mozilla.org/pipermail/es-discuss/',
+ {progress: false, cache: true})
+ .pipe(require('./lib/pipermail-filters').spam())
+ .pipe(require('./lib/pipermail-filters').fixSubjects())
+ .pipe(require('./lib/pipermail-filters').fixDates())
+ .pipe(require('./lib/pipermail-filters').after('40 days'))
+ .pipe(require('./lib/pipermail-filters').notExists())
+ .pipe(require('./lib/pipermail-output')({
+ user: {type: 'basic', username: 'user', password: 'password'},
+ organisation: 'esdiscuss',
+ team: '337802'
+ }))
+// .pipe(stringify())
+ .pipe(process.stdout);
+//stream.on('error', def.reject.bind(def));
+//stream.on('end', def.resolve.bind(def, null));
+
+function stringify() {
+ return require('through')(function (message) {
+ var date = normaliseDate(message.header.date);
+ var messageID = message.header.messageID.replace(/\</g, '').replace(/\>/g, '');
+ var path = 'https://raw.github.com/esdiscuss/' + date + '/master/' + encodeURIComponent(messageID);
+ this.queue(path + '/header.json' + '\n\n');
+ })
+}
+
+function normaliseDate(date) {
+ var month = '' + (date.getMonth() + 1);
+ if (month.length === 1) month = '0' + month;
+ return date.getFullYear() + '-' + month;
+}
View
14 lib/pipermail-filters/after.js
@@ -0,0 +1,14 @@
+var through = require('through');
+var ms = require('ms');
+
+module.exports = after;
+function after(timespan) {
+ var minimum = Date.now() - (typeof timespan === 'string' ? ms(timespan) : timespan);
+ if (typeof minimum != 'number' || isNaN(minimum)) throw new TypeError('timespan must be a number');
+ minimum = new Date(minimum);
+ return through(function (message) {
+ if (message.header.date > minimum) {
+ this.queue(message);
+ }
+ })
+}
View
9 lib/pipermail-filters/fix-dates.js
@@ -0,0 +1,9 @@
+var through = require('through');
+
+module.exports = fixDates;
+function fixDates() {
+ return through(function (item) {
+ item.header.date = new Date(item.header.date);
+ this.queue(item);
+ });
+}
View
18 lib/pipermail-filters/fix-subjects.js
@@ -0,0 +1,18 @@
+var through = require('through');
+
+module.exports = fixSubjects;
+function fixSubjects() {
+ var subjects = {};
+ var convoIDs = {};
+ return through(function (item) {
+ item.header.subject =
+ (convoIDs[item.header.messageID] =
+ convoIDs[item.header.inReplyTo] ||
+ subjects[tag(item.header.subject)] ||
+ (subjects[tag(item.header.subject)] = item.header.subject));
+ this.queue(item);
+ });
+ function tag(subject) {
+ return subject.replace(/[^a-z]+/gi, '').replace(/fwd?/gi, '').replace(/re/gi, '');
+ }
+}
View
14 lib/pipermail-filters/index.js
@@ -0,0 +1,14 @@
+var through = require('through');
+
+exports.spam = require('./spam');
+exports.fixSubjects = require('./fix-subjects');
+exports.fixDates = require('./fix-dates');
+exports.after = require('./after');
+exports.notExists = require('./not-exists');
+
+exports.selectSubject = selectSubject;
+function selectSubject() {
+ return through(function (item) {
+ this.queue([item.header.messageID, item.header.subject]);
+ });
+}
View
80 lib/pipermail-filters/not-exists.js
@@ -0,0 +1,80 @@
+var Q = require('q');
+var through = require('through');
+var request = require('request');
+
+module.exports = notExistsStream;
+
+function notExistsStream() {
+ var inProgress = 0;
+ var ended = false;
+ return through(function (message) {
+ var self = this;
+ if (inProgress === 0) self.pause();
+ inProgress++;
+
+ var date = normaliseDate(message.header.date);
+ var messageID = message.header.messageID.replace(/\</g, '').replace(/\>/g, '');
+
+ exists(date, messageID)
+ .done(function (exists) {
+ if (!exists) self.queue(message);
+ if (0 === --inProgress) self.resume();
+ if (ended && inProgress === 0) self.queue(null);
+ });
+ }, function () {
+ if (inProgress === 0) this.queue(null);
+ else ended = true;
+ });
+}
+
+function exists(date, message) {
+ var self = this;
+ var path = 'https://raw.github.com/esdiscuss/'
+ + date + '/master/'
+ + encodeURIComponent(message);
+ return pathExists(path + '/header.json')
+ .then(function (exists) {
+ return exists || pathExists(path + '/edited.md');
+ })
+ .then(function (exists) {
+ return exists || pathExists(path + '/original.md');
+ })
+};
+
+function pathExists(path) {
+ function exists() {
+ return Q.nfcall(request.head, path)
+ .spread(function (res) {
+ if (res.statusCode != 200 && res.statusCode != 404) {
+ throw error(path, res);
+ } else if (res.statusCode === 404) {
+ return false;
+ } else {
+ return true;
+ }
+ });
+ }
+ return retry(exists, 5, 0, path);
+}
+
+function error(path, res) {
+ return new Error('Server responded with ' + res.statusCode + ' to ' + JSON.stringify(path));
+}
+
+function retry(fn, attempts, delay, name) {
+ delay = delay || 0;
+ return attempts < 2 ? fn() : (fn()
+ .fail(function () {
+ console.log('retrying ' + name + ': ' + (delay + 1));
+ return Q.delay(Math.pow(2, delay) * 1000)
+ .then(function () {
+ return retry(fn, attempts - 1, delay + 1);
+ });
+ }))
+}
+
+function normaliseDate(date) {
+ var month = '' + (date.getMonth() + 1);
+ if (month.length === 1) month = '0' + month;
+ return date.getFullYear() + '-' + month;
+}
View
11 lib/pipermail-filters/spam.js
@@ -0,0 +1,11 @@
+var through = require('through');
+
+module.exports = spam;
+function spam() {
+ return through(function (item) {
+ if (!/spam/i.test(item.header.subject) &&
+ !/no subject/i.test(item.header.subject)) {
+ this.queue(item);
+ }
+ });
+}
View
162 lib/pipermail-output/github.js
@@ -0,0 +1,162 @@
+var Client = new require('github');
+var request = require('request');
+var Q = require('q');
+
+module.exports = GitHub;
+
+function GitHub(settings) {
+ this.settings = settings;
+ this.client = new Client({version: '3.0.0', debug: true});
+ this.client.authenticate(settings.user);
+
+ this.createdRepos = {};
+}
+
+GitHub.prototype.createRepo = function (repo) {
+ if (this.createdRepos[repo]) return this.createdRepos[repo];
+ var def = Q.defer();
+ var self = this;
+ var oldLog = require('github/util').log;
+ require('github/util').log = function () {};
+ function create() {
+ console.log('========Create Repo=========');
+ return Q.nfcall(self.client.repos.createFromOrg, {
+ org: self.settings.organisation,
+ name: repo,
+ auto_init: true,
+ team_id: self.settings.team
+ })
+ .fail(function (err) {
+ if (err && !(err.message && /already exists/.test(err.message))) {
+ throw err;
+ }
+ });
+ }
+ return this.createdRepos[repo] = retry(create, 5)
+ .then(function () {
+ require('github/util').log = oldLog;
+ }, function (err) {
+ require('github/util').log = oldLog;
+ self.createdRepos[repo] = false;
+ throw err;
+ });
+};
+
+GitHub.prototype.createCommit = function (repo) {
+ return new Commit(this.client, this.settings, repo);
+};
+
+function Commit(client, settings, repo) {
+ this.completed = false;
+ this.repo = repo;
+ this.client = client;
+ this.settings = settings;
+
+ console.log('========Get Sha Latest Commit==========');
+ this.shaLatestCommit = retry(Q.nfbind(client.gitdata.getReference, ({
+ user: settings.organisation,
+ repo: repo,
+ ref: 'heads/master'
+ })), 3).get('object').get('sha');
+ this.shaBaseTree = this.shaLatestCommit
+ .then(function (shaLatestCommit) {
+ console.log('=========Get Latest Commit=========');
+ return retry(Q.nfbind(client.gitdata.getCommit, {
+ user: settings.organisation,
+ repo: repo,
+ sha: shaLatestCommit
+ }), 3);
+ }).get('tree').get('sha');
+ this.shaNewTree = this.shaBaseTree;
+}
+
+Commit.prototype.addFile = function (path, content) {
+ return this.addFiles([{
+ path: path,
+ content: content
+ }]);
+};
+
+Commit.prototype.addFiles = function (files) {
+ this.assertLive();
+ var settings = this.settings;
+ var repo = this.repo;
+ var client = this.client;
+ var res = this.shaNewTree.then(function (shaBaseTree) {
+ console.log('==========Create New Tree===========');
+ return retry(Q.nfbind(client.gitdata.createTree, {
+ user: settings.organisation,
+ repo: repo,
+ tree: files.map(function (file) {
+ return {
+ path: file.path,
+ mode: '100644',
+ type: 'blob',
+ content: file.content,
+ sha: sha1(file.content)
+ }
+ }),
+ base_tree: shaBaseTree
+ }), 3);
+ });
+ this.shaNewTree = res.get('sha');
+ return res;
+};
+function sha1(str) {
+ var crypto = require('crypto')
+ , shasum = crypto.createHash('sha1');
+ shasum.update(str);
+ return shasum.digest('hex');
+}
+
+Commit.prototype.complete = function (message) {
+ this.assertLive();
+ this.completed = true;
+ var settings = this.settings;
+ var repo = this.repo;
+ var client = this.client;
+ return Q.all([this.shaLatestCommit, this.shaNewTree])
+ .spread(function (shaLatestCommit, shaNewTree) {
+ console.log('=============================');
+ console.log('creating commit');
+ return retry(Q.nfbind(client.gitdata.createCommit, {
+ user: settings.organisation,
+ repo: repo,
+ message: message,
+ tree: shaNewTree,
+ parents: [shaLatestCommit]
+ }), 3);
+ })
+ .get('sha')
+ .then(function (shaNewCommit) {
+ console.log('=============================');
+ console.log('pushing commit');
+ return retry(Q.nfbind(client.gitdata.createReference, {
+ user: settings.organisation,
+ repo: repo,
+ ref: 'heads/master',
+ sha: shaNewCommit
+ }), 3)
+ .fail(function (err) {
+ if (err instanceof Error) throw err;
+ else throw new Error(err);
+ });
+ });
+};
+
+Commit.prototype.assertLive = function () {
+ if (this.completed)
+ throw new Error('You can\'t interact with the commit after it\'s completed');
+};
+
+function retry(fn, attempts, delay) {
+ delay = delay || 0;
+ return attempts < 2 ? fn() : fn()
+ .fail(function () {
+ console.log('retrying: ' + (delay + 1));
+ return Q.delay(Math.pow(2, delay) * 1000)
+ .then(function () {
+ return retry(fn, attempts - 1, delay + 1);
+ });
+ })
+}
View
81 lib/pipermail-output/index.js
@@ -0,0 +1,81 @@
+var through = require('through');
+var GitHub = new require('./github');
+var Q = require('q');
+
+module.exports = outputToGitHub;
+function outputToGitHub(settings) {
+ var client = new GitHub(settings);
+ var commits = {};
+ var commitsToPush = [];
+ var i = 0;
+
+ var inProgress = 0;
+
+ return through(function (item) {
+ var self = this;
+
+ if (inProgress === 0) self.pause();
+ inProgress++
+
+ var date = normaliseDate(item.header.date);
+
+ var messageID = item.header.messageID.replace(/\</g, '').replace(/\>/g, '');
+ //console.log(date + '/' + messageID);
+ client.createRepo(date)
+ .then(function (exists) {
+ var commit = commits[date];
+ if (!commit) {
+ commit = commits[date] = client.createCommit(date);
+ commitsToPush.push(commit);
+ }
+
+ return commit.addFiles([
+ {
+ path: messageID + '/header.json',
+ content: JSON.stringify(item.header, null, 2)
+ },
+ {
+ path: messageID + '/original.md',
+ content: item.body
+ },
+ {
+ path: messageID + '/edited.md',
+ content: item.body
+ }]);
+ })
+ .timeout(600000)
+ .done(function () {
+ self.queue(date + '/' + messageID + ' -> created\n');
+ if (0 === --inProgress) self.resume();
+ }, function (err) {
+ console.error(err.stack || err.message || err);
+ if (0 === --inProgress) self.resume();
+ });
+ }, function (finish) {
+ var self = this;
+ checkForFinish();
+ function checkForFinish() {
+ if (inProgress) return setTimeout(checkForFinish, 100);
+ var current = Q.resolve(null);
+ commitsToPush.forEach(function (commit) {
+ current = current.then(function () {
+ return commit.complete('Add Messages');
+ });
+ });
+ current
+ .done(function () {
+ self.queue(null);
+ }, function (err) {
+ console.error(err.stack || err.message || err);
+ self.queue(null);
+ });
+ }
+
+ }, true);
+}
+
+function normaliseDate(date) {
+ var month = '' + (date.getMonth() + 1);
+ if (month.length === 1) month = '0' + month;
+ return date.getFullYear() + '-' + month;
+}
View
23 package.json
@@ -0,0 +1,23 @@
+{
+ "name": "esdiscuss-bot",
+ "version": "0.0.0",
+ "description": "The esdiscuss bot",
+ "main": "index.js",
+ "scripts": {
+ "test": "mocha -R spec"
+ },
+ "repository": {
+ "type": "git",
+ "url": "https://github.com/esdiscuss/bot.git"
+ },
+ "author": "ForbesLindesay",
+ "license": "MIT",
+ "readmeFilename": "README.md",
+ "gitHead": "e0fb3de077e9c0df98dffc87d48b029c0216c876",
+ "dependencies": {
+ "through": "~2.2.7",
+ "q": "~0.9.2",
+ "pipermail": "~1.0.1",
+ "request": "~2.16.2"
+ }
+}

0 comments on commit 0c22cf2

Please sign in to comment.