Skip to content
Permalink
Browse files

Refactor build data construction

  • Loading branch information...
mhart committed Jul 3, 2016
1 parent 19d0109 commit 2724d9f545c42035c5a7ea8b6fd20bf48f306d75
Showing with 139 additions and 124 deletions.
  1. +55 −8 actions/build.js
  2. +2 −23 actions/index.js
  3. +15 −0 actions/rebuild.js
  4. +7 −3 db/index.js
  5. +1 −1 html/build.html.js
  6. +4 −4 index.js
  7. +13 −16 sources/github.js
  8. +6 −38 sources/sns.js
  9. +27 −0 test/fixtures/push.tag.json
  10. +9 −31 test/parsers.js
@@ -14,12 +14,10 @@ var ecs = new AWS.ECS()

module.exports = runBuild

function runBuild(build, context, cb) {
// TODO: figure out whether to use mergeable or not
function runBuild(buildData, context, cb) {
// TODO: figure out whether to use mergeable flags on GH events or not

build.requestId = context.awsRequestId
build.logGroupName = context.logGroupName
build.logStreamName = context.logStreamName
var build = new BuildInfo(buildData, context)

// Sometimes errors will occur that we don't catch, and Lambda will retry those requests,
// so check if we've seen this request ID before, and if so, ignore it
@@ -31,8 +29,8 @@ function runBuild(build, context, cb) {
if (err) return cb(err)

if (data.retry) {
log.info(`Ignoring retry request for build #${data.buildNum}`)
return cb() // TODO: Ensure Github/Slack statuses are 'finished' too
log.info(`Ignoring retry request for build #${data.retry.buildNum}`)
return cb() // TODO: Ensure Github/Slack statuses are 'finished' too?
}

var config = configUtils.initConfig(data.configs, build)
@@ -52,7 +50,6 @@ function runBuild(build, context, cb) {
function cloneAndBuild(build, config, cb) {

build.token = config.secretEnv.GITHUB_TOKEN
build.cloneDir = path.join(configUtils.BASE_BUILD_DIR, build.repo)

clone(build, config, function(err) {
if (err) return cb(err)
@@ -303,3 +300,53 @@ function prepareDockerConfig(config) {
return utils.merge(defaultDockerConfig, config)
}

function BuildInfo(buildData, context) {
this.startedAt = new Date()
this.endedAt = null

this.status = 'pending'
this.statusEmitter = new (require('events'))()

this.project = buildData.project
this.buildNum = buildData.buildNum || 0

this.repo = buildData.repo || this.project.replace(/^gh\//, '')

if (buildData.trigger) {
var triggerPieces = buildData.trigger.split('/')
this.trigger = buildData.trigger
this.eventType = triggerPieces[0] == 'pr' ? 'pull_request' : 'push'
this.prNum = triggerPieces[0] == 'pr' ? +triggerPieces[1] : 0
this.branch = triggerPieces[0] == 'push' ? triggerPieces[1] : (buildData.branch || 'master')
} else {
this.eventType = buildData.eventType
this.prNum = buildData.prNum
this.branch = buildData.branch
this.trigger = this.prNum ? `pr/${this.prNum}` : `push/${this.branch}`
}

this.event = buildData.event
this.isPrivate = buildData.isPrivate

this.branch = buildData.branch
this.cloneRepo = buildData.cloneRepo || this.repo
this.checkoutBranch = buildData.checkoutBranch || this.branch
this.commit = buildData.commit
this.baseCommit = buildData.baseCommit
this.comment = buildData.comment
this.user = buildData.user

this.committers = buildData.committers

this.isFork = this.cloneRepo != this.repo

this.requestId = context.awsRequestId
this.logGroupName = context.logGroupName
this.logStreamName = context.logStreamName

this.cloneDir = path.join(configUtils.BASE_BUILD_DIR, this.repo)

this.token = ''
this.logUrl = ''
}

@@ -1,33 +1,12 @@
var config = require('../utils/config')
var db = require('../db')
var runBuild = require('./build')

// NOTE: All function exports from this module can be executed by Lambda

exports.build = runBuild

exports.version = function(event, context, cb) {
return cb(null, config.VERSION)
}

exports.rebuild = function(event, context, cb) {
if (!event.repo || !event.buildNum) {
return cb(new Error('Rebuild action missing repo or buildNum'))
}
db.getBuild(`gh/${event.repo}`, event.buildNum, function(err, build) {
if (err) return cb(err)
if (!build) return cb(new Error(`No build #${event.buildNum} found for repo ${event.repo}`))
var triggerPieces = build.trigger.split('/')
build.repo = event.repo
build.eventType = triggerPieces[0] == 'pr' ? 'pull_request' : 'push'
build.startedAt = new Date()
build.status = 'pending'
build.statusEmitter = new (require('events'))()
build.committers = build.committers.values.reduce((obj, key) => { obj[key] = key; return obj }, Object.create(null))
build.isFork = build.cloneRepo != build.repo
build.prNum = build.eventType == 'pull_request' ? +triggerPieces[1] : 0
exports.build = require('./build')

runBuild(build, context, cb)
})
}
exports.rebuild = require('./rebuild')

@@ -0,0 +1,15 @@
var db = require('../db')
var build = require('./build')

module.exports = rebuild

function rebuild(event, context, cb) {
if (!event.project || !event.buildNum) {
return cb(new Error('Rebuild action missing project or buildNum'))
}
db.getBuild(event.project, event.buildNum, function(err, buildData) {
if (err) return cb(err)
if (!buildData) return cb(new Error(`No build #${event.buildNum} found for project ${event.project}`))
build(buildData, context, cb)
})
}
@@ -73,7 +73,12 @@ exports.getBuild = function(project, buildNum, cb) {
Key: {project, buildNum},
}, function(err, data) {
if (err) return cb(friendlyErr(table, err))
cb(null, data && data.Item)
var buildData = (data || {}).Item
// Convert DynamoDB sets to JS sets
if (buildData && buildData.committers && buildData.committers.values) {
buildData.committers = new Set(buildData.committers.values)
}
cb(null, buildData)
})
}

@@ -82,7 +87,6 @@ exports.initBuild = function(build, cb) {
exports.getBuildNum(build.project, function(err, buildNum) {
if (err) return cb(err)
build.buildNum = buildNum
var committers = new Set(Object.keys(build.committers || {}).map(key => build.committers[key]))
client.put({
TableName: table,
Item: {
@@ -100,7 +104,7 @@ exports.initBuild = function(build, cb) {
baseCommit: build.baseCommit,
comment: build.comment,
user: build.user,
committers: committers.size ? client.createSet(Array.from(committers)) : undefined,
committers: (build.committers || {}).size ? client.createSet(Array.from(build.committers)) : undefined,
},
}, function(err) {
if (err) return cb(friendlyErr(table, err))
@@ -29,7 +29,7 @@ function render(params) {
var userTitle = build.prNum ? 'Pull request opener' : 'Branch pusher'

// Could calculate gravatar imgs here from email
var users = build.prNum ? [build.repo.split('/')[0]] : Object.keys(build.committers || {}).slice(0, 10).map(key => build.committers[key])
var users = build.prNum ? [build.repo.split('/')[0]] : Array.from(build.committers).slice(0, 10)
var usersIcon = users.length > 1 ? 'fa-users' : 'fa-user'
var usersStr = users.map(username => `<a href="https://github.com/${username}">${username}</a>`).join(', ')
var usersTitle = build.prNum ? 'Base repo user/organization' : 'Committers and authors'
@@ -38,16 +38,16 @@ function snsBuild(snsEvent, context, cb) {
cb(null, data)
})

sns.parseEvent(snsEvent, function(err, build) {
sns.parseEvent(snsEvent, function(err, buildData) {
if (err) return done(err)

if (build.ignore) {
log.info(build.ignore)
if (buildData.ignore) {
log.info(buildData.ignore)
log.info('Not running build')
return done()
}

actions.build(build, context, done)
actions.build(buildData, context, done)
})
}

@@ -155,7 +155,7 @@ GithubClient.prototype.request = function(options, cb) {
})
}

exports.parseEvent = function(event, eventType, build) {
exports.parseEvent = function(event, eventType) {

// Remove redundant fields like urls
exports.trimEvent(event)
@@ -177,13 +177,13 @@ exports.parseEvent = function(event, eventType, build) {
throw new Error('repository field is missing from GitHub event')
}

build = build || {}

build.event = event
build.eventType = eventType
build.repo = event.repository.full_name
build.isPrivate = event.repository.private
build.project = `gh/${build.repo}`
var build = {
event,
eventType,
repo: event.repository.full_name,
project: `gh/${event.repository.full_name}`,
isPrivate: event.repository.private,
}

if (eventType == 'pull_request') {
// https://developer.github.com/v3/activity/events/types/#pullrequestevent
@@ -209,7 +209,6 @@ exports.parseEvent = function(event, eventType, build) {
throw new Error(`base repo ${baseRepo.full_name} is different from event repo ${build.repo}`)
}

build.trigger = `pr/${prNum}`
build.branch = (base.ref || '').replace(/^refs\/heads\//, '')
build.cloneRepo = headRepo.full_name
build.checkoutBranch = (head.ref || '').replace(/^refs\/heads\//, '')
@@ -227,7 +226,7 @@ exports.parseEvent = function(event, eventType, build) {
var branchMatch = (event.ref || '').match(/^refs\/heads\/(.+)$/)

if (!branchMatch) {
return {ignore: `Ref does not match branch: ${event.ref}`}
return {ignore: `Ref does not match any branches: ${event.ref}`}
}

var branch = branchMatch[1]
@@ -236,7 +235,6 @@ exports.parseEvent = function(event, eventType, build) {
return {ignore: `Branch ${branch} was deleted`}
}

build.trigger = `push/${branch}`
build.branch = branch
build.cloneRepo = build.repo
build.checkoutBranch = branch
@@ -245,12 +243,11 @@ exports.parseEvent = function(event, eventType, build) {
build.comment = (event.head_commit || {}).message || ''
build.user = (event.pusher || {}).name

build.committers = (event.commits || []).concat(event.head_commit).reduce((committers, commit) => {
var author = commit.author || {}, committer = commit.committer || {}
committers[author.email] = author.username
committers[committer.email] = committer.username
build.committers = new Set((event.commits || []).concat(event.head_commit || {}).reduce((committers, commit) => {
if (commit.author) committers.push(commit.author.username)
if (commit.committer) committers.push(commit.committer.username)
return committers
}, Object.create(null))
}, []))
}

if (!/^[a-zA-Z0-9-_.]+\/[a-zA-Z0-9-_.]+$/.test(build.repo)) {
@@ -1,10 +1,7 @@
var EventEmitter = require('events')
var github = require('./github')
var db = require('../db')

exports.parseEvent = function(snsEvent, cb) {
var build = new BuildInfo()

var innerEvent

try {
@@ -25,49 +22,20 @@ exports.parseEvent = function(snsEvent, cb) {
if (!fullEvent) {
return cb(null, {ignore: 'Only received a partial event'})
}
parseGithubEvent(fullEvent, eventType, build, cb)
parseGithubEvent(fullEvent, eventType, cb)
})
}

parseGithubEvent(innerEvent, eventType, build, cb)
parseGithubEvent(innerEvent, eventType, cb)
}

function parseGithubEvent(event, eventType, build, cb) {
function parseGithubEvent(event, eventType, cb) {
var buildData
try {
build = github.parseEvent(event, eventType, build)
buildData = github.parseEvent(event, eventType)
} catch (e) {
return cb(e)
}
cb(null, build)
}

function BuildInfo(startedAt) {
this.startedAt = startedAt || new Date()

this.status = 'pending'
this.statusEmitter = new EventEmitter()

this.project = ''
this.buildNum = 0

this.ignore = false
this.event = null
this.eventType = ''
this.repo = ''
this.isPrivate = true

this.trigger = ''
this.branch = ''
this.cloneRepo = ''
this.checkoutBranch = ''
this.commit = ''
this.baseCommit = ''
this.comment = ''
this.user = ''

this.committers = null

this.isFork = false
this.prNum = 0
cb(null, buildData)
}

@@ -0,0 +1,27 @@
{
"Records": [
{
"EventSource": "aws:sns",
"EventVersion": "1.0",
"EventSubscriptionArn": "arn:aws:sns:us-east-1:999000111222:lambci-InvokeTopic-1234ABCD:2555e238-c1f1-455d-a414-add26ecdcf6a",
"Sns": {
"Type": "Notification",
"MessageId": "19999169-1e48-573e-bd29-dca8ba4b65de",
"TopicArn": "arn:aws:sns:us-east-1:999000111222:lambci-InvokeTopic-1234ABCD",
"Subject": null,
"Message": "{\"ref\":\"refs/tags/whatever\",\"before\":\"0000000000000000000000000000000000000000\",\"after\":\"ab64f6ecd8775edb64af24d9adfbbe9ce54f030a\",\"created\":true,\"deleted\":false,\"forced\":true,\"base_ref\":\"refs/heads/master\",\"compare\":\"https://github.com/mhart/test-ci-project/compare/whatever\",\"commits\":[],\"head_commit\":{\"id\":\"ab64f6ecd8775edb64af24d9adfbbe9ce54f030a\",\"tree_id\":\"6516e23572fd5137cce4e877ceafc9576f084c7d\",\"distinct\":true,\"message\":\"fix cmd\",\"timestamp\":\"2016-06-27T22:57:51-04:00\",\"author\":{\"name\":\"Michael Hart\",\"email\":\"michael.hart.au@gmail.com\",\"username\":\"mhart\"},\"committer\":{\"name\":\"Michael Hart\",\"email\":\"michael.hart.au@gmail.com\",\"username\":\"mhart\"},\"added\":[],\"removed\":[],\"modified\":[\"package.json\"]},\"repository\":{\"id\":34931472,\"name\":\"test-ci-project\",\"full_name\":\"mhart/test-ci-project\",\"owner\":{\"name\":\"mhart\",\"email\":\"michael.hart.au@gmail.com\"},\"private\":false,\"description\":null,\"fork\":false,\"created_at\":1430529913,\"updated_at\":\"2016-06-02T22:36:43Z\",\"pushed_at\":1467083098,\"clone_url\":\"https://github.com/mhart/test-ci-project.git\",\"homepage\":null,\"size\":325,\"stargazers_count\":0,\"watchers_count\":0,\"language\":\"JavaScript\",\"has_issues\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"forks_count\":1,\"open_issues_count\":1,\"forks\":1,\"open_issues\":1,\"watchers\":0,\"default_branch\":\"master\",\"stargazers\":0,\"master_branch\":\"master\"},\"pusher\":{\"name\":\"mhart\",\"email\":\"michael.hart.au@gmail.com\"},\"sender\":{\"login\":\"mhart\",\"id\":367936,\"avatar_url\":\"https://avatars.githubusercontent.com/u/367936?v=3\",\"gravatar_id\":\"\",\"type\":\"User\",\"site_admin\":false}}",
"Timestamp": "2015-05-04T16:43:47.657Z",
"SignatureVersion": "1",
"Signature": "nGIanfewZfjGx0pAKIY+vCT5Eow3ZAtqYeNz0aiAAgtCNl2uLTf84UCxgwgqhfF1elzoXpitQuIb6U5rZhbcXBSGh29YWgcW1aVXXOhjnPWvWhljVix8HIp4jT+K3SN/y2RneJkcHAL9rYBVpOuGMs/XOjXcSWDUQAZqylVsv9Gn/DFB1wbdLsrES57kb1j1J8plIyiLvSIIATpjBTZ/DDNh8v3umJYGG3xe79fdcmVYXG0UoqYjgXWOCTBg3e1r2cRqvilue1D9piJHgyG4UTMo54D/ouSsJJ4ZBeVtgi1YH8BUMQPtOVqQe3K4gP6nkFIx0RWIXmnp+LhwYBzn4w==",
"SigningCertUrl": "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-d6d679a1d18e95c2f9ffcf11f4f9e198.pem",
"UnsubscribeUrl": "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:999000111222:lambci-InvokeTopic-1234ABCD:2555e238-c1f1-455d-a414-add26ecdcf6a",
"MessageAttributes": {
"X-Github-Event": {
"Type": "String",
"Value": "push"
}
}
}
}
]
}

0 comments on commit 2724d9f

Please sign in to comment.
You can’t perform that action at this time.