Skip to content
Permalink
Browse files

Add an awesome Slack webhook when builds finish

  • Loading branch information...
ThatJoeMoore committed Jun 15, 2018
1 parent 5f83e3c commit 76517da166c2835a3751ecc23861da2671b4986d
@@ -169,11 +169,20 @@ Resources:
Fn::ImportValue: !Sub "${AccountStackName}-AssemblerRepositoryUri"
EnvironmentVariables:
- Name: DESTINATION_S3_BUCKET
Type: PLAINTEXT
Value: !Ref CdnContentBucket
- Name: BUILD_ENV
Type: PLAINTEXT
Value: !Ref Environment
- Name: CDN_HOST
Type: PLAINTEXT
Value: !Ref RootDNS
- Name: SLACK_WEBHOOK_URL
Type: PARAMETER_STORE
Value: !Sub "${CDNName}.${Environment}.slack-webhook"
- Name: SLACK_CHANNEL
Type: PARAMETER_STORE
Value: !Sub "${CDNName}.${Environment}.slack-channel"
ServiceRole:
Fn::ImportValue: !Sub "${AccountStackName}-BuilderRole"
Tags:
@@ -64,6 +64,8 @@ const args = require('yargs')
alias: 'w',
describe: 'Directory in which to assemble the CDN contents. Must be empty.'
})
.option('slack-url', {describe: 'Slack Webhook URL for build notifications'})
.option('slack-channel', {describe: 'Slack Channel for build notifications'})
.describe('verbose', 'turns on verbose logging')
.alias('verbose', 'v')
.boolean('verbose')
@@ -104,7 +106,9 @@ async function runAssembler(args) {
githubCredentials: await loadGithubCredentials(args),
env: args.env,
cdnHost: args.cdnHost,
forceBuild: args.forceBuild
forceBuild: args.forceBuild,
slackUrl: args.slackUrl,
slackChannel: args.slackChannel,
})
}

@@ -35,7 +35,7 @@ const buildMeta = require('./src/build-meta');
const {uploadFiles2} = require('./src/upload-files');
const buildLayout = require('./src/build-layout');
const constants = require('./src/constants');

const {NoopMessager, SlackMessager} = require('./src/messagers');

module.exports = async function cdnAssembler(config, targetBucket, opts) {
let {workDir, githubCredentials, dryRun, env, forceBuild, cdnHost} = (opts || {});
@@ -55,54 +55,83 @@ module.exports = async function cdnAssembler(config, targetBucket, opts) {
let sourceDir = path.join(workDir, 'sources');
let assembledDir = path.join(workDir, 'assembled');

log.info("----- Getting current manifest -----");
let oldManifest = await getOldManifest(targetBucket, cdnHost);

log.info("----- Building new manifest -----");
let newManifest = await assembleManifest(config, cdnHost);
const messages = initMessager(opts);

const buildContext = {
config,
targetBucket,
dryRun,
forceBuild,
directories: {
workDir,
sourceDir,
assembledDir,
},
cdnHost,
env,
messages,
started: new Date(),
};

await fs.writeJson(path.join(workDir, 'manifest.json'), newManifest, {
spaces: 1,
replacer: (key, value) => {
if (key === 'config') {
return undefined;
try {
log.info("----- Getting current manifest -----");
let oldManifest = await getOldManifest(buildContext);

log.info("----- Building new manifest -----");
let newManifest = await assembleManifest(buildContext, oldManifest);

await fs.writeJson(path.join(workDir, 'manifest.json'), newManifest, {
spaces: 1,
replacer: (key, value) => {
if (key === 'config') {
return undefined;
}
return value;
}
return value;
}
});
});

log.info("----- Planning Actions -----");
let actions = planActions(oldManifest, newManifest, forceBuild);
log.info("----- Planning Actions -----");
let actions = planActions(buildContext, oldManifest, newManifest);

if (!hasPlannedActions(actions)) {
log.info("No planned actions. Exiting.");
return;
}
if (!hasPlannedActions(actions)) {
log.info("No planned actions. Exiting.");
return;
}

logPlannedActions(buildContext, actions, oldManifest, newManifest);

logPlannedActions(actions);
log.info("----- Downloading Sources -----");
let sourceDirs = await downloadSources(buildContext, newManifest, actions);

log.info("----- Downloading Sources -----");
let sourceDirs = await downloadSources(newManifest, actions, sourceDir);
log.info("----- Assembling Artifacts -----");
await assembleArtifacts(buildContext, newManifest, actions, sourceDirs);

log.info("----- Assembling Artifacts -----");
await assembleArtifacts(newManifest, actions, sourceDirs, assembledDir);
log.info("----- Building CDN Layout -----");
const filesystem = await buildLayout(buildContext, oldManifest, newManifest, actions, sourceDirs);

log.info("----- Building CDN Layout -----");
const filesystem = await buildLayout(oldManifest, newManifest, actions, sourceDirs, cdnHost);
await fs.writeJson('./filesystem.json', filesystem, {spaces: 2});

await fs.writeJson('./filesystem.json', filesystem, {spaces: 2});
// log.info("----- Building Library Meta Files -----");
// const versionManifests = await buildMeta(newManifest, assembledDir);
//
log.info("----- Uploading Files -----");
await uploadFiles2(buildContext, filesystem, actions, newManifest);
// await uploadFiles(oldManifest, newManifest, versionManifests, actions, targetBucket, assembledDir, cdnHost, dryRun);

// log.info("----- Building Library Meta Files -----");
// const versionManifests = await buildMeta(newManifest, assembledDir);
//
log.info("----- Uploading Files -----");
await uploadFiles2(targetBucket, filesystem, actions, newManifest, cdnHost, dryRun);
// await uploadFiles(oldManifest, newManifest, versionManifests, actions, targetBucket, assembledDir, cdnHost, dryRun);
await messages.sendSuccess(buildContext);
} catch (err) {
await messages.sendError(buildContext, err);
process.exit(1);
}
};

async function getOldManifest(bucket, cdnHost) {
const found = (await getManifestFromS3(bucket)) || (await getManifestViaHTTP(cdnHost));
async function getOldManifest({messages, targetBucket, cdnHost}) {
const found = (await getManifestFromS3(targetBucket)) || (await getManifestViaHTTP(cdnHost));

if (found) {
return found;
}
messages.warning({message: `Unable to find an old version of the manifest. Using an empty one.`});
return found || getEmptyManifest();
}

@@ -142,19 +171,40 @@ function getEmptyManifest() {
};
}

function logPlannedActions(actions) {
function logPlannedActions(buildContext, actions, oldManifest, newManifest) {
const {messages} = buildContext;
log.info("----- Planned Actions: -----");

Object.entries(actions).forEach(([id, acts]) => {
const oldLib = oldManifest.libraries[id];
const newLib = newManifest.libraries[id];
log.info(` * ${id}`);
if (acts.deleteLib) {
log.info(' ## DELETE LIBRARY ##');
messages.deletedLib({
libId: id,
libName: oldLib.name,
libLink: oldLib.links.source,
});
return;
}
if (acts.add.length === 0 && acts.update.length === 0 && acts.remove.length === 0) {
log.info(' No Changes');
return;
}

const libInfo = {
libId: id,
libName: newLib.name,
libLink: newLib.links.source,
};

if (!oldLib && newLib) {
messages.addedLib(libInfo);
} else {
messages.updatedLib(libInfo);
}

if (acts.add.length > 0) {
log.info(' Add ' + acts.add)
}
@@ -164,9 +214,46 @@ function logPlannedActions(actions) {
if (acts.remove.length > 0) {
log.info(' Remove ' + acts.remove)
}

for (const ver of newLib.versions) {
const verInfo = {
libId: id,
versionId: ver.name,
versionLink: ver.link,
};
if (acts.add.includes(ver.name)) {
messages.newVersion(verInfo);
} else if (acts.update.includes(ver.name)) {
messages.updatedVersion(verInfo);
}
}
for (const ver of oldLib.versions) {
const verInfo = {
libId: id,
versionId: ver.name,
versionLink: ver.link,
};
if (acts.remove.includes(ver.name)) {
messages.removedVersion(verInfo);
}
}

const changedAliases = oldLib ? findChangedAliases(oldLib.aliases, newLib.aliases) : Object.entries(newLib.aliases);

changedAliases.forEach(([alias, target]) => {
messages.updatedAlias({libId: id, aliasName: alias, aliasTarget: target});
});
});
}

function findChangedAliases(oldAliases, newAliases) {
return Object.entries(newAliases)
.filter(([alias, target]) => {
const old = oldAliases[alias];
return old !== target;
});
}

function hasPlannedActions(actions) {
return Object.values(actions).some(acts => {
return acts.removeLib || acts.add.length > 0 || acts.update.length > 0 || acts.remove.length > 0;
@@ -184,3 +271,11 @@ async function setupGithubCredentials(credentials, env) {

await GithubProvider.setCredentials(actual.user, actual.token);
}

function initMessager({slackUrl, slackChannel}) {
if (slackUrl) {
return new SlackMessager({webhookUrl: slackUrl, channel: slackChannel});
} else {
return new NoopMessager();
}
}
@@ -19,6 +19,7 @@
"aws-sdk": "^2.89.0",
"axios": "^0.18.0",
"chai-fs": "^1.0.0",
"chalk": "^2.4.1",
"child-process-promise": "^2.2.1",
"common-tags": "^1.7.2",
"cpx": "^1.5.0",
@@ -25,14 +25,15 @@ const repoConfig = require('./repo-config');
const moment = require('moment-timezone');
const log = require('winston');

const { URL } = require('url');
const {URL} = require('url');

const processBasicUsage = require('./util/basic-usage-processor');

module.exports = async function assembleManifest(mainConfig, cdnHost) {
log.info('Assembling new manifest from config:', JSON.stringify(mainConfig, null, 2));
let libs = await Promise.all(Object.entries(mainConfig.libraries).map(async function ([id, defn]) {
return [id, await loadLib(id, defn, cdnHost)];
module.exports = async function assembleManifest(buildContext, oldManifest) {
const {config, cdnHost} = buildContext;
log.info('Assembling new manifest from config:', JSON.stringify(config, null, 2));
let libs = await Promise.all(Object.entries(config.libraries).map(async function ([id, defn]) {
return [id, await loadLib(buildContext, id, defn, cdnHost, oldManifest.libraries[id])];
}));

let libraries = {};
@@ -48,7 +49,12 @@ module.exports = async function assembleManifest(mainConfig, cdnHost) {
};
};

async function loadLib(id, defn, cdnHost) {
function refMustBePreserved(version) {
return version.type === 'release';
}

async function loadLib(buildContext, id, defn, cdnHost, oldLib) {
if (!oldLib) oldLib = {versions: []};
log.debug(`Loading library ${id}`);
const cdnBase = 'https://' + cdnHost;

@@ -58,11 +64,34 @@ async function loadLib(id, defn, cdnHost) {

let refs = await provider.listRefs();

const versionNames = refs.map(it => it.name);
const versionNames = [...new Set([
...refs.map(it => it.name),
...oldLib.versions.filter(refMustBePreserved).map(it => it.name)
])];

const versions = await Promise.all(versionNames.map(
name => {
const ref = refs.find(it => it.name === name);
const oldRef = oldLib.versions.find(it => it.name === name);
if (ref) {
return refToVersion(id, defn, mainConfig, ref, cdnBase);
} else if (oldRef) {
buildContext.messages.warning({
message: `${id} - protected ref ${oldRef.ref} was removed from ${oldLib.source}, but will not be removed from the CDN.`
});
oldRef.missing_source = true;
return oldRef;
} else {
// We shouldn't ever get here
throw `Unable to find ref ${id}@${name}`;
}
}));

const libAliases = aliases(versionNames);

const versions = await Promise.all(refs.map(ref => postProcessRef(id, defn, mainConfig, ref, cdnBase, libAliases)));
versions.forEach(version => {
version.aliases = buildVersionAliases(id, defn, version.name, libAliases);
});

const deprecated = !!mainConfig.deprecated;
let deprecationMessage = undefined;
@@ -97,7 +126,7 @@ async function loadLib(id, defn, cdnHost) {
return libDefinition;
}

async function postProcessRef(libId, libDefn, libConfig, ref, cdnBase, libAliases) {
async function refToVersion(libId, libDefn, libConfig, ref, cdnBase) {
const path = `/${libId}/${versionPath(ref.name, ref.type)}/`;

const absoluteUrl = new URL(path, cdnBase).toString();
@@ -113,8 +142,6 @@ async function postProcessRef(libId, libDefn, libConfig, ref, cdnBase, libAliase
result.basic_usage = basic_usage;
}

result.aliases = buildVersionAliases(libId, libDefn, ref.name, libAliases);

return result;
}

@@ -44,7 +44,7 @@ const CACHE_CONTROL_ONE_MINUTE = 'public, max-age=60, s-maxage=0';

const REDIRECTS_PATH = '/.cdn-infra/redirects.json';

module.exports = async function buildLayout(oldManifest, newManifest, actions, sourceDirs, cdnHost) {
module.exports = async function buildLayout(buildContext, oldManifest, newManifest, actions, sourceDirs) {
const files = [];

for (const [libId, lib] of Object.entries(newManifest.libraries)) {
@@ -24,7 +24,9 @@ const globs = require('./util/globs');

const log = require('winston');

module.exports = async function assembleArtifacts(manifest, actions, sourceDirs, assembledDir) {
module.exports = async function assembleArtifacts(buildContext, manifest, actions, sourceDirs) {
const { assembledDir } = buildContext.directories;

await fsp.emptyDir(assembledDir);

let promises = Object.entries(manifest.libraries).map(async function ([id, defn]) {

0 comments on commit 76517da

Please sign in to comment.
You can’t perform that action at this time.