Skip to content
Permalink
Browse files

Push Contents to S3

All content is now being pushed to S3. Huzzah!
  • Loading branch information...
ThatJoeMoore committed Feb 1, 2017
1 parent d8d2c19 commit 68d6250e0068c3d68f1d6d9cacdf0bd02de4000a
Showing with 246 additions and 6 deletions.
  1. +7 −0 _aws/s3.yml
  2. +5 −0 lib/constants.js
  3. +5 −1 lib/steps/build-filesystem.js
  4. +11 −0 lib/steps/commit-content.js
  5. +58 −0 lib/steps/push-to-s3.js
  6. +7 −2 new-run.js
  7. +3 −0 package.json
  8. +1 −1 todos.md
  9. +149 −2 yarn.lock
@@ -22,6 +22,8 @@ Resources:
BucketName: !FindInMap [Bucket, !Ref Environment, name]
VersioningConfiguration:
Status: Enabled
WebsiteConfiguration:
IndexDocument: index.html
Tags:
- Key: Name
Value: !Join [ " ", [ !FindInMap [ Bucket, !Ref Environment, name ], Web Community CDN Bucket ] ]
@@ -40,4 +42,9 @@ Outputs:
Value: !Ref ContentBucket
Export:
Name: !Sub "${AWS::StackName}-Bucket"
WebsiteUrl:
Description: S3 Website URL
Value: !GetAtt [ContentBucket, WebsiteURL]
Export:
Name: !Sub "${AWS::StackName}-WebsiteUrl"

@@ -24,6 +24,11 @@ module.exports = freezer({
GITHUB_REPO: 'web-cdn',
CONTENT_BRANCH: 'content',
USER_AGENT: 'BYU-Web-Community-CDN ' + packageVersion
},
S3: {
BUCKET: {
PROD: 'byu-web-community-cdn-prod'
}
}
});

@@ -21,6 +21,7 @@ const zlib = require('zlib');
* @property {string[]} deleted
* @property {string[]} modified
* @property {string[]} unchanged
* @property {boolean} onlyManifestChanged
*/

/**
@@ -260,11 +261,14 @@ module.exports = function buildFilesystem(config, contentPath, workPath) {
.filter(f => !added.includes(f))
.filter(f => !modified.includes(f));

let onlyManifestChanged =!added.length && !deleted.length && modified.length === 1 && modified[0] === 'manifest.json';

return {
added: added,
modified: modified,
deleted: deleted,
unchanged: unchanged
unchanged: unchanged,
onlyManifestChanged: onlyManifestChanged
};
});
}
@@ -5,6 +5,7 @@ const klaw = require('klaw');
const ghClient = require('./../github/github-client');
const constants = require('./../constants');
const path = require('path');
const log = require('winston');

/**
*
@@ -14,6 +15,12 @@ const path = require('path');
* @returns {Promise.<*>}
*/
module.exports = function commitContent(config, fileChanges, contentPath) {
log.info('-------------- Committing File Contents --------------');

if (fileChanges.onlyManifestChanged) {
log.info('Skipping commit; only manifest file changed');
return Promise.resolve();
}

return Promise.all([
_fetchBlobs(fileChanges.unchanged),
@@ -28,6 +35,7 @@ module.exports = function commitContent(config, fileChanges, contentPath) {


function _fetchBlobs(files) {
log.info('-------------- Fetching Current Tree --------------');
// While we could theoretically calculate these ourselves, line breaks make it...tricky. So we'll rely on Github.
return readTree(config.contentInfo.treeSha, '');

@@ -52,6 +60,7 @@ module.exports = function commitContent(config, fileChanges, contentPath) {
}

function _createNewBlobs(files) {
log.info('-------------- Creating New Blobs --------------');
let promises = files.map(f => {
let fullPath = path.join(contentPath, f);
return ghClient.addBlob(
@@ -66,6 +75,7 @@ module.exports = function commitContent(config, fileChanges, contentPath) {
}

function _createTree(blobs) {
log.info('-------------- Creating New Tree --------------');
let tree = blobs.map(b => {
return {
path: b.path,
@@ -81,6 +91,7 @@ module.exports = function commitContent(config, fileChanges, contentPath) {
}

function _createCommit(treeSha) {
log.info('-------------- Creating Commit --------------');
let commit = {
message: 'Update CDN Contents',
tree: treeSha,
@@ -0,0 +1,58 @@
/**
* Created by ThatJoeMoore on 2/1/17
*/
"use strict";
const AWS = require('aws-sdk');
const log = require('winston');
const fs = require('fs-extra-p');
const constants = require('../constants');

const s3Options = {
s3Client: new AWS.S3()
};
const s3 = require('s3').createClient(s3Options);

/**
*
* @param {string} contentPath
* @param {string} stagingPath
* @param {FilesystemChanges} filesystemChanges
* @returns {Promise.<*>}
*/
module.exports = function pushToS3(contentPath, stagingPath, filesystemChanges) {
log.info('-------------- Pushing to Amazon S3 --------------');
if (filesystemChanges.onlyManifestChanged) {
log.info('Only manifest changed; skipping push to S3');
return Promise.resolve();
}

return fs.copy(contentPath, stagingPath, {
overwrite: true,
dereference: true,
preserveTimestamps: true
}).then(() => _uploadDir(stagingPath));

function _uploadDir(dir) {
return new Promise((resolve, reject) => {
let uploader = s3.uploadDir({
localDir: dir,
deleteRemoved: true,
followSymlinks: false,
s3Params: {
Bucket: constants.S3.BUCKET.PROD,
ACL: 'public-read'
}
});
uploader.on('progress', () => {
if (uploader.progressAmount > 0)
log.info(`S3 Progress: ${(uploader.progressAmount / uploader.progressTotal) * 100}%`);
});
uploader.on('error', reject);
uploader.on('end', resolve);
});

}
};



@@ -1,16 +1,17 @@
"use strict";

const CdnConfig = require('./lib/cdn-config');
const fsp = require('fs-extra-p');
const path = require('path');
const buildFilesystem = require('./lib/steps/build-filesystem');
const commitContent = require('./lib/steps/commit-content');
const pushToS3 = require('./lib/steps/push-to-s3');

const SCRATCH_DIR_NAME = '.tmp';

const scratchPath = path.join(process.cwd(), SCRATCH_DIR_NAME);
const contentPath = path.join(scratchPath, 'content');
const workPath = path.join(scratchPath, 'work');
const stagingPath = path.join(scratchPath, 's3-staging');


/*
@@ -44,7 +45,11 @@ configPromise.then(cfg => {
// fsp.writeJsonSync('filesystem.json', changes);
// return changes;
// })
.then(changes => commitContent(cfg, changes, contentPath));
.then(changes => {
return commitContent(cfg, changes, contentPath)
.then(() => pushToS3(contentPath, stagingPath, changes))
})
;
}).catch(err => {
console.error(err);
});
@@ -9,6 +9,7 @@
"author": "",
"license": "Apache-2.0",
"dependencies": {
"aws-sdk": "^2.9.0",
"chai": "^3.5.0",
"chai-as-promised": "^6.0.0",
"cpx": "^1.5.0",
@@ -18,10 +19,12 @@
"fs-extra": "^1.0.0",
"fs-extra-p": "^3.1.0",
"klaw": "^1.3.1",
"mime": "^1.3.4",
"mocha": "^3.2.0",
"node-yaml": "^3.0.3",
"request": "^2.79.0",
"request-promise-native": "^1.0.3",
"s3": "^4.4.0",
"semver": "^5.3.0",
"winston": "^2.3.1"
}
@@ -3,6 +3,6 @@
* [x] Read and assemble configuration, including library repos
* [x] Lay out filesystem
* [x] Push contents to Github
* [ ] Push contents to Amazon S3
* [x] Push contents to Amazon S3
* [ ] Generate website
* [ ] Bundle into Lambda

0 comments on commit 68d6250

Please sign in to comment.
You can’t perform that action at this time.