From c5e5cad5790928ac1611264c3dad7f238f9908dc Mon Sep 17 00:00:00 2001 From: Nils Knappmeier Date: Sat, 5 Aug 2023 15:11:00 +0200 Subject: [PATCH] implement publish workflow --- tasks/aws-s3-builds-page/createS3Client.js | 23 ------ tasks/aws-s3-builds-page/index.js | 5 -- tasks/aws-s3-builds-page/publish-test.js | 47 ++++++++++++ tasks/aws-s3-builds-page/publish.js | 37 ++++++++++ tasks/aws-s3-builds-page/s3-test.js | 65 ----------------- .../{ => s3client}/deleteFile.js | 6 +- .../aws-s3-builds-page/s3client/fetchFile.js | 10 +++ tasks/aws-s3-builds-page/s3client/index.js | 37 ++++++++++ .../{ => s3client}/listFiles.js | 4 +- .../s3client/requireEnvVar.js | 8 ++ tasks/aws-s3-builds-page/s3client/s3-test.js | 47 ++++++++++++ .../{ => s3client}/uploadFile.js | 6 +- .../aws-s3-builds-page/test-utils/runTest.js | 33 +++++++++ tasks/publish-to-aws.js | 73 +------------------ 14 files changed, 226 insertions(+), 175 deletions(-) delete mode 100644 tasks/aws-s3-builds-page/createS3Client.js delete mode 100644 tasks/aws-s3-builds-page/index.js create mode 100644 tasks/aws-s3-builds-page/publish-test.js create mode 100644 tasks/aws-s3-builds-page/publish.js delete mode 100644 tasks/aws-s3-builds-page/s3-test.js rename tasks/aws-s3-builds-page/{ => s3client}/deleteFile.js (56%) create mode 100644 tasks/aws-s3-builds-page/s3client/fetchFile.js create mode 100644 tasks/aws-s3-builds-page/s3client/index.js rename tasks/aws-s3-builds-page/{ => s3client}/listFiles.js (79%) create mode 100644 tasks/aws-s3-builds-page/s3client/requireEnvVar.js create mode 100644 tasks/aws-s3-builds-page/s3client/s3-test.js rename tasks/aws-s3-builds-page/{ => s3client}/uploadFile.js (63%) create mode 100644 tasks/aws-s3-builds-page/test-utils/runTest.js diff --git a/tasks/aws-s3-builds-page/createS3Client.js b/tasks/aws-s3-builds-page/createS3Client.js deleted file mode 100644 index 4ff79cdec..000000000 --- a/tasks/aws-s3-builds-page/createS3Client.js +++ /dev/null @@ -1,23 +0,0 @@ -const { S3Client } = require('@aws-sdk/client-s3'); - -function requiredEnvVar(name) { - if (!process.env[name]) { - throw new Error(`Environment variable "${name}" is required.`); - } - return process.env[name]; -} - -function createS3Client() { - // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html - requiredEnvVar('AWS_ACCESS_KEY_ID'); - requiredEnvVar('AWS_SECRET_ACCESS_KEY'); - - return { - bucket: requiredEnvVar('S3_BUCKET_NAME'), - s3Client: new S3Client({ - region: 'us-east-1' - }) - }; -} - -module.exports = { createS3Client }; diff --git a/tasks/aws-s3-builds-page/index.js b/tasks/aws-s3-builds-page/index.js deleted file mode 100644 index a18ffb0a2..000000000 --- a/tasks/aws-s3-builds-page/index.js +++ /dev/null @@ -1,5 +0,0 @@ -const { listFiles } = require('./listFiles'); -const { uploadFile } = require('./uploadFile'); -const { deleteFile } = require('./deleteFile'); - -module.exports = { listFiles, uploadFile, deleteFile }; diff --git a/tasks/aws-s3-builds-page/publish-test.js b/tasks/aws-s3-builds-page/publish-test.js new file mode 100644 index 000000000..5dd26bfef --- /dev/null +++ b/tasks/aws-s3-builds-page/publish-test.js @@ -0,0 +1,47 @@ +const crypto = require('crypto'); +const { publishWithSuffixes } = require('./publish'); +const { runTest } = require('./test-utils/runTest'); +const { createS3Client } = require('./s3client'); +const fs = require('node:fs/promises'); + +// This is a test file. It is intended to be run manually with the proper environment variables set +// +// Run it from the project root using "node tasks/aws-s3-builds-page/publish-test.js" + +const s3Client = createS3Client(); + +runTest(async ({ log }) => { + const suffix1 = `-test-file-` + crypto.randomUUID(); + const suffix2 = `-test-file-` + crypto.randomUUID(); + log(`Publish ${suffix1} and ${suffix2}`); + await publishWithSuffixes([suffix1, suffix2]); + await compareAndDeleteFiles(suffix1, log); + await compareAndDeleteFiles(suffix2, log); +}); + +async function compareAndDeleteFiles(suffix, log) { + const pairs = [ + ['dist/handlebars.js', `handlebars${suffix}.js`], + ['dist/handlebars.min.js', `handlebars.min${suffix}.js`], + ['dist/handlebars.runtime.js', `handlebars.runtime${suffix}.js`], + ['dist/handlebars.runtime.min.js', `handlebars.runtime.min${suffix}.js`] + ]; + for (const [localFile, remoteFile] of pairs) { + await expectSameContents(localFile, remoteFile, log); + log(`Deleting "${remoteFile}"`); + await s3Client.deleteFile(remoteFile); + } +} + +async function expectSameContents(localFile, remoteFile, log) { + log( + `Checking file contents "${localFile}" vs "${s3Client.fileUrl(remoteFile)}"` + ); + const remoteContents = await s3Client.fetchFile(remoteFile); + const localContents = await fs.readFile(localFile, 'utf-8'); + if (remoteContents !== localContents) { + throw new Error( + `Files do not match: ${localFile}" vs "${s3Client.fileUrl(remoteFile)}"` + ); + } +} diff --git a/tasks/aws-s3-builds-page/publish.js b/tasks/aws-s3-builds-page/publish.js new file mode 100644 index 000000000..b5f1b986d --- /dev/null +++ b/tasks/aws-s3-builds-page/publish.js @@ -0,0 +1,37 @@ +/* eslint-disable no-console */ +const { createS3Client } = require('./s3client'); + +const filenames = [ + 'handlebars.js', + 'handlebars.min.js', + 'handlebars.runtime.js', + 'handlebars.runtime.min.js' +]; + +async function publishWithSuffixes(suffixes) { + const s3Client = createS3Client(); + const publishPromises = suffixes.map(suffix => + publishSuffix(s3Client, suffix) + ); + return Promise.all(publishPromises); +} + +async function publishSuffix(s3client, suffix) { + const publishPromises = filenames.map(async filename => { + const nameInBucket = getNameInBucket(filename, suffix); + const localFile = getLocalFile(filename); + await s3client.uploadFile(localFile, nameInBucket); + console.log(`Published ${localFile} to build server (${nameInBucket})`); + }); + return Promise.all(publishPromises); +} + +function getNameInBucket(filename, suffix) { + return filename.replace(/\.js$/, suffix + '.js'); +} + +function getLocalFile(filename) { + return 'dist/' + filename; +} + +module.exports = { publishWithSuffixes }; diff --git a/tasks/aws-s3-builds-page/s3-test.js b/tasks/aws-s3-builds-page/s3-test.js deleted file mode 100644 index af72d2e38..000000000 --- a/tasks/aws-s3-builds-page/s3-test.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable no-console */ -const { listFiles, uploadFile, deleteFile } = require('./index'); - -const crypto = require('crypto'); -const uuid = crypto.randomUUID(); - -const BUCKET_BASE_URL = `https://s3.amazonaws.com/${process.env.S3_BUCKET_NAME}`; - -async function run() { - const filename = `test-file-${uuid}`; - console.log(`Starting test with target file "${filename}"`); - - console.log(`Uploading "${filename}"`); - await uploadFile('package.json', filename); - - console.log(`Check if uploaded "${filename}"`); - const listing = await listFiles(); - if (!listing.includes(filename)) { - throw new Error(`File "${filename}" has not been uploaded`); - } - - console.log(`Check contents of "${filename}"`); - const uploadedContents = await ( - await fetch(BUCKET_BASE_URL + '/' + filename) - ).text(); - expectStringContains('"name": "handlebars"', uploadedContents); - - console.log(`Delete "${filename}"`); - await deleteFile(filename); - - console.log(`Check if deleted "${filename}"`); - const listingAfterDelete = await listFiles(); - if (listingAfterDelete.includes(filename)) { - throw new Error(`File "${filename}" has not been deleted`); - } -} - -run() - .finally(logTestFilesInBucket) - .catch(error => { - console.error(error); - process.exit(1); - }); - -function expectStringContains(needle, haystack) { - if (!haystack.includes(needle)) { - throw new Error(`Expecting to find "${needle}" in string "${haystack}"`); - } -} - -async function logTestFilesInBucket() { - const listing = await listFiles(); - const testFilesInBucket = listing.filter(name => - name.startsWith('test-file-') - ); - for (const filename of testFilesInBucket) { - console.log(`Detected surplus file "${filename}"`); - if (process.argv[2] === '--delete-surplus') { - await deleteFile(filename); - } else { - console.log(`run with --delete-surplus to delete it`); - } - } - console.log('DONE'); -} diff --git a/tasks/aws-s3-builds-page/deleteFile.js b/tasks/aws-s3-builds-page/s3client/deleteFile.js similarity index 56% rename from tasks/aws-s3-builds-page/deleteFile.js rename to tasks/aws-s3-builds-page/s3client/deleteFile.js index 8207aefc8..a667581e9 100644 --- a/tasks/aws-s3-builds-page/deleteFile.js +++ b/tasks/aws-s3-builds-page/s3client/deleteFile.js @@ -1,11 +1,9 @@ -const { createS3Client } = require('./createS3Client'); const { DeleteObjectCommand } = require('@aws-sdk/client-s3'); -async function deleteFile(name) { - const { s3Client, bucket } = createS3Client(); +async function deleteFile(s3Client, bucket, remoteName) { const command = new DeleteObjectCommand({ Bucket: bucket, - Key: name + Key: remoteName }); await s3Client.send(command); } diff --git a/tasks/aws-s3-builds-page/s3client/fetchFile.js b/tasks/aws-s3-builds-page/s3client/fetchFile.js new file mode 100644 index 000000000..d9ebdd264 --- /dev/null +++ b/tasks/aws-s3-builds-page/s3client/fetchFile.js @@ -0,0 +1,10 @@ +async function fetchFile(bucket, remoteName) { + return (await fetch(fileUrl(bucket, remoteName))).text(); +} + +function fileUrl(bucket, remoteName) { + const bucketUrl = `https://s3.amazonaws.com/${bucket}`; + return `${bucketUrl}/${remoteName}`; +} + +module.exports = { fetchFile, fileUrl }; diff --git a/tasks/aws-s3-builds-page/s3client/index.js b/tasks/aws-s3-builds-page/s3client/index.js new file mode 100644 index 000000000..62771dff8 --- /dev/null +++ b/tasks/aws-s3-builds-page/s3client/index.js @@ -0,0 +1,37 @@ +const { listFiles } = require('./listFiles'); +const { uploadFile } = require('./uploadFile'); +const { deleteFile } = require('./deleteFile'); +const { S3Client } = require('@aws-sdk/client-s3'); +const { requireEnvVar } = require('./requireEnvVar'); +const { fetchFile, fileUrl } = require('./fetchFile'); + +module.exports = { createS3Client }; + +function createS3Client() { + // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html + requireEnvVar('AWS_ACCESS_KEY_ID'); + requireEnvVar('AWS_SECRET_ACCESS_KEY'); + + const bucket = requireEnvVar('S3_BUCKET_NAME'); + const s3Client = new S3Client({ + region: 'us-east-1' + }); + + return { + async listFiles() { + return listFiles(s3Client, bucket); + }, + async uploadFile(localName, remoteName) { + await uploadFile(s3Client, bucket, localName, remoteName); + }, + async deleteFile(remoteName) { + await deleteFile(s3Client, bucket, remoteName); + }, + async fetchFile(remoteName) { + return fetchFile(bucket, remoteName); + }, + fileUrl(remoteName) { + return fileUrl(bucket, remoteName); + } + }; +} diff --git a/tasks/aws-s3-builds-page/listFiles.js b/tasks/aws-s3-builds-page/s3client/listFiles.js similarity index 79% rename from tasks/aws-s3-builds-page/listFiles.js rename to tasks/aws-s3-builds-page/s3client/listFiles.js index 55fd248df..8796e2eda 100644 --- a/tasks/aws-s3-builds-page/listFiles.js +++ b/tasks/aws-s3-builds-page/s3client/listFiles.js @@ -1,8 +1,6 @@ const { ListObjectsV2Command } = require('@aws-sdk/client-s3'); -const { createS3Client } = require('./createS3Client'); -async function listFiles() { - const { s3Client, bucket } = createS3Client(); +async function listFiles(s3Client, bucket) { const command = new ListObjectsV2Command({ Bucket: bucket }); diff --git a/tasks/aws-s3-builds-page/s3client/requireEnvVar.js b/tasks/aws-s3-builds-page/s3client/requireEnvVar.js new file mode 100644 index 000000000..d1b1e9bd1 --- /dev/null +++ b/tasks/aws-s3-builds-page/s3client/requireEnvVar.js @@ -0,0 +1,8 @@ +function requireEnvVar(name) { + if (!process.env[name]) { + throw new Error(`Environment variable "${name}" is required.`); + } + return process.env[name]; +} + +module.exports = { requireEnvVar }; diff --git a/tasks/aws-s3-builds-page/s3client/s3-test.js b/tasks/aws-s3-builds-page/s3client/s3-test.js new file mode 100644 index 000000000..b54dc33d4 --- /dev/null +++ b/tasks/aws-s3-builds-page/s3client/s3-test.js @@ -0,0 +1,47 @@ +/* eslint-disable no-console */ +const { createS3Client } = require('./index'); +const crypto = require('crypto'); +const { runTest } = require('../test-utils/runTest'); + +// This is a test file. It is intended to be run manually +// with the proper environment variables set +// It tests whether the upload/list/delete methods in this directory +// work properly. +// +// Run it from the project root using "node tasks/aws-s3-builds-page/s3client/s3-test.js" + +const client = createS3Client(); + +runTest(async ({ log }) => { + const uuid = crypto.randomUUID(); + const filename = `test-file-${uuid}`; + log(`Starting test with target file "${filename}"`); + + log(`Uploading "${filename}"`); + await client.uploadFile('package.json', filename); + + log(`Check if uploaded "${filename}"`); + const listing = await client.listFiles(); + if (!listing.includes(filename)) { + throw new Error(`File "${filename}" has not been uploaded`); + } + + log(`Check contents of "${filename}"`); + const uploadedContents = await client.fetchFile(filename); + expectStringContains('"name": "handlebars"', uploadedContents); + + log(`Delete "${filename}"`); + await client.deleteFile(filename); + + log(`Check if deleted "${filename}"`); + const listingAfterDelete = await client.listFiles(); + if (listingAfterDelete.includes(filename)) { + throw new Error(`File "${filename}" has not been deleted`); + } +}); + +function expectStringContains(needle, haystack) { + if (!haystack.includes(needle)) { + throw new Error(`Expecting to find "${needle}" in string "${haystack}"`); + } +} diff --git a/tasks/aws-s3-builds-page/uploadFile.js b/tasks/aws-s3-builds-page/s3client/uploadFile.js similarity index 63% rename from tasks/aws-s3-builds-page/uploadFile.js rename to tasks/aws-s3-builds-page/s3client/uploadFile.js index 000959563..231b8cd3e 100644 --- a/tasks/aws-s3-builds-page/uploadFile.js +++ b/tasks/aws-s3-builds-page/s3client/uploadFile.js @@ -1,13 +1,11 @@ -const { createS3Client } = require('./createS3Client'); const { PutObjectCommand } = require('@aws-sdk/client-s3'); const fs = require('node:fs/promises'); -async function uploadFile(localName, targetName) { - const { s3Client, bucket } = createS3Client(); +async function uploadFile(s3Client, bucket, localName, remoteName) { const fileContents = await fs.readFile(localName); const command = new PutObjectCommand({ Bucket: bucket, - Key: targetName, + Key: remoteName, Body: fileContents }); await s3Client.send(command); diff --git a/tasks/aws-s3-builds-page/test-utils/runTest.js b/tasks/aws-s3-builds-page/test-utils/runTest.js new file mode 100644 index 000000000..d6cff124c --- /dev/null +++ b/tasks/aws-s3-builds-page/test-utils/runTest.js @@ -0,0 +1,33 @@ +/* eslint-disable no-console */ +const { createS3Client } = require('../s3client/index'); + +const s3Client = createS3Client(); + +function runTest(asyncFn) { + asyncFn({ log: console.log.bind(console) }) + .finally(detectSurplusFiles) + .catch(error => { + console.error(error); + process.exit(1); + }); +} + +async function detectSurplusFiles() { + const listing = await s3Client.listFiles(); + let surplusFileDetected = false; + const testFilesInBucket = listing.filter(name => name.includes('test-file')); + for (const filename of testFilesInBucket) { + if (process.argv[2] === '--delete-surplus') { + await s3Client.deleteFile(filename); + } else { + console.log(`Detected surplus file "${filename}"`); + surplusFileDetected = true; + } + } + if (surplusFileDetected) { + console.log(`run with --delete-surplus to delete surplus files`); + } + console.log('DONE'); +} + +module.exports = { runTest }; diff --git a/tasks/publish-to-aws.js b/tasks/publish-to-aws.js index 49655fa17..522ef0f81 100644 --- a/tasks/publish-to-aws.js +++ b/tasks/publish-to-aws.js @@ -1,7 +1,7 @@ -const AWS = require('aws-sdk'); const git = require('./util/git'); const { createRegisterAsyncTaskFn } = require('./util/async-grunt-task'); const semver = require('semver'); +const { publishWithSuffixes } = require('./aws-s3-builds-page/publish'); module.exports = function(grunt) { const registerAsyncTask = createRegisterAsyncTaskFn(grunt); @@ -27,76 +27,7 @@ module.exports = function(grunt) { } if (suffixes.length > 0) { - initSDK(); - grunt.log.writeln( - 'publishing file-suffixes: ' + JSON.stringify(suffixes) - ); - await publish(suffixes); + await publishWithSuffixes(suffixes); } }); - - function initSDK() { - const bucket = process.env.S3_BUCKET_NAME, - key = process.env.S3_ACCESS_KEY_ID, - secret = process.env.S3_SECRET_ACCESS_KEY; - - if (!bucket || !key || !secret) { - throw new Error('Missing S3 config values'); - } - - AWS.config.update({ accessKeyId: key, secretAccessKey: secret }); - } - - async function publish(suffixes) { - const publishPromises = suffixes.map(suffix => publishSuffix(suffix)); - return Promise.all(publishPromises); - } - - async function publishSuffix(suffix) { - const filenames = [ - 'handlebars.js', - 'handlebars.min.js', - 'handlebars.runtime.js', - 'handlebars.runtime.min.js' - ]; - const publishPromises = filenames.map(async filename => { - const nameInBucket = getNameInBucket(filename, suffix); - const localFile = getLocalFile(filename); - await uploadToBucket(localFile, nameInBucket); - grunt.log.writeln( - `Published ${localFile} to build server (${nameInBucket})` - ); - }); - return Promise.all(publishPromises); - } - - async function uploadToBucket(localFile, nameInBucket) { - const bucket = process.env.S3_BUCKET_NAME; - const uploadParams = { - Bucket: bucket, - Key: nameInBucket, - Body: grunt.file.read(localFile) - }; - return s3PutObject(uploadParams); - } }; - -function s3PutObject(uploadParams) { - const s3 = new AWS.S3(); - return new Promise((resolve, reject) => { - s3.putObject(uploadParams, err => { - if (err != null) { - return reject(err); - } - resolve(); - }); - }); -} - -function getNameInBucket(filename, suffix) { - return filename.replace(/\.js$/, suffix + '.js'); -} - -function getLocalFile(filename) { - return 'dist/' + filename; -}