diff --git a/Dockerfile b/Dockerfile index eb4d7f0c0..0371ab077 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ USER docsworker-xlarge WORKDIR /home/docsworker-xlarge # get shared.mk -RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk +RUN curl https://raw.githubusercontent.com/madelinezec/docs-worker-pool/DOP-2000-Makefiles/makefiles/shared.mk -o shared.mk # install snooty parser RUN python3 -m pip uninstall -y snooty diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index 2dd0844e9..4e44b6526 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -80,6 +80,7 @@ class S3PublishClass { //as defined in githubJob.js if (manifestPrefix) deployCommands[deployCommands.length - 1] += ` MANIFEST_PREFIX=${manifestPrefix} GLOBAL_SEARCH_FLAG=${this.GitHubJob.currentJob.payload.stableBranch}`; } + // deploy site try { const exec = workerUtils.getExecPromise(); @@ -93,57 +94,38 @@ class S3PublishClass { ); throw new Error(`Failed pushing to prod: ${stderr}`) } - // check for json string output from mut - const validateJsonOutput = stdout ? stdout.substr(0, stdout.lastIndexOf(']}') + 2) : ''; - // check if json was returned from mut try { - const stdoutJSON = JSON.parse(validateJsonOutput); - const urls = stdoutJSON.urls; - // pass in urls to fastly function to purge cache - this.fastly.purgeCache(urls).then(function (data) { - logger.save(`${'(prod)'.padEnd(15)}Fastly finished purging URL's`); - logger.sendSlackMsg('Fastly Summary: The following pages were purged from cache for your deploy'); - // when finished purging - // batch urls to send as single slack message - let batchedUrls = []; - for (let i = 0; i < urls.length; i++) { - const purgedUrl = urls[i]; - if (purgedUrl && purgedUrl.indexOf('.html') !== -1) { - batchedUrls.push(purgedUrl); - } - // if over certain length, send as a single slack message and reset the array - if (batchedUrls.length > 20 || i >= (urls.length - 1)) { - logger.sendSlackMsg(`${batchedUrls.join('\n')}`); - batchedUrls = []; - } - } - }); - } catch (e) { - // if not JSON, then it's a normal string output from mut - // get only last part of message which includes # of files changes + s3 link - if (stdout.indexOf('Summary') !== -1) { - stdoutMod = stdout.substr(stdout.indexOf('Summary')); - } - } + const makefileOutput = stdout.replace(/\r/g, '').split(/\n/); + // the URLS are always third line returned bc of the makefile target + const stdoutJSON = JSON.parse(makefileOutput[2]); + //contains URLs corresponding to files updated via our push to S3 + const updatedURLsArray = stdoutJSON.urls; + // purgeCache purges the now stale content and requests the URLs to warm the cache for our users + await this.fastly.purgeCache(updatedURLsArray); + //save purged URLs to job object + workerUtils.updateJobWithPurgedURLs(this.GitHubJob.currentJob, updatedURLsArray); - return new Promise((resolve) => { - logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); - logger.save( - `${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}` - ); - resolve({ - status: 'success', - stdout: stdoutMod - }); - }); - } catch (errResult) { - logger.save(`${'(prod)'.padEnd(15)}stdErr: ${errResult.stderr}`); - throw errResult; + return new Promise((resolve) => { + logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); + logger.save(`${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}`); + resolve({ + status: 'success', + stdout: stdoutMod, + }); + },); + } catch (error) { + console.trace(error) + throw(error) } + } + catch (errResult) { + logger.save(`${'(prod)'.padEnd(15)}stdErr: ${errResult.stderr}`); + throw errResult; +} } } module.exports = { - S3PublishClass + S3PublishClass, }; diff --git a/worker/jobTypes/githubJob.js b/worker/jobTypes/githubJob.js index 5bacf4b82..86af370fb 100644 --- a/worker/jobTypes/githubJob.js +++ b/worker/jobTypes/githubJob.js @@ -130,7 +130,8 @@ class GitHubJobClass { // our maintained directory of makefiles async downloadMakefile() { - const makefileLocation = `https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/Makefile.${this.currentJob.payload.repoName}`; + // TODO: remove call to my fork and branch + const makefileLocation = `https://raw.githubusercontent.com/madelinezec/docs-worker-pool/DOP-2000-Makefiles/makefiles/Makefile.${this.currentJob.payload.repoName}`; const returnObject = {}; return new Promise(function(resolve, reject) { request(makefileLocation, function(error, response, body) { diff --git a/worker/package-lock.json b/worker/package-lock.json index 45153aaa1..52f5a9417 100644 --- a/worker/package-lock.json +++ b/worker/package-lock.json @@ -4171,11 +4171,18 @@ "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" }, "axios": { - "version": "0.19.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz", - "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==", + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", + "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", "requires": { - "follow-redirects": "1.5.10" + "follow-redirects": "^1.10.0" + }, + "dependencies": { + "follow-redirects": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz", + "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==" + } } }, "axobject-query": { @@ -6365,6 +6372,14 @@ "type-fest": "0.15.1" }, "dependencies": { + "axios": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz", + "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==", + "requires": { + "follow-redirects": "1.5.10" + } + }, "type-fest": { "version": "0.15.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.15.1.tgz", @@ -10507,6 +10522,14 @@ } } }, + "axios": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz", + "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==", + "requires": { + "follow-redirects": "1.5.10" + } + }, "binary-extensions": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz", @@ -22663,6 +22686,11 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" }, + "toml": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", + "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==" + }, "toposort": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/toposort/-/toposort-2.0.2.tgz", diff --git a/worker/package.json b/worker/package.json index 7694f7e5f..aefdd7e54 100644 --- a/worker/package.json +++ b/worker/package.json @@ -19,6 +19,7 @@ "@babel/polyfill": "^7.4.4", "async-retry": "^1.2.3", "aws-sdk": "^2.383.0", + "axios": "^0.21.1", "core-js": "^3.1.4", "express": "^4.16.4", "fastly": "^2.2.1", @@ -34,10 +35,10 @@ "remote-file-size": "^3.0.5", "simple-git": "^1.107.0", "supertest": "^4.0.2", + "toml": "^3.0.0", "typescript": "^4.0.2", "validator": "^10.11.0", - "xmlhttprequest": "^1.8.0", - "toml": "^3.0.0" + "xmlhttprequest": "^1.8.0" }, "devDependencies": { "aws-sdk-mock": "^4.3.0", diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index de23ee536..21ebd295b 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -1,92 +1,128 @@ -const request = require('request'); -const utils = require('../utils/utils'); +const axios = require('axios').default; const environment = require('../utils/environment').EnvironmentClass; const fastly = require('fastly')(environment.getFastlyToken()); +const utils = require('../utils/utils'); +const Logger = require('../utils/logger').LoggerClass; + +const fastlyServiceId = environment.getFastlyServiceId(); +const headers = { + 'Fastly-Key': environment.getFastlyToken(), + 'Accept': 'application/json', + 'Content-Type': 'application/json', + 'Fastly-Debug': 1, +}; + class FastlyJobClass { - // pass in a job payload to setup class - constructor(currentJob) { - this.currentJob = currentJob; - if (fastly === undefined) { - utils.logInMongo(currentJob, 'fastly connectivity not found'); + // pass in a job payload to setup class + constructor(currentJob) { + this.currentJob = currentJob; + this.logger = new Logger(currentJob); + if (fastly === undefined) { + utils.logInMongo(currentJob, 'fastly connectivity not found'); + } } - } - // takes in an array of urls and purges cache for each - async purgeCache(urlArray) { - if (!Array.isArray(urlArray)) { - throw new Error('Parameter `urlArray` needs to be an array of urls'); + // takes in an array of surrogate keys and purges cache for each + async purgeCache(urlArray) { + if (!Array.isArray(urlArray)) { + throw new Error('Parameter `urlArray` needs to be an array of urls'); + } + + try { + //retrieve surrogate key associated with each URL/file updated in push to S3 + const surrogateKeyPromises = urlArray.map(url => this.retrieveSurrogateKey(url)); + const surrogateKeyArray = await Promise.all(surrogateKeyPromises) + + //purge each surrogate key + const purgeRequestPromises = surrogateKeyArray.map(surrogateKey => this.requestPurgeOfSurrogateKey(surrogateKey)); + await Promise.all(purgeRequestPromises); + + // GET request the URLs to warm cache for our users + const warmCachePromises = urlArray.map(url => this.warmCache(url)); + await Promise.all(warmCachePromises) + } catch (error) { + this.logger.save(`${'(prod)'.padEnd(15)}error in purge cache: ${error}`); + throw error + } + } - let that = this; - let urlCounter = urlArray.length; - let purgeMessages = []; - - // the 1 is just "some" value needed for this header: https://docs.fastly.com/en/guides/soft-purges - const headers = { - 'fastly-key': environment.getFastlyToken(), - 'accept': 'application/json', - 'Fastly-Soft-Purge': '1', - }; - - return new Promise((resolve, reject) => { - for (let i = 0; i < urlArray.length; i++) { - // perform request to purge - request({ - method: 'PURGE', - url: urlArray[i], - headers: headers, - }, function(err, response, body) { - // url was not valid to purge - if (!response) { - utils.logInMongo(that.currentJob, `Error: service for this url does not exist in fastly for purging ${urlArray[i]}`); - purgeMessages.push({ - 'status': 'failure', - 'message': `service with url ${urlArray[i]} does not exist in fastly` + async retrieveSurrogateKey(url) { + + try { + return axios({ + method: 'HEAD', + url: url, + headers: headers, + }).then(response => { + if (response.status === 200) { + return response.headers['surrogate-key']; + } }); - } else if (response.headers['content-type'].indexOf('application/json') === 0) { - try { - body = JSON.parse(body); - purgeMessages.push(body); - } catch(er) { - utils.logInMongo(that.currentJob, `Error: failed parsing output from fastly for url ${urlArray[i]}`); - console.log(`Error: failed parsing output from fastly for url ${urlArray[i]}`); - } - } - // when we are done purging all urls - // this is outside of the conditional above because if some url fails to purge - // we do not want to actually have this entire build fail, just show warning - urlCounter--; - if (urlCounter <= 0) { - resolve({ - 'status': 'success', - 'fastlyMessages': purgeMessages, + } catch (error) { + this.logger.save(`${'(prod)'.padEnd(15)}error in retrieveSurrogateKey: ${error}`); + throw error + } + + } + + async requestPurgeOfSurrogateKey(surrogateKey) { + headers['Surrogate-Key'] = surrogateKey + + try { + return axios({ + method: `POST`, + url: `https://api.fastly.com/service/${fastlyServiceId}/purge/${surrogateKey}`, + path: `/service/${fastlyServiceId}/purge${surrogateKey}`, + headers: headers, + }) + .then(response => { + if (response.status === 200) { + return true + } + }); + } catch (error) { + this.logger.save(`${'(prod)'.padEnd(15)}error in requestPurgeOfSurrogateKey: ${error}`); + throw error; + } + } + + // request urls of updated content to "warm" the cache for our customers + async warmCache(updatedUrl) { + + try { + return axios.get(updatedUrl) + .then(response => { + if (response.status === 200) { + return true; + } + }) + } catch (error) { + this.logger.save(`${'(prod)'.padEnd(15)}stdErr: ${error}`); + throw error; + } + } + + // upserts {source: target} mappings + // to the fastly edge dictionary + async connectAndUpsert(map) { + const options = { + item_value: map.target, + }; + const connectString = `/service/${fastlyServiceId}/dictionary/${environment.getDochubMap()}/item/${ + map.source + }`; + + return new Promise((resolve, reject) => { + fastly.request('PUT', connectString, options, (err, obj) => { + if (err) reject(err); + resolve(obj); }); - } - }); - } - }) - } - - // upserts {source: target} mappings - // to the fastly edge dictionary - async connectAndUpsert(map) { - const options = { - item_value: map.target - }; - const connectString = `/service/${environment.getFastlyServiceId()}/dictionary/${environment.getDochubMap()}/item/${ - map.source - }`; - - return new Promise((resolve, reject) => { - fastly.request('PUT', connectString, options, function(err, obj) { - if (err) reject(err); - resolve(obj); - }); - }) - } + }) + } } module.exports = { - FastlyJobClass: FastlyJobClass -}; + FastlyJobClass, +}; \ No newline at end of file diff --git a/worker/utils/mongo.js b/worker/utils/mongo.js index c2c4c2d8d..0ba118e6e 100644 --- a/worker/utils/mongo.js +++ b/worker/utils/mongo.js @@ -130,6 +130,25 @@ module.exports = { } }, + async updateJobWithPurgedURLs(currentJob, urlArray) { + const queueCollection = module.exports.getCollection(); + if (queueCollection) { + const query = { _id: currentJob._id }; + const update = { + $push: { ['purgedURLs']: urlArray }, + }; + + try { + await queueCollection.updateOne(query, update); + } catch (err) { + console.log(`Error in updateJobWithPurgedURLs(): ${err}`); + throw err + } + } else { + console.log('Error in logInMongo(): queueCollection does not exist'); + } + }, + // Adds Log Message To Job In The Queue async logMessageInMongo(currentJob, message) { const queueCollection = module.exports.getCollection(); diff --git a/worker/utils/utils.js b/worker/utils/utils.js index a9da33ef0..9baa8d669 100644 --- a/worker/utils/utils.js +++ b/worker/utils/utils.js @@ -134,6 +134,11 @@ module.exports = { getExecPromise() { return exec; }, + + // save array of purged URLs to job object + async updateJobWithPurgedURLs(currentJob, urlArray) { + await mongo.updateJobWithPurgedURLs(currentJob, urlArray); + }, // Adds log message (message) to current job in queue at spot (currentJob.numFailures) async logInMongo(currentJob, message) {