From 000bb42a43b40d3f7bd77f2d84073769facab2d7 Mon Sep 17 00:00:00 2001 From: madelinezec Date: Thu, 11 Mar 2021 17:20:55 -0500 Subject: [PATCH 01/24] testing --- worker/jobTypes/S3Publish.js | 21 ++++------ worker/jobTypes/githubJob.js | 2 +- worker/utils/fastlyJob.js | 81 +++++++++++++++++------------------- 3 files changed, 48 insertions(+), 56 deletions(-) diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index 2dd0844e9..cc8a49c41 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -94,18 +94,18 @@ class S3PublishClass { throw new Error(`Failed pushing to prod: ${stderr}`) } // check for json string output from mut - const validateJsonOutput = stdout ? stdout.substr(0, stdout.lastIndexOf(']}') + 2) : ''; - // check if json was returned from mut try { - const stdoutJSON = JSON.parse(validateJsonOutput); + + const makefileOutput = stdout.replace(/\r/g, "").split(/\n/); + // the surrogate keys are always third line returned bc of the makefile target + const stdoutJSON = JSON.parse(makefileOutput[2]); const urls = stdoutJSON.urls; - // pass in urls to fastly function to purge cache this.fastly.purgeCache(urls).then(function (data) { logger.save(`${'(prod)'.padEnd(15)}Fastly finished purging URL's`); logger.sendSlackMsg('Fastly Summary: The following pages were purged from cache for your deploy'); // when finished purging - // batch urls to send as single slack message + // batch surrogate keys to send as single slack message let batchedUrls = []; for (let i = 0; i < urls.length; i++) { const purgedUrl = urls[i]; @@ -119,14 +119,11 @@ class S3PublishClass { } } }); - } catch (e) { - // if not JSON, then it's a normal string output from mut - // get only last part of message which includes # of files changes + s3 link - if (stdout.indexOf('Summary') !== -1) { - stdoutMod = stdout.substr(stdout.indexOf('Summary')); - } + } catch (error) { + console.trace(error) + throw(error) } - + return new Promise((resolve) => { logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); logger.save( diff --git a/worker/jobTypes/githubJob.js b/worker/jobTypes/githubJob.js index 746652152..20f8920d4 100644 --- a/worker/jobTypes/githubJob.js +++ b/worker/jobTypes/githubJob.js @@ -130,7 +130,7 @@ class GitHubJobClass { // our maintained directory of makefiles async downloadMakefile() { - const makefileLocation = `https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/Makefile.${this.currentJob.payload.repoName}`; + const makefileLocation = `https://raw.githubusercontent.com/madelinezec/docs-worker-pool/DOP-2000-Makefiles/makefiles/Makefile.${this.currentJob.payload.repoName}`; const returnObject = {}; return new Promise(function(resolve, reject) { request(makefileLocation, function(error, response, body) { diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index de23ee536..1326b4f72 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -2,6 +2,7 @@ const request = require('request'); const utils = require('../utils/utils'); const environment = require('../utils/environment').EnvironmentClass; const fastly = require('fastly')(environment.getFastlyToken()); +const https = require('https') class FastlyJobClass { // pass in a job payload to setup class @@ -13,60 +14,54 @@ class FastlyJobClass { } // takes in an array of urls and purges cache for each - async purgeCache(urlArray) { - if (!Array.isArray(urlArray)) { - throw new Error('Parameter `urlArray` needs to be an array of urls'); + async purgeCache(surrogateKeyArray) { + + if (!Array.isArray(surrogateKeyArray)) { + throw new Error('Parameter `surrogateKeyArray` needs to be an array of urls'); } let that = this; - let urlCounter = urlArray.length; + let urlCounter = surrogateKeyArray.length; let purgeMessages = []; + const fastly_service_id = environment.getFastlyServiceId(); - // the 1 is just "some" value needed for this header: https://docs.fastly.com/en/guides/soft-purges const headers = { - 'fastly-key': environment.getFastlyToken(), - 'accept': 'application/json', - 'Fastly-Soft-Purge': '1', + 'Fastly-Key': environment.getFastlyToken(), + 'Accept': 'application/json', + 'Content-Type': 'application/json' }; return new Promise((resolve, reject) => { - for (let i = 0; i < urlArray.length; i++) { + + + for (let i = 0; i < surrogateKeyArray.length; i++) { // perform request to purge - request({ - method: 'PURGE', - url: urlArray[i], - headers: headers, - }, function(err, response, body) { - // url was not valid to purge - if (!response) { - utils.logInMongo(that.currentJob, `Error: service for this url does not exist in fastly for purging ${urlArray[i]}`); - purgeMessages.push({ - 'status': 'failure', - 'message': `service with url ${urlArray[i]} does not exist in fastly` - }); - } else if (response.headers['content-type'].indexOf('application/json') === 0) { - try { - body = JSON.parse(body); - purgeMessages.push(body); - } catch(er) { - utils.logInMongo(that.currentJob, `Error: failed parsing output from fastly for url ${urlArray[i]}`); - console.log(`Error: failed parsing output from fastly for url ${urlArray[i]}`); + try { + headers['Surrogate-Key'] = surrogateKeyArray[i] + request({ + method: `POST`, + url: `https://api.fastly.com/service/${fastly_service_id}/purge${surrogateKeyArray[i]}`, + path: `/service/${fastly_service_id}/purge${surrogateKeyArray[i]}`, + headers: headers, + }, function(err, response, body) { + // surrogate key was not valid to purge + if (err){ + console.trace(err) } - } - // when we are done purging all urls - // this is outside of the conditional above because if some url fails to purge - // we do not want to actually have this entire build fail, just show warning - urlCounter--; - if (urlCounter <= 0) { - resolve({ - 'status': 'success', - 'fastlyMessages': purgeMessages, - }); - } - }); - } - }) - } + + /* capture the purge request id in case we still see stale content on site, + contact Fastly for further assistance with purge id and resource + see https://docs.fastly.com/en/guides/single-purges */ + console.log(body) + }) + } catch (error) { + console.log(error) + throw error + } + + } + }) +} // upserts {source: target} mappings // to the fastly edge dictionary From fa9e9671270e35b5b81192ab40d9723589d9cb9b Mon Sep 17 00:00:00 2001 From: madelinezec Date: Thu, 11 Mar 2021 17:20:55 -0500 Subject: [PATCH 02/24] testing --- worker/jobTypes/GatsbyAdapter.js | 5 +- worker/jobTypes/S3Publish.js | 59 ++++++++++++-- worker/jobTypes/githubJob.js | 2 +- worker/utils/fastlyJob.js | 128 +++++++++++++++++++++---------- worker/utils/mongo.js | 15 ++++ 5 files changed, 160 insertions(+), 49 deletions(-) diff --git a/worker/jobTypes/GatsbyAdapter.js b/worker/jobTypes/GatsbyAdapter.js index 6c93e8364..effaae370 100644 --- a/worker/jobTypes/GatsbyAdapter.js +++ b/worker/jobTypes/GatsbyAdapter.js @@ -1,5 +1,6 @@ const workerUtils = require('../utils/utils'); const fs = require('fs-extra'); +const { Logger } = require('mongodb'); class GatsbyAdapterClass { constructor(GitHubJob) { @@ -23,13 +24,15 @@ class GatsbyAdapterClass { const envVars = await this.constructEnvVars(); fs.writeFile(`repos/${this.GitHubJob.currentJob.payload.repoName}/.env.production`, envVars, { encoding: 'utf8', flag: 'w' }, function(err) { + console.log("we are writing the .env.production file!") if(err) { console.log(`error writing .env.production file: ${err.stderr}`); throw err; } }); } catch (error) { - console.log(error) + + Logger.save(error) throw error } } diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index 2dd0844e9..0d8427a7f 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -94,13 +94,14 @@ class S3PublishClass { throw new Error(`Failed pushing to prod: ${stderr}`) } // check for json string output from mut - const validateJsonOutput = stdout ? stdout.substr(0, stdout.lastIndexOf(']}') + 2) : ''; - // check if json was returned from mut try { - const stdoutJSON = JSON.parse(validateJsonOutput); + + const makefileOutput = stdout.replace(/\r/g, "").split(/\n/); + // the URLS are always third line returned bc of the makefile target + const stdoutJSON = JSON.parse(makefileOutput[2]); const urls = stdoutJSON.urls; - // pass in urls to fastly function to purge cache + this.fastly.purgeCache(urls).then(function (data) { logger.save(`${'(prod)'.padEnd(15)}Fastly finished purging URL's`); logger.sendSlackMsg('Fastly Summary: The following pages were purged from cache for your deploy'); @@ -108,6 +109,8 @@ class S3PublishClass { // batch urls to send as single slack message let batchedUrls = []; for (let i = 0; i < urls.length; i++) { + console.log(urls[i]) + console.log(urls[i].split('.')[0]) const purgedUrl = urls[i]; if (purgedUrl && purgedUrl.indexOf('.html') !== -1) { batchedUrls.push(purgedUrl); @@ -119,15 +122,55 @@ class S3PublishClass { } } }); - } catch (e) { - // if not JSON, then it's a normal string output from mut - // get only last part of message which includes # of files changes + s3 link + } catch (error) { + console.log("there is an error!!! we never caught it lmao") + console.log(error) if (stdout.indexOf('Summary') !== -1) { stdoutMod = stdout.substr(stdout.indexOf('Summary')); + console.log("except caught but not recorded!") } } - + + + // try { + // console.log("this enters at all!") + // const stdoutJSON = JSON.parse(validateJsonOutput); + // console.log("we are inside the try catch!") + // console.log("this is stdout JSON: ", stdoutJSON) + // const urls = stdoutJSON.urls; + // console.log("these are the URLS: ", urls) + // // pass in urls to fastly function to purge cache + // this.fastly.purgeCache(urls).then(function (data) { + // logger.save(`${'(prod)'.padEnd(15)}Fastly finished purging URL's`); + // logger.sendSlackMsg('Fastly Summary: The following pages were purged from cache for your deploy'); + // // when finished purging + // // batch urls to send as single slack message + // let batchedUrls = []; + // for (let i = 0; i < urls.length; i++) { + // const purgedUrl = urls[i]; + // if (purgedUrl && purgedUrl.indexOf('.html') !== -1) { + // batchedUrls.push(purgedUrl); + // } + // // if over certain length, send as a single slack message and reset the array + // if (batchedUrls.length > 20 || i >= (urls.length - 1)) { + // logger.sendSlackMsg(`${batchedUrls.join('\n')}`); + // batchedUrls = []; + // } + // } + // }); + // } catch (e) { + // // if not JSON, then it's a normal string output from mut + // // get only last part of message which includes # of files changes + s3 link + // console.log("there is an error!!! we never caught it lmao") + // console.log(e) + // if (stdout.indexOf('Summary') !== -1) { + // stdoutMod = stdout.substr(stdout.indexOf('Summary')); + // console.log("except caught but not recorded!") + // } + // } + return new Promise((resolve) => { + console.log("we are going to return!!!") logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); logger.save( `${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}` diff --git a/worker/jobTypes/githubJob.js b/worker/jobTypes/githubJob.js index 746652152..20f8920d4 100644 --- a/worker/jobTypes/githubJob.js +++ b/worker/jobTypes/githubJob.js @@ -130,7 +130,7 @@ class GitHubJobClass { // our maintained directory of makefiles async downloadMakefile() { - const makefileLocation = `https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/Makefile.${this.currentJob.payload.repoName}`; + const makefileLocation = `https://raw.githubusercontent.com/madelinezec/docs-worker-pool/DOP-2000-Makefiles/makefiles/Makefile.${this.currentJob.payload.repoName}`; const returnObject = {}; return new Promise(function(resolve, reject) { request(makefileLocation, function(error, response, body) { diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index de23ee536..0f53e7281 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -2,6 +2,7 @@ const request = require('request'); const utils = require('../utils/utils'); const environment = require('../utils/environment').EnvironmentClass; const fastly = require('fastly')(environment.getFastlyToken()); +const https = require('https') class FastlyJobClass { // pass in a job payload to setup class @@ -14,6 +15,7 @@ class FastlyJobClass { // takes in an array of urls and purges cache for each async purgeCache(urlArray) { + console.log("purge cache is called!!!!") if (!Array.isArray(urlArray)) { throw new Error('Parameter `urlArray` needs to be an array of urls'); } @@ -21,49 +23,97 @@ class FastlyJobClass { let that = this; let urlCounter = urlArray.length; let purgeMessages = []; - - // the 1 is just "some" value needed for this header: https://docs.fastly.com/en/guides/soft-purges - const headers = { - 'fastly-key': environment.getFastlyToken(), - 'accept': 'application/json', - 'Fastly-Soft-Purge': '1', - }; + const fastly_service_id = environment.getFastlyServiceId(); + const options = { + method:'POST', + port: 80, + host: 'docs-mongodbcom-integration.corp.mongodb.com', + headers : { + 'Fastly-Key': environment.getFastlyToken(), + 'Accept': 'application/json', + 'Content-Type': 'application/json', + 'Host': 'www.docs-mongodbcom-integration.corp.mongodb.com' + } + } + // const headers = { + // 'Fastly-Key': environment.getFastlyToken(), + // 'Accept': 'application/json', + // 'Content-Type': 'application/json' + // }; return new Promise((resolve, reject) => { + + + + for (let i = 0; i < urlArray.length; i++) { // perform request to purge - request({ - method: 'PURGE', - url: urlArray[i], - headers: headers, - }, function(err, response, body) { - // url was not valid to purge - if (!response) { - utils.logInMongo(that.currentJob, `Error: service for this url does not exist in fastly for purging ${urlArray[i]}`); - purgeMessages.push({ - 'status': 'failure', - 'message': `service with url ${urlArray[i]} does not exist in fastly` - }); - } else if (response.headers['content-type'].indexOf('application/json') === 0) { - try { - body = JSON.parse(body); - purgeMessages.push(body); - } catch(er) { - utils.logInMongo(that.currentJob, `Error: failed parsing output from fastly for url ${urlArray[i]}`); - console.log(`Error: failed parsing output from fastly for url ${urlArray[i]}`); - } - } - // when we are done purging all urls - // this is outside of the conditional above because if some url fails to purge - // we do not want to actually have this entire build fail, just show warning - urlCounter--; - if (urlCounter <= 0) { - resolve({ - 'status': 'success', - 'fastlyMessages': purgeMessages, - }); - } - }); + console.log(urlArray[i]) + // console.log(`service/${fastly_service_id}/purge/${urlArray[i]}`) + + options['path'] = `/service/${fastly_service_id}/purge` + options['service-id'] = fastly_service_id + options['Surrogate-Key'] = urlArray[i] + const req = https.request(options, res => { + console.log(`statusCode: ${res.statusCode}`) + + res.on('data', d => { + process.stdout.write(d) + }) + }) + + req.on('error', error => { + console.error(error) + }) + + req.write(data) + req.end() + + + // request({ + // method: `POST`, + // // method: 'PURGE', + // path: `/service/${fastly_service_id}/purge${urlArray[i]}`, + // headers: headers, + // }, function(err, response, body) { + // // url was not valid to purge + // if (err){ + // console.log("is there an err???") + // console.log(err) + // } + // console.log(response) + + // console.log("\n") + // // console.log(JSON.parse(body)) + // console.log(body) + // if (!response) { + // console.log("this is an error in the response!!!") + // utils.logInMongo(that.currentJob, `Error: service for this url does not exist in fastly for purging ${urlArray[i]}`); + // purgeMessages.push({ + // 'status': 'failure', + // 'message': `service with url ${urlArray[i]} does not exist in fastly` + // }); + // } else if (response.headers['content-type'].indexOf('application/json') === 0) { + // try { + // body = JSON.parse(body); + // purgeMessages.push(body); + // } catch(er) { + // console.log("hey!!!") + // utils.logInMongo(that.currentJob, `Error: failed parsing output from fastly for url ${urlArray[i]}`); + // console.log(`Error: failed parsing output from fastly for url ${urlArray[i]}`); + // } + // } + // // when we are done purging all urls + // // this is outside of the conditional above because if some url fails to purge + // // we do not want to actually have this entire build fail, just show warning + // urlCounter--; + // if (urlCounter <= 0) { + // resolve({ + // 'status': 'success', + // 'fastlyMessages': purgeMessages, + // }); + // } + // }); } }) } diff --git a/worker/utils/mongo.js b/worker/utils/mongo.js index c2c4c2d8d..69a208301 100644 --- a/worker/utils/mongo.js +++ b/worker/utils/mongo.js @@ -9,8 +9,14 @@ const runXlarge = EnvironmentClass.getXlarge(); const url = `mongodb+srv://${username}:${password}@cluster0-ylwlz.mongodb.net/admin?retryWrites=true`; // Collection information +<<<<<<< HEAD const DB_NAME = EnvironmentClass.getDB(); // Database name for autobuilder in MongoDB Atlas const COLL_NAME = EnvironmentClass.getCollection(); // Collection name in MongoDB Atlas +======= +const DB_NAME = EnvironmentClass.getDB(); // Database name of the queue in MongoDB Atlas +const COLL_NAME = EnvironmentClass.getCollection(); // Collection name of the queue in MongoDB Atlas +console.log("collection name inside mongo.js ", COLL_NAME) +>>>>>>> testing const META_NAME = 'meta'; const MONITOR_NAME = 'monitor'; const ENTITLEMENTS_NAME = 'entitlements'; @@ -92,11 +98,20 @@ module.exports = { const update = { $set: { startTime: new Date(), status: 'inProgress' } }; const options = { sort: { priority: -1, createdTime: 1 }, returnNewDocument: true }; +<<<<<<< HEAD try { return queueCollection.findOneAndUpdate(query, update, options); } catch (error) { console.trace(error) +======= + //await queueCollection.findOne({ status: 'inQueue' }); + //console.log(await queueCollection.findOne({ status: 'inQueue' })) + try { + return queueCollection.findOneAndUpdate(query, update, options); + } catch (error) { + console.log(error) +>>>>>>> testing throw error } }, From c68c1999a7ff1f64e8ea79d538c2c6d1edc8190c Mon Sep 17 00:00:00 2001 From: madelinezec Date: Thu, 11 Mar 2021 17:20:55 -0500 Subject: [PATCH 03/24] testing --- worker/jobTypes/S3Publish.js | 50 ++------------- worker/utils/fastlyJob.js | 121 ++++++++++------------------------- 2 files changed, 37 insertions(+), 134 deletions(-) diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index 0d8427a7f..909deb4d4 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -106,7 +106,7 @@ class S3PublishClass { logger.save(`${'(prod)'.padEnd(15)}Fastly finished purging URL's`); logger.sendSlackMsg('Fastly Summary: The following pages were purged from cache for your deploy'); // when finished purging - // batch urls to send as single slack message + // batch surrogate keys to send as single slack message let batchedUrls = []; for (let i = 0; i < urls.length; i++) { console.log(urls[i]) @@ -123,52 +123,10 @@ class S3PublishClass { } }); } catch (error) { - console.log("there is an error!!! we never caught it lmao") - console.log(error) - if (stdout.indexOf('Summary') !== -1) { - stdoutMod = stdout.substr(stdout.indexOf('Summary')); - console.log("except caught but not recorded!") - } + console.trace(error) + throw(error) } - - - // try { - // console.log("this enters at all!") - // const stdoutJSON = JSON.parse(validateJsonOutput); - // console.log("we are inside the try catch!") - // console.log("this is stdout JSON: ", stdoutJSON) - // const urls = stdoutJSON.urls; - // console.log("these are the URLS: ", urls) - // // pass in urls to fastly function to purge cache - // this.fastly.purgeCache(urls).then(function (data) { - // logger.save(`${'(prod)'.padEnd(15)}Fastly finished purging URL's`); - // logger.sendSlackMsg('Fastly Summary: The following pages were purged from cache for your deploy'); - // // when finished purging - // // batch urls to send as single slack message - // let batchedUrls = []; - // for (let i = 0; i < urls.length; i++) { - // const purgedUrl = urls[i]; - // if (purgedUrl && purgedUrl.indexOf('.html') !== -1) { - // batchedUrls.push(purgedUrl); - // } - // // if over certain length, send as a single slack message and reset the array - // if (batchedUrls.length > 20 || i >= (urls.length - 1)) { - // logger.sendSlackMsg(`${batchedUrls.join('\n')}`); - // batchedUrls = []; - // } - // } - // }); - // } catch (e) { - // // if not JSON, then it's a normal string output from mut - // // get only last part of message which includes # of files changes + s3 link - // console.log("there is an error!!! we never caught it lmao") - // console.log(e) - // if (stdout.indexOf('Summary') !== -1) { - // stdoutMod = stdout.substr(stdout.indexOf('Summary')); - // console.log("except caught but not recorded!") - // } - // } - + return new Promise((resolve) => { console.log("we are going to return!!!") logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index 0f53e7281..282565718 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -21,102 +21,47 @@ class FastlyJobClass { } let that = this; - let urlCounter = urlArray.length; + let urlCounter = surrogateKeyArray.length; let purgeMessages = []; const fastly_service_id = environment.getFastlyServiceId(); - const options = { - method:'POST', - port: 80, - host: 'docs-mongodbcom-integration.corp.mongodb.com', - headers : { - 'Fastly-Key': environment.getFastlyToken(), - 'Accept': 'application/json', - 'Content-Type': 'application/json', - 'Host': 'www.docs-mongodbcom-integration.corp.mongodb.com' - } - } - // const headers = { - // 'Fastly-Key': environment.getFastlyToken(), - // 'Accept': 'application/json', - // 'Content-Type': 'application/json' - // }; + + const headers = { + 'Fastly-Key': environment.getFastlyToken(), + 'Accept': 'application/json', + 'Content-Type': 'application/json' + }; return new Promise((resolve, reject) => { - - - - for (let i = 0; i < urlArray.length; i++) { + + for (let i = 0; i < surrogateKeyArray.length; i++) { // perform request to purge - console.log(urlArray[i]) - // console.log(`service/${fastly_service_id}/purge/${urlArray[i]}`) - - options['path'] = `/service/${fastly_service_id}/purge` - options['service-id'] = fastly_service_id - options['Surrogate-Key'] = urlArray[i] - const req = https.request(options, res => { - console.log(`statusCode: ${res.statusCode}`) - - res.on('data', d => { - process.stdout.write(d) - }) - }) - - req.on('error', error => { - console.error(error) + try { + headers['Surrogate-Key'] = surrogateKeyArray[i] + request({ + method: `POST`, + url: `https://api.fastly.com/service/${fastly_service_id}/purge${surrogateKeyArray[i]}`, + path: `/service/${fastly_service_id}/purge${surrogateKeyArray[i]}`, + headers: headers, + }, function(err, response, body) { + // surrogate key was not valid to purge + if (err){ + console.trace(err) + } + + /* capture the purge request id in case we still see stale content on site, + contact Fastly for further assistance with purge id and resource + see https://docs.fastly.com/en/guides/single-purges */ + console.log(body) }) - - req.write(data) - req.end() + } catch (error) { + console.log(error) + throw error + } - - // request({ - // method: `POST`, - // // method: 'PURGE', - // path: `/service/${fastly_service_id}/purge${urlArray[i]}`, - // headers: headers, - // }, function(err, response, body) { - // // url was not valid to purge - // if (err){ - // console.log("is there an err???") - // console.log(err) - // } - // console.log(response) - - // console.log("\n") - // // console.log(JSON.parse(body)) - // console.log(body) - // if (!response) { - // console.log("this is an error in the response!!!") - // utils.logInMongo(that.currentJob, `Error: service for this url does not exist in fastly for purging ${urlArray[i]}`); - // purgeMessages.push({ - // 'status': 'failure', - // 'message': `service with url ${urlArray[i]} does not exist in fastly` - // }); - // } else if (response.headers['content-type'].indexOf('application/json') === 0) { - // try { - // body = JSON.parse(body); - // purgeMessages.push(body); - // } catch(er) { - // console.log("hey!!!") - // utils.logInMongo(that.currentJob, `Error: failed parsing output from fastly for url ${urlArray[i]}`); - // console.log(`Error: failed parsing output from fastly for url ${urlArray[i]}`); - // } - // } - // // when we are done purging all urls - // // this is outside of the conditional above because if some url fails to purge - // // we do not want to actually have this entire build fail, just show warning - // urlCounter--; - // if (urlCounter <= 0) { - // resolve({ - // 'status': 'success', - // 'fastlyMessages': purgeMessages, - // }); - // } - // }); - } - }) - } + } + }) +} // upserts {source: target} mappings // to the fastly edge dictionary From ba1155e30141436871248570ce5cbe07d269aa40 Mon Sep 17 00:00:00 2001 From: madelinezec Date: Wed, 28 Apr 2021 15:35:25 -0400 Subject: [PATCH 04/24] save changes --- worker/jobTypes/S3Publish.js | 24 +++++++++++++++++------- worker/utils/fastlyJob.js | 25 ++++++++++++++++++++----- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index 909deb4d4..097ed61ae 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -95,7 +95,6 @@ class S3PublishClass { } // check for json string output from mut // check if json was returned from mut - try { const makefileOutput = stdout.replace(/\r/g, "").split(/\n/); // the URLS are always third line returned bc of the makefile target @@ -121,14 +120,25 @@ class S3PublishClass { batchedUrls = []; } } - }); - } catch (error) { - console.trace(error) - throw(error) - } + // if over certain length, send as a single slack message and reset the array + if (batchedUrls.length > 20 || i >= (surrogateKeyArray.length - 1)) { + logger.sendSlackMsg(`${batchedUrls.join('\n')}`); + batchedUrls = []; + } + } + }); + } catch (error) { + console.trace(error) + throw(error) + } + + try { + this.fastly.warmCache(st) + } catch (error) { + + } return new Promise((resolve) => { - console.log("we are going to return!!!") logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); logger.save( `${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}` diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index 282565718..ffc042f86 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -13,11 +13,26 @@ class FastlyJobClass { } } - // takes in an array of urls and purges cache for each - async purgeCache(urlArray) { - console.log("purge cache is called!!!!") - if (!Array.isArray(urlArray)) { - throw new Error('Parameter `urlArray` needs to be an array of urls'); + // request urls of updated content to "warm" the cache for our customers + async warmCache(updatedContentURLS) { + if (!Array.isArray(updatedContentURLS)) { + throw new Error('Parameter `updatedContentURLS` needs to be an array of urls'); + } + + + for (let i = 0; i < updatedContentURLS.length; i++) { + request(updatedContentURLS[i], function (error, response, body) { + if (!error && response.statusCode == 200) { + console.log(body); + } + }) + } + } + // takes in an array of surrogate keys and purges cache for each + async purgeCache(surrogateKeyArray) { + + if (!Array.isArray(surrogateKeyArray)) { + throw new Error('Parameter `surrogateKeyArray` needs to be an array of urls'); } let that = this; From 72aa3bc823e99e39690bfc2a941c0f65f2011cea Mon Sep 17 00:00:00 2001 From: madelinezec Date: Fri, 30 Apr 2021 15:41:10 -0400 Subject: [PATCH 05/24] test --- Dockerfile | 4 +- worker/jobTypes/S3Publish.js | 33 +++++----- worker/jobTypes/githubJob.js | 2 +- worker/utils/fastlyJob.js | 117 +++++++++++++++++++++++------------ 4 files changed, 97 insertions(+), 59 deletions(-) diff --git a/Dockerfile b/Dockerfile index 404b01369..b7b6f11c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,14 +42,14 @@ RUN python3 -m pip install pip==20.2 flit==3.0.0 RUN git clone https://github.com/mongodb/snooty-parser.git && \ cd snooty-parser && \ git fetch --tags && \ - git checkout && \ + git checkout v0.9.6 && \ FLIT_ROOT_INSTALL=1 python3 -m flit install # install snooty front-end RUN git clone https://github.com/mongodb/snooty.git snooty RUN cd snooty && \ git fetch --all && \ - git checkout && \ + git checkout v0.9.8 && \ npm install && \ git clone https://github.com/mongodb/docs-tools.git docs-tools && \ mkdir -p ./static/images && \ diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index 097ed61ae..ab1694406 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -121,40 +121,43 @@ class S3PublishClass { } } // if over certain length, send as a single slack message and reset the array - if (batchedUrls.length > 20 || i >= (surrogateKeyArray.length - 1)) { + if (batchedUrls.length > 20 || i >= (urlArray.length - 1)) { logger.sendSlackMsg(`${batchedUrls.join('\n')}`); batchedUrls = []; } } - }); + ); } catch (error) { console.trace(error) throw(error) } try { - this.fastly.warmCache(st) + this.fastly.warmCache(urlArray) } catch (error) { - + //do something with error } - - return new Promise((resolve) => { - logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); - logger.save( - `${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}` - ); - resolve({ - status: 'success', - stdout: stdoutMod + try{ + + return new Promise((resolve) => { + logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); + logger.save( + `${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}` + ); + resolve({ + status: 'success', + stdout: stdoutMod + }); }); - }); - } catch (errResult) { + } + catch (errResult) { logger.save(`${'(prod)'.padEnd(15)}stdErr: ${errResult.stderr}`); throw errResult; } } } + module.exports = { S3PublishClass }; diff --git a/worker/jobTypes/githubJob.js b/worker/jobTypes/githubJob.js index 20f8920d4..746652152 100644 --- a/worker/jobTypes/githubJob.js +++ b/worker/jobTypes/githubJob.js @@ -130,7 +130,7 @@ class GitHubJobClass { // our maintained directory of makefiles async downloadMakefile() { - const makefileLocation = `https://raw.githubusercontent.com/madelinezec/docs-worker-pool/DOP-2000-Makefiles/makefiles/Makefile.${this.currentJob.payload.repoName}`; + const makefileLocation = `https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/Makefile.${this.currentJob.payload.repoName}`; const returnObject = {}; return new Promise(function(resolve, reject) { request(makefileLocation, function(error, response, body) { diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index ffc042f86..90e13e39d 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -3,6 +3,14 @@ const utils = require('../utils/utils'); const environment = require('../utils/environment').EnvironmentClass; const fastly = require('fastly')(environment.getFastlyToken()); const https = require('https') +const fastly_service_id = environment.getFastlyServiceId(); +const headers = { + 'Fastly-Key': environment.getFastlyToken(), + 'Accept': 'application/json', + 'Content-Type': 'application/json', + 'Fastly-Debug': 1 +}; + class FastlyJobClass { // pass in a job payload to setup class @@ -15,21 +23,23 @@ class FastlyJobClass { // request urls of updated content to "warm" the cache for our customers async warmCache(updatedContentURLS) { - if (!Array.isArray(updatedContentURLS)) { - throw new Error('Parameter `updatedContentURLS` needs to be an array of urls'); - } + console.log("warm cache called") + // if (!Array.isArray(updatedContentURLS)) { + // throw new Error('Parameter `updatedContentURLS` needs to be an array of urls'); + // } - for (let i = 0; i < updatedContentURLS.length; i++) { - request(updatedContentURLS[i], function (error, response, body) { - if (!error && response.statusCode == 200) { - console.log(body); - } - }) - } + // for (let i = 0; i < updatedContentURLS.length; i++) { + // request(updatedContentURLS[i], function (error, response, body) { + // if (!error && response.statusCode == 200) { + // console.log(body); + // } + // }) + // } } // takes in an array of surrogate keys and purges cache for each - async purgeCache(surrogateKeyArray) { + async purgeCache(urlArray) { + if (!Array.isArray(surrogateKeyArray)) { throw new Error('Parameter `surrogateKeyArray` needs to be an array of urls'); @@ -38,46 +48,71 @@ class FastlyJobClass { let that = this; let urlCounter = surrogateKeyArray.length; let purgeMessages = []; - const fastly_service_id = environment.getFastlyServiceId(); - - const headers = { - 'Fastly-Key': environment.getFastlyToken(), - 'Accept': 'application/json', - 'Content-Type': 'application/json' - }; return new Promise((resolve, reject) => { - for (let i = 0; i < surrogateKeyArray.length; i++) { + for (let i = 0; i < urlArray.length; i++) { + //retrieve surrogate key + const surrogateKey = this.retrieveSurrogateKey(urlArray[i]) + // perform request to purge - try { - headers['Surrogate-Key'] = surrogateKeyArray[i] - request({ - method: `POST`, - url: `https://api.fastly.com/service/${fastly_service_id}/purge${surrogateKeyArray[i]}`, - path: `/service/${fastly_service_id}/purge${surrogateKeyArray[i]}`, - headers: headers, - }, function(err, response, body) { - // surrogate key was not valid to purge - if (err){ - console.trace(err) - } - - /* capture the purge request id in case we still see stale content on site, - contact Fastly for further assistance with purge id and resource - see https://docs.fastly.com/en/guides/single-purges */ - console.log(body) - }) - } catch (error) { - console.log(error) - throw error - } + this.requestPurgeOfSurrogateKey(surrogateKey) } }) } +async retrieveSurrogateKey(url) { + try { + request({ + method: `GET`, + url: url, + headers: headers, + }, function(err, response, body) { + // surrogate key was not valid to purge + if (err){ + console.trace(err) + } + console.log(request.headers) + /* capture the purge request id in case we still see stale content on site, + contact Fastly for further assistance with purge id and resource + see https://docs.fastly.com/en/guides/single-purges */ + console.log(body) + + }) + } catch (error) { + + } +} + + async requestPurgeOfSurrogateKey(surrogateKey){ + console.log("this is the surrogate key") + try { + headers['Surrogate-Key'] = surrogateKey + request({ + method: `POST`, + url: `https://api.fastly.com/service/${fastly_service_id}/purge${surrogateKeyArray}`, + path: `/service/${fastly_service_id}/purge${surrogateKeyArray}`, + headers: headers, + }, function(err, response, body) { + // surrogate key was not valid to purge + if (err){ + console.trace(err) + } + + /* capture the purge request id in case we still see stale content on site, + contact Fastly for further assistance with purge id and resource + see https://docs.fastly.com/en/guides/single-purges */ + console.log(body) + } + ) + } catch (error) { + console.log(error) + throw error + } + } + // upserts {source: target} mappings // to the fastly edge dictionary async connectAndUpsert(map) { From a49cd5732a2a207c8446e2d8c0c403f4d774166a Mon Sep 17 00:00:00 2001 From: madelinezec Date: Mon, 3 May 2021 11:56:00 -0400 Subject: [PATCH 06/24] commit before switch branch --- worker/jobTypes/S3Publish.js | 33 ++++++++++++++------------------- worker/utils/fastlyJob.js | 25 ++++++++++++------------- worker/utils/mongo.js | 15 --------------- 3 files changed, 26 insertions(+), 47 deletions(-) diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index ab1694406..aa1871076 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -80,6 +80,7 @@ class S3PublishClass { //as defined in githubJob.js if (manifestPrefix) deployCommands[deployCommands.length - 1] += ` MANIFEST_PREFIX=${manifestPrefix} GLOBAL_SEARCH_FLAG=${this.GitHubJob.currentJob.payload.stableBranch}`; } + // deploy site try { const exec = workerUtils.getExecPromise(); @@ -95,7 +96,7 @@ class S3PublishClass { } // check for json string output from mut // check if json was returned from mut - + try{ const makefileOutput = stdout.replace(/\r/g, "").split(/\n/); // the URLS are always third line returned bc of the makefile target const stdoutJSON = JSON.parse(makefileOutput[2]); @@ -131,24 +132,16 @@ class S3PublishClass { console.trace(error) throw(error) } - - try { - this.fastly.warmCache(urlArray) - } catch (error) { - //do something with error - } - try{ - - return new Promise((resolve) => { - logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); - logger.save( - `${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}` - ); - resolve({ - status: 'success', - stdout: stdoutMod - }); - }); + return new Promise((resolve) => { + logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); + logger.save( + `${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}` + ); + resolve({ + status: 'success', + stdout: stdoutMod + }); + }); } catch (errResult) { logger.save(`${'(prod)'.padEnd(15)}stdErr: ${errResult.stderr}`); @@ -158,6 +151,8 @@ class S3PublishClass { } + + module.exports = { S3PublishClass }; diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index 90e13e39d..a612cdbd9 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -41,12 +41,12 @@ class FastlyJobClass { async purgeCache(urlArray) { - if (!Array.isArray(surrogateKeyArray)) { - throw new Error('Parameter `surrogateKeyArray` needs to be an array of urls'); + if (!Array.isArray(urlArray)) { + throw new Error('Parameter `url array` needs to be an array of urls'); } let that = this; - let urlCounter = surrogateKeyArray.length; + let urlCounter = urlArray.length; let purgeMessages = []; return new Promise((resolve, reject) => { @@ -57,7 +57,7 @@ class FastlyJobClass { const surrogateKey = this.retrieveSurrogateKey(urlArray[i]) // perform request to purge - this.requestPurgeOfSurrogateKey(surrogateKey) + // this.requestPurgeOfSurrogateKey(surrogateKey) } }) @@ -66,7 +66,7 @@ class FastlyJobClass { async retrieveSurrogateKey(url) { try { request({ - method: `GET`, + method: `HEAD`, url: url, headers: headers, }, function(err, response, body) { @@ -74,26 +74,25 @@ async retrieveSurrogateKey(url) { if (err){ console.trace(err) } - console.log(request.headers) + console.log("this is the url: ", url) + console.log("these are the response headers: ", response.headers) + console.log("this is the surrogate key, ", response.headers['surrogate-key']) /* capture the purge request id in case we still see stale content on site, contact Fastly for further assistance with purge id and resource see https://docs.fastly.com/en/guides/single-purges */ - console.log(body) - }) } catch (error) { - + console.log("error in retrieval: ", error) } } async requestPurgeOfSurrogateKey(surrogateKey){ - console.log("this is the surrogate key") try { headers['Surrogate-Key'] = surrogateKey request({ method: `POST`, - url: `https://api.fastly.com/service/${fastly_service_id}/purge${surrogateKeyArray}`, - path: `/service/${fastly_service_id}/purge${surrogateKeyArray}`, + url: `https://api.fastly.com/service/${fastly_service_id}/purge${surrogateKey}`, + path: `/service/${fastly_service_id}/purge${surrogateKey}`, headers: headers, }, function(err, response, body) { // surrogate key was not valid to purge @@ -104,7 +103,7 @@ async retrieveSurrogateKey(url) { /* capture the purge request id in case we still see stale content on site, contact Fastly for further assistance with purge id and resource see https://docs.fastly.com/en/guides/single-purges */ - console.log(body) + // console.log(body) } ) } catch (error) { diff --git a/worker/utils/mongo.js b/worker/utils/mongo.js index 69a208301..c2c4c2d8d 100644 --- a/worker/utils/mongo.js +++ b/worker/utils/mongo.js @@ -9,14 +9,8 @@ const runXlarge = EnvironmentClass.getXlarge(); const url = `mongodb+srv://${username}:${password}@cluster0-ylwlz.mongodb.net/admin?retryWrites=true`; // Collection information -<<<<<<< HEAD const DB_NAME = EnvironmentClass.getDB(); // Database name for autobuilder in MongoDB Atlas const COLL_NAME = EnvironmentClass.getCollection(); // Collection name in MongoDB Atlas -======= -const DB_NAME = EnvironmentClass.getDB(); // Database name of the queue in MongoDB Atlas -const COLL_NAME = EnvironmentClass.getCollection(); // Collection name of the queue in MongoDB Atlas -console.log("collection name inside mongo.js ", COLL_NAME) ->>>>>>> testing const META_NAME = 'meta'; const MONITOR_NAME = 'monitor'; const ENTITLEMENTS_NAME = 'entitlements'; @@ -98,20 +92,11 @@ module.exports = { const update = { $set: { startTime: new Date(), status: 'inProgress' } }; const options = { sort: { priority: -1, createdTime: 1 }, returnNewDocument: true }; -<<<<<<< HEAD try { return queueCollection.findOneAndUpdate(query, update, options); } catch (error) { console.trace(error) -======= - //await queueCollection.findOne({ status: 'inQueue' }); - //console.log(await queueCollection.findOne({ status: 'inQueue' })) - try { - return queueCollection.findOneAndUpdate(query, update, options); - } catch (error) { - console.log(error) ->>>>>>> testing throw error } }, From ca347305c15b96efec5e53564d074cd5e443aea2 Mon Sep 17 00:00:00 2001 From: madelinezec Date: Mon, 3 May 2021 12:09:17 -0400 Subject: [PATCH 07/24] commit before switch --- worker/jobTypes/githubJob.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/jobTypes/githubJob.js b/worker/jobTypes/githubJob.js index 746652152..708d52fe2 100644 --- a/worker/jobTypes/githubJob.js +++ b/worker/jobTypes/githubJob.js @@ -130,7 +130,7 @@ class GitHubJobClass { // our maintained directory of makefiles async downloadMakefile() { - const makefileLocation = `https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/Makefile.${this.currentJob.payload.repoName}`; + const makefileLocation = `https://raw.githubusercontent.com/madelinezec/docs-worker-pool/meta/makefiles/Makefile.${this.currentJob.payload.repoName}`; const returnObject = {}; return new Promise(function(resolve, reject) { request(makefileLocation, function(error, response, body) { From ac3590265cafc45ed97bd4a163f4c19a818c72f1 Mon Sep 17 00:00:00 2001 From: madelinezec Date: Mon, 3 May 2021 14:19:53 -0400 Subject: [PATCH 08/24] temp --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b7b6f11c9..b0efd6b76 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ USER docsworker-xlarge WORKDIR /home/docsworker-xlarge # get shared.mk -RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk +RUN curl https://raw.githubusercontent.com/madelinezec/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk # install snooty parser RUN python3 -m pip uninstall -y snooty From a92ee5f073b2995e5e8cd8a9f14e3dd925945334 Mon Sep 17 00:00:00 2001 From: madelinezec Date: Mon, 3 May 2021 18:10:23 -0400 Subject: [PATCH 09/24] test --- worker/utils/fastlyJob.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index a612cdbd9..1d36c4742 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -67,22 +67,24 @@ async retrieveSurrogateKey(url) { try { request({ method: `HEAD`, - url: url, + url: 'https://' + url, headers: headers, }, function(err, response, body) { // surrogate key was not valid to purge if (err){ console.trace(err) + throw err } console.log("this is the url: ", url) - console.log("these are the response headers: ", response.headers) - console.log("this is the surrogate key, ", response.headers['surrogate-key']) + console.log("these are the response ", response) + // console.log("this is the surrogate key, ", response.headers['surrogate-key']) /* capture the purge request id in case we still see stale content on site, contact Fastly for further assistance with purge id and resource see https://docs.fastly.com/en/guides/single-purges */ }) } catch (error) { - console.log("error in retrieval: ", error) + console.trace("error in retrieval: ", error) + throw error } } From b907bd6489d253161a5d73d56fd2f9f634304e2f Mon Sep 17 00:00:00 2001 From: madelinezec Date: Mon, 10 May 2021 16:04:08 -0400 Subject: [PATCH 10/24] test --- worker/jobTypes/S3Publish.js | 1 + worker/jobTypes/githubJob.js | 4 -- worker/utils/fastlyJob.js | 124 +++++++++++++++++------------------ 3 files changed, 63 insertions(+), 66 deletions(-) diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index aa1871076..ebf8c1eb1 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -98,6 +98,7 @@ class S3PublishClass { // check if json was returned from mut try{ const makefileOutput = stdout.replace(/\r/g, "").split(/\n/); + console.log("this is the makefile output: \n", makefileOutput) // the URLS are always third line returned bc of the makefile target const stdoutJSON = JSON.parse(makefileOutput[2]); const urls = stdoutJSON.urls; diff --git a/worker/jobTypes/githubJob.js b/worker/jobTypes/githubJob.js index 9ffbc96b2..20f8920d4 100644 --- a/worker/jobTypes/githubJob.js +++ b/worker/jobTypes/githubJob.js @@ -130,11 +130,7 @@ class GitHubJobClass { // our maintained directory of makefiles async downloadMakefile() { -<<<<<<< HEAD - const makefileLocation = `https://raw.githubusercontent.com/madelinezec/docs-worker-pool/meta/makefiles/Makefile.${this.currentJob.payload.repoName}`; -======= const makefileLocation = `https://raw.githubusercontent.com/madelinezec/docs-worker-pool/DOP-2000-Makefiles/makefiles/Makefile.${this.currentJob.payload.repoName}`; ->>>>>>> 000bb42a43b40d3f7bd77f2d84073769facab2d7 const returnObject = {}; return new Promise(function(resolve, reject) { request(makefileLocation, function(error, response, body) { diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index 381560fa5..17c177b58 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -22,95 +22,95 @@ class FastlyJobClass { } // request urls of updated content to "warm" the cache for our customers - async warmCache(updatedContentURLS) { +warmCache(updatedUrl) { console.log("warm cache called") - // if (!Array.isArray(updatedContentURLS)) { - // throw new Error('Parameter `updatedContentURLS` needs to be an array of urls'); - // } - - - // for (let i = 0; i < updatedContentURLS.length; i++) { - // request(updatedContentURLS[i], function (error, response, body) { - // if (!error && response.statusCode == 200) { - // console.log(body); - // } - // }) - // } + try { + return new Promise(function (resolve) { + request.get(updatedUrl, function (err, response) { + if (!err && response.statusCode === 200) { + console.log("status code of warming the cache: ", response.statusCode) + resolve(response.statusCode) + } + }) + }) + } catch (error) { + console.trace(error) + throw error + } } // takes in an array of surrogate keys and purges cache for each async purgeCache(urlArray) { - - if (!Array.isArray(urlArray)) { throw new Error('Parameter `url array` needs to be an array of urls'); } let that = this; - let urlCounter = surrogateKeyArray.length; + let urlCounter = urlArray.length; let purgeMessages = []; const fastly_service_id = environment.getFastlyServiceId(); - return new Promise((resolve, reject) => { - - + // return new Promise((resolve, reject) => { + for (let i = 0; i < urlArray.length; i++) { //retrieve surrogate key - const surrogateKey = this.retrieveSurrogateKey(urlArray[i]) - - // perform request to purge - // this.requestPurgeOfSurrogateKey(surrogateKey) + try { + const surrogateKey = await this.retrieveSurrogateKey(urlArray[i]) + // perform request to purge + await this.requestPurgeOfSurrogateKey(surrogateKey) + //warm cache + await this.warmCache(urlArray[i]) + } catch (error) { + console.trace(error) + throw error + } } - }) + // }) } -async retrieveSurrogateKey(url) { + retrieveSurrogateKey(url) { try { - request({ - method: `HEAD`, - url: 'https://' + url, - headers: headers, - }, function(err, response, body) { - // surrogate key was not valid to purge - if (err){ - console.trace(err) - throw err - } - console.log("this is the url: ", url) - console.log("these are the response ", response) - // console.log("this is the surrogate key, ", response.headers['surrogate-key']) - /* capture the purge request id in case we still see stale content on site, - contact Fastly for further assistance with purge id and resource - see https://docs.fastly.com/en/guides/single-purges */ - }) + return new Promise(function (resolve) { + request({ + method: `HEAD`, + url: url, + headers: headers, + }, function(err, response, body) { + if (!error && response.statusCode == 200) { + console.log("these are the surrogate: ", response.headers['surrogate-key']) + resolve(response.headers['surrogate-key']); + } + }) + }) + } catch (error) { - console.trace("error in retrieval: ", error) + console.trace(error) throw error } + } - async requestPurgeOfSurrogateKey(surrogateKey){ + requestPurgeOfSurrogateKey(surrogateKey){ + try { - headers['Surrogate-Key'] = surrogateKey - request({ - method: `POST`, - url: `https://api.fastly.com/service/${fastly_service_id}/purge${surrogateKey}`, - path: `/service/${fastly_service_id}/purge${surrogateKey}`, - headers: headers, - }, function(err, response, body) { - // surrogate key was not valid to purge - if (err){ - console.trace(err) - } - - /* capture the purge request id in case we still see stale content on site, - contact Fastly for further assistance with purge id and resource - see https://docs.fastly.com/en/guides/single-purges */ - // console.log(body) + return new Promise(function (resolve) { + headers['Surrogate-Key'] = surrogateKey + request({ + method: `POST`, + url: `https://api.fastly.com/service/${fastly_service_id}/purge/${surrogateKey}`, + path: `/service/${fastly_service_id}/purge${surrogateKey}`, + headers: headers, + }, function(err, response, body) { + if (!error && response.statusCode == 200){ + console.log("this is the response from purging!!! ", response.statusCode) + resolve(response.statusCode) + } + } - ) + ) + }) } catch (error) { - console.log(error) + console.trace(error) throw error } } From 37204fdaa1c7ddc51abd120fe104537383880e86 Mon Sep 17 00:00:00 2001 From: madelinezec Date: Mon, 10 May 2021 16:58:06 -0400 Subject: [PATCH 11/24] test --- worker/jobTypes/S3Publish.js | 1 - worker/utils/fastlyJob.js | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index ebf8c1eb1..aa1871076 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -98,7 +98,6 @@ class S3PublishClass { // check if json was returned from mut try{ const makefileOutput = stdout.replace(/\r/g, "").split(/\n/); - console.log("this is the makefile output: \n", makefileOutput) // the URLS are always third line returned bc of the makefile target const stdoutJSON = JSON.parse(makefileOutput[2]); const urls = stdoutJSON.urls; diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index 17c177b58..5e051c7d8 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -76,7 +76,8 @@ warmCache(updatedUrl) { url: url, headers: headers, }, function(err, response, body) { - if (!error && response.statusCode == 200) { + if (!err && response.statusCode == 200) { + console.log("this is the url ", url) console.log("these are the surrogate: ", response.headers['surrogate-key']) resolve(response.headers['surrogate-key']); } @@ -101,12 +102,11 @@ warmCache(updatedUrl) { path: `/service/${fastly_service_id}/purge${surrogateKey}`, headers: headers, }, function(err, response, body) { - if (!error && response.statusCode == 200){ + if (!err && response.statusCode == 200){ console.log("this is the response from purging!!! ", response.statusCode) resolve(response.statusCode) } - - } + } ) }) } catch (error) { From bae663b7b87d4e860adfb7f4f9ce08bc5e26b55f Mon Sep 17 00:00:00 2001 From: Cassidy Schaufele Date: Tue, 11 May 2021 09:07:49 -0600 Subject: [PATCH 12/24] 5-11-21 Release version bump (#417) --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ea95f7b82..8de07b542 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,14 +42,14 @@ RUN python3 -m pip install pip==20.2 flit==3.0.0 RUN git clone https://github.com/mongodb/snooty-parser.git && \ cd snooty-parser && \ git fetch --tags && \ - git checkout v0.9.7 && \ + git checkout v0.9.8 && \ FLIT_ROOT_INSTALL=1 python3 -m flit install # install snooty front-end RUN git clone https://github.com/mongodb/snooty.git snooty RUN cd snooty && \ git fetch --all && \ - git checkout v0.9.9 && \ + git checkout v0.9.10 && \ npm install && \ git clone https://github.com/mongodb/docs-tools.git docs-tools && \ mkdir -p ./static/images && \ From 9084172b0e1d32590eecbf7c418e73e3859998b5 Mon Sep 17 00:00:00 2001 From: madelinezec Date: Tue, 11 May 2021 11:54:46 -0400 Subject: [PATCH 13/24] clean up --- worker/jobTypes/S3Publish.js | 49 +++++++++++------------------------- worker/utils/fastlyJob.js | 14 +++++------ worker/utils/mongo.js | 2 ++ worker/worker.js | 2 +- 4 files changed, 24 insertions(+), 43 deletions(-) diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index aa1871076..63744d717 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -102,46 +102,27 @@ class S3PublishClass { const stdoutJSON = JSON.parse(makefileOutput[2]); const urls = stdoutJSON.urls; - this.fastly.purgeCache(urls).then(function (data) { - logger.save(`${'(prod)'.padEnd(15)}Fastly finished purging URL's`); - logger.sendSlackMsg('Fastly Summary: The following pages were purged from cache for your deploy'); - // when finished purging - // batch surrogate keys to send as single slack message - let batchedUrls = []; - for (let i = 0; i < urls.length; i++) { - console.log(urls[i]) - console.log(urls[i].split('.')[0]) - const purgedUrl = urls[i]; - if (purgedUrl && purgedUrl.indexOf('.html') !== -1) { - batchedUrls.push(purgedUrl); - } - // if over certain length, send as a single slack message and reset the array - if (batchedUrls.length > 20 || i >= (urls.length - 1)) { - logger.sendSlackMsg(`${batchedUrls.join('\n')}`); - batchedUrls = []; - } - } - // if over certain length, send as a single slack message and reset the array - if (batchedUrls.length > 20 || i >= (urlArray.length - 1)) { - logger.sendSlackMsg(`${batchedUrls.join('\n')}`); - batchedUrls = []; - } + await this.fastly.purgeCache(urls); + logger.save(`${'(prod)'.padEnd(15)}Fastly finished purging URL's`); + logger.save('Fastly Summary: The following pages were purged from cache for your deploy: \n',urls); + // when finished purging + // batch surrogate keys to send as single slack message + return new Promise((resolve) => { + logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); + logger.save( + `${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}` + ); + resolve({ + status: 'success', + stdout: stdoutMod + }); } ); } catch (error) { console.trace(error) throw(error) } - return new Promise((resolve) => { - logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); - logger.save( - `${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}` - ); - resolve({ - status: 'success', - stdout: stdoutMod - }); - }); + } catch (errResult) { logger.save(`${'(prod)'.padEnd(15)}stdErr: ${errResult.stderr}`); diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index 5e051c7d8..176594360 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -23,12 +23,10 @@ class FastlyJobClass { // request urls of updated content to "warm" the cache for our customers warmCache(updatedUrl) { - console.log("warm cache called") try { return new Promise(function (resolve) { request.get(updatedUrl, function (err, response) { if (!err && response.statusCode === 200) { - console.log("status code of warming the cache: ", response.statusCode) resolve(response.statusCode) } }) @@ -48,8 +46,6 @@ warmCache(updatedUrl) { let urlCounter = urlArray.length; let purgeMessages = []; const fastly_service_id = environment.getFastlyServiceId(); - - // return new Promise((resolve, reject) => { for (let i = 0; i < urlArray.length; i++) { //retrieve surrogate key @@ -65,6 +61,11 @@ warmCache(updatedUrl) { throw error } } + console.trace("we finished purging the array of urls") + return new Promise((resolve) => { + resolve(true); + }); + //what should be condition for rejecting?? // }) } @@ -77,8 +78,6 @@ warmCache(updatedUrl) { headers: headers, }, function(err, response, body) { if (!err && response.statusCode == 200) { - console.log("this is the url ", url) - console.log("these are the surrogate: ", response.headers['surrogate-key']) resolve(response.headers['surrogate-key']); } }) @@ -92,7 +91,6 @@ warmCache(updatedUrl) { } requestPurgeOfSurrogateKey(surrogateKey){ - try { return new Promise(function (resolve) { headers['Surrogate-Key'] = surrogateKey @@ -103,9 +101,9 @@ warmCache(updatedUrl) { headers: headers, }, function(err, response, body) { if (!err && response.statusCode == 200){ - console.log("this is the response from purging!!! ", response.statusCode) resolve(response.statusCode) } + //else what to do if there is err? } ) }) diff --git a/worker/utils/mongo.js b/worker/utils/mongo.js index c2c4c2d8d..537e14051 100644 --- a/worker/utils/mongo.js +++ b/worker/utils/mongo.js @@ -103,6 +103,8 @@ module.exports = { // Sends Job To completed Status and Sets End Time async finishJobWithResult(queueCollection, job, result) { + console.log("finishJobWithResult called!!!") + const query = { _id: job._id }; const update = { $set: { diff --git a/worker/worker.js b/worker/worker.js index 962109780..45dfa5c30 100644 --- a/worker/worker.js +++ b/worker/worker.js @@ -185,7 +185,7 @@ module.exports = { jobTypeToFunc[currentJob.payload.jobType].function(currentJob), `Worker Timeout Error: Timed out performing ${currentJob.payload.jobType} for jobId: ${currentJob._id}` ); - + console.trace('job has finished we are in worker.js') // Update the job to be successful await workerUtils .promiseTimeoutS( From 7fc097b4cef31a2ced443e8e3a209f3a901eefdc Mon Sep 17 00:00:00 2001 From: madelinezec Date: Tue, 11 May 2021 11:55:56 -0400 Subject: [PATCH 14/24] clean --- worker/utils/fastlyJob.js | 3 +-- worker/utils/mongo.js | 2 -- worker/worker.js | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index 176594360..15529f023 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -58,15 +58,14 @@ warmCache(updatedUrl) { await this.warmCache(urlArray[i]) } catch (error) { console.trace(error) + //should return reject here? throw error } } - console.trace("we finished purging the array of urls") return new Promise((resolve) => { resolve(true); }); //what should be condition for rejecting?? - // }) } retrieveSurrogateKey(url) { diff --git a/worker/utils/mongo.js b/worker/utils/mongo.js index 537e14051..c2c4c2d8d 100644 --- a/worker/utils/mongo.js +++ b/worker/utils/mongo.js @@ -103,8 +103,6 @@ module.exports = { // Sends Job To completed Status and Sets End Time async finishJobWithResult(queueCollection, job, result) { - console.log("finishJobWithResult called!!!") - const query = { _id: job._id }; const update = { $set: { diff --git a/worker/worker.js b/worker/worker.js index 45dfa5c30..f749d0d10 100644 --- a/worker/worker.js +++ b/worker/worker.js @@ -185,7 +185,6 @@ module.exports = { jobTypeToFunc[currentJob.payload.jobType].function(currentJob), `Worker Timeout Error: Timed out performing ${currentJob.payload.jobType} for jobId: ${currentJob._id}` ); - console.trace('job has finished we are in worker.js') // Update the job to be successful await workerUtils .promiseTimeoutS( From 2b4fcbaf5021786e0259f3b04d8117bbf7039e1e Mon Sep 17 00:00:00 2001 From: madelinezec Date: Tue, 11 May 2021 12:09:34 -0400 Subject: [PATCH 15/24] clean up --- Dockerfile | 6 +++--- worker/jobTypes/GatsbyAdapter.js | 5 +---- worker/worker.js | 1 + 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index b0efd6b76..ea95f7b82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ USER docsworker-xlarge WORKDIR /home/docsworker-xlarge # get shared.mk -RUN curl https://raw.githubusercontent.com/madelinezec/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk +RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk # install snooty parser RUN python3 -m pip uninstall -y snooty @@ -42,14 +42,14 @@ RUN python3 -m pip install pip==20.2 flit==3.0.0 RUN git clone https://github.com/mongodb/snooty-parser.git && \ cd snooty-parser && \ git fetch --tags && \ - git checkout v0.9.6 && \ + git checkout v0.9.7 && \ FLIT_ROOT_INSTALL=1 python3 -m flit install # install snooty front-end RUN git clone https://github.com/mongodb/snooty.git snooty RUN cd snooty && \ git fetch --all && \ - git checkout v0.9.8 && \ + git checkout v0.9.9 && \ npm install && \ git clone https://github.com/mongodb/docs-tools.git docs-tools && \ mkdir -p ./static/images && \ diff --git a/worker/jobTypes/GatsbyAdapter.js b/worker/jobTypes/GatsbyAdapter.js index effaae370..6c93e8364 100644 --- a/worker/jobTypes/GatsbyAdapter.js +++ b/worker/jobTypes/GatsbyAdapter.js @@ -1,6 +1,5 @@ const workerUtils = require('../utils/utils'); const fs = require('fs-extra'); -const { Logger } = require('mongodb'); class GatsbyAdapterClass { constructor(GitHubJob) { @@ -24,15 +23,13 @@ class GatsbyAdapterClass { const envVars = await this.constructEnvVars(); fs.writeFile(`repos/${this.GitHubJob.currentJob.payload.repoName}/.env.production`, envVars, { encoding: 'utf8', flag: 'w' }, function(err) { - console.log("we are writing the .env.production file!") if(err) { console.log(`error writing .env.production file: ${err.stderr}`); throw err; } }); } catch (error) { - - Logger.save(error) + console.log(error) throw error } } diff --git a/worker/worker.js b/worker/worker.js index f749d0d10..962109780 100644 --- a/worker/worker.js +++ b/worker/worker.js @@ -185,6 +185,7 @@ module.exports = { jobTypeToFunc[currentJob.payload.jobType].function(currentJob), `Worker Timeout Error: Timed out performing ${currentJob.payload.jobType} for jobId: ${currentJob._id}` ); + // Update the job to be successful await workerUtils .promiseTimeoutS( From 6bf9cb7ac67efd1bfd1d34ffad85bd5dc180f09e Mon Sep 17 00:00:00 2001 From: madelinezec Date: Tue, 11 May 2021 12:11:42 -0400 Subject: [PATCH 16/24] clean --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ea95f7b82..8de07b542 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,14 +42,14 @@ RUN python3 -m pip install pip==20.2 flit==3.0.0 RUN git clone https://github.com/mongodb/snooty-parser.git && \ cd snooty-parser && \ git fetch --tags && \ - git checkout v0.9.7 && \ + git checkout v0.9.8 && \ FLIT_ROOT_INSTALL=1 python3 -m flit install # install snooty front-end RUN git clone https://github.com/mongodb/snooty.git snooty RUN cd snooty && \ git fetch --all && \ - git checkout v0.9.9 && \ + git checkout v0.9.10 && \ npm install && \ git clone https://github.com/mongodb/docs-tools.git docs-tools && \ mkdir -p ./static/images && \ From 6b7e6054da5e19fa69fe0f04b5ca9ce415da661c Mon Sep 17 00:00:00 2001 From: madelinezec Date: Tue, 11 May 2021 12:12:30 -0400 Subject: [PATCH 17/24] clean --- worker/jobTypes/S3Publish.js | 3 --- worker/utils/fastlyJob.js | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index 63744d717..ed54474c1 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -131,9 +131,6 @@ class S3PublishClass { } } - - - module.exports = { S3PublishClass }; diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index 15529f023..4b4f754cc 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -39,7 +39,7 @@ warmCache(updatedUrl) { // takes in an array of surrogate keys and purges cache for each async purgeCache(urlArray) { if (!Array.isArray(urlArray)) { - throw new Error('Parameter `url array` needs to be an array of urls'); + throw new Error('Parameter `urlArray` needs to be an array of urls'); } let that = this; From 47d1695ce60d00551ff6924dbf20d8453196ba95 Mon Sep 17 00:00:00 2001 From: madelinezec Date: Tue, 11 May 2021 12:12:30 -0400 Subject: [PATCH 18/24] clean --- worker/jobTypes/S3Publish.js | 9 ++------- worker/utils/fastlyJob.js | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index 63744d717..55838b1be 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -94,8 +94,7 @@ class S3PublishClass { ); throw new Error(`Failed pushing to prod: ${stderr}`) } - // check for json string output from mut - // check if json was returned from mut + try{ const makefileOutput = stdout.replace(/\r/g, "").split(/\n/); // the URLS are always third line returned bc of the makefile target @@ -105,8 +104,7 @@ class S3PublishClass { await this.fastly.purgeCache(urls); logger.save(`${'(prod)'.padEnd(15)}Fastly finished purging URL's`); logger.save('Fastly Summary: The following pages were purged from cache for your deploy: \n',urls); - // when finished purging - // batch surrogate keys to send as single slack message + return new Promise((resolve) => { logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); logger.save( @@ -131,9 +129,6 @@ class S3PublishClass { } } - - - module.exports = { S3PublishClass }; diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index 15529f023..4b4f754cc 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -39,7 +39,7 @@ warmCache(updatedUrl) { // takes in an array of surrogate keys and purges cache for each async purgeCache(urlArray) { if (!Array.isArray(urlArray)) { - throw new Error('Parameter `url array` needs to be an array of urls'); + throw new Error('Parameter `urlArray` needs to be an array of urls'); } let that = this; From 30bd7668b69ba181c7bf4460e2b06ad07a27d19f Mon Sep 17 00:00:00 2001 From: madelinezec Date: Tue, 11 May 2021 13:32:05 -0400 Subject: [PATCH 19/24] lint --- worker/jobTypes/S3Publish.js | 34 +++--- worker/package-lock.json | 59 +++++----- worker/package.json | 5 +- worker/utils/fastlyJob.js | 212 +++++++++++++++++------------------ 4 files changed, 153 insertions(+), 157 deletions(-) diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index 55838b1be..1f23a3f6b 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -95,40 +95,36 @@ class S3PublishClass { throw new Error(`Failed pushing to prod: ${stderr}`) } - try{ - const makefileOutput = stdout.replace(/\r/g, "").split(/\n/); + try { + const makefileOutput = stdout.replace(/\r/g, '').split(/\n/); // the URLS are always third line returned bc of the makefile target const stdoutJSON = JSON.parse(makefileOutput[2]); const urls = stdoutJSON.urls; - + await this.fastly.purgeCache(urls); logger.save(`${'(prod)'.padEnd(15)}Fastly finished purging URL's`); - logger.save('Fastly Summary: The following pages were purged from cache for your deploy: \n',urls); + logger.save('Fastly Summary: The following pages were purged from cache for your deploy: \n', urls); return new Promise((resolve) => { logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); - logger.save( - `${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}` - ); + logger.save(`${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}`); resolve({ status: 'success', - stdout: stdoutMod - }); - } - ); - } catch (error) { + stdout: stdoutMod, + }); + },); + } catch (error) { console.trace(error) throw(error) } - - } - catch (errResult) { - logger.save(`${'(prod)'.padEnd(15)}stdErr: ${errResult.stderr}`); - throw errResult; - } + } + catch (errResult) { + logger.save(`${'(prod)'.padEnd(15)}stdErr: ${errResult.stderr}`); + throw errResult; +} } } module.exports = { - S3PublishClass + S3PublishClass, }; diff --git a/worker/package-lock.json b/worker/package-lock.json index 45153aaa1..4f59cf5e9 100644 --- a/worker/package-lock.json +++ b/worker/package-lock.json @@ -12643,12 +12643,30 @@ "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", - "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", "requires": { - "ajv": "^6.5.5", + "ajv": "^6.12.3", "har-schema": "^2.0.0" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + } } }, "has": { @@ -20254,9 +20272,9 @@ "integrity": "sha1-3mMSg3P8v3w8z6TeWkgMRaZ5WOs=" }, "request": { - "version": "2.88.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", - "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", "requires": { "aws-sign2": "~0.7.0", "aws4": "^1.8.0", @@ -20265,7 +20283,7 @@ "extend": "~3.0.2", "forever-agent": "~0.6.1", "form-data": "~2.3.2", - "har-validator": "~5.1.0", + "har-validator": "~5.1.3", "http-signature": "~1.2.0", "is-typedarray": "~1.0.0", "isstream": "~0.1.2", @@ -20275,7 +20293,7 @@ "performance-now": "^2.1.0", "qs": "~6.5.2", "safe-buffer": "^5.1.2", - "tough-cookie": "~2.4.3", + "tough-cookie": "~2.5.0", "tunnel-agent": "^0.6.0", "uuid": "^3.3.2" }, @@ -20290,24 +20308,10 @@ "mime-types": "^2.1.12" } }, - "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" - }, "qs": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" - }, - "tough-cookie": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", - "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", - "requires": { - "psl": "^1.1.24", - "punycode": "^1.4.1" - } } } }, @@ -22663,6 +22667,11 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" }, + "toml": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", + "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==" + }, "toposort": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/toposort/-/toposort-2.0.2.tgz", @@ -22672,7 +22681,6 @@ "version": "2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", - "dev": true, "requires": { "psl": "^1.1.28", "punycode": "^2.1.1" @@ -22681,8 +22689,7 @@ "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", - "dev": true + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" } } }, diff --git a/worker/package.json b/worker/package.json index 7694f7e5f..bcb0009d7 100644 --- a/worker/package.json +++ b/worker/package.json @@ -32,12 +32,13 @@ "react": "^16.7.0", "react-dom": "^16.7.0", "remote-file-size": "^3.0.5", + "request": "^2.88.2", "simple-git": "^1.107.0", "supertest": "^4.0.2", + "toml": "^3.0.0", "typescript": "^4.0.2", "validator": "^10.11.0", - "xmlhttprequest": "^1.8.0", - "toml": "^3.0.0" + "xmlhttprequest": "^1.8.0" }, "devDependencies": { "aws-sdk-mock": "^4.3.0", diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index 4b4f754cc..d8025879b 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -1,136 +1,128 @@ const request = require('request'); -const utils = require('../utils/utils'); const environment = require('../utils/environment').EnvironmentClass; const fastly = require('fastly')(environment.getFastlyToken()); -const https = require('https') -const fastly_service_id = environment.getFastlyServiceId(); +const utils = require('../utils/utils'); + +const fastlyServiceId = environment.getFastlyServiceId(); const headers = { - 'Fastly-Key': environment.getFastlyToken(), - 'Accept': 'application/json', - 'Content-Type': 'application/json', - 'Fastly-Debug': 1 + 'Fastly-Key': environment.getFastlyToken(), + 'Accept': 'application/json', + 'Content-Type': 'application/json', + 'Fastly-Debug': 1, }; class FastlyJobClass { - // pass in a job payload to setup class - constructor(currentJob) { - this.currentJob = currentJob; - if (fastly === undefined) { - utils.logInMongo(currentJob, 'fastly connectivity not found'); + // pass in a job payload to setup class + constructor(currentJob) { + this.currentJob = currentJob; + if (fastly === undefined) { + utils.logInMongo(currentJob, 'fastly connectivity not found'); + } } - } - // request urls of updated content to "warm" the cache for our customers -warmCache(updatedUrl) { - try { - return new Promise(function (resolve) { - request.get(updatedUrl, function (err, response) { - if (!err && response.statusCode === 200) { - resolve(response.statusCode) + // takes in an array of surrogate keys and purges cache for each + async purgeCache(urlArray) { + if (!Array.isArray(urlArray)) { + throw new Error('Parameter `urlArray` needs to be an array of urls'); + } + + for (let i = 0; i < urlArray.length; i++) { + // retrieve surrogate key + try { + const surrogateKey = await this.retrieveSurrogateKey(urlArray[i]); + // perform request to purge + await this.requestPurgeOfSurrogateKey(surrogateKey); + + // warm cache + await this.warmCache(urlArray[i]); + } catch (error) { + console.trace(error); + // should return reject here? + throw error; + } } - }) - }) - } catch (error) { - console.trace(error) - throw error - } - } - // takes in an array of surrogate keys and purges cache for each - async purgeCache(urlArray) { - if (!Array.isArray(urlArray)) { - throw new Error('Parameter `urlArray` needs to be an array of urls'); + return new Promise((resolve) => { + resolve(true); + }); + // what should be condition for rejecting?? } - let that = this; - let urlCounter = urlArray.length; - let purgeMessages = []; - const fastly_service_id = environment.getFastlyServiceId(); - - for (let i = 0; i < urlArray.length; i++) { - //retrieve surrogate key + retrieveSurrogateKey(url) { try { - const surrogateKey = await this.retrieveSurrogateKey(urlArray[i]) - // perform request to purge - await this.requestPurgeOfSurrogateKey(surrogateKey) - - //warm cache - await this.warmCache(urlArray[i]) + return new Promise((resolve) => { + request({ + method: 'HEAD', + url: url, + headers: headers, + }, (err, response) => { + if (!err && response.statusCode === 200) { + resolve(response.headers['surrogate-key']); + } + }); + }); } catch (error) { - console.trace(error) - //should return reject here? - throw error + console.trace(error); + throw error; } } - return new Promise((resolve) => { - resolve(true); - }); - //what should be condition for rejecting?? -} - - retrieveSurrogateKey(url) { - try { - return new Promise(function (resolve) { - request({ - method: `HEAD`, - url: url, - headers: headers, - }, function(err, response, body) { - if (!err && response.statusCode == 200) { - resolve(response.headers['surrogate-key']); - } - }) - }) - - } catch (error) { - console.trace(error) - throw error - } -} + requestPurgeOfSurrogateKey(surrogateKey){ + try { + return new Promise((resolve) => { + headers['Surrogate-Key'] = surrogateKey + request({ + method: `POST`, + url: `https://api.fastly.com/service/${fastlyServiceId}/purge/${surrogateKey}`, + path: `/service/${fastlyServiceId}/purge${surrogateKey}`, + headers: headers, + }, (err, response) => { + if (!err && response.statusCode === 200){ + resolve(response.statusCode); + } + }); + }); + } catch (error) { + console.trace(error); + throw error; + } + } - requestPurgeOfSurrogateKey(surrogateKey){ - try { - return new Promise(function (resolve) { - headers['Surrogate-Key'] = surrogateKey - request({ - method: `POST`, - url: `https://api.fastly.com/service/${fastly_service_id}/purge/${surrogateKey}`, - path: `/service/${fastly_service_id}/purge${surrogateKey}`, - headers: headers, - }, function(err, response, body) { - if (!err && response.statusCode == 200){ - resolve(response.statusCode) - } - //else what to do if there is err? + // request urls of updated content to "warm" the cache for our customers + warmCache(updatedUrl) { + try { + return new Promise((resolve) => { + request.get(updatedUrl, (err, response) => { + if (!err && response.statusCode === 200) { + resolve(response.statusCode); + } + }); + }); + } catch (error) { + console.trace(error); + throw error; } - ) - }) - } catch (error) { - console.trace(error) - throw error - } - } + } - // upserts {source: target} mappings - // to the fastly edge dictionary - async connectAndUpsert(map) { - const options = { - item_value: map.target - }; - const connectString = `/service/${environment.getFastlyServiceId()}/dictionary/${environment.getDochubMap()}/item/${ - map.source - }`; + // upserts {source: target} mappings + // to the fastly edge dictionary + async connectAndUpsert(map) { + const options = { + item_value: map.target, + }; + const connectString = `/service/${fastlyServiceId}/dictionary/${environment.getDochubMap()}/item/${ + map.source + }`; - return new Promise((resolve, reject) => { - fastly.request('PUT', connectString, options, function(err, obj) { - if (err) reject(err); - resolve(obj); - }); - }) + return new Promise((resolve, reject) => { + fastly.request('PUT', connectString, options, (err, obj) => { + if (err) reject(err); + resolve(obj); + }); + }) + } } -} module.exports = { - FastlyJobClass: FastlyJobClass + FastlyJobClass, }; From 8a52dcdf99d723847434789350dd72f75008c3cb Mon Sep 17 00:00:00 2001 From: madelinezec Date: Tue, 11 May 2021 17:53:59 -0400 Subject: [PATCH 20/24] refactor --- worker/jobTypes/S3Publish.js | 11 +-- worker/utils/fastlyJob.js | 144 ++++++++++++++++++----------------- worker/utils/mongo.js | 19 +++++ worker/utils/utils.js | 5 ++ 4 files changed, 104 insertions(+), 75 deletions(-) diff --git a/worker/jobTypes/S3Publish.js b/worker/jobTypes/S3Publish.js index 1f23a3f6b..4e44b6526 100644 --- a/worker/jobTypes/S3Publish.js +++ b/worker/jobTypes/S3Publish.js @@ -99,11 +99,12 @@ class S3PublishClass { const makefileOutput = stdout.replace(/\r/g, '').split(/\n/); // the URLS are always third line returned bc of the makefile target const stdoutJSON = JSON.parse(makefileOutput[2]); - const urls = stdoutJSON.urls; - - await this.fastly.purgeCache(urls); - logger.save(`${'(prod)'.padEnd(15)}Fastly finished purging URL's`); - logger.save('Fastly Summary: The following pages were purged from cache for your deploy: \n', urls); + //contains URLs corresponding to files updated via our push to S3 + const updatedURLsArray = stdoutJSON.urls; + // purgeCache purges the now stale content and requests the URLs to warm the cache for our users + await this.fastly.purgeCache(updatedURLsArray); + //save purged URLs to job object + workerUtils.updateJobWithPurgedURLs(this.GitHubJob.currentJob, updatedURLsArray); return new Promise((resolve) => { logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`); diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index d8025879b..e16a74407 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -1,7 +1,8 @@ -const request = require('request'); +const axios = require('axios').default; const environment = require('../utils/environment').EnvironmentClass; const fastly = require('fastly')(environment.getFastlyToken()); const utils = require('../utils/utils'); +const Logger = require('../utils/logger').LoggerClass; const fastlyServiceId = environment.getFastlyServiceId(); const headers = { @@ -16,8 +17,9 @@ class FastlyJobClass { // pass in a job payload to setup class constructor(currentJob) { this.currentJob = currentJob; + this.logger = new Logger(currentJob); if (fastly === undefined) { - utils.logInMongo(currentJob, 'fastly connectivity not found'); + utils.logInMongo(currentJob, 'fastly connectivity not found'); } } @@ -27,102 +29,104 @@ class FastlyJobClass { throw new Error('Parameter `urlArray` needs to be an array of urls'); } - for (let i = 0; i < urlArray.length; i++) { - // retrieve surrogate key - try { - const surrogateKey = await this.retrieveSurrogateKey(urlArray[i]); - // perform request to purge - await this.requestPurgeOfSurrogateKey(surrogateKey); - - // warm cache - await this.warmCache(urlArray[i]); - } catch (error) { - console.trace(error); - // should return reject here? - throw error; - } - } - return new Promise((resolve) => { - resolve(true); - }); - // what should be condition for rejecting?? + try { + //retrieve surrogate key associated with each URL/file updated in push to S3 + const surrogateKeyPromises = urlArray.map(url => this.retrieveSurrogateKey(url)); + const surrogateKeyArray = await Promise.all(surrogateKeyPromises).catch(console.log); ; + console.log(surrogateKeyArray) + //purge each surrogate key + // const purgeRequestPromises = surrogateKeyArray.map(surrogateKey => this.requestPurgeOfSurrogateKey(surrogateKey)); + // await Promise.all(purgeRequestPromises); + + // // GET request the URLs to warm cache for our users + // const warmCachePromises = urlArray.map(url => this.warmCache(url)); + // await Promise.all(warmCachePromises) + } catch (error) { + logger.save(`${'(prod)'.padEnd(15)}error in purge cache: ${error}`); + throw error + } + } - retrieveSurrogateKey(url) { + async retrieveSurrogateKey(url) { + try { - return new Promise((resolve) => { - request({ + axios({ method: 'HEAD', url: url, headers: headers, - }, (err, response) => { - if (!err && response.statusCode === 200) { - resolve(response.headers['surrogate-key']); - } - }); + }).then(response => { + console.log("this is the response status: ", response.status) + if (response.status === 200) { + console.log("this is the surrogate key!! ", response.headers['surrogate-key']) + return response.headers['surrogate-key']; + } + // console.log("this is the response inside retrieveSurrogateKey: ", response) }); } catch (error) { - console.trace(error); - throw error; + logger.save(`${'(prod)'.padEnd(15)}error in retrieveSurrogateKey: ${error}`); + throw error } + } - requestPurgeOfSurrogateKey(surrogateKey){ + async requestPurgeOfSurrogateKey(surrogateKey) { + headers['Surrogate-Key'] = surrogateKey + try { - return new Promise((resolve) => { - headers['Surrogate-Key'] = surrogateKey - request({ - method: `POST`, - url: `https://api.fastly.com/service/${fastlyServiceId}/purge/${surrogateKey}`, - path: `/service/${fastlyServiceId}/purge${surrogateKey}`, - headers: headers, - }, (err, response) => { - if (!err && response.statusCode === 200){ - resolve(response.statusCode); - } - }); - }); + axios({ + method: `POST`, + url: `https://api.fastly.com/service/${fastlyServiceId}/purge/${surrogateKey}`, + path: `/service/${fastlyServiceId}/purge${surrogateKey}`, + headers: headers, + }) + .then(response => { + console.log("the status code for purging!! ", response.status) + if (response.status === 200) { + return true + } + }); } catch (error) { - console.trace(error); - throw error; + logger.save(`${'(prod)'.padEnd(15)}error in requestPurgeOfSurrogateKey: ${error}`); + throw error; } - } + } // request urls of updated content to "warm" the cache for our customers - warmCache(updatedUrl) { + async warmCache(updatedUrl) { + try { - return new Promise((resolve) => { - request.get(updatedUrl, (err, response) => { - if (!err && response.statusCode === 200) { - resolve(response.statusCode); - } - }); - }); + axios.get(updatedUrl) + .then(response => { + if (response.status === 200) { + return true; + } + }) } catch (error) { - console.trace(error); - throw error; + logger.save(`${'(prod)'.padEnd(15)}stdErr: ${error}`); + throw error; } - } + } - // upserts {source: target} mappings - // to the fastly edge dictionary - async connectAndUpsert(map) { + // upserts {source: target} mappings + // to the fastly edge dictionary + async connectAndUpsert(map) { const options = { - item_value: map.target, + item_value: map.target, }; const connectString = `/service/${fastlyServiceId}/dictionary/${environment.getDochubMap()}/item/${ map.source }`; return new Promise((resolve, reject) => { - fastly.request('PUT', connectString, options, (err, obj) => { - if (err) reject(err); - resolve(obj); - }); + fastly.request('PUT', connectString, options, (err, obj) => { + if (err) reject(err); + resolve(obj); + }); }) - } - } + } +} module.exports = { FastlyJobClass, -}; +}; \ No newline at end of file diff --git a/worker/utils/mongo.js b/worker/utils/mongo.js index c2c4c2d8d..0ba118e6e 100644 --- a/worker/utils/mongo.js +++ b/worker/utils/mongo.js @@ -130,6 +130,25 @@ module.exports = { } }, + async updateJobWithPurgedURLs(currentJob, urlArray) { + const queueCollection = module.exports.getCollection(); + if (queueCollection) { + const query = { _id: currentJob._id }; + const update = { + $push: { ['purgedURLs']: urlArray }, + }; + + try { + await queueCollection.updateOne(query, update); + } catch (err) { + console.log(`Error in updateJobWithPurgedURLs(): ${err}`); + throw err + } + } else { + console.log('Error in logInMongo(): queueCollection does not exist'); + } + }, + // Adds Log Message To Job In The Queue async logMessageInMongo(currentJob, message) { const queueCollection = module.exports.getCollection(); diff --git a/worker/utils/utils.js b/worker/utils/utils.js index a9da33ef0..9baa8d669 100644 --- a/worker/utils/utils.js +++ b/worker/utils/utils.js @@ -134,6 +134,11 @@ module.exports = { getExecPromise() { return exec; }, + + // save array of purged URLs to job object + async updateJobWithPurgedURLs(currentJob, urlArray) { + await mongo.updateJobWithPurgedURLs(currentJob, urlArray); + }, // Adds log message (message) to current job in queue at spot (currentJob.numFailures) async logInMongo(currentJob, message) { From 9ef822aee85422f7672523e4bd3d9b3020b6eb05 Mon Sep 17 00:00:00 2001 From: madelinezec Date: Tue, 11 May 2021 19:18:01 -0400 Subject: [PATCH 21/24] refactor --- worker/utils/fastlyJob.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index e16a74407..55bae3b04 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -32,15 +32,15 @@ class FastlyJobClass { try { //retrieve surrogate key associated with each URL/file updated in push to S3 const surrogateKeyPromises = urlArray.map(url => this.retrieveSurrogateKey(url)); - const surrogateKeyArray = await Promise.all(surrogateKeyPromises).catch(console.log); ; - console.log(surrogateKeyArray) + const surrogateKeyArray = await Promise.all(surrogateKeyPromises).catch(console.log); + //purge each surrogate key - // const purgeRequestPromises = surrogateKeyArray.map(surrogateKey => this.requestPurgeOfSurrogateKey(surrogateKey)); - // await Promise.all(purgeRequestPromises); + const purgeRequestPromises = surrogateKeyArray.map(surrogateKey => this.requestPurgeOfSurrogateKey(surrogateKey)); + await Promise.all(purgeRequestPromises); - // // GET request the URLs to warm cache for our users - // const warmCachePromises = urlArray.map(url => this.warmCache(url)); - // await Promise.all(warmCachePromises) + // GET request the URLs to warm cache for our users + const warmCachePromises = urlArray.map(url => this.warmCache(url)); + await Promise.all(warmCachePromises) } catch (error) { logger.save(`${'(prod)'.padEnd(15)}error in purge cache: ${error}`); throw error @@ -51,7 +51,7 @@ class FastlyJobClass { async retrieveSurrogateKey(url) { try { - axios({ + return axios({ method: 'HEAD', url: url, headers: headers, @@ -74,7 +74,7 @@ class FastlyJobClass { headers['Surrogate-Key'] = surrogateKey try { - axios({ + return axios({ method: `POST`, url: `https://api.fastly.com/service/${fastlyServiceId}/purge/${surrogateKey}`, path: `/service/${fastlyServiceId}/purge${surrogateKey}`, @@ -96,7 +96,7 @@ class FastlyJobClass { async warmCache(updatedUrl) { try { - axios.get(updatedUrl) + return axios.get(updatedUrl) .then(response => { if (response.status === 200) { return true; From 4dcf5db120f53cd931a905543f480288fee967e1 Mon Sep 17 00:00:00 2001 From: madelinezec Date: Wed, 12 May 2021 13:17:10 -0400 Subject: [PATCH 22/24] refactor --- worker/utils/fastlyJob.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index 55bae3b04..6871caaef 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -42,7 +42,7 @@ class FastlyJobClass { const warmCachePromises = urlArray.map(url => this.warmCache(url)); await Promise.all(warmCachePromises) } catch (error) { - logger.save(`${'(prod)'.padEnd(15)}error in purge cache: ${error}`); + this.logger.save(`${'(prod)'.padEnd(15)}error in purge cache: ${error}`); throw error } From 248a4ef2f804351cde1a9caf1d89b009aba4a292 Mon Sep 17 00:00:00 2001 From: madelinezec Date: Wed, 12 May 2021 13:25:34 -0400 Subject: [PATCH 23/24] clean --- worker/utils/fastlyJob.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index 6871caaef..66ee64c28 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -32,7 +32,7 @@ class FastlyJobClass { try { //retrieve surrogate key associated with each URL/file updated in push to S3 const surrogateKeyPromises = urlArray.map(url => this.retrieveSurrogateKey(url)); - const surrogateKeyArray = await Promise.all(surrogateKeyPromises).catch(console.log); + const surrogateKeyArray = await Promise.all(surrogateKeyPromises) //purge each surrogate key const purgeRequestPromises = surrogateKeyArray.map(surrogateKey => this.requestPurgeOfSurrogateKey(surrogateKey)); @@ -56,12 +56,9 @@ class FastlyJobClass { url: url, headers: headers, }).then(response => { - console.log("this is the response status: ", response.status) if (response.status === 200) { - console.log("this is the surrogate key!! ", response.headers['surrogate-key']) return response.headers['surrogate-key']; } - // console.log("this is the response inside retrieveSurrogateKey: ", response) }); } catch (error) { logger.save(`${'(prod)'.padEnd(15)}error in retrieveSurrogateKey: ${error}`); @@ -81,7 +78,6 @@ class FastlyJobClass { headers: headers, }) .then(response => { - console.log("the status code for purging!! ", response.status) if (response.status === 200) { return true } From 4d8ba5018b782276da4134f03e233c46822f8721 Mon Sep 17 00:00:00 2001 From: madelinezec Date: Wed, 12 May 2021 13:34:01 -0400 Subject: [PATCH 24/24] add axios --- Dockerfile | 2 +- worker/jobTypes/githubJob.js | 1 + worker/package-lock.json | 85 ++++++++++++++++++++++-------------- worker/package.json | 2 +- worker/utils/fastlyJob.js | 6 +-- 5 files changed, 59 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8de07b542..d72185714 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ USER docsworker-xlarge WORKDIR /home/docsworker-xlarge # get shared.mk -RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk +RUN curl https://raw.githubusercontent.com/madelinezec/docs-worker-pool/DOP-2000-Makefiles/makefiles/shared.mk -o shared.mk # install snooty parser RUN python3 -m pip uninstall -y snooty diff --git a/worker/jobTypes/githubJob.js b/worker/jobTypes/githubJob.js index 9aafd04b2..86af370fb 100644 --- a/worker/jobTypes/githubJob.js +++ b/worker/jobTypes/githubJob.js @@ -130,6 +130,7 @@ class GitHubJobClass { // our maintained directory of makefiles async downloadMakefile() { + // TODO: remove call to my fork and branch const makefileLocation = `https://raw.githubusercontent.com/madelinezec/docs-worker-pool/DOP-2000-Makefiles/makefiles/Makefile.${this.currentJob.payload.repoName}`; const returnObject = {}; return new Promise(function(resolve, reject) { diff --git a/worker/package-lock.json b/worker/package-lock.json index 4f59cf5e9..52f5a9417 100644 --- a/worker/package-lock.json +++ b/worker/package-lock.json @@ -4171,11 +4171,18 @@ "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" }, "axios": { - "version": "0.19.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz", - "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==", + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", + "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", "requires": { - "follow-redirects": "1.5.10" + "follow-redirects": "^1.10.0" + }, + "dependencies": { + "follow-redirects": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz", + "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==" + } } }, "axobject-query": { @@ -6365,6 +6372,14 @@ "type-fest": "0.15.1" }, "dependencies": { + "axios": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz", + "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==", + "requires": { + "follow-redirects": "1.5.10" + } + }, "type-fest": { "version": "0.15.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.15.1.tgz", @@ -10507,6 +10522,14 @@ } } }, + "axios": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz", + "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==", + "requires": { + "follow-redirects": "1.5.10" + } + }, "binary-extensions": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz", @@ -12643,30 +12666,12 @@ "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { - "version": "5.1.5", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", - "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", "requires": { - "ajv": "^6.12.3", + "ajv": "^6.5.5", "har-schema": "^2.0.0" - }, - "dependencies": { - "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "requires": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" - } } }, "has": { @@ -20272,9 +20277,9 @@ "integrity": "sha1-3mMSg3P8v3w8z6TeWkgMRaZ5WOs=" }, "request": { - "version": "2.88.2", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", - "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", "requires": { "aws-sign2": "~0.7.0", "aws4": "^1.8.0", @@ -20283,7 +20288,7 @@ "extend": "~3.0.2", "forever-agent": "~0.6.1", "form-data": "~2.3.2", - "har-validator": "~5.1.3", + "har-validator": "~5.1.0", "http-signature": "~1.2.0", "is-typedarray": "~1.0.0", "isstream": "~0.1.2", @@ -20293,7 +20298,7 @@ "performance-now": "^2.1.0", "qs": "~6.5.2", "safe-buffer": "^5.1.2", - "tough-cookie": "~2.5.0", + "tough-cookie": "~2.4.3", "tunnel-agent": "^0.6.0", "uuid": "^3.3.2" }, @@ -20308,10 +20313,24 @@ "mime-types": "^2.1.12" } }, + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + }, "qs": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + }, + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + } } } }, @@ -22681,6 +22700,7 @@ "version": "2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, "requires": { "psl": "^1.1.28", "punycode": "^2.1.1" @@ -22689,7 +22709,8 @@ "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "dev": true } } }, diff --git a/worker/package.json b/worker/package.json index bcb0009d7..aefdd7e54 100644 --- a/worker/package.json +++ b/worker/package.json @@ -19,6 +19,7 @@ "@babel/polyfill": "^7.4.4", "async-retry": "^1.2.3", "aws-sdk": "^2.383.0", + "axios": "^0.21.1", "core-js": "^3.1.4", "express": "^4.16.4", "fastly": "^2.2.1", @@ -32,7 +33,6 @@ "react": "^16.7.0", "react-dom": "^16.7.0", "remote-file-size": "^3.0.5", - "request": "^2.88.2", "simple-git": "^1.107.0", "supertest": "^4.0.2", "toml": "^3.0.0", diff --git a/worker/utils/fastlyJob.js b/worker/utils/fastlyJob.js index 66ee64c28..21ebd295b 100644 --- a/worker/utils/fastlyJob.js +++ b/worker/utils/fastlyJob.js @@ -61,7 +61,7 @@ class FastlyJobClass { } }); } catch (error) { - logger.save(`${'(prod)'.padEnd(15)}error in retrieveSurrogateKey: ${error}`); + this.logger.save(`${'(prod)'.padEnd(15)}error in retrieveSurrogateKey: ${error}`); throw error } @@ -83,7 +83,7 @@ class FastlyJobClass { } }); } catch (error) { - logger.save(`${'(prod)'.padEnd(15)}error in requestPurgeOfSurrogateKey: ${error}`); + this.logger.save(`${'(prod)'.padEnd(15)}error in requestPurgeOfSurrogateKey: ${error}`); throw error; } } @@ -99,7 +99,7 @@ class FastlyJobClass { } }) } catch (error) { - logger.save(`${'(prod)'.padEnd(15)}stdErr: ${error}`); + this.logger.save(`${'(prod)'.padEnd(15)}stdErr: ${error}`); throw error; } }