Cache purge changes

worker/jobTypes/S3Publish.js

@@ -80,6 +80,7 @@ class S3PublishClass {
       //as defined in githubJob.js
       if (manifestPrefix) deployCommands[deployCommands.length - 1] +=  ` MANIFEST_PREFIX=${manifestPrefix} GLOBAL_SEARCH_FLAG=${this.GitHubJob.currentJob.payload.stableBranch}`;
     }
+
     // deploy site
     try {
       const exec = workerUtils.getExecPromise();
@@ -93,57 +94,38 @@ class S3PublishClass {
         );
         throw new Error(`Failed pushing to prod: ${stderr}`)
       }
-      // check for json string output from mut
-      const validateJsonOutput = stdout ? stdout.substr(0, stdout.lastIndexOf(']}') + 2) : '';
 
-      // check if json was returned from mut
       try {
-        const stdoutJSON = JSON.parse(validateJsonOutput);
-        const urls = stdoutJSON.urls;
-        // pass in urls to fastly function to purge cache
-        this.fastly.purgeCache(urls).then(function (data) {
-          logger.save(`${'(prod)'.padEnd(15)}Fastly finished purging URL's`);
-          logger.sendSlackMsg('Fastly Summary: The following pages were purged from cache for your deploy');
-          // when finished purging
-          // batch urls to send as single slack message
-          let batchedUrls = [];
-          for (let i = 0; i < urls.length; i++) {
-            const purgedUrl = urls[i];
-            if (purgedUrl && purgedUrl.indexOf('.html') !== -1) {
-              batchedUrls.push(purgedUrl);
-            }
-            // if over certain length, send as a single slack message and reset the array
-            if (batchedUrls.length > 20 || i >= (urls.length - 1)) {
-              logger.sendSlackMsg(`${batchedUrls.join('\n')}`);
-              batchedUrls = [];
-            }
-          }
-        });
-      } catch (e) {
-        // if not JSON, then it's a normal string output from mut
-        // get only last part of message which includes # of files changes + s3 link
-        if (stdout.indexOf('Summary') !== -1) {
-          stdoutMod = stdout.substr(stdout.indexOf('Summary'));
-        }
-      }
+        const makefileOutput = stdout.replace(/\r/g, '').split(/\n/);
+        // the URLS are always third line returned bc of the makefile target
+        const stdoutJSON = JSON.parse(makefileOutput[2]);
+        //contains URLs corresponding to files updated via our push to S3
+        const updatedURLsArray = stdoutJSON.urls;
+        // purgeCache purges the now stale content and requests the URLs to warm the cache for our users
+        await this.fastly.purgeCache(updatedURLsArray);
+        //save purged URLs to job object
+        workerUtils.updateJobWithPurgedURLs(this.GitHubJob.currentJob, updatedURLsArray);
 
-      return new Promise((resolve) => {
-        logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`);
-        logger.save(
-          `${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}`
-        );
-        resolve({
-          status: 'success',
-          stdout: stdoutMod
-        });
-      });
-    } catch (errResult) {
-      logger.save(`${'(prod)'.padEnd(15)}stdErr: ${errResult.stderr}`);
-      throw errResult;
+        return new Promise((resolve) => {
+          logger.save(`${'(prod)'.padEnd(15)}Finished pushing to production`);
+          logger.save(`${'(prod)'.padEnd(15)}Deploy details:\n\n${stdoutMod}`);
+          resolve({
+            status: 'success',
+            stdout: stdoutMod,
+            });
+          },);
+      } catch (error) {
+      console.trace(error)
+      throw(error)
     }
+  }     
+  catch (errResult) {
+  logger.save(`${'(prod)'.padEnd(15)}stdErr: ${errResult.stderr}`);
+  throw errResult;
+}
   }
 }
 
 module.exports = {
-  S3PublishClass
+  S3PublishClass,
 };

worker/package.json

@@ -19,6 +19,7 @@
     "@babel/polyfill": "^7.4.4",
     "async-retry": "^1.2.3",
     "aws-sdk": "^2.383.0",
+    "axios": "^0.21.1",
     "core-js": "^3.1.4",
     "express": "^4.16.4",
     "fastly": "^2.2.1",
@@ -34,10 +35,10 @@
     "remote-file-size": "^3.0.5",
     "simple-git": "^1.107.0",
     "supertest": "^4.0.2",
+    "toml": "^3.0.0",
     "typescript": "^4.0.2",
     "validator": "^10.11.0",
-    "xmlhttprequest": "^1.8.0", 
-    "toml": "^3.0.0"
+    "xmlhttprequest": "^1.8.0"
   },
   "devDependencies": {
     "aws-sdk-mock": "^4.3.0",

worker/utils/fastlyJob.js

@@ -1,92 +1,128 @@
-const request = require('request');
-const utils = require('../utils/utils');
+const axios = require('axios').default;
 const environment = require('../utils/environment').EnvironmentClass;
 const fastly = require('fastly')(environment.getFastlyToken());
+const utils = require('../utils/utils');
+const Logger = require('../utils/logger').LoggerClass;
+
+const fastlyServiceId = environment.getFastlyServiceId();
+const headers = {
+    'Fastly-Key': environment.getFastlyToken(),
+    'Accept': 'application/json',
+    'Content-Type': 'application/json',
+    'Fastly-Debug': 1,
+};
+
 
 class FastlyJobClass {
-  // pass in a job payload to setup class
-  constructor(currentJob) {
-    this.currentJob = currentJob;
-    if (fastly === undefined) {
-      utils.logInMongo(currentJob, 'fastly connectivity not found');
+    // pass in a job payload to setup class
+    constructor(currentJob) {
+        this.currentJob = currentJob;
+        this.logger = new Logger(currentJob);
+        if (fastly === undefined) {
+            utils.logInMongo(currentJob, 'fastly connectivity not found');
+        }
     }
-  }
 
-  // takes in an array of urls and purges cache for each
-  async purgeCache(urlArray) {
-    if (!Array.isArray(urlArray)) {
-      throw new Error('Parameter `urlArray` needs to be an array of urls');
+    // takes in an array of surrogate keys and purges cache for each
+    async purgeCache(urlArray) {
+        if (!Array.isArray(urlArray)) {
+            throw new Error('Parameter `urlArray` needs to be an array of urls');
+        }
+
+        try {
+            //retrieve surrogate key associated with each URL/file updated in push to S3
+            const surrogateKeyPromises = urlArray.map(url => this.retrieveSurrogateKey(url));
+            const surrogateKeyArray = await Promise.all(surrogateKeyPromises)
+
+            //purge each surrogate key
+            const purgeRequestPromises = surrogateKeyArray.map(surrogateKey => this.requestPurgeOfSurrogateKey(surrogateKey));
+            await Promise.all(purgeRequestPromises);
+
+            // GET request the URLs to warm cache for our users
+            const warmCachePromises = urlArray.map(url => this.warmCache(url));
+            await Promise.all(warmCachePromises)
+        } catch (error) {
+            this.logger.save(`${'(prod)'.padEnd(15)}error in purge cache: ${error}`);
+            throw error
+        }
+
     }
 
-    let that = this;
-    let urlCounter = urlArray.length;
-    let purgeMessages = [];
-
-    // the 1 is just "some" value needed for this header: https://docs.fastly.com/en/guides/soft-purges
-    const headers = {
-      'fastly-key': environment.getFastlyToken(),
-      'accept': 'application/json',
-      'Fastly-Soft-Purge': '1',
-    };
-
-    return new Promise((resolve, reject) => {
-      for (let i = 0; i < urlArray.length; i++) {
-        // perform request to purge
-        request({
-          method: 'PURGE',
-          url: urlArray[i],
-          headers: headers,
-        }, function(err, response, body) {
-          // url was not valid to purge
-          if (!response) {
-            utils.logInMongo(that.currentJob, `Error: service for this url does not exist in fastly for purging ${urlArray[i]}`);
-            purgeMessages.push({
-              'status': 'failure',
-              'message': `service with url ${urlArray[i]} does not exist in fastly`
+    async retrieveSurrogateKey(url) {
+
+        try {
+            return axios({
+                method: 'HEAD',
+                url: url,
+                headers: headers,
+            }).then(response => {
+                if (response.status === 200) {
+                    return response.headers['surrogate-key'];
+                }
             });
-          } else if (response.headers['content-type'].indexOf('application/json') === 0) {
-            try {
-              body = JSON.parse(body);
-              purgeMessages.push(body);
-            } catch(er) {
-              utils.logInMongo(that.currentJob, `Error: failed parsing output from fastly for url ${urlArray[i]}`);
-              console.log(`Error: failed parsing output from fastly for url ${urlArray[i]}`);
-            }
-          }
-          // when we are done purging all urls
-          // this is outside of the conditional above because if some url fails to purge
-          // we do not want to actually have this entire build fail, just show warning
-          urlCounter--;
-          if (urlCounter <= 0) {
-            resolve({
-              'status': 'success',
-              'fastlyMessages': purgeMessages,
+        } catch (error) {
+            this.logger.save(`${'(prod)'.padEnd(15)}error in retrieveSurrogateKey: ${error}`);
+            throw error
+        }
+
+    }
+
+    async requestPurgeOfSurrogateKey(surrogateKey) {
+        headers['Surrogate-Key'] = surrogateKey
+
+        try {
+            return axios({
+                    method: `POST`,
+                    url: `https://api.fastly.com/service/${fastlyServiceId}/purge/${surrogateKey}`,
+                    path: `/service/${fastlyServiceId}/purge${surrogateKey}`,
+                    headers: headers,
+                })
+                .then(response => {
+                    if (response.status === 200) {
+                        return true
+                    }
+                });
+        } catch (error) {
+            this.logger.save(`${'(prod)'.padEnd(15)}error in requestPurgeOfSurrogateKey: ${error}`);
+            throw error;
+        }
+    }
+
+    // request urls of updated content to "warm" the cache for our customers
+    async warmCache(updatedUrl) {
+
+        try {
+            return axios.get(updatedUrl)
+                .then(response => {
+                    if (response.status === 200) {
+                        return true;
+                    }
+                })
+        } catch (error) {
+            this.logger.save(`${'(prod)'.padEnd(15)}stdErr: ${error}`);
+            throw error;
+        }
+    }
+
+    // upserts {source: target} mappings
+    // to the fastly edge dictionary
+    async connectAndUpsert(map) {
+        const options = {
+            item_value: map.target,
+        };
+        const connectString = `/service/${fastlyServiceId}/dictionary/${environment.getDochubMap()}/item/${
+          map.source
+          }`;
+
+        return new Promise((resolve, reject) => {
+            fastly.request('PUT', connectString, options, (err, obj) => {
+                if (err) reject(err);
+                resolve(obj);
             });
-          }
-        });
-      }
-    })
-  }
-
-  // upserts {source: target} mappings
-  // to the fastly edge dictionary
-  async connectAndUpsert(map) {
-    const options = {
-      item_value: map.target
-    };
-    const connectString = `/service/${environment.getFastlyServiceId()}/dictionary/${environment.getDochubMap()}/item/${
-      map.source
-      }`;
-
-    return new Promise((resolve, reject) => {
-      fastly.request('PUT', connectString, options, function(err, obj) {
-        if (err) reject(err);
-        resolve(obj);
-      });
-    })
-  }
+        })
+    }
 }
 
 module.exports = {
-  FastlyJobClass: FastlyJobClass
-};
+    FastlyJobClass,
+};