Initial proof-of-concept to have a static cache

Author: AndrewFarley

lib/pip.js

@@ -6,17 +6,19 @@ const {spawnSync} = require('child_process');
 const {quote} = require('shell-quote');
 const values = require('lodash.values');
 const {buildImage, getBindPath} = require('./docker');
+const md5File = require('md5-file')
+
 
 /**
- * Install requirements described in requirementsPath to targetPath
+ * Just generate the requirements file in the .serverless folder
  * @param {string} requirementsPath
  * @param {string} targetFolder
  * @param {Object} serverless
  * @param {string} servicePath
  * @param {Object} options
  * @return {undefined}
  */
-function installRequirements(requirementsPath, targetFolder, serverless, servicePath, options) {
+function installRequirementsFile(requirementsPath, targetFolder, serverless, servicePath, options) {
   // Create target folder if it does not exist
   const targetRequirementsFolder = path.join(targetFolder, 'requirements');
   fse.ensureDirSync(targetRequirementsFolder);
@@ -27,8 +29,24 @@ function installRequirements(requirementsPath, targetFolder, serverless, service
   } else {
     generateRequirementsFile(requirementsPath, dotSlsReqs, options);
   }
+  
+  serverless.cli.log(`Generated requirements of ${requirementsPath} in ${targetFolder}...`);
+}
 
-  serverless.cli.log(`Installing requirements of ${requirementsPath} in ${targetFolder}...`);
+
+/**
+ * Install requirements described in requirementsPath to targetPath
+ * @param {string} targetFolder
+ * @param {Object} serverless
+ * @param {string} servicePath
+ * @param {Object} options
+ * @return {undefined}
+ */
+function installRequirements(targetFolder, serverless, servicePath, options) {
+  const targetRequirementsFolder = path.join(targetFolder, 'requirements');
+  const dotSlsReqs = path.join(targetFolder, 'requirements.txt');
+  
+  serverless.cli.log(`Installing requirements from ${dotSlsReqs} ...`);
 
   let cmd;
   let cmdOptions;
@@ -149,7 +167,13 @@ function generateRequirementsFile(source, target, options) {
  * @return {undefined}
  */
 function installAllRequirements() {
+  // Added a static cache folder and try to read the cached sum
+  const requirementsStaticCacheFolder = path.join(this.servicePath, '.serverless-python-requirements-static-cache');
+  fse.ensureDirSync(requirementsStaticCacheFolder);
+  try { cached_sum = fse.readFileSync(path.join(requirementsStaticCacheFolder, '.requirements.txt.md5')); } catch (exception) { cached_sum = false; }
+
   fse.ensureDirSync(path.join(this.servicePath, '.serverless'));
+
   if (this.serverless.service.package.individually) {
     let doneModules = [];
     values(this.serverless.service.functions)
@@ -158,24 +182,54 @@ function installAllRequirements() {
           set(f, ['module'], '.');
         }
         if (!doneModules.includes(f.module)) {
-          installRequirements(
+          installRequirementsFile(
             path.join(f.module, this.options.fileName),
             path.join('.serverless', f.module),
             this.serverless,
             this.servicePath,
             this.options
           );
+          if ( cached_sum && cached_sum == md5File.sync(path.join(this.servicePath, '.serverless/requirements.txt')) ) {
+            this.serverless.cli.log(`Using static cache...`);
+            fse.copySync(requirementsStaticCacheFolder, path.join('.serverless', f.module, 'requirements/'))
+            fse.removeSync(path.join('.serverless', f.module, 'requirements/.requirements.txt.md5'))
+          } else {
+            installRequirements(
+              path.join('.serverless', f.module),
+              this.serverless,
+              this.servicePath,
+              this.options
+            );
+          }
+          // Write our MD5 hash of our requirements.txt to a file to detect if we need to re-run installRequirements
+          fse.writeFileSync(path.join(requirementsStaticCacheFolder, '.requirements.txt.md5'), md5File.sync(path.join(this.servicePath, '.serverless/requirements.txt')));
           doneModules.push(f.module);
         }
       });
   } else {
-    installRequirements(
+    installRequirementsFile(
       this.options.fileName,
       '.serverless',
       this.serverless,
       this.servicePath,
       this.options
     );
+    if ( cached_sum && cached_sum == md5File.sync(path.join(this.servicePath, '.serverless/requirements.txt')) ) {
+      this.serverless.cli.log(`Using static cache...`);
+      fse.copySync(requirementsStaticCacheFolder, '.serverless/requirements/')
+      fse.removeSync('.serverless/requirements/.requirements.txt.md5')
+    } else {
+      installRequirements(
+        '.serverless',
+        this.serverless,
+        this.servicePath,
+        this.options
+      );
+    }
+    // Backup our static files to cache...
+    fse.copySync('.serverless/requirements', requirementsStaticCacheFolder)
+    // Write our MD5 hash of our requirements.txt to a file to detect if we need to re-run installRequirements
+    fse.writeFileSync(path.join(requirementsStaticCacheFolder, '.requirements.txt.md5'), md5File.sync(path.join(this.servicePath, '.serverless/requirements.txt')));
   }
 };
 

package.json

@@ -51,6 +51,7 @@
     "glob-all": "^3.1.0",
     "is-wsl": "^1.1.0",
     "jszip": "^2.5.0",
+    "md5-file": "^3.2.3",
     "lodash.get": "^4.4.2",
     "lodash.set": "^4.3.2",
     "lodash.values": "^4.3.0",