feat(AWS Deploy): Introduce new version of hashing algorithm (#8661)

docs/deprecations.md

@@ -6,6 +6,22 @@ layout: Doc
 
 # Serverless Framework Deprecations
 
+<a name="LAMBDA_HASHING_VERSION_V2"><div>&nbsp;</div></a>
+
+## Default `provider.lambdaHashingVersion`
+
+Starting with v3.0.0, the default value of `lambdaHashingVersion` will be equal to `20201221`. You can adapt to this behavior now, by setting `provider.lambdaHashingVersion` to `20201221`.
+
+When trying to `sls deploy` for the first time after migration to new `lambdaHashingVersion`, you might encounter an error, similar to the one below:
+
+```
+  Serverless Error ---------------------------------------
+
+  An error occurred: FooLambdaVersion3IV5NZ3sE5T2UFimCOai2Tc6eCaW7yIYOP786U0Oc - A version for this Lambda function exists ( 11 ). Modify the function to create a new version..
+```
+
+It is an expected behavior, to avoid it, you need to modify your function(s) code and try to redeploy it again. One common approach is to modify an utility function that is used by all/most of your Lambda functions.
+
 <a name="LOAD_VARIABLES_FROM_ENV_FILES"><div>&nbsp;</div></a>
 
 ## Automatic loading environment variables from .env and .env.{stage} files

docs/providers/aws/guide/serverless.yml.md

@@ -59,6 +59,7 @@ provider:
   deploymentPrefix: serverless # The S3 prefix under which deployed artifacts should be stored. Default is serverless
   role: arn:aws:iam::XXXXXX:role/role # Overwrite the default IAM role which is used for all functions
   rolePermissionsBoundary: arn:aws:iam::XXXXXX:policy/policy # ARN of an Permissions Boundary for the role.
+  lambdaHashingVersion: 20201221 # optional, version of hashing algorithm that should be used by the framework
   cfnRole: arn:aws:iam::XXXXXX:role/role # ARN of an IAM role for CloudFormation service. If specified, CloudFormation uses the role's credentials
   cloudFront:
     myCachePolicy1: # used as a reference in function.events[].cloudfront.cachePolicy.name

lib/plugins/aws/lib/normalizeFiles.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const deepSortObjectByKey = require('../../../utils/deepSortObjectByKey');
 const _ = require('lodash');
 
 module.exports = {
@@ -31,6 +32,12 @@ module.exports = {
       }
     });
 
+    // Sort resources and outputs to ensure consistent hashing
+    normalizedTemplate.Resources = deepSortObjectByKey(normalizedTemplate.Resources);
+    if (normalizedTemplate.Outputs) {
+      normalizedTemplate.Outputs = deepSortObjectByKey(normalizedTemplate.Outputs);
+    }
+
     return normalizedTemplate;
   },
 };

lib/plugins/aws/package/compile/functions.js

@@ -6,6 +6,8 @@ const crypto = require('crypto');
 const fs = require('fs');
 const _ = require('lodash');
 const path = require('path');
+const deepSortObjectByKey = require('../../../../utils/deepSortObjectByKey');
+const getHashForFilePath = require('../lib/getHashForFilePath');
 
 class AwsCompileFunctions {
   constructor(serverless, options) {
@@ -26,7 +28,7 @@ class AwsCompileFunctions {
     }
 
     this.hooks = {
-      'intialize': () => {
+      'initialize': () => {
         if (_.get(this.serverless.service.serviceObject, 'awsKmsKeyArn')) {
           this.serverless._logDeprecation(
             'AWS_KMS_KEY_ARN',
@@ -43,6 +45,19 @@ class AwsCompileFunctions {
               '"awsKmsKeyArn" function property will be replaced by "kmsKeyArn"'
           );
         }
+        if (
+          !this.serverless.service.provider.lambdaHashingVersion &&
+          (this.serverless.service.provider.versionFunctions ||
+            Object.values(this.serverless.service.functions).some(
+              ({ versionFunction }) => versionFunction
+            ))
+        ) {
+          this.serverless._logDeprecation(
+            'LAMBDA_HASHING_VERSION_V2',
+            'Starting with next major version, ' +
+              'default value of provider.lambdaHashingVersion will be equal to "20201221"'
+          );
+        }
       },
       'package:compileFunctions': () =>
         BbPromise.bind(this)
@@ -114,6 +129,16 @@ class AwsCompileFunctions {
     });
   }
 
+  async addFileToHash(filePath, hash) {
+    const lambdaHashingVersion = this.serverless.service.provider.lambdaHashingVersion;
+    if (lambdaHashingVersion) {
+      const filePathHash = await getHashForFilePath(filePath);
+      hash.write(filePathHash);
+    } else {
+      await addFileContentsToHashes(filePath, [hash]);
+    }
+  }
+
   async compileFunction(functionName) {
     const cfTemplate = this.serverless.service.provider.compiledCloudFormationTemplate;
     const functionResource = this.cfLambdaFunctionTemplate();
@@ -407,11 +432,10 @@ class AwsCompileFunctions {
           functionObject.image.lastIndexOf('@sha256:') + '@sha256:'.length
         );
       } else {
-        const fileHash = crypto.createHash('sha256');
-        fileHash.setEncoding('base64');
-        await addFileContentsToHashes(artifactFilePath, [fileHash, versionHash]);
-        fileHash.end();
-        versionResource.Properties.CodeSha256 = fileHash.read();
+        const fileHash = await getHashForFilePath(artifactFilePath);
+        versionResource.Properties.CodeSha256 = fileHash;
+
+        await this.addFileToHash(artifactFilePath, versionHash);
       }
       // Include all referenced layer code in the version id hash
       const layerArtifactPaths = [];
@@ -421,31 +445,41 @@ class AwsCompileFunctions {
       });
 
       for (const layerArtifactPath of layerArtifactPaths.sort()) {
-        await addFileContentsToHashes(layerArtifactPath, [versionHash]);
+        await this.addFileToHash(layerArtifactPath, versionHash);
       }
 
       // Include function and layer configuration details in the version id hash
       for (const layerConfig of layerConfigurations) {
         delete layerConfig.properties.Content.S3Key;
       }
 
-      // sort the layer conifigurations for hash consistency
-      const sortedLayerConfigurations = {};
-      const byKey = ([key1], [key2]) => key1.localeCompare(key2);
-      for (const { name, properties: layerProperties } of layerConfigurations) {
-        sortedLayerConfigurations[name] = _.fromPairs(Object.entries(layerProperties).sort(byKey));
-      }
-
       const functionProperties = _.cloneDeep(functionResource.Properties);
       // In `image` case, we assume it's path to ECR image digest
       if (!functionObject.image) delete functionProperties.Code;
       // Properties applied to function globally (not specific to version or alias)
       delete functionProperties.ReservedConcurrentExecutions;
       delete functionProperties.Tags;
-      functionProperties.layerConfigurations = sortedLayerConfigurations;
 
-      const sortedFunctionProperties = _.fromPairs(Object.entries(functionProperties).sort(byKey));
-      versionHash.write(JSON.stringify(sortedFunctionProperties));
+      const lambdaHashingVersion = this.serverless.service.provider.lambdaHashingVersion;
+      if (lambdaHashingVersion) {
+        functionProperties.layerConfigurations = layerConfigurations;
+        versionHash.write(JSON.stringify(deepSortObjectByKey(functionProperties)));
+      } else {
+        // sort the layer conifigurations for hash consistency
+        const sortedLayerConfigurations = {};
+        const byKey = ([key1], [key2]) => key1.localeCompare(key2);
+        for (const { name, properties: layerProperties } of layerConfigurations) {
+          sortedLayerConfigurations[name] = _.fromPairs(
+            Object.entries(layerProperties).sort(byKey)
+          );
+        }
+        functionProperties.layerConfigurations = sortedLayerConfigurations;
+        const sortedFunctionProperties = _.fromPairs(
+          Object.entries(functionProperties).sort(byKey)
+        );
+
+        versionHash.write(JSON.stringify(sortedFunctionProperties));
+      }
 
       versionHash.end();
       const versionDigest = versionHash.read();
@@ -609,7 +643,7 @@ class AwsCompileFunctions {
 
   compileFunctions() {
     const allFunctions = this.serverless.service.getAllFunctions();
-    return BbPromise.each(allFunctions, functionName => this.compileFunction(functionName));
+    return Promise.all(allFunctions.map(functionName => this.compileFunction(functionName)));
   }
 
   cfLambdaFunctionTemplate() {

lib/plugins/aws/package/lib/getHashForFilePath.js

@@ -0,0 +1,33 @@
+'use strict';
+
+const memoize = require('memoizee');
+const crypto = require('crypto');
+const fs = require('fs');
+
+const getHashForFilePath = memoize(
+  filePath => {
+    const fileHash = crypto.createHash('sha256');
+    fileHash.setEncoding('base64');
+    return new Promise((resolve, reject) => {
+      const readStream = fs.createReadStream(filePath);
+      readStream
+        .on('data', chunk => {
+          fileHash.write(chunk);
+        })
+        .on('close', () => {
+          fileHash.end();
+          resolve(fileHash.read());
+        })
+        .on('error', error => {
+          reject(
+            new Error(
+              `Error: ${error} encountered during hash calculation for provided filePath: ${filePath}`
+            )
+          );
+        });
+    });
+  },
+  { promise: true }
+);
+
+module.exports = getHashForFilePath;

lib/plugins/aws/provider.js

@@ -742,6 +742,10 @@ class AwsProvider {
             iamManagedPolicies: { type: 'array', items: { $ref: '#/definitions/awsArnString' } },
             iamRoleStatements: { $ref: '#/definitions/awsIamPolicyStatements' },
             kmsKeyArn: { $ref: '#/definitions/awsKmsArn' },
+            lambdaHashingVersion: {
+              type: 'string',
+              enum: ['20201221'],
+            },
             layers: { $ref: '#/definitions/awsLambdaLayers' },
             logRetentionInDays: {
               enum: [1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 3653],

lib/utils/deepSortObjectByKey.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const _ = require('lodash');
+
+const deepSortObjectByKey = obj => {
+  if (Array.isArray(obj)) {
+    return obj.map(deepSortObjectByKey);
+  }
+
+  if (_.isPlainObject(obj)) {
+    return _.fromPairs(
+      Object.entries(obj)
+        .sort(([key], [otherKey]) => key.localeCompare(otherKey))
+        .map(([key, value]) => [key, deepSortObjectByKey(value)])
+    );
+  }
+
+  return obj;
+};
+
+module.exports = deepSortObjectByKey;

test/unit/lib/plugins/aws/deploy/lib/checkForChanges.test.js

@@ -1092,7 +1092,7 @@ describe('checkForChanges #2', () => {
                   'serverless/test-package-artifact/dev/1589988704359-2020-05-20T15:31:44.359Z/compiled-cloudformation-template.json',
               })
               .returns({
-                Metadata: { filesha256: 'P8d0U46tohyBFJp06rTa3SvmUzpfkkVDTxE/jssbqYM=' },
+                Metadata: { filesha256: 'p2wLB86RTnPkFQLaGCUQFdk6/nwyVGiX2mGJl2m0bD0=' },
               });
 
             headObjectStub

test/unit/lib/plugins/aws/lib/normalizeFiles.test.js

@@ -176,5 +176,47 @@ describe('normalizeFiles', () => {
         },
       });
     });
+
+    it('should sort resources and outputs alphabetically', () => {
+      const input = {
+        Resources: {
+          ResourceThatShouldBeLast: {
+            Type: 'AWS::XXX::XXX',
+          },
+          ResourceThatShouldBeFirst: {
+            Type: 'AWS::XXX::XXX',
+          },
+        },
+        Outputs: {
+          OutputThatShouldBeLast: {
+            Value: 'SomeValue',
+          },
+          OutputThatShouldBeFirst: {
+            Value: 'AnotherValue',
+          },
+        },
+      };
+
+      const result = normalizeFiles.normalizeCloudFormationTemplate(input);
+
+      expect(result).to.deep.equal({
+        Resources: {
+          ResourceThatShouldBeFirst: {
+            Type: 'AWS::XXX::XXX',
+          },
+          ResourceThatShouldBeLast: {
+            Type: 'AWS::XXX::XXX',
+          },
+        },
+        Outputs: {
+          OutputThatShouldBeLast: {
+            Value: 'SomeValue',
+          },
+          OutputThatShouldBeFirst: {
+            Value: 'AnotherValue',
+          },
+        },
+      });
+    });
   });
 });