From 86ff8c84d46770727a7e3e4e5439a6ca7380db67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Barthelet?= Date: Sun, 28 Nov 2021 20:57:30 +0100 Subject: [PATCH 1/8] Add event filtering patterns to stream event type --- lib/plugins/aws/package/compile/events/stream.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/plugins/aws/package/compile/events/stream.js b/lib/plugins/aws/package/compile/events/stream.js index 03264140bcb..bc0e5888354 100644 --- a/lib/plugins/aws/package/compile/events/stream.js +++ b/lib/plugins/aws/package/compile/events/stream.js @@ -69,6 +69,12 @@ class AwsCompileStreamEvents { required: ['onFailure'], }, tumblingWindowInSeconds: { type: 'integer', minimum: 0, maximum: 900 }, + patterns: { + type: 'array', + minItems: 1, + maxItems: 5, + items: { type: 'object' }, + }, }, additionalProperties: false, anyOf: [ @@ -273,6 +279,14 @@ class AwsCompileStreamEvents { }; } + if (event.stream.patterns != null) { + streamResource.Properties.FilterCriteria = { + Filters: event.stream.patterns.map((pattern) => ({ + Pattern: JSON.stringify(pattern), + })), + }; + } + const newStreamObject = { [streamLogicalId]: streamResource, }; From 28ecf113f5dc5023643a5aa2da4cd54da017f066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Barthelet?= Date: Sun, 28 Nov 2021 21:09:02 +0100 Subject: [PATCH 2/8] Update stream documentation --- docs/providers/aws/events/streams.md | 19 +++++++++++++++++++ docs/providers/aws/guide/serverless.yml.md | 2 ++ 2 files changed, 21 insertions(+) diff --git a/docs/providers/aws/events/streams.md b/docs/providers/aws/events/streams.md index e4dada34fa6..8c6f82388c3 100644 --- a/docs/providers/aws/events/streams.md +++ b/docs/providers/aws/events/streams.md @@ -329,3 +329,22 @@ functions: arn: arn:aws:dynamodb:region:XXXXXX:table/foo/stream/1970-01-01T00:00:00.000 tumblingWindowInSeconds: 30 ``` + +## Setting FilterCriteria + +This configuration allows customers to filter event before lambda invocation. It accepts up to 5 filter criterion. If one event matches at least 1 pattern, lambda will process it. + +For more information and examples, read the [AWS event filtering documentation](https://docs.aws.amazon.com/lambda/latest/dg/invocation-eventfiltering.html) + +Note: Serverless only sets this property if you explicitly add it to the stream configuration (see example below). The following exemple will only process inserted item in the DynamoDB table (not the removed and modified items). + +```yml +functions: + handleInsertedDynamoDBItem: + handler: handler.preprocess + events: + - stream: + arn: arn:aws:dynamodb:region:XXXXXX:table/foo/stream/1970-01-01T00:00:00.000 + patterns: + - eventName: [INSERT] +``` diff --git a/docs/providers/aws/guide/serverless.yml.md b/docs/providers/aws/guide/serverless.yml.md index a4ba3714b1b..99297c219ca 100644 --- a/docs/providers/aws/guide/serverless.yml.md +++ b/docs/providers/aws/guide/serverless.yml.md @@ -458,6 +458,8 @@ functions: startingPosition: LATEST enabled: true functionResponseType: ReportBatchItemFailures + patterns: + - partitionKey: [ 1 ] - msk: arn: arn:aws:kafka:us-east-1:111111111111:cluster/ClusterName/a1a1a1a1a1a1a1a1a # ARN of MSK Cluster topic: kafkaTopic # name of Kafka topic to consume from From 8a0bfa02bff7449fde87d5491ae0a6a078f828f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Barthelet?= Date: Mon, 29 Nov 2021 12:16:45 +0100 Subject: [PATCH 3/8] Update max patterns to 10 to match quota increase possibility --- lib/plugins/aws/package/compile/events/stream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plugins/aws/package/compile/events/stream.js b/lib/plugins/aws/package/compile/events/stream.js index bc0e5888354..5d55dfda8d7 100644 --- a/lib/plugins/aws/package/compile/events/stream.js +++ b/lib/plugins/aws/package/compile/events/stream.js @@ -72,7 +72,7 @@ class AwsCompileStreamEvents { patterns: { type: 'array', minItems: 1, - maxItems: 5, + maxItems: 10, items: { type: 'object' }, }, }, From 6669610ae901a199967165c5d880a31c593e7cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Barthelet?= Date: Mon, 29 Nov 2021 12:19:03 +0100 Subject: [PATCH 4/8] Simplify check of patterns existence --- lib/plugins/aws/package/compile/events/stream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plugins/aws/package/compile/events/stream.js b/lib/plugins/aws/package/compile/events/stream.js index 5d55dfda8d7..12a3c6bc75b 100644 --- a/lib/plugins/aws/package/compile/events/stream.js +++ b/lib/plugins/aws/package/compile/events/stream.js @@ -279,7 +279,7 @@ class AwsCompileStreamEvents { }; } - if (event.stream.patterns != null) { + if (event.stream.patterns) { streamResource.Properties.FilterCriteria = { Filters: event.stream.patterns.map((pattern) => ({ Pattern: JSON.stringify(pattern), From 443ea08f01000f7f045356101bbb2bb1c9541618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Barthelet?= Date: Mon, 29 Nov 2021 12:40:22 +0100 Subject: [PATCH 5/8] Add unit tests for stream patterns property --- .../aws/package/compile/events/stream.test.js | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test/unit/lib/plugins/aws/package/compile/events/stream.test.js b/test/unit/lib/plugins/aws/package/compile/events/stream.test.js index cc894e2df08..2661d56c7e8 100644 --- a/test/unit/lib/plugins/aws/package/compile/events/stream.test.js +++ b/test/unit/lib/plugins/aws/package/compile/events/stream.test.js @@ -1724,4 +1724,43 @@ describe('AwsCompileStreamEvents #2', () => { ); }); }); + describe('with patterns', () => { + let eventSourceMappingResource; + + before(async () => { + const { awsNaming, cfTemplate } = await runServerless({ + fixture: 'function', + configExt: { + functions: { + basic: { + events: [ + { + stream: { + arn: 'arn:aws:dynamodb:region:account:table/foo/stream/1', + patterns: [{ eventName: ['INSERT'] }, { eventName: ['MODIFY'] }], + }, + }, + ], + }, + }, + }, + command: 'package', + }); + const streamLogicalId = awsNaming.getStreamLogicalId('basic', 'dynamodb', 'foo'); + eventSourceMappingResource = cfTemplate.Resources[streamLogicalId]; + }); + + it('should wrap patterns within FilterCriteria property', () => { + expect(eventSourceMappingResource.Properties.FilterCriteria).to.deep.equal({ + Filters: [ + { + Pattern: JSON.stringify({ eventName: ['INSERT'] }), + }, + { + Pattern: JSON.stringify({ eventName: ['MODIFY'] }), + }, + ], + }); + }); + }); }); From 327f08549b1cb6792f95e18723519f7bc7a2517e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Barthelet?= Date: Mon, 29 Nov 2021 16:46:41 +0100 Subject: [PATCH 6/8] Mention quota extension for filters quantity in docs --- docs/providers/aws/events/streams.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/providers/aws/events/streams.md b/docs/providers/aws/events/streams.md index 8c6f82388c3..213ce4bd723 100644 --- a/docs/providers/aws/events/streams.md +++ b/docs/providers/aws/events/streams.md @@ -332,7 +332,7 @@ functions: ## Setting FilterCriteria -This configuration allows customers to filter event before lambda invocation. It accepts up to 5 filter criterion. If one event matches at least 1 pattern, lambda will process it. +This configuration allows customers to filter event before lambda invocation. It accepts up to 5 filter criterion by default and up to 10 with quota extension. If one event matches at least 1 pattern, lambda will process it. For more information and examples, read the [AWS event filtering documentation](https://docs.aws.amazon.com/lambda/latest/dg/invocation-eventfiltering.html) From a1cbdb28be21e235f39aa91638f0625d3a7796d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Barthelet?= Date: Mon, 29 Nov 2021 16:47:40 +0100 Subject: [PATCH 7/8] Improve exemple readability --- docs/providers/aws/events/streams.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/providers/aws/events/streams.md b/docs/providers/aws/events/streams.md index 213ce4bd723..5f482567300 100644 --- a/docs/providers/aws/events/streams.md +++ b/docs/providers/aws/events/streams.md @@ -334,9 +334,9 @@ functions: This configuration allows customers to filter event before lambda invocation. It accepts up to 5 filter criterion by default and up to 10 with quota extension. If one event matches at least 1 pattern, lambda will process it. -For more information and examples, read the [AWS event filtering documentation](https://docs.aws.amazon.com/lambda/latest/dg/invocation-eventfiltering.html) +For more details and examples of filter patterns, please see the [AWS event filtering documentation](https://docs.aws.amazon.com/lambda/latest/dg/invocation-eventfiltering.html) -Note: Serverless only sets this property if you explicitly add it to the stream configuration (see example below). The following exemple will only process inserted item in the DynamoDB table (not the removed and modified items). +Note: Serverless only sets this property if you explicitly add it to the stream configuration (see an example below). The following example will only process inserted items in the DynamoDB table (it will skip removed and modified items). ```yml functions: From a4bcb31ef454600d18f16218454cb2b89fc6fe2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Barthelet?= Date: Mon, 29 Nov 2021 22:35:33 +0100 Subject: [PATCH 8/8] Change patterns to filterPatterns --- docs/providers/aws/events/streams.md | 4 ++-- docs/providers/aws/guide/serverless.yml.md | 2 +- lib/plugins/aws/package/compile/events/stream.js | 6 +++--- .../lib/plugins/aws/package/compile/events/stream.test.js | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/providers/aws/events/streams.md b/docs/providers/aws/events/streams.md index 5f482567300..b0c615e0d87 100644 --- a/docs/providers/aws/events/streams.md +++ b/docs/providers/aws/events/streams.md @@ -330,7 +330,7 @@ functions: tumblingWindowInSeconds: 30 ``` -## Setting FilterCriteria +## Setting filter patterns This configuration allows customers to filter event before lambda invocation. It accepts up to 5 filter criterion by default and up to 10 with quota extension. If one event matches at least 1 pattern, lambda will process it. @@ -345,6 +345,6 @@ functions: events: - stream: arn: arn:aws:dynamodb:region:XXXXXX:table/foo/stream/1970-01-01T00:00:00.000 - patterns: + filterPatterns: - eventName: [INSERT] ``` diff --git a/docs/providers/aws/guide/serverless.yml.md b/docs/providers/aws/guide/serverless.yml.md index 99297c219ca..6bd1506282c 100644 --- a/docs/providers/aws/guide/serverless.yml.md +++ b/docs/providers/aws/guide/serverless.yml.md @@ -458,7 +458,7 @@ functions: startingPosition: LATEST enabled: true functionResponseType: ReportBatchItemFailures - patterns: + filterPatterns: - partitionKey: [ 1 ] - msk: arn: arn:aws:kafka:us-east-1:111111111111:cluster/ClusterName/a1a1a1a1a1a1a1a1a # ARN of MSK Cluster diff --git a/lib/plugins/aws/package/compile/events/stream.js b/lib/plugins/aws/package/compile/events/stream.js index 12a3c6bc75b..21377addd94 100644 --- a/lib/plugins/aws/package/compile/events/stream.js +++ b/lib/plugins/aws/package/compile/events/stream.js @@ -69,7 +69,7 @@ class AwsCompileStreamEvents { required: ['onFailure'], }, tumblingWindowInSeconds: { type: 'integer', minimum: 0, maximum: 900 }, - patterns: { + filterPatterns: { type: 'array', minItems: 1, maxItems: 10, @@ -279,9 +279,9 @@ class AwsCompileStreamEvents { }; } - if (event.stream.patterns) { + if (event.stream.filterPatterns) { streamResource.Properties.FilterCriteria = { - Filters: event.stream.patterns.map((pattern) => ({ + Filters: event.stream.filterPatterns.map((pattern) => ({ Pattern: JSON.stringify(pattern), })), }; diff --git a/test/unit/lib/plugins/aws/package/compile/events/stream.test.js b/test/unit/lib/plugins/aws/package/compile/events/stream.test.js index 2661d56c7e8..6b640a4122d 100644 --- a/test/unit/lib/plugins/aws/package/compile/events/stream.test.js +++ b/test/unit/lib/plugins/aws/package/compile/events/stream.test.js @@ -1724,7 +1724,7 @@ describe('AwsCompileStreamEvents #2', () => { ); }); }); - describe('with patterns', () => { + describe('with filterPatterns', () => { let eventSourceMappingResource; before(async () => { @@ -1737,7 +1737,7 @@ describe('AwsCompileStreamEvents #2', () => { { stream: { arn: 'arn:aws:dynamodb:region:account:table/foo/stream/1', - patterns: [{ eventName: ['INSERT'] }, { eventName: ['MODIFY'] }], + filterPatterns: [{ eventName: ['INSERT'] }, { eventName: ['MODIFY'] }], }, }, ],