From 741b80f5d035eb735b28135bb2f29fafbbebdb29 Mon Sep 17 00:00:00 2001
From: Parro <29497+Parro@users.noreply.github.com>
Date: Tue, 19 Jul 2022 18:01:36 +0200
Subject: [PATCH 1/7] Add cli parameter to set the percentage of durations to
discard
---
README-INPUT-OUTPUT.md | 1 +
lambda/executor.js | 20 +++++++++++++++++---
lambda/utils.js | 10 +++++-----
test/unit/test-utils.js | 6 +++++-
4 files changed, 28 insertions(+), 9 deletions(-)
diff --git a/README-INPUT-OUTPUT.md b/README-INPUT-OUTPUT.md
index ca2b3509..2392ca68 100644
--- a/README-INPUT-OUTPUT.md
+++ b/README-INPUT-OUTPUT.md
@@ -20,6 +20,7 @@ The state machine accepts the following intput parameters:
* **dryRun** (false by default): if true, the state machine will execute the input function only once and it will disable every functionality related to logs analysis, auto-tuning, and visualization; the dry-run mode is intended for testing purposes, for example to verify that IAM permissions are set up correctly
* **preProcessorARN** (string): it must be the ARN of a Lambda function; if provided, the function will be invoked before every invocation of `lambdaARN`; more details below in the [Pre/Post-processing functions section](#user-content-prepost-processing-functions)
* **postProcessorARN** (string): it must be the ARN of a Lambda function; if provided, the function will be invoked after every invocation of `lambdaARN`; more details below in the [Pre/Post-processing functions section](#user-content-prepost-processing-functions)
+* **discardTopBottom** (number between 0.0 and 0.4, by default is 0.2): By default, the state machine will discard the top/bottom 20% of "outliers" (the fastest and slowest), to filter out the effects of cold starts that would bias the overall averages. This behaviour could be changed with this parameter, that could take a value between 0 and 0.4, with 0 meaning "no trimmed mean at all" and 0.4 meaning "trim 40% of the top/bottom results" (resulting in only 20% of the results being considered).
## State machine configuration (at deployment time)
diff --git a/lambda/executor.js b/lambda/executor.js
index 221df4d9..b3efe681 100644
--- a/lambda/executor.js
+++ b/lambda/executor.js
@@ -25,6 +25,7 @@ module.exports.handler = async(event, context) => {
dryRun,
preProcessorARN,
postProcessorARN,
+ discardTopBottom,
} = await extractDataFromInput(event);
validateInput(lambdaARN, value, num); // may throw
@@ -54,7 +55,7 @@ module.exports.handler = async(event, context) => {
// get base cost for Lambda
const baseCost = utils.lambdaBaseCost(utils.regionFromARN(lambdaARN), architecture);
- return computeStatistics(baseCost, results, value);
+ return computeStatistics(baseCost, results, value, discardTopBottom);
};
const validateInput = (lambdaARN, value, num) => {
@@ -78,9 +79,21 @@ const extractPayloadValue = async(input) => {
return null;
};
+
+const extractDiscardTopBottomValue = (event) => {
+ // extract discardTopBottom used to trim values from average duration
+ let discardTopBottom = event.extractDiscardTopBottom;
+ if (typeof discardTopBottom === 'undefined') {
+ discardTopBottom = 0.2;
+ }
+ // discardTopBottom must be between 0 and 0.4
+ return Math.min(Math.max(discardTopBottom, 0.0), 0.4);
+};
+
const extractDataFromInput = async(event) => {
const input = event.input; // original state machine input
const payload = await extractPayloadValue(input);
+ const discardTopBottom = extractDiscardTopBottomValue(input);
return {
value: parseInt(event.value, 10),
lambdaARN: input.lambdaARN,
@@ -90,6 +103,7 @@ const extractDataFromInput = async(event) => {
dryRun: input.dryRun === true,
preProcessorARN: input.preProcessorARN,
postProcessorARN: input.postProcessorARN,
+ discardTopBottom: discardTopBottom,
};
};
@@ -123,12 +137,12 @@ const runInSeries = async(num, lambdaARN, lambdaAlias, payloads, preARN, postARN
return results;
};
-const computeStatistics = (baseCost, results, value) => {
+const computeStatistics = (baseCost, results, value, discardTopBottom) => {
// use results (which include logs) to compute average duration ...
const durations = utils.parseLogAndExtractDurations(results);
- const averageDuration = utils.computeAverageDuration(durations);
+ const averageDuration = utils.computeAverageDuration(durations, discardTopBottom);
console.log('Average duration: ', averageDuration);
// ... and overall statistics
diff --git a/lambda/utils.js b/lambda/utils.js
index a1cf943b..67ffbfd1 100644
--- a/lambda/utils.js
+++ b/lambda/utils.js
@@ -450,20 +450,20 @@ module.exports.computeTotalCost = (minCost, minRAM, value, durations) => {
/**
* Copute average duration
*/
-module.exports.computeAverageDuration = (durations) => {
+module.exports.computeAverageDuration = (durations, discardTopBottom) => {
if (!durations || !durations.length) {
return 0;
}
- // 20% of durations will be discarted (trimmed mean)
- const toBeDiscarded = parseInt(durations.length * 20 / 100, 10);
+ // a percentage of durations will be discarded (trimmed mean)
+ const toBeDiscarded = parseInt(durations.length * discardTopBottom, 10);
const newN = durations.length - 2 * toBeDiscarded;
- // compute trimmed mean (discard 20% of low/high values)
+ // compute trimmed mean (discard a percentage of low/high values)
const averageDuration = durations
.sort(function(a, b) { return a - b; }) // sort numerically
- .slice(toBeDiscarded, -toBeDiscarded) // discard first/last values
+ .slice(toBeDiscarded, toBeDiscarded > 0 ? -toBeDiscarded : durations.length) // discard first/last values
.reduce((a, b) => a + b, 0) // sum all together
/ newN
;
diff --git a/test/unit/test-utils.js b/test/unit/test-utils.js
index ddb1595a..065caf98 100644
--- a/test/unit/test-utils.js
+++ b/test/unit/test-utils.js
@@ -214,9 +214,13 @@ describe('Lambda Utils', () => {
];
it('should return the average duration', () => {
- const duration = utils.computeAverageDuration(durations);
+ const duration = utils.computeAverageDuration(durations, 0.2);
expect(duration).to.be(2);
});
+ it('should return the average duration with no trimmed value', () => {
+ const duration = utils.computeAverageDuration(durations, 0);
+ expect(duration).to.be(401.4);
+ });
it('should return 0 if empty results', () => {
const duration = utils.computeAverageDuration([]);
expect(duration).to.be(0);
From 4d76acfe9acf66439e87a0f7a0baa8fc1a505194 Mon Sep 17 00:00:00 2001
From: Parro <29497+Parro@users.noreply.github.com>
Date: Tue, 19 Jul 2022 18:02:17 +0200
Subject: [PATCH 2/7] Fix typos
---
README-ADVANCED.md | 2 +-
README-DEPLOY.md | 10 +++++-----
README-INPUT-OUTPUT.md | 6 +++---
README-SAR.md | 8 ++++----
lambda/executor.js | 2 +-
lambda/utils.js | 4 ++--
6 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/README-ADVANCED.md b/README-ADVANCED.md
index 8255bee2..e4903f0c 100644
--- a/README-ADVANCED.md
+++ b/README-ADVANCED.md
@@ -5,7 +5,7 @@ This section describes some advanced features of this project, as well as some c
## Error handling
-If something goes wrong during the initialization or execution states, the `CleanUpOnError` step will be executed. All temporary versions and alises will be deleted as expected (the same happens in the `Cleaner` step).
+If something goes wrong during the initialization or execution states, the `CleanUpOnError` step will be executed. All temporary versions and aliases will be deleted as expected (the same happens in the `Cleaner` step).
You can customize the `totalExecutionTimeout` parameter at deploy time (up to 15min). This parameter will be used both for Lambda function timeouts and Step Function tasks timeouts. In case the `Executor` raises a timeout error, you will see a `States.Timeout` error. Keep in mind that the timeout you configure will vary whether you're setting `parallelInvocation` to `true` or `false`. When you enable parallel invocation, all the function executions will run concurrently (rather than in series) so that you can keep that timeout lower and your overall state machine execution faster.
diff --git a/README-DEPLOY.md b/README-DEPLOY.md
index 102229d2..4ebdb8f3 100644
--- a/README-DEPLOY.md
+++ b/README-DEPLOY.md
@@ -2,9 +2,9 @@
There are multiple options to deploy the tool.
-If you are familiar with Infrastructure as Code, there are 4 ways for you to create all of the resources neccesary for Lambda Power Tuning.
+If you are familiar with Infrastructure as Code, there are 4 ways for you to create all of the resources necessary for Lambda Power Tuning.
-The following three options utilize [AWS CloudFormation](https://aws.amazon.com/cloudformation/) on your behalf to create the neccessary resources. Each will create a new CloudFormation stack in your AWS account containing all the resources for the Lambda Power Tuning tool.
+The following three options utilize [AWS CloudFormation](https://aws.amazon.com/cloudformation/) on your behalf to create the necessary resources. Each will create a new CloudFormation stack in your AWS account containing all the resources for the Lambda Power Tuning tool.
1. The easiest way is to [deploy the app via the AWS Serverless Application Repository (SAR)](#option1)
1. Manually [using the AWS SAM CLI](#option2)
1. Manually [using the AWS CDK](#option3)
@@ -46,15 +46,15 @@ You can also integrate the SAR app in your existing CloudFormation stacks - chec
[`sam build -u`](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-cli-command-reference-sam-build.html) will run SAM build using a Docker container image that provides an environment similar to that which your function would run in. SAM build in-turn looks at your AWS SAM template file for information about Lambda functions and layers in this project.
Once the build has completed you should see output that states `Build Succeeded`. If not there will be error messages providing guidance on what went wrong.
-1. Deploy the applicaiton using the SAM deploy "guided" mode:
+1. Deploy the application using the SAM deploy "guided" mode:
```bash
$ sam deploy -g
```
- [`sam deploy -g`](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-cli-command-reference-sam-deploy.html) will provide simple prompts to walk you through the process of deploying the tool. Provide a unique name for the 'Stack Name' and supply the [AWS Region](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html#Concepts.RegionsAndAvailabilityZones.Regions) you want to run the tool in and then you can select the defaults for testing of this tool. After accepting the promted questions with a "Y" you can optionally save your application configuration.
+ [`sam deploy -g`](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-cli-command-reference-sam-deploy.html) will provide simple prompts to walk you through the process of deploying the tool. Provide a unique name for the 'Stack Name' and supply the [AWS Region](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html#Concepts.RegionsAndAvailabilityZones.Regions) you want to run the tool in and then you can select the defaults for testing of this tool. After accepting the prompted questions with a "Y" you can optionally save your application configuration.
After that the SAM CLI will run the required commands to create the resources for the Lambda Power Tuning tool. The CloudFormation outputs shown will highlight any issues or failures.
- If there are no issues, once complete you will see the stack ouputs and a `Successfully created/updated stack` message.
+ If there are no issues, once complete you will see the stack outputs and a `Successfully created/updated stack` message.
## Option 3: Deploy the AWS SAR app with AWS CDK
diff --git a/README-INPUT-OUTPUT.md b/README-INPUT-OUTPUT.md
index 2392ca68..46f38294 100644
--- a/README-INPUT-OUTPUT.md
+++ b/README-INPUT-OUTPUT.md
@@ -5,12 +5,12 @@ Each execution of the state machine will require an input and will provide the c
## State machine input (at execution time)
-The state machine accepts the following intput parameters:
+The state machine accepts the following input parameters:
* **lambdaARN** (required, string): unique identifier of the Lambda function you want to optimize
* **powerValues** (optional, string or list of integers): the list of power values to be tested; if not provided, the default values configured at deploy-time are used (by default: 128MB, 256MB, 512MB, 1024MB, 1536MB, and 3008MB); you can provide any power values between 128MB and 10,240MB
* **num** (required, integer): the # of invocations for each power configuration (minimum 5, recommended: between 10 and 100)
-* **payload** (string, object, or list): the static payload that will be used for every invocation (object or string); when using a list, a weighted payload is expected in the shape of `[{"payload": {...}, "weight": X }, {"payload": {...}, "weight": Y }, {"payload": {...}, "weight": Z }]`, where the weights `X`, `Y`, and `Z` are treated as relative weights (not perentages); more details below in the [Weighted Payloads section](#user-content-weighted-payloads)
+* **payload** (string, object, or list): the static payload that will be used for every invocation (object or string); when using a list, a weighted payload is expected in the shape of `[{"payload": {...}, "weight": X }, {"payload": {...}, "weight": Y }, {"payload": {...}, "weight": Z }]`, where the weights `X`, `Y`, and `Z` are treated as relative weights (not percentages); more details below in the [Weighted Payloads section](#user-content-weighted-payloads)
* **payloadS3** (string): a reference to Amazon S3 for large payloads (>256KB), formatted as `s3://bucket/key`; it requires read-only IAM permissions, see `payloadS3Bucket` and `payloadS3Key` below and find more details in the [S3 payloads section](#user-content-s3-payloads)
* **parallelInvocation** (false by default): if true, all the invocations will be executed in parallel (note: depending on the value of `num`, you may experience throttling when setting `parallelInvocation` to true)
* **strategy** (string): it can be `"cost"` or `"speed"` or `"balanced"` (the default value is `"cost"`); if you use `"cost"` the state machine will suggest the cheapest option (disregarding its performance), while if you use `"speed"` the state machine will suggest the fastest option (disregarding its cost). When using `"balanced"` the state machine will choose a compromise between `"cost"` and `"speed"` according to the parameter `"balancedWeight"`
@@ -35,7 +35,7 @@ The CloudFormation template accepts the following parameters:
* **payloadS3Bucket** (string): the S3 bucket name used for large payloads (>256KB); if provided, it's added to a custom managed IAM policy that grants read-only permission to the S3 bucket; more details below in the [S3 payloads section](#user-content-s3-payloads)
* **payloadS3Key** (string, default=`*`): they S3 object key used for large payloads (>256KB); the default value grants access to all S3 objects in the bucket specified with `payloadS3Bucket`; more details below in the [S3 payloads section](#user-content-s3-payloads)
-Please note that the total execution time should stay below 300 seconds (5 min), which is the default timeout. You can easily estimate the total execution timout based on the average duration of your functions. For example, if your function's average execution time is 5 seconds and you haven't enabled `parallelInvocation`, you should set `totalExecutionTimeout` to at least `num * 5`: 50 seconds if `num=10`, 500 seconds if `num=100`, and so on. If you have enabled `parallelInvocation`, usually you don't need to tune the value of `totalExecutionTimeout` unless your average execution time is above 5 min.
+Please note that the total execution time should stay below 300 seconds (5 min), which is the default timeout. You can easily estimate the total execution timeout based on the average duration of your functions. For example, if your function's average execution time is 5 seconds and you haven't enabled `parallelInvocation`, you should set `totalExecutionTimeout` to at least `num * 5`: 50 seconds if `num=10`, 500 seconds if `num=100`, and so on. If you have enabled `parallelInvocation`, usually you don't need to tune the value of `totalExecutionTimeout` unless your average execution time is above 5 min.
### Usage in CI/CD pipelines
diff --git a/README-SAR.md b/README-SAR.md
index d4a89100..3c857cec 100644
--- a/README-SAR.md
+++ b/README-SAR.md
@@ -37,12 +37,12 @@ Once the execution has completed, you will find the execution results in the "**
## State machine input (at execution time)
-The state machine accepts the following intput parameters:
+The state machine accepts the following input parameters:
* **lambdaARN** (required, string): unique identifier of the Lambda function you want to optimize
* **powerValues** (optional, string or list of integers): the list of power values to be tested; if not provided, the default values configured at deploy-time are used (by default: 128MB, 256MB, 512MB, 1024MB, 1536MB, and 3008MB); you can provide any power values between 128MB and 10,240MB
* **num** (required, integer): the # of invocations for each power configuration (minimum 5, recommended: between 10 and 100)
-* **payload** (string, object, or list): the static payload that will be used for every invocation (object or string); when using a list, a weighted payload is expected in the shape of `[{"payload": {...}, "weight": X }, {"payload": {...}, "weight": Y }, {"payload": {...}, "weight": Z }]`, where the weights `X`, `Y`, and `Z` are treated as relative weights (not perentages); more details below in the Weighted Payloads section
+* **payload** (string, object, or list): the static payload that will be used for every invocation (object or string); when using a list, a weighted payload is expected in the shape of `[{"payload": {...}, "weight": X }, {"payload": {...}, "weight": Y }, {"payload": {...}, "weight": Z }]`, where the weights `X`, `Y`, and `Z` are treated as relative weights (not percentages); more details below in the Weighted Payloads section
* **payloadS3** (string): a reference to Amazon S3 for large payloads (>256KB), formatted as `s3://bucket/key`; it requires read-only IAM permissions, see `payloadS3Bucket` and `payloadS3Key` below and find more details in the S3 payloads section
* **parallelInvocation** (false by default): if true, all the invocations will be executed in parallel (note: depending on the value of `num`, you may experience throttling when setting `parallelInvocation` to true)
* **strategy** (string): it can be `"cost"` or `"speed"` or `"balanced"` (the default value is `"cost"`); if you use `"cost"` the state machine will suggest the cheapest option (disregarding its performance), while if you use `"speed"` the state machine will suggest the fastest option (disregarding its cost). When using `"balanced"` the state machine will choose a compromise between `"cost"` and `"speed"` according to the parameter `"balancedWeight"`
@@ -66,7 +66,7 @@ The CloudFormation template accepts the following parameters:
* **payloadS3Bucket** (string): the S3 bucket name used for large payloads (>256KB); if provided, it's added to a custom managed IAM policy that grants read-only permission to the S3 bucket; more details below in the [S3 payloads section](#user-content-s3-payloads)
* **payloadS3Key** (string, default=`*`): they S3 object key used for large payloads (>256KB); the default value grants access to all S3 objects in the bucket specified with `payloadS3Bucket`; more details below in the [S3 payloads section](#user-content-s3-payloads)
-Please note that the total execution time should stay below 300 seconds (5 min), which is the default timeout. You can easily estimate the total execution timout based on the average duration of your functions. For example, if your function's average execution time is 5 seconds and you haven't enabled `parallelInvocation`, you should set `totalExecutionTimeout` to at least `num * 5`: 50 seconds if `num=10`, 500 seconds if `num=100`, and so on. If you have enabled `parallelInvocation`, usually you don't need to tune the value of `totalExecutionTimeout` unless your average execution time is above 5 min.
+Please note that the total execution time should stay below 300 seconds (5 min), which is the default timeout. You can easily estimate the total execution timeout based on the average duration of your functions. For example, if your function's average execution time is 5 seconds and you haven't enabled `parallelInvocation`, you should set `totalExecutionTimeout` to at least `num * 5`: 50 seconds if `num=10`, 500 seconds if `num=100`, and so on. If you have enabled `parallelInvocation`, usually you don't need to tune the value of `totalExecutionTimeout` unless your average execution time is above 5 min.
### Usage in CI/CD pipelines
@@ -223,7 +223,7 @@ There are three main costs associated with AWS Lambda Power Tuning:
## Error handling
-If something goes wrong during the initialization or execution states, the `CleanUpOnError` step will be executed. All temporary versions and alises will be deleted as expected (the same happens in the `Cleaner` step).
+If something goes wrong during the initialization or execution states, the `CleanUpOnError` step will be executed. All temporary versions and aliases will be deleted as expected (the same happens in the `Cleaner` step).
You can customize the `totalExecutionTimeout` parameter at deploy time (up to 15min). This parameter will be used both for Lambda function timeouts and Step Function tasks timeouts. In case the `Executor` raises a timeout error, you will see a `States.Timeout` error. Keep in mind that the timeout you configure will vary whether you're setting `parallelInvocation` to `true` or `false`. When you enable parallel invocation, all the function executions will run concurrently (rather than in series) so that you can keep that timeout lower and your overall state machine execution faster.
diff --git a/lambda/executor.js b/lambda/executor.js
index b3efe681..dadc2135 100644
--- a/lambda/executor.js
+++ b/lambda/executor.js
@@ -12,7 +12,7 @@ const minRAM = parseInt(process.env.minRAM, 10);
/**
* Execute the given function N times in series or in parallel.
- * Then compute execution statistics (averate cost and duration).
+ * Then compute execution statistics (average cost and duration).
*/
module.exports.handler = async(event, context) => {
// read input from event
diff --git a/lambda/utils.js b/lambda/utils.js
index 67ffbfd1..8a294766 100644
--- a/lambda/utils.js
+++ b/lambda/utils.js
@@ -440,7 +440,7 @@ module.exports.computeTotalCost = (minCost, minRAM, value, durations) => {
return 0;
}
- // compute corresponding cost for each durationo
+ // compute corresponding cost for each duration
const costs = durations.map(duration => utils.computePrice(minCost, minRAM, value, duration));
// sum all together
@@ -448,7 +448,7 @@ module.exports.computeTotalCost = (minCost, minRAM, value, durations) => {
};
/**
- * Copute average duration
+ * Compute average duration
*/
module.exports.computeAverageDuration = (durations, discardTopBottom) => {
if (!durations || !durations.length) {
From 62239993feceb65a7b72cf9e5740c30ded971447 Mon Sep 17 00:00:00 2001
From: Parro <29497+Parro@users.noreply.github.com>
Date: Wed, 20 Jul 2022 14:50:10 +0200
Subject: [PATCH 3/7] Fix wrong parameter name
---
lambda/executor.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lambda/executor.js b/lambda/executor.js
index dadc2135..0fff1eba 100644
--- a/lambda/executor.js
+++ b/lambda/executor.js
@@ -82,7 +82,7 @@ const extractPayloadValue = async(input) => {
const extractDiscardTopBottomValue = (event) => {
// extract discardTopBottom used to trim values from average duration
- let discardTopBottom = event.extractDiscardTopBottom;
+ let discardTopBottom = event.discardTopBottom;
if (typeof discardTopBottom === 'undefined') {
discardTopBottom = 0.2;
}
From 897ffddbe2519b5d00c0dad9eb425f0b66e16995 Mon Sep 17 00:00:00 2001
From: Parro <29497+Parro@users.noreply.github.com>
Date: Wed, 20 Jul 2022 14:50:16 +0200
Subject: [PATCH 4/7] Add tests
---
test/unit/test-lambda.js | 107 +++++++++++++++++++++++++++++++++++++++
test/unit/test-utils.js | 11 ++--
2 files changed, 115 insertions(+), 3 deletions(-)
diff --git a/test/unit/test-lambda.js b/test/unit/test-lambda.js
index e3060b81..01499b5e 100644
--- a/test/unit/test-lambda.js
+++ b/test/unit/test-lambda.js
@@ -1123,6 +1123,113 @@ describe('Lambda Functions', async() => {
expect(getLambdaArchitectureCounter).to.be(1);
});
+ const discardTopBottomValues = [
+ // set to 0.4, maximum value
+ 0.7,
+ 0.4,
+ // default value
+ 0.2,
+ // no trimming
+ 0,
+ ];
+ const trimmedDurationsValues = [
+ 3.5,
+ 3.5,
+ 3.5833333333333335,
+ 27.21,
+ ];
+
+ discardTopBottomValues.forEach((discardTopBottomValue, forEachIndex) => {
+ console.log('extractDiscardTopBottomValue', discardTopBottomValue);
+ it(`should discard ${discardTopBottomValue * 100}% of durations`, async() => {
+ const logResults = [
+ // 0.1s
+ {
+ StatusCode: 200,
+ LogResult: 'U1RBUlQgUmVxdWVzdElkOiA0NzlmYjUxYy0xZTM4LTExZTctOTljYS02N2JmMTYzNjA4ZWQgVmVyc2lvbjogOTkKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTEgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTIgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTMgPSB1bmRlZmluZWQKRU5EIFJlcXVlc3RJZDogNDc5ZmI1MWMtMWUzOC0xMWU3LTk5Y2EtNjdiZjE2MzYwOGVkClJFUE9SVCBSZXF1ZXN0SWQ6IDQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAlEdXJhdGlvbjogMC4xIG1zCUJpbGxlZCBEdXJhdGlvbjogMTAwIG1zCUluaXQgRHVyYXRpb246IDAuMSBtcyAJTWVtb3J5IFNpemU6IDEyOCBNQglNYXggTWVtb3J5IFVzZWQ6IDE1IE1CCQo=',
+ Payload: 'null',
+ },
+ // 0.5s
+ {
+ StatusCode: 200,
+ LogResult: 'U1RBUlQgUmVxdWVzdElkOiA0NzlmYjUxYy0xZTM4LTExZTctOTljYS02N2JmMTYzNjA4ZWQgVmVyc2lvbjogOTkKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTEgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTIgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTMgPSB1bmRlZmluZWQKRU5EIFJlcXVlc3RJZDogNDc5ZmI1MWMtMWUzOC0xMWU3LTk5Y2EtNjdiZjE2MzYwOGVkClJFUE9SVCBSZXF1ZXN0SWQ6IDQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAlEdXJhdGlvbjogMC41IG1zCUJpbGxlZCBEdXJhdGlvbjogMTAwIG1zIAlNZW1vcnkgU2l6ZTogMTI4IE1CCU1heCBNZW1vcnkgVXNlZDogMTUgTUIJCg==',
+ Payload: 'null',
+ },
+ // 2s
+ {
+ StatusCode: 200,
+ LogResult: 'U1RBUlQgUmVxdWVzdElkOiA0NzlmYjUxYy0xZTM4LTExZTctOTljYS02N2JmMTYzNjA4ZWQgVmVyc2lvbjogOTkKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTEgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTIgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTMgPSB1bmRlZmluZWQKRU5EIFJlcXVlc3RJZDogNDc5ZmI1MWMtMWUzOC0xMWU3LTk5Y2EtNjdiZjE2MzYwOGVkClJFUE9SVCBSZXF1ZXN0SWQ6IDQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAlEdXJhdGlvbjogMi4wIG1zCUJpbGxlZCBEdXJhdGlvbjogMTAwIG1zIAlNZW1vcnkgU2l6ZTogMTI4IE1CCU1heCBNZW1vcnkgVXNlZDogMTUgTUIJCg==',
+ Payload: 'null',
+ },
+ // 3s
+ {
+ StatusCode: 200,
+ LogResult: 'U1RBUlQgUmVxdWVzdElkOiA0NzlmYjUxYy0xZTM4LTExZTctOTljYS02N2JmMTYzNjA4ZWQgVmVyc2lvbjogOTkKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTEgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTIgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTMgPSB1bmRlZmluZWQKRU5EIFJlcXVlc3RJZDogNDc5ZmI1MWMtMWUzOC0xMWU3LTk5Y2EtNjdiZjE2MzYwOGVkClJFUE9SVCBSZXF1ZXN0SWQ6IDQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAlEdXJhdGlvbjogMy4wIG1zCUJpbGxlZCBEdXJhdGlvbjogMTAwIG1zIAlNZW1vcnkgU2l6ZTogMTI4IE1CCU1heCBNZW1vcnkgVXNlZDogMTUgTUIJCg==',
+ Payload: 'null',
+ },
+ // 3s
+ {
+ StatusCode: 200,
+ LogResult: 'U1RBUlQgUmVxdWVzdElkOiA0NzlmYjUxYy0xZTM4LTExZTctOTljYS02N2JmMTYzNjA4ZWQgVmVyc2lvbjogOTkKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTEgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTIgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTMgPSB1bmRlZmluZWQKRU5EIFJlcXVlc3RJZDogNDc5ZmI1MWMtMWUzOC0xMWU3LTk5Y2EtNjdiZjE2MzYwOGVkClJFUE9SVCBSZXF1ZXN0SWQ6IDQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAlEdXJhdGlvbjogMy4wIG1zCUJpbGxlZCBEdXJhdGlvbjogMTAwIG1zIAlNZW1vcnkgU2l6ZTogMTI4IE1CCU1heCBNZW1vcnkgVXNlZDogMTUgTUIJCg==',
+ Payload: 'null',
+ },
+ // 4s
+ {
+ StatusCode: 200,
+ LogResult: 'U1RBUlQgUmVxdWVzdElkOiA0NzlmYjUxYy0xZTM4LTExZTctOTljYS02N2JmMTYzNjA4ZWQgVmVyc2lvbjogOTkKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTEgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTIgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTMgPSB1bmRlZmluZWQKRU5EIFJlcXVlc3RJZDogNDc5ZmI1MWMtMWUzOC0xMWU3LTk5Y2EtNjdiZjE2MzYwOGVkClJFUE9SVCBSZXF1ZXN0SWQ6IDQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAlEdXJhdGlvbjogNC4wIG1zCUJpbGxlZCBEdXJhdGlvbjogMTAwIG1zIAlNZW1vcnkgU2l6ZTogMTI4IE1CCU1heCBNZW1vcnkgVXNlZDogMTUgTUIJCg==',
+ Payload: 'null',
+ },
+ // 4.5s
+ {
+ StatusCode: 200,
+ LogResult: 'U1RBUlQgUmVxdWVzdElkOiA0NzlmYjUxYy0xZTM4LTExZTctOTljYS02N2JmMTYzNjA4ZWQgVmVyc2lvbjogOTkKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTEgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTIgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTMgPSB1bmRlZmluZWQKRU5EIFJlcXVlc3RJZDogNDc5ZmI1MWMtMWUzOC0xMWU3LTk5Y2EtNjdiZjE2MzYwOGVkClJFUE9SVCBSZXF1ZXN0SWQ6IDQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAlEdXJhdGlvbjogNC41IG1zCUJpbGxlZCBEdXJhdGlvbjogMTAwIG1zIAlNZW1vcnkgU2l6ZTogMTI4IE1CCU1heCBNZW1vcnkgVXNlZDogMTUgTUIJCg==',
+ Payload: 'null',
+ },
+ // 5s
+ {
+ StatusCode: 200,
+ LogResult: 'U1RBUlQgUmVxdWVzdElkOiA0NzlmYjUxYy0xZTM4LTExZTctOTljYS02N2JmMTYzNjA4ZWQgVmVyc2lvbjogOTkKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTEgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTIgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTMgPSB1bmRlZmluZWQKRU5EIFJlcXVlc3RJZDogNDc5ZmI1MWMtMWUzOC0xMWU3LTk5Y2EtNjdiZjE2MzYwOGVkClJFUE9SVCBSZXF1ZXN0SWQ6IDQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAlEdXJhdGlvbjogNS4wIG1zCUJpbGxlZCBEdXJhdGlvbjogMTAwIG1zIAlNZW1vcnkgU2l6ZTogMTI4IE1CCU1heCBNZW1vcnkgVXNlZDogMTUgTUIJCg==',
+ Payload: 'null',
+ },
+ // 50s
+ {
+ StatusCode: 200,
+ LogResult: 'U1RBUlQgUmVxdWVzdElkOiA0NzlmYjUxYy0xZTM4LTExZTctOTljYS02N2JmMTYzNjA4ZWQgVmVyc2lvbjogOTkKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTEgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTIgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTMgPSB1bmRlZmluZWQKRU5EIFJlcXVlc3RJZDogNDc5ZmI1MWMtMWUzOC0xMWU3LTk5Y2EtNjdiZjE2MzYwOGVkClJFUE9SVCBSZXF1ZXN0SWQ6IDQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAlEdXJhdGlvbjogNTAuMCBtcwlCaWxsZWQgRHVyYXRpb246IDEwMCBtcyAJTWVtb3J5IFNpemU6IDEyOCBNQglNYXggTWVtb3J5IFVzZWQ6IDE1IE1CCQo=',
+ Payload: 'null',
+ },
+ // 200s
+ {
+ StatusCode: 200,
+ LogResult: 'U1RBUlQgUmVxdWVzdElkOiA0NzlmYjUxYy0xZTM4LTExZTctOTljYS02N2JmMTYzNjA4ZWQgVmVyc2lvbjogOTkKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTEgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTIgPSB1bmRlZmluZWQKMjAxNy0wNC0xMFQyMTo1NDozMi42ODNaCTQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAl2YWx1ZTMgPSB1bmRlZmluZWQKRU5EIFJlcXVlc3RJZDogNDc5ZmI1MWMtMWUzOC0xMWU3LTk5Y2EtNjdiZjE2MzYwOGVkClJFUE9SVCBSZXF1ZXN0SWQ6IDQ3OWZiNTFjLTFlMzgtMTFlNy05OWNhLTY3YmYxNjM2MDhlZAlEdXJhdGlvbjogMjAwLjAgbXMJQmlsbGVkIER1cmF0aW9uOiAxMDAgbXMgCU1lbW9yeSBTaXplOiAxMjggTUIJTWF4IE1lbW9yeSBVc2VkOiAxNSBNQgkK',
+ Payload: 'null',
+ },
+ ];
+
+ let invokeCounter = 0;
+ invokeLambdaStub && invokeLambdaStub.restore();
+ invokeLambdaStub = sandBox.stub(utils, 'invokeLambda')
+ .callsFake(async(_arn, _alias, payload) => {
+ invokeLambdaPayloads.push(payload);
+ const logResult = logResults[invokeCounter];
+ invokeCounter++;
+
+ return logResult;
+ });
+
+ const response = await invokeForSuccess(handler, {
+ value: '128',
+ input: {
+ lambdaARN: 'arnOK',
+ num: 10,
+ discardTopBottom: discardTopBottomValue,
+ },
+ });
+
+ console.log('response', response);
+
+ expect(response.averageDuration).to.be(trimmedDurationsValues[forEachIndex]);
+ });
+ });
});
describe('analyzer', () => {
diff --git a/test/unit/test-utils.js b/test/unit/test-utils.js
index 065caf98..2dc99703 100644
--- a/test/unit/test-utils.js
+++ b/test/unit/test-utils.js
@@ -210,16 +210,21 @@ describe('Lambda Utils', () => {
describe('computeAverageDuration', () => {
const durations = [
- 1, 1, 2, 3, 2000,
+ 1, 1, 2, 3, 6, 2000,
];
it('should return the average duration', () => {
const duration = utils.computeAverageDuration(durations, 0.2);
- expect(duration).to.be(2);
+ expect(duration).to.be(3);
+ });
+
+ it('should return the average duration custom trimming', () => {
+ const duration = utils.computeAverageDuration(durations, 0.4);
+ expect(duration).to.be(2.5);
});
it('should return the average duration with no trimmed value', () => {
const duration = utils.computeAverageDuration(durations, 0);
- expect(duration).to.be(401.4);
+ expect(duration).to.be(335.5);
});
it('should return 0 if empty results', () => {
const duration = utils.computeAverageDuration([]);
From ca69f1bd4fc509b2f6854feb94fd06f32201fd2d Mon Sep 17 00:00:00 2001
From: Alex Casalboni
Date: Mon, 8 Aug 2022 14:41:23 +0200
Subject: [PATCH 5/7] Minor doc update for discardTopBottom
---
README-INPUT-OUTPUT.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README-INPUT-OUTPUT.md b/README-INPUT-OUTPUT.md
index 46f38294..f022e766 100644
--- a/README-INPUT-OUTPUT.md
+++ b/README-INPUT-OUTPUT.md
@@ -20,7 +20,7 @@ The state machine accepts the following input parameters:
* **dryRun** (false by default): if true, the state machine will execute the input function only once and it will disable every functionality related to logs analysis, auto-tuning, and visualization; the dry-run mode is intended for testing purposes, for example to verify that IAM permissions are set up correctly
* **preProcessorARN** (string): it must be the ARN of a Lambda function; if provided, the function will be invoked before every invocation of `lambdaARN`; more details below in the [Pre/Post-processing functions section](#user-content-prepost-processing-functions)
* **postProcessorARN** (string): it must be the ARN of a Lambda function; if provided, the function will be invoked after every invocation of `lambdaARN`; more details below in the [Pre/Post-processing functions section](#user-content-prepost-processing-functions)
-* **discardTopBottom** (number between 0.0 and 0.4, by default is 0.2): By default, the state machine will discard the top/bottom 20% of "outliers" (the fastest and slowest), to filter out the effects of cold starts that would bias the overall averages. This behaviour could be changed with this parameter, that could take a value between 0 and 0.4, with 0 meaning "no trimmed mean at all" and 0.4 meaning "trim 40% of the top/bottom results" (resulting in only 20% of the results being considered).
+* **discardTopBottom** (number between 0.0 and 0.4, by default is 0.2): By default, the state machine will discard the top/bottom 20% of "outliers" (the fastest and slowest), to filter out the effects of cold starts that would bias the overall averages. You can customize this parameter by providing a value between 0 and 0.4, with 0 meaning no results are discarded and 0.4 meaning that 40% of the top/bottom results are discarded (i.e. only 20% of the results are considered).
## State machine configuration (at deployment time)
From 4799ff4b3ea61b56c7a9c0bc8a1df13c1a892023 Mon Sep 17 00:00:00 2001
From: Alex Casalboni
Date: Mon, 8 Aug 2022 14:43:24 +0200
Subject: [PATCH 6/7] Add discardTopBottom input parameter to SAR doc
---
README-SAR.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/README-SAR.md b/README-SAR.md
index 3c857cec..f9f0b771 100644
--- a/README-SAR.md
+++ b/README-SAR.md
@@ -52,6 +52,7 @@ The state machine accepts the following input parameters:
* **dryRun** (false by default): if true, the state machine will execute the input function only once and it will disable every functionality related to logs analysis, auto-tuning, and visualization; the dry-run mode is intended for testing purposes, for example to verify that IAM permissions are set up correctly
* **preProcessorARN** (string): it must be the ARN of a Lambda function; if provided, the function will be invoked before every invocation of `lambdaARN`; more details below in the Pre/Post-processing functions section
* **postProcessorARN** (string): it must be the ARN of a Lambda function; if provided, the function will be invoked after every invocation of `lambdaARN`; more details below in the Pre/Post-processing functions section
+* **discardTopBottom** (number between 0.0 and 0.4, by default is 0.2): By default, the state machine will discard the top/bottom 20% of "outliers" (the fastest and slowest), to filter out the effects of cold starts that would bias the overall averages. You can customize this parameter by providing a value between 0 and 0.4, with 0 meaning no results are discarded and 0.4 meaning that 40% of the top/bottom results are discarded (i.e. only 20% of the results are considered).
## State machine configuration (at deployment time)
From 8b9212688d3f902f731c916a8a984e0841ac91a3 Mon Sep 17 00:00:00 2001
From: Alex Casalboni
Date: Mon, 8 Aug 2022 15:07:21 +0200
Subject: [PATCH 7/7] Update computeAverageDuration tests and improved logging
---
lambda/utils.js | 7 +++++++
test/unit/test-utils.js | 14 ++++++++++----
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/lambda/utils.js b/lambda/utils.js
index 8a294766..a03fcfad 100644
--- a/lambda/utils.js
+++ b/lambda/utils.js
@@ -458,6 +458,13 @@ module.exports.computeAverageDuration = (durations, discardTopBottom) => {
// a percentage of durations will be discarded (trimmed mean)
const toBeDiscarded = parseInt(durations.length * discardTopBottom, 10);
+ if (discardTopBottom > 0 && toBeDiscarded === 0) {
+ // not an error, but worth logging
+ // this happens when you have less than 5 invocations
+ // (only happens if dryrun or in tests)
+ console.log("not enough results to discard");
+ }
+
const newN = durations.length - 2 * toBeDiscarded;
// compute trimmed mean (discard a percentage of low/high values)
diff --git a/test/unit/test-utils.js b/test/unit/test-utils.js
index 2dc99703..98a7e929 100644
--- a/test/unit/test-utils.js
+++ b/test/unit/test-utils.js
@@ -210,7 +210,9 @@ describe('Lambda Utils', () => {
describe('computeAverageDuration', () => {
const durations = [
- 1, 1, 2, 3, 6, 2000,
+ // keep 5 values because it's the minimum length
+ // `num` can't be smaller than 5, unless it's a dryrun
+ 1, 2, 3, 4, 2000,
];
it('should return the average duration', () => {
@@ -220,14 +222,18 @@ describe('Lambda Utils', () => {
it('should return the average duration custom trimming', () => {
const duration = utils.computeAverageDuration(durations, 0.4);
- expect(duration).to.be(2.5);
+ expect(duration).to.be(3);
});
it('should return the average duration with no trimmed value', () => {
const duration = utils.computeAverageDuration(durations, 0);
- expect(duration).to.be(335.5);
+ expect(duration).to.be(402);
+ });
+ it('should return the average duration even if not enough results to discard', () => {
+ const duration = utils.computeAverageDuration([1], 0.4);
+ expect(duration).to.be(1);
});
it('should return 0 if empty results', () => {
- const duration = utils.computeAverageDuration([]);
+ const duration = utils.computeAverageDuration([], 0.2);
expect(duration).to.be(0);
});
});