test(fargate): add aws fargate tests to cli repo (#2407)

* test(fargate): add simple bom test

* test(fargate): add mixed hierarchy test

* test(fargate): add ensure test

* test(fargate): add ensure and expect failure test

* test(fargate): add expect failure test

* test(fargate): add kitchen sink test with multiple features

* test(fargate): add lots of output test

* test(fargate): add memory hog test

* test(fargate): increase think time in memory-hog

* test(fargate): increase test timeout due to aws tests being long

* test(cli): create report abstraction and use in fargate tests

* test(fargate): record fargate tests to cloud dashboard

* test(fargate): rename fargate scenarios to improve debugging

* test(fargate): increase deps for mixed-hierarchy test

* test(fargate): use 51 workers as in original test

packages/artillery/package.json

@@ -46,7 +46,7 @@
     "test:unit": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --timeout=420 --color test/unit/*.test.js",
     "test:acceptance": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --timeout=420 test/cli/*.test.js && bash test/lib/run.sh && tap --no-coverage --color test/testcases/plugins/*.test.js",
     "test": "npm run test:unit && npm run test:acceptance",
-    "test:aws": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --color --timeout=600 test/cloud-e2e/**/*.test.js",
+    "test:aws": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --color --timeout=3600 test/cloud-e2e/**/*.test.js",
     "lint": "eslint --ext \".js,.ts,.tsx\" .",
     "lint-fix": "npm run lint -- --fix"
   },

packages/artillery/test/cli/_helpers.js

@@ -5,6 +5,7 @@ const path = require('path');
 const os = require('os');
 const { getBinPathSync } = require('get-bin-path');
 const a9path = getBinPathSync();
+const { createHash } = require('crypto');
 
 async function execute(args, options) {
   try {
@@ -30,6 +31,14 @@ async function getRootPath(filename) {
   return path.resolve(__dirname, '..', '..', filename);
 }
 
+function generateTmpReportPath(testName, extension) {
+  return returnTmpPath(
+    `report-${createHash('md5')
+      .update(testName)
+      .digest('hex')}-${Date.now()}.${extension}`
+  );
+}
+
 function getTestTags(additionalTags) {
   const actorTag = `actor:${process.env.GITHUB_ACTOR || 'localhost'}`;
   const repoTag = `repo:${process.env.GITHUB_REPO || 'artilleryio/artillery'}`;
@@ -43,5 +52,6 @@ module.exports = {
   deleteFile,
   getRootPath,
   returnTmpPath,
+  generateTmpReportPath,
   getTestTags
 };

packages/artillery/test/cli/command-report.test.js

@@ -1,8 +1,8 @@
 const tap = require('tap');
-const { execute, returnTmpPath } = require('../cli/_helpers.js');
+const { execute, generateTmpReportPath } = require('../cli/_helpers.js');
 
 tap.test('If we report specifying output, no browser is opened', async (t) => {
-  const outputFilePath = returnTmpPath('report.html');
+  const outputFilePath = generateTmpReportPath(t.name, 'html');
 
   const [exitCode] = await execute([
     'report',

packages/artillery/test/cli/command-run.test.js

@@ -3,20 +3,16 @@ const {
   execute,
   deleteFile,
   getRootPath,
-  returnTmpPath
+  returnTmpPath,
+  generateTmpReportPath
 } = require('../cli/_helpers.js');
 const fs = require('fs');
 const path = require('path');
 const execa = require('execa');
-const { createHash } = require('crypto');
 
 let reportFilePath;
 tap.beforeEach(async (t) => {
-  reportFilePath = returnTmpPath(
-    `report-${createHash('md5')
-      .update(t.name)
-      .digest('hex')}-${Date.now()}.json`
-  );
+  reportFilePath = generateTmpReportPath(t.name, 'json');
 });
 
 tap.test('Run a simple script', async (t) => {

packages/artillery/test/cli/run-typescript.test.js

@@ -1,15 +1,10 @@
 const tap = require('tap');
-const { execute, returnTmpPath } = require('../cli/_helpers.js');
-const { createHash } = require('crypto');
+const { execute, generateTmpReportPath } = require('../cli/_helpers.js');
 const fs = require('fs');
 
 let reportFilePath;
 tap.beforeEach(async (t) => {
-  reportFilePath = returnTmpPath(
-    `report-${createHash('md5')
-      .update(t.name)
-      .digest('hex')}-${Date.now()}.json`
-  );
+  reportFilePath = generateTmpReportPath(t.name, 'json');
 });
 
 tap.test('Can run a Typescript processor', async (t) => {

packages/artillery/test/cloud-e2e/fargate/fixtures/cli-exit-conditions/with-expect-ensure.yml

@@ -0,0 +1,18 @@
+config:
+  target: "http://asciiart.artillery.io:8080"
+  plugins:
+    expect: {}
+    ensure:
+      p99: 10000
+      thresholds:
+        - "http.response_time.p95": 1
+  phases:
+    - duration: 5
+      arrivalRate: 1
+scenarios:
+  - name: expect-ensure-exit-condition-test
+    flow:
+      - get:
+          url: "/"
+          expect:
+            - statusCode: 300

packages/artillery/test/cloud-e2e/fargate/fixtures/cli-exit-conditions/with-expect.yml

@@ -0,0 +1,14 @@
+config:
+  target: "http://asciiart.artillery.io:8080"
+  plugins:
+    expect: {}
+  phases:
+    - duration: 5
+      arrivalRate: 1
+scenarios:
+  - name: expect-exit-condition-test
+    flow:
+      - get:
+          url: "/"
+          expect:
+            - statusCode: 300

packages/artillery/test/cloud-e2e/fargate/fixtures/cli-kitchen-sink/kitchen-sink-env

@@ -0,0 +1,2 @@
+DOTENV1=/
+DOTENV2=/dino

packages/artillery/test/cloud-e2e/fargate/fixtures/cli-kitchen-sink/kitchen-sink.yml

@@ -0,0 +1,22 @@
+config:
+  target: http://asciiart.artillery.io:8080
+  plugins:
+    ensure: {}
+  phases:
+    - duration: 20
+      arrivalRate: 1
+  ensure:
+    p99: 10000
+    thresholds:
+      - "http.response_time.p99": 10000
+scenarios:
+  - name: load homepage
+    flow:
+      - get:
+          url: "{{$processEnvironment.SECRET1}}"
+      - get:
+          url: "{{$processEnvironment.SECRET2}}"
+      - get:
+          url: "{{$processEnvironment.DOTENV1}}"
+      - get:
+          url: "{{$processEnvironment.DOTENV2}}"

packages/artillery/test/cloud-e2e/fargate/fixtures/large-output/lots-of-output.yml

@@ -0,0 +1,14 @@
+config:
+  target: http://asciiart.artillery.io:8080
+  phases:
+    - arrivalRate: 1
+      duration: 60
+  defaults:
+    headers:
+      user-agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36"
+  processor: "./processor.js"
+scenarios:
+  - flow:
+    - get:
+        url: "/"
+        afterResponse: logOutput

packages/artillery/test/cloud-e2e/fargate/fixtures/large-output/processor.js

@@ -0,0 +1,19 @@
+function logOutput(req, res, userContext, events, done) {
+  for (let i = 0; i < 10; i++) {
+    events.emit(
+      'counter',
+      `very.very.long.name.for.a.counter.metric.so.that.we.generate.a.lot.of.console.output.${Date.now()}${i}`,
+      1
+    );
+    events.emit(
+      'histogram',
+      `very.very.long.name.for.a.histogram.metric.so.that.we.generate.a.lot.of.console.output.${Date.now()}${i}`,
+      100
+    );
+  }
+  return done();
+}
+
+module.exports = {
+  logOutput
+};

packages/artillery/test/cloud-e2e/fargate/fixtures/memory-hog/memory-hog.yml

@@ -0,0 +1,20 @@
+# 10 VUs, making 10 calls each to a function that allocates 100MB -> ~10GB total
+# This will fail with default configuration, which is:
+# - 4GB RAM given to workers on Fargate
+# - Node.js memory limit of 4GB
+
+config:
+  target: http://asciiart.artillery.io:8080
+  phases:
+    - arrivalRate: 1
+      duration: 10
+  processor: './processor.js'
+
+scenarios:
+  - flow:
+      - loop:
+          - get:
+              url: "/armadillo"
+              beforeRequest: hogSomeRam
+          - think: 1
+        count: 10

packages/artillery/test/cloud-e2e/fargate/fixtures/memory-hog/processor.js

@@ -0,0 +1,18 @@
+let data = [];
+
+console.log('NODE_OPTIONS:');
+console.log(process.env.NODE_OPTIONS);
+
+function hogSomeRam(req, context, events, next) {
+  // Allocate 100MB
+  data.push(Buffer.alloc(1024 * 1024 * 100, 1));
+
+  console.log(new Date(), 'allocated more data');
+  console.log('RSS (MB):', process.memoryUsage().rss / 1024 / 1024);
+
+  return next();
+}
+
+module.exports = {
+  hogSomeRam
+};

packages/artillery/test/cloud-e2e/fargate/fixtures/mixed-hierarchy/code/functions.js

@@ -0,0 +1,7 @@
+const bc = require('@babel/core');
+const uuid = require('uuid');
+const client = require('aws-sdk/clients/lambda');
+
+module.exports = {
+  setUrl: require('./set-url')
+};

packages/artillery/test/cloud-e2e/fargate/fixtures/mixed-hierarchy/code/lib/signer.js

@@ -0,0 +1,2 @@
+const meow = require('../meow');
+module.exports = require('aws4');

packages/artillery/test/cloud-e2e/fargate/fixtures/mixed-hierarchy/code/meow.js

@@ -0,0 +1,2 @@
+const lodash = require('lodash');
+module.export = 'meow';

packages/artillery/test/cloud-e2e/fargate/fixtures/mixed-hierarchy/code/set-url.js

@@ -0,0 +1,9 @@
+module.exports = setUrl;
+
+const AWS = require('aws-sdk');
+const signer = require('./lib/signer');
+
+function setUrl(req, res, ctx, ee, done) {
+  req.url = '/';
+  return done();
+}

packages/artillery/test/cloud-e2e/fargate/fixtures/mixed-hierarchy/config/config.yml

@@ -0,0 +1,22 @@
+config:
+  target: http://asciiart.artillery.io:8080
+  processor: "../code/functions.js"
+  environments:
+    main:
+      phases:
+        - duration: 20
+          arrivalRate: 1
+      payload:
+        - path: "../data/variables.csv" # this is resolved relative to the main script for now, NOT this file
+          fields: ["username", "email"]
+      plugins:
+        metrics-by-endpoint: {}
+    stage:
+      plugins:
+        datadog: {}
+  plugins:
+    http-file-uploads:
+      filePaths:
+        - ../data/files
+    publish-metrics:
+      - type: statsd

packages/artillery/test/cloud-e2e/fargate/fixtures/mixed-hierarchy/data/variables.csv

@@ -0,0 +1,4 @@
+bob,bob@example.com
+jane,jane@example.com
+aditya,aditya@example.com
+aadhya,aadhya@example.com