Merge pull request #1 from hassy/feat/templates

packages/artillery-plugin-expect/README.md

@@ -14,6 +14,8 @@ Add expectations to your HTTP scenarios for functional API testing with Artiller
 npm install -g artillery-plugin-expect
 ```
 
+**Important**: this plugin requires Artillery `v1.6.0-26` or higher.
+
 ### Enable the plugin in the config section
 
 ```yaml
@@ -43,6 +45,53 @@ scenarios:
               - "{{ name }}"
 ```
 
+### Run your test & see results
+
+Run your script that uses expectations with:
+
+```
+artillery run --quiet my-script.yaml
+```
+
+The `--quiet` option is to stop Artillery from printing its default reports to the console.
+
+Failed expectations provide request and response details:
+
+![artillery expectations plugin screenshot](./docs/expect-output.png)
+
+### Re-using scenarios as load tests or functional tests
+
+This plugin allows for the same scenario to be re-used for either load testing or functional testing of an API. (The only real difference between the two, of course, is how many virtual users you run -- only one for functional tests, and `$BIG_NUMBER` for a load test.)
+
+In practical terms, what you probably want to do is use the [`environments` functionality](https://artillery.io/docs/script-reference/#environments) in Artillery to create a separate "environment" with configuration for functional testing. Something like:
+
+```yaml
+config:
+  target: "https://my.api.internal"
+  environments:
+    #
+    # This is our load testing profile, where we create a lot of virtual users.
+    # Note that we don't load the plugin here, so that we don't see the output
+    # from the plugin.
+    #
+    load:
+      phases:
+        - duration: 600
+          arrivalRate: 10
+    #
+    # This is our functional testing profile, with a single virtual user, and
+    # the plugin enabled.
+    #
+    functional:
+      phases:
+        - duration: 1
+          arrivalCount: 1
+      plugins:
+        expect: {}
+scenarios:
+  # Your scenario definitions go here.
+```
+
 ## Expectations
 
 ### `statusCode`

packages/artillery-plugin-expect/lib/expectations.js

@@ -6,7 +6,7 @@
 
 const debug = require('debug')('plugin:expect');
 const chalk = require('chalk');
-const renderVariables = require('artillery/util').renderVariables;
+const template = global.artillery ? global.artillery.util.template : require('artillery/util').template;
 const _ = require('lodash');
 
 module.exports = {
@@ -16,9 +16,6 @@ module.exports = {
   equals: expectEquals
 };
 
-// FIXME: Current implementation only works with primitive values,
-// and forces everything to a string. Objects, lists, and type checks
-// can be implemented with template() exported from artillery/util.
 function expectEquals(expectation, body, req, res, userContext) {
   debug('check equals');
   debug('expectation:', expectation);
@@ -31,7 +28,7 @@ function expectEquals(expectation, body, req, res, userContext) {
   };
 
   const values = _.map(expectation.equals, (str) => {
-    return String(renderVariables(String(str), userContext.vars));
+    return String(template(String(str), userContext.vars));
   });
 
   const unique = _.uniq(values);
@@ -46,13 +43,14 @@ function expectContentType(expectation, body, req, res, userContext) {
   debug('expectation:', expectation);
   debug('body:', typeof body);
 
+  const expectedContentType = template(expectation.contentType, userContext);
   let result = {
     ok: false,
-    expected: expectation.contentType,
+    expected: expectedContentType,
     type: 'contentType'
   };
 
-  if (expectation.contentType === 'json') {
+  if (expectedContentType === 'json') {
     if (
       typeof body === 'object' &&
       res.headers['content-type'].indexOf('application/json') !== -1
@@ -69,7 +67,7 @@ function expectContentType(expectation, body, req, res, userContext) {
       return result;
     }
   } else {
-    result.ok = res.headers['content-type'] && res.headers['content-type'].toLowerCase() === expectation.contentType.toLowerCase();
+    result.ok = res.headers['content-type'] && res.headers['content-type'].toLowerCase() === expectedContentType.toLowerCase();
     result.got = res.headers['content-type'] || 'content-type header not set';
     return result;
   }
@@ -78,33 +76,36 @@ function expectContentType(expectation, body, req, res, userContext) {
 function expectStatusCode(expectation, body, req, res, userContext) {
   debug('check statusCode');
 
+  const expectedStatusCode = template(expectation.statusCode, userContext);
+
   let result = {
     ok: false,
-    expected: expectation.statusCode,
+    expected: expectedStatusCode,
     type: 'statusCode'
   };
 
-  result.ok = res.statusCode === expectation.statusCode;
+  result.ok = Number(res.statusCode) === Number(expectedStatusCode);
   result.got = res.statusCode;
   return result;
 }
 
 function expectHasProperty(expectation, body, req, res, userContext) {
   debug('check hasProperty');
 
+  const expectedProperty = template(expectation.hasProperty, userContext);
   let result = {
     ok: false,
-    expected: expectation.hasProperty,
+    expected: expectedProperty,
     type: 'hasProperty'
   };
 
   if (typeof body === 'object') {
-    if (_.has(body, expectation.hasProperty)) {
+    if (_.has(body, expectedProperty)) {
       result.ok = true;
-      result.got = `${body[expectation.hasProperty]}`;
+      result.got = expectedProperty;
       return result;
     } else {
-      result.got = `response body has no ${expectation.hasProperty} property`;
+      result.got = `response body has no ${expectedProperty} property`;
       return result;
     }
   } else {

packages/artillery-plugin-expect/package.json

@@ -17,7 +17,10 @@
   "devDependencies": {
     "@commitlint/cli": "^7.0.0",
     "@commitlint/config-conventional": "^7.0.1",
-    "husky": "^1.0.0-rc.13"
+    "artillery": "^1.6.0-26",
+    "ava": "^0.25.0",
+    "husky": "^1.0.0-rc.13",
+    "shelljs": "^0.8.3"
   },
   "husky": {
     "hooks": {

packages/artillery-plugin-expect/test/index.js

@@ -0,0 +1,79 @@
+'use strict';
+
+import test from 'ava';
+import createDebug from 'debug';
+const debug = createDebug('expect-plugin:test');
+import EventEmitter from 'events';
+
+const shelljs = require('shelljs');
+const path = require('path');
+
+//
+// We only need this when running unit tests. When the plugin actually runs inside
+// a recent version of Artillery, the appropriate object is already set up.
+//
+global.artillery = {
+  util: {
+    template: require('artillery/util').template
+  }
+};
+
+test('Basic interface checks', async t => {
+  const script = {
+    config: {},
+    scenarios: []
+  };
+
+  const ExpectationsPlugin = require('../index');
+  const events = new EventEmitter();
+  const plugin = new ExpectationsPlugin.Plugin(script, events);
+
+  t.true(typeof ExpectationsPlugin.Plugin === 'function');
+  t.true(typeof plugin === 'object');
+
+  t.pass();
+});
+
+test('Expectation: statusCode', async (t) => {
+  const expectations = require('../lib/expectations');
+
+  const data = [
+    // expectation - value received - user context - expected result
+    [ '{{ expectedStatus }}', 200, { vars: { expectedStatus: 200 }}, true ],
+    [ 200, 200, { vars: {}}, true ],
+    [ '200', 200, { vars: {}}, true ],
+    [ 200, '200', { vars: {}}, true ],
+    [ '200', '200', { vars: {}}, true ],
+
+    [ '{{ expectedStatus }}', 200, { vars: { expectedStatus: 202 }}, false ],
+    [ '{{ expectedStatus }}', '200', { vars: {}}, false ],
+    [ 301, '200', { vars: {}}, false ],
+  ];
+
+  data.forEach((e) => {
+    const result = expectations.statusCode(
+      { statusCode: e[0] }, // expectation
+      {}, // body
+      {}, // req
+      { statusCode: e[1] }, // res
+      e[2] // userContext
+    );
+
+    t.true(result.ok === e[3]);
+  });
+});
+
+test('Integration with Artillery', async (t) => {
+  const output = shelljs.exec(
+    `${__dirname}/../node_modules/.bin/artillery run --quiet ${__dirname}/pets-test.yaml`,
+  {
+    env: {
+      ARTILLERY_PLUGIN_PATH: path.resolve(__dirname, '..', '..'),
+      PATH: process.env.PATH
+    },
+    silent: true
+  }).stdout;
+
+  t.true(output.indexOf('ok contentType json') > -1);
+  t.true(output.indexOf('ok statusCode 404') > -1);
+});

packages/artillery-plugin-expect/test/mock-pets-server.yaml

@@ -0,0 +1,25 @@
+- request:
+    uri: /pets
+    method: POST
+    body: "*"
+  response:
+    code: 202
+    body: '{"success:"true"}'
+    headers:
+      content-type: application/json
+- request:
+    uri: /pets
+    method: GET
+  response:
+    code: 200
+    body: '{"result": [{"name":"Tiki", "species":"pony"}]}'
+    headers:
+      content-type: application/json
+- request:
+    uri: /pets/1
+    method: GET
+  response:
+    code: 200
+    body: '{"result": [{"name": "Luna", "species": "dog"}]}'
+    headers:
+      content-type: application/json

packages/artillery-plugin-expect/test/pets-test.yaml

@@ -0,0 +1,47 @@
+config:
+  target: http://localhost:9090
+  phases:
+    - duration: 1
+      arrivalCount: 1
+  plugins:
+    expect: {}
+  payload:
+    - path: "./urls.csv"
+      fields:
+        - url
+        - statusCode
+scenarios:
+  - flow:
+      - get:
+          url: "/pets"
+          expect:
+            - contentType: json
+            - statusCode: 200
+            - hasProperty: result
+      - post:
+          url: "/pets"
+          json:
+            name: Tiki
+            species: pony
+          expect:
+            statusCode: 202
+      - get:
+          url: "/pets/1"
+          capture:
+            - json: "$.result.0.name"
+              as: "name"
+          expect:
+            statusCode: 200
+            contentType: json
+            equals:
+              - "{{ name }}"
+              - "Luna"
+      - get:
+          url: "/pets/2"
+          expect:
+            statusCode: 404
+      - get:
+          name: "CSV-driven expectation"
+          url: "{{ url }}"
+          expect:
+            statusCode: "{{ statusCode }}"

packages/artillery-plugin-expect/test/run.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+typeset -r DIR=$(cd "$(dirname "$0")" && pwd)
+
+docker run --rm -p 9090:9090 -v "$DIR":/data quii/mockingjay-server:1.10.7 --config /data/mock-pets-server.yaml &
+docker_pid=$!
+docker_status=$?
+
+if [[ $docker_status -ne 0 ]] ; then
+    echo "Could not start mock server"
+    exit 1
+fi
+
+sleep 5
+
+test_status=$("$DIR"/../node_modules/.bin/ava $DIR/index.js)
+
+kill $docker_pid
+sleep 5
+
+exit $test_status

packages/artillery-plugin-expect/test/urls.csv

@@ -0,0 +1,3 @@
+/pets,200
+/pets/1,200
+/pets/100,404