diff --git a/packages/artillery-plugin-expect/README.md b/packages/artillery-plugin-expect/README.md index 67a9a02aa6..e37568afba 100644 --- a/packages/artillery-plugin-expect/README.md +++ b/packages/artillery-plugin-expect/README.md @@ -14,6 +14,8 @@ Add expectations to your HTTP scenarios for functional API testing with Artiller npm install -g artillery-plugin-expect ``` +**Important**: this plugin requires Artillery `v1.6.0-26` or higher. + ### Enable the plugin in the config section ```yaml @@ -43,6 +45,53 @@ scenarios: - "{{ name }}" ``` +### Run your test & see results + +Run your script that uses expectations with: + +``` +artillery run --quiet my-script.yaml +``` + +The `--quiet` option is to stop Artillery from printing its default reports to the console. + +Failed expectations provide request and response details: + +![artillery expectations plugin screenshot](./docs/expect-output.png) + +### Re-using scenarios as load tests or functional tests + +This plugin allows for the same scenario to be re-used for either load testing or functional testing of an API. (The only real difference between the two, of course, is how many virtual users you run -- only one for functional tests, and `$BIG_NUMBER` for a load test.) + +In practical terms, what you probably want to do is use the [`environments` functionality](https://artillery.io/docs/script-reference/#environments) in Artillery to create a separate "environment" with configuration for functional testing. Something like: + +```yaml +config: + target: "https://my.api.internal" + environments: + # + # This is our load testing profile, where we create a lot of virtual users. + # Note that we don't load the plugin here, so that we don't see the output + # from the plugin. + # + load: + phases: + - duration: 600 + arrivalRate: 10 + # + # This is our functional testing profile, with a single virtual user, and + # the plugin enabled. + # + functional: + phases: + - duration: 1 + arrivalCount: 1 + plugins: + expect: {} +scenarios: + # Your scenario definitions go here. +``` + ## Expectations ### `statusCode` diff --git a/packages/artillery-plugin-expect/docs/expect-output.png b/packages/artillery-plugin-expect/docs/expect-output.png new file mode 100644 index 0000000000..ca77f13959 Binary files /dev/null and b/packages/artillery-plugin-expect/docs/expect-output.png differ diff --git a/packages/artillery-plugin-expect/lib/expectations.js b/packages/artillery-plugin-expect/lib/expectations.js index 08ec8457ad..fa3ae92f2b 100644 --- a/packages/artillery-plugin-expect/lib/expectations.js +++ b/packages/artillery-plugin-expect/lib/expectations.js @@ -6,7 +6,7 @@ const debug = require('debug')('plugin:expect'); const chalk = require('chalk'); -const renderVariables = require('artillery/util').renderVariables; +const template = global.artillery ? global.artillery.util.template : require('artillery/util').template; const _ = require('lodash'); module.exports = { @@ -16,9 +16,6 @@ module.exports = { equals: expectEquals }; -// FIXME: Current implementation only works with primitive values, -// and forces everything to a string. Objects, lists, and type checks -// can be implemented with template() exported from artillery/util. function expectEquals(expectation, body, req, res, userContext) { debug('check equals'); debug('expectation:', expectation); @@ -31,7 +28,7 @@ function expectEquals(expectation, body, req, res, userContext) { }; const values = _.map(expectation.equals, (str) => { - return String(renderVariables(String(str), userContext.vars)); + return String(template(String(str), userContext.vars)); }); const unique = _.uniq(values); @@ -46,13 +43,14 @@ function expectContentType(expectation, body, req, res, userContext) { debug('expectation:', expectation); debug('body:', typeof body); + const expectedContentType = template(expectation.contentType, userContext); let result = { ok: false, - expected: expectation.contentType, + expected: expectedContentType, type: 'contentType' }; - if (expectation.contentType === 'json') { + if (expectedContentType === 'json') { if ( typeof body === 'object' && res.headers['content-type'].indexOf('application/json') !== -1 @@ -69,7 +67,7 @@ function expectContentType(expectation, body, req, res, userContext) { return result; } } else { - result.ok = res.headers['content-type'] && res.headers['content-type'].toLowerCase() === expectation.contentType.toLowerCase(); + result.ok = res.headers['content-type'] && res.headers['content-type'].toLowerCase() === expectedContentType.toLowerCase(); result.got = res.headers['content-type'] || 'content-type header not set'; return result; } @@ -78,13 +76,15 @@ function expectContentType(expectation, body, req, res, userContext) { function expectStatusCode(expectation, body, req, res, userContext) { debug('check statusCode'); + const expectedStatusCode = template(expectation.statusCode, userContext); + let result = { ok: false, - expected: expectation.statusCode, + expected: expectedStatusCode, type: 'statusCode' }; - result.ok = res.statusCode === expectation.statusCode; + result.ok = Number(res.statusCode) === Number(expectedStatusCode); result.got = res.statusCode; return result; } @@ -92,19 +92,20 @@ function expectStatusCode(expectation, body, req, res, userContext) { function expectHasProperty(expectation, body, req, res, userContext) { debug('check hasProperty'); + const expectedProperty = template(expectation.hasProperty, userContext); let result = { ok: false, - expected: expectation.hasProperty, + expected: expectedProperty, type: 'hasProperty' }; if (typeof body === 'object') { - if (_.has(body, expectation.hasProperty)) { + if (_.has(body, expectedProperty)) { result.ok = true; - result.got = `${body[expectation.hasProperty]}`; + result.got = expectedProperty; return result; } else { - result.got = `response body has no ${expectation.hasProperty} property`; + result.got = `response body has no ${expectedProperty} property`; return result; } } else { diff --git a/packages/artillery-plugin-expect/package.json b/packages/artillery-plugin-expect/package.json index 0c9a76a1de..eaf9f7bf4f 100644 --- a/packages/artillery-plugin-expect/package.json +++ b/packages/artillery-plugin-expect/package.json @@ -17,7 +17,10 @@ "devDependencies": { "@commitlint/cli": "^7.0.0", "@commitlint/config-conventional": "^7.0.1", - "husky": "^1.0.0-rc.13" + "artillery": "^1.6.0-26", + "ava": "^0.25.0", + "husky": "^1.0.0-rc.13", + "shelljs": "^0.8.3" }, "husky": { "hooks": { diff --git a/packages/artillery-plugin-expect/test/index.js b/packages/artillery-plugin-expect/test/index.js new file mode 100644 index 0000000000..5e16d7b6dd --- /dev/null +++ b/packages/artillery-plugin-expect/test/index.js @@ -0,0 +1,79 @@ +'use strict'; + +import test from 'ava'; +import createDebug from 'debug'; +const debug = createDebug('expect-plugin:test'); +import EventEmitter from 'events'; + +const shelljs = require('shelljs'); +const path = require('path'); + +// +// We only need this when running unit tests. When the plugin actually runs inside +// a recent version of Artillery, the appropriate object is already set up. +// +global.artillery = { + util: { + template: require('artillery/util').template + } +}; + +test('Basic interface checks', async t => { + const script = { + config: {}, + scenarios: [] + }; + + const ExpectationsPlugin = require('../index'); + const events = new EventEmitter(); + const plugin = new ExpectationsPlugin.Plugin(script, events); + + t.true(typeof ExpectationsPlugin.Plugin === 'function'); + t.true(typeof plugin === 'object'); + + t.pass(); +}); + +test('Expectation: statusCode', async (t) => { + const expectations = require('../lib/expectations'); + + const data = [ + // expectation - value received - user context - expected result + [ '{{ expectedStatus }}', 200, { vars: { expectedStatus: 200 }}, true ], + [ 200, 200, { vars: {}}, true ], + [ '200', 200, { vars: {}}, true ], + [ 200, '200', { vars: {}}, true ], + [ '200', '200', { vars: {}}, true ], + + [ '{{ expectedStatus }}', 200, { vars: { expectedStatus: 202 }}, false ], + [ '{{ expectedStatus }}', '200', { vars: {}}, false ], + [ 301, '200', { vars: {}}, false ], + ]; + + data.forEach((e) => { + const result = expectations.statusCode( + { statusCode: e[0] }, // expectation + {}, // body + {}, // req + { statusCode: e[1] }, // res + e[2] // userContext + ); + + t.true(result.ok === e[3]); + }); +}); + +test('Integration with Artillery', async (t) => { + const output = shelljs.exec( + `${__dirname}/../node_modules/.bin/artillery run --quiet ${__dirname}/pets-test.yaml`, + { + env: { + ARTILLERY_PLUGIN_PATH: path.resolve(__dirname, '..', '..'), + PATH: process.env.PATH + }, + silent: true + }).stdout; + + t.true(output.indexOf('ok contentType json') > -1); + t.true(output.indexOf('ok statusCode 404') > -1); +}); diff --git a/packages/artillery-plugin-expect/test/mock-pets-server.yaml b/packages/artillery-plugin-expect/test/mock-pets-server.yaml new file mode 100644 index 0000000000..ad395e3ec6 --- /dev/null +++ b/packages/artillery-plugin-expect/test/mock-pets-server.yaml @@ -0,0 +1,25 @@ +- request: + uri: /pets + method: POST + body: "*" + response: + code: 202 + body: '{"success:"true"}' + headers: + content-type: application/json +- request: + uri: /pets + method: GET + response: + code: 200 + body: '{"result": [{"name":"Tiki", "species":"pony"}]}' + headers: + content-type: application/json +- request: + uri: /pets/1 + method: GET + response: + code: 200 + body: '{"result": [{"name": "Luna", "species": "dog"}]}' + headers: + content-type: application/json diff --git a/packages/artillery-plugin-expect/test/pets-test.yaml b/packages/artillery-plugin-expect/test/pets-test.yaml new file mode 100644 index 0000000000..e4b07e21e9 --- /dev/null +++ b/packages/artillery-plugin-expect/test/pets-test.yaml @@ -0,0 +1,47 @@ +config: + target: http://localhost:9090 + phases: + - duration: 1 + arrivalCount: 1 + plugins: + expect: {} + payload: + - path: "./urls.csv" + fields: + - url + - statusCode +scenarios: + - flow: + - get: + url: "/pets" + expect: + - contentType: json + - statusCode: 200 + - hasProperty: result + - post: + url: "/pets" + json: + name: Tiki + species: pony + expect: + statusCode: 202 + - get: + url: "/pets/1" + capture: + - json: "$.result.0.name" + as: "name" + expect: + statusCode: 200 + contentType: json + equals: + - "{{ name }}" + - "Luna" + - get: + url: "/pets/2" + expect: + statusCode: 404 + - get: + name: "CSV-driven expectation" + url: "{{ url }}" + expect: + statusCode: "{{ statusCode }}" diff --git a/packages/artillery-plugin-expect/test/run.sh b/packages/artillery-plugin-expect/test/run.sh new file mode 100755 index 0000000000..157be2be9c --- /dev/null +++ b/packages/artillery-plugin-expect/test/run.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -eu -o pipefail +typeset -r DIR=$(cd "$(dirname "$0")" && pwd) + +docker run --rm -p 9090:9090 -v "$DIR":/data quii/mockingjay-server:1.10.7 --config /data/mock-pets-server.yaml & +docker_pid=$! +docker_status=$? + +if [[ $docker_status -ne 0 ]] ; then + echo "Could not start mock server" + exit 1 +fi + +sleep 5 + +test_status=$("$DIR"/../node_modules/.bin/ava $DIR/index.js) + +kill $docker_pid +sleep 5 + +exit $test_status diff --git a/packages/artillery-plugin-expect/test/urls.csv b/packages/artillery-plugin-expect/test/urls.csv new file mode 100644 index 0000000000..cbb745a930 --- /dev/null +++ b/packages/artillery-plugin-expect/test/urls.csv @@ -0,0 +1,3 @@ +/pets,200 +/pets/1,200 +/pets/100,404