fix(types): implies default "http" engine, fixes request "cookies" an…

…d "match" (#2054)

* fix: support "http" as the implicit engine

* fix: support "match" on http requests

* fix: annotate request "cookies" as free-form object

* chore: run build before tests

* test: add example-based tests for types

* fix: mark request "match" as deprecated

* fix: make root-level "scenarios" optional

* feat: annotate expect plugin

* test: add additional example tests

* fix(expect): "notStatusCode" can be a number

* feat(expect): document "useOnlyRequestNames" option

packages/types/definitions.ts

@@ -1,3 +1,5 @@
+import { ExpectPluginConfig, ExpectPluginMetrics } from './plugins/expect';
+
 export type TestScript = {
   /**
    * @title Configuration
@@ -21,7 +23,7 @@ export type TestScript = {
   /**
    * @title Scenarios
    */
-  scenarios: Scenarios;
+  scenarios?: Scenarios;
 };
 
 export type Config = {
@@ -55,6 +57,7 @@ export type Config = {
    */
   plugins?: {
     [key: string]: any;
+    expect?: ExpectPluginConfig;
   };
   ensure?: {
     [key: string]: any;
@@ -286,7 +289,7 @@ export type Scenario = {
       /**
        * @title HTTP engine
        */
-      engine: 'http';
+      engine?: 'http';
       /**
        * @title Scenario flow
        */
@@ -387,6 +390,11 @@ export type BaseFlow =
       function: string;
     };
 
+export type HttpResponseMatch = {
+  json: any;
+  value: string;
+};
+
 export type HttpFlow =
   | BaseFlow
   | {
@@ -452,10 +460,7 @@ export type SocketIoFlow =
         };
         acknowledge?: {
           data?: string;
-          match?: {
-            json: any;
-            value: string;
-          };
+          match?: HttpResponseMatch;
         };
       };
     };
@@ -472,7 +477,9 @@ export type DefaultHttpRequest = {
   /**
    * @title Cookie
    */
-  cookie?: Record<string, string>;
+  cookie?: {
+    [name: string]: string;
+  };
   /**
    * @title Query string
    */
@@ -487,6 +494,14 @@ export type DefaultHttpRequest = {
    * @title Capture
    */
   capture?: TestPhaseCapture | Array<TestPhaseCapture>;
+  /**
+   * (Deprecated) Response validation criteria.
+   * Please use the expectations plugin instead:
+   * https://www.artillery.io/docs/reference/extensions/expect
+   * @deprecated true
+   * @title Match
+   */
+  match?: HttpResponseMatch;
   /**
    * Automatically set the "Accept-Encoding" request header
    * and decode compressed responses encoded with gzip.
@@ -521,6 +536,16 @@ export type DefaultHttpRequest = {
    * @title Request condition
    */
   ifTrue?: string;
+
+  /**
+   * Plugin-specific properties.
+   */
+
+  /**
+   * https://www.artillery.io/docs/reference/extensions/expect#expectations
+   * @title Expect plugin expectations
+   */
+  expect?: ExpectPluginMetrics;
 };
 
 export type HttpRequestWithBody = DefaultHttpRequest &

packages/types/package.json

@@ -7,6 +7,7 @@
   ],
   "license": "MPL-2.0",
   "scripts": {
+    "pretest": "npm run build",
     "test": "tap --ts --no-coverage --color --timeout 60 test/**/*.test.ts",
     "build": "typescript-json-schema ./tsconfig.schema.json \"TestScript\" --required --noExtraProps -o ./schema.json",
     "prepare": "npm run build",

packages/types/plugins/expect.ts

@@ -0,0 +1,100 @@
+export type ExpectPluginConfig = {
+  /**
+   * @title Output format
+   */
+  outputFormat?: 'pretty' | 'json' | 'prettyError' | 'silent';
+  /**
+   * (Deprecated) Formatter
+   * Please use the `outputFormat` option instead.
+   * @deprecated true
+   * @title Formatter
+   */
+  formatter?: ExpectPluginConfig['outputFormat'];
+  /**
+   * Reports failures from expect plugin as errors
+   * in the test report.
+   * @default false
+   * @title Report failures as errors
+   */
+  reportFailuresAsErrors?: boolean;
+  /**
+   * Use request name instead of the URL path when
+   * logging requests.
+   * @title Use only request names
+   */
+  useOnlyRequestNames?: boolean;
+  /**
+   * Sets a 200 OK status code expectation for all requests.
+   * @default false
+   * @title Expect 200 by default
+   */
+  expectDefault200?: boolean;
+};
+
+export type ExpectPluginMetrics = {
+  /**
+   * Check that the response status code.
+   * If the list of status codes is provided, check that the response
+   * status code is present in the list.
+   * https://www.artillery.io/docs/reference/extensions/expect#statuscode
+   * @title Status code
+   */
+  statusCode?: number | Array<number>;
+  /**
+   * Check that the response status code does not equal the given status.
+   * If the list of status codes is provided, check that the response
+   * status code is not present in the list.
+   * https://www.artillery.io/docs/reference/extensions/expect#notstatuscode
+   * @title Not status code
+   */
+  notStatusCode?: number | Array<number>;
+  /**
+   * Check that the value of the `Content-Type` response header.
+   * https://www.artillery.io/docs/reference/extensions/expect#contenttype
+   * @title Content type
+   */
+  contentType?: string;
+  /**
+   * Check that the response object has the given property.
+   * https://www.artillery.io/docs/reference/extensions/expect#hasproperty-and-nothasproperty
+   * @title Has property
+   */
+  hasProperty?: string;
+  /**
+   * Check that the response object doesn't have the given property.
+   * https://www.artillery.io/docs/reference/extensions/expect#hasproperty-and-nothasproperty
+   * @title Not has property
+   */
+  notHasProperty?: string;
+  /**
+   * Check that two or more values are the same.
+   * https://www.artillery.io/docs/reference/extensions/expect#equals
+   * @title Equals
+   */
+  equals?: Array<string>;
+  /**
+   * Check that the response contains the given header.
+   * https://www.artillery.io/docs/reference/extensions/expect#hasheader
+   * @title Has header
+   */
+  hasHeader?: string;
+  /**
+   * Check that the response contains a header and its value
+   * matches is present in the list.
+   * https://www.artillery.io/docs/reference/extensions/expect#headerequals
+   * @title Header equals
+   */
+  headerEquals?: Array<string>;
+  /**
+   * Check that the response matches a regular expression.
+   * https://www.artillery.io/docs/reference/extensions/expect#matchesregexp
+   * @title
+   */
+  matchesRegexp?: string;
+  /**
+   * Check the presence of a cache hit/miss header from a CDN.
+   * https://www.artillery.io/docs/reference/extensions/expect#cdnhit
+   * @title CDN hit
+   */
+  cdnHit?: boolean;
+};

packages/types/test/engine.http.test.ts

@@ -0,0 +1,46 @@
+import * as tap from 'tap';
+import { validateTestScript } from './helpers';
+
+tap.test(
+  'uses "http" engine when no explicit scenario engine is provided',
+  (tap) => {
+    tap.same(
+      validateTestScript(`
+scenarios:
+  - name: My HTTP scenario
+    flow:
+      - get:
+          url: /resource
+      - think: 5
+`),
+      []
+    );
+
+    tap.ok(
+      validateTestScript(`
+scenarios:
+  - flow:
+      - send: Oops, not WebSocket!
+`).length > 0
+    );
+
+    tap.end();
+  }
+);
+
+tap.test('understands explicit "http" scenario engine', (tap) => {
+  tap.same(
+    validateTestScript(`
+scenarios:
+  - name: My HTTP scenario
+    engine: http
+    flow:
+      - get:
+          url: /resource
+      - think: 5
+`),
+    []
+  );
+
+  tap.end();
+});

packages/types/test/engine.socketio.test.ts

@@ -0,0 +1,65 @@
+import * as tap from 'tap';
+import { validateTestScript } from './helpers';
+
+tap.test('validates scenario flow when using "socketio" engine', (tap) => {
+  tap.same(
+    validateTestScript(`
+scenarios:
+  - engine: socketio
+    flow:
+      - emit:
+          channel: myChannel
+          data: Hello world
+    `),
+    []
+  );
+
+  tap.end();
+});
+
+tap.test('allows general flow properties', (tap) => {
+  tap.same(
+    validateTestScript(`
+scenarios:
+  - engine: socketio
+    flow:
+      - emit:
+          channel: myChannel
+          data: Hello world
+      - think: 5
+      - log: Debug here
+    `),
+    []
+  );
+
+  tap.end();
+});
+
+tap.test('supports HTTP flow properties for "socketio" engine', (tap) => {
+  const errors = validateTestScript(`
+config:
+  target: http://localhost:3000
+  phases:
+    - duration: 10
+      rampTo: 50
+scenarios:
+  - engine: socketio
+    flow:
+      - get:
+          url: /resource
+      - think: 500
+      - emit:
+          channel: "echoResponse"
+          data: "hello"
+      - loop:
+          - post:
+              url: /resource
+          - emit:
+              channel: "anotherChannel"
+              data: "world"
+        count: 5
+`);
+
+  tap.same(errors, []);
+  tap.end();
+});

packages/types/test/engine.websocket.test.ts

@@ -0,0 +1,49 @@
+import * as tap from 'tap';
+import { validateTestScript } from './helpers';
+
+tap.test('validates scenario flow using "websocket" scenario engine', (tap) => {
+  tap.same(
+    validateTestScript(`
+scenarios:
+  - engine: websocket
+    flow:
+      - send: Hello world
+  `),
+    []
+  );
+
+  tap.end();
+});
+
+tap.test('supports general scenario flow properties', (tap) => {
+  tap.same(
+    validateTestScript(`
+scenarios:
+  - engine: websocket
+    flow:
+      - log: Debug here
+      - think: 5
+  `),
+    []
+  );
+
+  tap.end();
+});
+
+tap.test(
+  'errors on http flow when using "websocket" scenario engine',
+  (tap) => {
+    tap.ok(
+      validateTestScript(`
+scenarios:
+  - name: My HTTP scenario
+    engine: websocket
+    flow:
+      - get:
+          url: /resource
+`).length > 0
+    );
+
+    tap.end();
+  }
+);