Skip to content

Commit

Permalink
add basic support for testing matrix
Browse files Browse the repository at this point in the history
  • Loading branch information
dominiqueclarke committed Feb 13, 2023
1 parent cbf4c4e commit bbfefe5
Show file tree
Hide file tree
Showing 7 changed files with 156 additions and 14 deletions.
2 changes: 1 addition & 1 deletion __tests__/core/runner.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -729,7 +729,7 @@ describe('runner', () => {
type: 'browser',
tags: [],
locations: ['united_kingdom'],
privaateLocations: undefined,
privateLocations: undefined,
schedule: 3,
params: undefined,
playwrightOptions: undefined,
Expand Down
7 changes: 7 additions & 0 deletions src/common_types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,7 @@ export type RunOptions = BaseArgs & {
environment?: string;
playwrightOptions?: PlaywrightOptions;
networkConditions?: NetworkConditions;
matrix?: Matrix;
reporter?: BuiltInReporterName | ReporterInstance;
};

Expand All @@ -254,12 +255,18 @@ export type ProjectSettings = {
space: string;
};

export type Matrix = {
values: Record<string, unknown[]>;
adjustments?: Array<Record<string, unknown>>;
}

export type PlaywrightOptions = LaunchOptions & BrowserContextOptions;
export type SyntheticsConfig = {
params?: Params;
playwrightOptions?: PlaywrightOptions;
monitor?: MonitorConfig;
project?: ProjectSettings;
matrix?: Matrix;
};

/** Runner Payload types */
Expand Down
34 changes: 29 additions & 5 deletions src/core/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,21 @@
* THE SOFTWARE.
*
*/

import { Journey, JourneyCallback, JourneyOptions } from '../dsl';
import Runner from './runner';
import { VoidCallback, HooksCallback, Location } from '../common_types';
import { normalizeOptions } from '../options';
import { wrapFnWithLocation } from '../helpers';
import { getCombinations, getCombinationName } from '../matrix';
import { log } from './logger';
import { MonitorConfig } from '../dsl/monitor';


/* TODO: Testing
* Local vs global matrix: Local matrix fully overwrites global matrix, rather than merging
* Adjustments: Duplicates in adjustments do not run extra journeys
*
/**
* Use a gloabl Runner which would be accessed by the runtime and
* required to handle the local vs global invocation through CLI
Expand All @@ -38,7 +45,6 @@ const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER');
if (!global[SYNTHETICS_RUNNER]) {
global[SYNTHETICS_RUNNER] = new Runner();
}

export const runner: Runner = global[SYNTHETICS_RUNNER];

export const journey = wrapFnWithLocation(
Expand All @@ -51,9 +57,27 @@ export const journey = wrapFnWithLocation(
if (typeof options === 'string') {
options = { name: options, id: options };
}
const j = new Journey(options, callback, location);
runner.addJourney(j);
return j;
const { matrix: globalMatrix } = normalizeOptions({});
const { matrix: journeyMatrix } = options;
if (!globalMatrix && !journeyMatrix) {
const j = new Journey(options, callback, location);
runner.addJourney(j);
return j;
}

// local journey matrix takes priority over global matrix
const matrix = journeyMatrix || globalMatrix;

if (!matrix.values) {
throw new Error('Please specify values for your testing matrix');
}

const combinations = getCombinations(matrix);
combinations.forEach(matrixParams => {
const name = getCombinationName((options as JourneyOptions)?.name, matrixParams);
const j = new Journey({...options as JourneyOptions, name}, callback, location, matrixParams);
runner.addJourney(j);
})
}
);

Expand Down
18 changes: 12 additions & 6 deletions src/core/runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -339,16 +339,18 @@ export default class Runner {

async runJourney(journey: Journey, options: RunOptions) {
const result: JourneyResult = { status: 'succeeded' };
const context = await Runner.createContext(options);
const params = {...options.params, ...journey.params};
const journeyOptions = { ...options, params};
const context = await Runner.createContext(journeyOptions);
log(`Runner: start journey (${journey.name})`);
try {
this.registerJourney(journey, context);
const hookArgs = {
env: options.environment,
params: options.params,
params: params,
};
await this.runBeforeHook(journey, hookArgs);
const stepResults = await this.runSteps(journey, context, options);
const stepResults = await this.runSteps(journey, context, journeyOptions);
/**
* Mark journey as failed if any intermediate step fails
*/
Expand All @@ -363,7 +365,7 @@ export default class Runner {
result.status = 'failed';
result.error = e;
} finally {
await this.endJourney(journey, { ...context, ...result }, options);
await this.endJourney(journey, { ...context, ...result }, journeyOptions);
await Gatherer.dispose(context.driver);
}
log(`Runner: end journey (${journey.name})`);
Expand Down Expand Up @@ -399,6 +401,7 @@ export default class Runner {
const { match, tags } = options;
const monitors: Monitor[] = [];
for (const journey of this.journeys) {
const params = { ...this.monitor.config?.params, ...options.params, ...journey.params };
if (!journey.isMatch(match, tags)) {
continue;
}
Expand All @@ -407,8 +410,11 @@ export default class Runner {
* Execute dummy callback to get all monitor specific
* configurations for the current journey
*/
journey.callback({ params: options.params } as any);
journey.monitor.update(this.monitor?.config);
journey.callback({ params: params } as any);
journey.monitor.update({
...this.monitor?.config,
params: Object.keys(params).length ? params : undefined
});
journey.monitor.validate();
monitors.push(journey.monitor);
}
Expand Down
8 changes: 6 additions & 2 deletions src/dsl/journey.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,14 @@ import {
} from 'playwright-chromium';
import micromatch, { isMatch } from 'micromatch';
import { Step } from './step';
import { VoidCallback, HooksCallback, Params, Location } from '../common_types';
import { VoidCallback, HooksCallback, Params, Location, Matrix } from '../common_types';
import { Monitor, MonitorConfig } from './monitor';

export type JourneyOptions = {
name: string;
id?: string;
tags?: string[];
matrix?: Matrix;
};

type HookType = 'before' | 'after';
Expand All @@ -61,18 +62,21 @@ export class Journey {
steps: Step[] = [];
hooks: Hooks = { before: [], after: [] };
monitor: Monitor;
params: Params = {};

constructor(
options: JourneyOptions,
callback: JourneyCallback,
location?: Location
location?: Location,
params?: Params,
) {
this.name = options.name;
this.id = options.id || options.name;
this.tags = options.tags;
this.callback = callback;
this.location = location;
this.updateMonitor({});
this.params = params;
}

addStep(name: string, callback: VoidCallback, location?: Location) {
Expand Down
96 changes: 96 additions & 0 deletions src/matrix.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
/**
* MIT License
*
* Copyright (c) 2020-present, Elastic NV
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*
*/

import { createHash } from 'crypto';
import type { Matrix } from './common_types';

export const getCombinations = (matrix: Matrix): Array<Record<string, unknown>> => {
const { values, adjustments } = matrix;
const matrixKeys = Object.keys(matrix.values);
const entries = Object.values(values);

let combinations = calculateCombinations(entries[0]);
for (let i = 1; i < entries.length; i++) {
combinations = calculateCombinations(combinations, entries[i]);
}

const matrixParams = combinations.map(combination => {
return getCombinationParams(matrixKeys, combination);
});

if (!adjustments) {
return matrixParams;
}

const currentCombinations = new Set(matrixParams.map(params => {
const hash = createHash('sha256');
const paramHash = hash.update(JSON.stringify(params)).digest('base64');
return paramHash;
}));

adjustments.forEach(adjustment => {
const hash = createHash('sha256');
const adjustmentHash = hash.update(JSON.stringify(adjustment)).digest('base64');
if (!currentCombinations.has(adjustmentHash)) {
matrixParams.push(adjustment);
}
});

return matrixParams;
}

export const calculateCombinations = (groupA: Array<unknown | unknown[]>, groupB?: Array<unknown>): Array<unknown[]> => {
const results = [];
groupA.forEach(optionA => {
if (!groupB) {
results.push([optionA]);
return;
}
groupB.forEach(optionB => {
if (Array.isArray(optionA)) {
return results.push([...optionA, optionB])
} else {
return results.push([optionA, optionB])
}
});
});
return results;
}

export const getCombinationName = (name: string, combinations: Record<string, unknown>) => {
const values = Object.values(combinations);
return values.reduce<string>((acc, combination) => {
const nameAdjustment = typeof combination === 'object' ? JSON.stringify(combination) : combination.toString();
acc += ` - ${nameAdjustment.toString()}`;
return acc;
}, name).trim();
}

export const getCombinationParams = (keys: string[], values: unknown[]): Record<string, unknown> => {
return keys.reduce<Record<string, unknown>>((acc, key, index) => {
acc[key] = values[index];
return acc;
}, {});
}
5 changes: 5 additions & 0 deletions src/options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,11 @@ export function normalizeOptions(cliArgs: CliArgs): RunOptions {
*/
options.params = Object.freeze(merge(config.params, cliArgs.params || {}));

/**
* Grab matrix only from config and not cliArgs
*/
options.matrix = Object.freeze(config.matrix);

/**
* Merge playwright options from CLI and Synthetics config
* and prefer individual options over other option
Expand Down

0 comments on commit bbfefe5

Please sign in to comment.