From e3a9054c63d80d20d947073766bf6fe32c179650 Mon Sep 17 00:00:00 2001 From: petruki <31597636+petruki@users.noreply.github.com> Date: Wed, 8 Feb 2023 21:07:31 -0800 Subject: [PATCH 1/2] Fixing flaky snapshot tests --- src/switcher-client.ts | 3 +- test/datemoment.test.ts | 4 +- test/helper/utils.ts | 2 +- test/strategy-operations/date.test.ts | 4 +- test/strategy-operations/network.test.ts | 4 +- test/strategy-operations/numeric.test.ts | 4 +- test/strategy-operations/payload.test.ts | 4 +- test/strategy-operations/regex.test.ts | 4 +- test/strategy-operations/time.test.ts | 4 +- test/strategy-operations/value.test.ts | 4 +- test/switcher-client.test.ts | 50 +++++++++++++----------- test/switcher-integrated.test.ts | 6 +-- test/switcher-snapshot.test.ts | 43 +++++++++++++------- test/switcher-watch-snapshot.test.ts | 4 +- test/timed-match.test.ts | 4 +- 15 files changed, 82 insertions(+), 62 deletions(-) diff --git a/src/switcher-client.ts b/src/switcher-client.ts index adcb7ed..8282da9 100644 --- a/src/switcher-client.ts +++ b/src/switcher-client.ts @@ -154,7 +154,8 @@ export class Switcher { return; } - Switcher._watcher = Deno.watchFs(`${Switcher._options.snapshotLocation}${Switcher._context.environment}.json`); + const snapshotFile = `${Switcher._options.snapshotLocation}${Switcher._context.environment}.json`; + Switcher._watcher = Deno.watchFs(snapshotFile); for await (const event of Switcher._watcher) { const dataString = JSON.stringify(event); if (Switcher._watchDebounce.has(dataString)) { diff --git a/test/datemoment.test.ts b/test/datemoment.test.ts index 1fd3ed6..d830b93 100644 --- a/test/datemoment.test.ts +++ b/test/datemoment.test.ts @@ -1,5 +1,5 @@ -import { assertThrows, assertFalse } from 'https://deno.land/std@0.176.0/testing/asserts.ts'; -import { describe, it } from 'https://deno.land/std@0.176.0/testing/bdd.ts'; +import { assertThrows, assertFalse } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; +import { describe, it } from 'https://deno.land/std@0.177.0/testing/bdd.ts'; import { assertTrue } from './helper/utils.ts'; import DateMoment from '../src/lib/utils/datemoment.ts'; diff --git a/test/helper/utils.ts b/test/helper/utils.ts index a407dd4..926de00 100644 --- a/test/helper/utils.ts +++ b/test/helper/utils.ts @@ -1,5 +1,5 @@ // deno-lint-ignore-file no-explicit-any -import { assertEquals } from "https://deno.land/std@0.176.0/testing/asserts.ts"; +import { assertEquals } from "https://deno.land/std@0.177.0/testing/asserts.ts"; import * as mf from 'https://deno.land/x/mock_fetch@0.3.0/mod.ts'; export function given(route: string, expect: any, status = 200) { diff --git a/test/strategy-operations/date.test.ts b/test/strategy-operations/date.test.ts index 921cfb0..3db7d32 100644 --- a/test/strategy-operations/date.test.ts +++ b/test/strategy-operations/date.test.ts @@ -1,5 +1,5 @@ -import { assertFalse } from 'https://deno.land/std@0.176.0/testing/asserts.ts'; -import { describe, it } from 'https://deno.land/std@0.176.0/testing/bdd.ts'; +import { assertFalse } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; +import { describe, it } from 'https://deno.land/std@0.177.0/testing/bdd.ts'; import { assertTrue } from '../helper/utils.ts'; import { OperationsType, diff --git a/test/strategy-operations/network.test.ts b/test/strategy-operations/network.test.ts index e67a417..976de95 100644 --- a/test/strategy-operations/network.test.ts +++ b/test/strategy-operations/network.test.ts @@ -1,5 +1,5 @@ -import { assertFalse } from 'https://deno.land/std@0.176.0/testing/asserts.ts'; -import { describe, it } from 'https://deno.land/std@0.176.0/testing/bdd.ts'; +import { assertFalse } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; +import { describe, it } from 'https://deno.land/std@0.177.0/testing/bdd.ts'; import { assertTrue } from '../helper/utils.ts'; import { OperationsType, diff --git a/test/strategy-operations/numeric.test.ts b/test/strategy-operations/numeric.test.ts index 7cd4714..0bbf980 100644 --- a/test/strategy-operations/numeric.test.ts +++ b/test/strategy-operations/numeric.test.ts @@ -1,5 +1,5 @@ -import { assertFalse } from 'https://deno.land/std@0.176.0/testing/asserts.ts'; -import { describe, it } from 'https://deno.land/std@0.176.0/testing/bdd.ts'; +import { assertFalse } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; +import { describe, it } from 'https://deno.land/std@0.177.0/testing/bdd.ts'; import { assertTrue } from '../helper/utils.ts'; import { OperationsType, diff --git a/test/strategy-operations/payload.test.ts b/test/strategy-operations/payload.test.ts index 523db29..beeda1a 100644 --- a/test/strategy-operations/payload.test.ts +++ b/test/strategy-operations/payload.test.ts @@ -1,5 +1,5 @@ -import { assertFalse, assertArrayIncludes } from 'https://deno.land/std@0.176.0/testing/asserts.ts'; -import { describe, it } from 'https://deno.land/std@0.176.0/testing/bdd.ts'; +import { assertFalse, assertArrayIncludes } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; +import { describe, it } from 'https://deno.land/std@0.177.0/testing/bdd.ts'; import { assertTrue } from '../helper/utils.ts'; import { OperationsType, diff --git a/test/strategy-operations/regex.test.ts b/test/strategy-operations/regex.test.ts index da91d92..f86d6db 100644 --- a/test/strategy-operations/regex.test.ts +++ b/test/strategy-operations/regex.test.ts @@ -1,5 +1,5 @@ -import { assertFalse } from 'https://deno.land/std@0.176.0/testing/asserts.ts'; -import { describe, it } from 'https://deno.land/std@0.176.0/testing/bdd.ts'; +import { assertFalse } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; +import { describe, it } from 'https://deno.land/std@0.177.0/testing/bdd.ts'; import { assertTrue } from '../helper/utils.ts'; import { OperationsType, diff --git a/test/strategy-operations/time.test.ts b/test/strategy-operations/time.test.ts index e653910..8c6fde3 100644 --- a/test/strategy-operations/time.test.ts +++ b/test/strategy-operations/time.test.ts @@ -1,5 +1,5 @@ -import { assertFalse } from 'https://deno.land/std@0.176.0/testing/asserts.ts'; -import { describe, it } from 'https://deno.land/std@0.176.0/testing/bdd.ts'; +import { assertFalse } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; +import { describe, it } from 'https://deno.land/std@0.177.0/testing/bdd.ts'; import { assertTrue } from '../helper/utils.ts'; import { OperationsType, diff --git a/test/strategy-operations/value.test.ts b/test/strategy-operations/value.test.ts index 2da7c62..a33519f 100644 --- a/test/strategy-operations/value.test.ts +++ b/test/strategy-operations/value.test.ts @@ -1,5 +1,5 @@ -import { assertFalse } from 'https://deno.land/std@0.176.0/testing/asserts.ts'; -import { describe, it } from 'https://deno.land/std@0.176.0/testing/bdd.ts'; +import { assertFalse } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; +import { describe, it } from 'https://deno.land/std@0.177.0/testing/bdd.ts'; import { assertTrue } from '../helper/utils.ts'; import { OperationsType, diff --git a/test/switcher-client.test.ts b/test/switcher-client.test.ts index eca0588..9d5e60c 100644 --- a/test/switcher-client.test.ts +++ b/test/switcher-client.test.ts @@ -1,5 +1,6 @@ -import { describe, it, afterAll, beforeEach, beforeAll } from 'https://deno.land/std@0.176.0/testing/bdd.ts'; -import { assertEquals, assertRejects, assertFalse } from 'https://deno.land/std@0.176.0/testing/asserts.ts'; +import { describe, it, afterAll, beforeEach, beforeAll } from 'https://deno.land/std@0.177.0/testing/bdd.ts'; +import { assertEquals, assertRejects, assertFalse, assertExists } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; +import { delay } from 'https://deno.land/std@0.177.0/async/delay.ts'; import { assertTrue } from './helper/utils.ts' import { StrategiesType } from '../src/lib/snapshot.ts'; @@ -11,7 +12,7 @@ import { checkRegex } from '../mod.ts'; -const testSettings = { sanitizeExit: false, sanitizeResources: false }; +const testSettings = { sanitizeOps: false, sanitizeResources: false, sanitizeExit: false }; describe('E2E test - Switcher offline:', function () { let switcher: Switcher; @@ -39,7 +40,7 @@ describe('E2E test - Switcher offline:', function () { switcher = Switcher.factory(); }); - it('should be valid - isItOn', async function () { + it('should be valid - isItOn', testSettings, async function () { await switcher.prepare('FF2FOR2020', [ checkValue('Japan'), checkNetwork('10.0.0.3') @@ -48,7 +49,7 @@ describe('E2E test - Switcher offline:', function () { assertTrue(await switcher.isItOn('FF2FOR2020')); }); - it('should be valid - No prepare function needed', async function () { + it('should be valid - No prepare function needed', testSettings, async function () { const result = await switcher.isItOn('FF2FOR2020', [ checkValue('Japan'), checkNetwork('10.0.0.3') @@ -57,22 +58,22 @@ describe('E2E test - Switcher offline:', function () { assertTrue(result); }); - it('should be valid - No prepare function needed (no input as well)', async function () { + it('should be valid - No prepare function needed (no input as well)', testSettings, async function () { const result = await switcher.isItOn('FF2FOR2030'); assertTrue(result); }); - it('should be valid - Switcher strategy disabled', async function () { + it('should be valid - Switcher strategy disabled', testSettings, async function () { const result = await switcher.isItOn('FF2FOR2021', [checkNetwork('192.168.0.1')]); assertTrue(result); }); - it('should be valid - No Switcher strategy', async function () { + it('should be valid - No Switcher strategy', testSettings, async function () { const result = await switcher.isItOn('FF2FOR2022'); assertTrue(result); }); - it('should be valid - JSON Payload matches all keys', async function () { + it('should be valid - JSON Payload matches all keys', testSettings, async function () { await switcher.prepare('FF2FOR2023', [ checkPayload(JSON.stringify({ id: 1, @@ -87,7 +88,7 @@ describe('E2E test - Switcher offline:', function () { assertTrue(result); }); - it('should be invalid - REGEX failed to perform in time', async function () { + it('should be invalid - REGEX failed to perform in time', testSettings, async function () { const getTimer = (timer: number) => (timer - Date.now()) * -1; let timer = Date.now(); @@ -99,7 +100,7 @@ describe('E2E test - Switcher offline:', function () { assertTrue(timer < 600); }); - it('should be invalid - JSON Payload does NOT match all keys', async function () { + it('should be invalid - JSON Payload does NOT match all keys', testSettings, async function () { await switcher.prepare('FF2FOR2023', [ checkPayload(JSON.stringify({ id: 1, @@ -114,7 +115,7 @@ describe('E2E test - Switcher offline:', function () { `Strategy '${StrategiesType.PAYLOAD}' does not agree`); }); - it('should be invalid - Input (IP) does not match', async function () { + it('should be invalid - Input (IP) does not match', testSettings, async function () { await switcher.prepare('FF2FOR2020', [ checkValue('Japan'), checkNetwork('192.168.0.2') @@ -125,25 +126,25 @@ describe('E2E test - Switcher offline:', function () { `Strategy '${StrategiesType.NETWORK}' does not agree`); }); - it('should be invalid - Input not provided', async function () { + it('should be invalid - Input not provided', testSettings, async function () { assertFalse(await switcher.isItOn('FF2FOR2020')); assertEquals(Switcher.getLogger('FF2FOR2020')[0].response.reason, `Strategy '${StrategiesType.NETWORK}' did not receive any input`); }); - it('should be invalid - Switcher config disabled', async function () { + it('should be invalid - Switcher config disabled', testSettings, async function () { assertFalse(await switcher.isItOn('FF2FOR2031')); assertEquals(Switcher.getLogger('FF2FOR2031')[0].response.reason, 'Config disabled'); }); - it('should be invalid - Switcher group disabled', async function () { + it('should be invalid - Switcher group disabled', testSettings, async function () { assertFalse(await switcher.isItOn('FF2FOR2040')); assertEquals(Switcher.getLogger('FF2FOR2040')[0].response.reason, 'Group disabled'); }); - it('should be valid assuming key to be false and then forgetting it', async function () { + it('should be valid assuming key to be false and then forgetting it', testSettings, async function () { await switcher.prepare('FF2FOR2020', [ checkValue('Japan'), checkNetwork('10.0.0.3') @@ -157,7 +158,7 @@ describe('E2E test - Switcher offline:', function () { assertTrue(await switcher.isItOn()); }); - it('should be valid assuming unknown key to be true', async function () { + it('should be valid assuming unknown key to be true', testSettings, async function () { await switcher.prepare('UNKNOWN', [ checkValue('Japan'), checkNetwork('10.0.0.3') @@ -172,7 +173,7 @@ describe('E2E test - Switcher offline:', function () { Error, 'Something went wrong: {"error":"Unable to load a key UNKNOWN"}'); }); - it('should enable test mode which will prevent a snapshot to be watchable', async function () { + it('should enable test mode which will prevent a snapshot to be watchable', testSettings, async function () { //given Switcher.buildContext({ url, apiKey, domain, component, environment }, { offline: true, logger: true @@ -188,6 +189,8 @@ describe('E2E test - Switcher offline:', function () { }); it('should be invalid - Offline mode cannot load snapshot from an invalid path', testSettings, async function () { + await delay(2000); + Switcher.buildContext({ url, apiKey, domain, component, environment }, { offline: true, snapshotLocation: '//somewhere/' @@ -199,14 +202,15 @@ describe('E2E test - Switcher offline:', function () { Error, 'Something went wrong: It was not possible to load the file at //somewhere/'); }); - it('should be valid - Offline mode', testSettings, function () { + it('should be valid - Offline mode', testSettings, async function () { + await delay(2000); + Switcher.buildContext({ url, apiKey, domain, component, environment }, { offline: true, snapshotLocation: 'generated-snapshots/' }); - - // Igored test: not working - // await Switcher.loadSnapshot(); - // assertExists(Switcher.snapshot); + + await Switcher.loadSnapshot(); + assertExists(Switcher.snapshot); }); }); \ No newline at end of file diff --git a/test/switcher-integrated.test.ts b/test/switcher-integrated.test.ts index 6d58a24..23caf55 100644 --- a/test/switcher-integrated.test.ts +++ b/test/switcher-integrated.test.ts @@ -1,7 +1,7 @@ // deno-lint-ignore-file no-explicit-any -import { describe, it, afterAll, afterEach, beforeEach } from 'https://deno.land/std@0.176.0/testing/bdd.ts'; -import { assertEquals, assertNotEquals, assertRejects, assertFalse } from 'https://deno.land/std@0.176.0/testing/asserts.ts'; -import { assertSpyCalls, spy } from 'https://deno.land/std@0.176.0/testing/mock.ts'; +import { describe, it, afterAll, afterEach, beforeEach } from 'https://deno.land/std@0.177.0/testing/bdd.ts'; +import { assertEquals, assertNotEquals, assertRejects, assertFalse } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; +import { assertSpyCalls, spy } from 'https://deno.land/std@0.177.0/testing/mock.ts'; import { given, givenError, tearDown, assertTrue, generateAuth, generateResult } from './helper/utils.ts' import { diff --git a/test/switcher-snapshot.test.ts b/test/switcher-snapshot.test.ts index c9081de..77b8310 100644 --- a/test/switcher-snapshot.test.ts +++ b/test/switcher-snapshot.test.ts @@ -1,12 +1,13 @@ // deno-lint-ignore-file no-explicit-any -import { describe, it, afterAll, beforeEach } from 'https://deno.land/std@0.176.0/testing/bdd.ts'; -import { assertRejects, assertFalse } from 'https://deno.land/std@0.176.0/testing/asserts.ts'; +import { describe, it, afterAll, beforeEach } from 'https://deno.land/std@0.177.0/testing/bdd.ts'; +import { assertRejects, assertFalse, assertExists } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; +import { delay } from 'https://deno.land/std@0.177.0/async/delay.ts'; import { existsSync } from 'https://deno.land/std@0.110.0/fs/mod.ts'; -import { given, givenError, tearDown, generateAuth, generateStatus } from './helper/utils.ts'; +import { given, givenError, tearDown, generateAuth, generateStatus, assertTrue } from './helper/utils.ts'; import { Switcher } from '../mod.ts'; -const testSettings = { sanitizeExit: false, sanitizeResources: false }; +const testSettings = { sanitizeOps: false, sanitizeResources: false, sanitizeExit: false }; describe('E2E test - Switcher offline - Snapshot:', function () { const token = '[token]'; @@ -16,6 +17,8 @@ describe('E2E test - Switcher offline - Snapshot:', function () { const dataJSON = dataBuffer.toString(); beforeEach(function() { + Switcher.unloadSnapshot(); + contextSettings = { url: 'http://localhost:3000', apiKey: '[api_key]', @@ -38,7 +41,9 @@ describe('E2E test - Switcher offline - Snapshot:', function () { Deno.removeSync('generated-snapshots/', { recursive: true }); }); - it('should update snapshot', testSettings, function () { + it('should update snapshot', testSettings, async function () { + await delay(2000); + //give given('POST@/criteria/auth', generateAuth(token, 5)); given('GET@/criteria/snapshot_check/:version', generateStatus(false)); @@ -50,15 +55,16 @@ describe('E2E test - Switcher offline - Snapshot:', function () { offline: true }); - // Ignored test: not working - // await Switcher.loadSnapshot(true); - // assertTrue(await Switcher.checkSnapshot()); + await Switcher.loadSnapshot(true); + assertTrue(await Switcher.checkSnapshot()); - // //restore state to avoid process leakage - // Switcher.unloadSnapshot(); + //restore state to avoid process leakage + Switcher.unloadSnapshot(); }); it('should NOT update snapshot', testSettings, async function () { + await delay(2000); + //given given('POST@/criteria/auth', generateAuth(token, 5)); given('GET@/criteria/snapshot_check/:version', generateStatus(true)); // No available update @@ -69,6 +75,8 @@ describe('E2E test - Switcher offline - Snapshot:', function () { }); it('should NOT update snapshot - check Snapshot Error', testSettings, async function () { + await delay(2000); + //given given('POST@/criteria/auth', generateAuth(token, 5)); givenError('GET@/criteria/snapshot_check/:version', 'ECONNREFUSED'); @@ -82,6 +90,8 @@ describe('E2E test - Switcher offline - Snapshot:', function () { }); it('should NOT update snapshot - resolve Snapshot Error', testSettings, async function () { + await delay(2000); + //given given('POST@/criteria/auth', generateAuth(token, 5)); given('GET@/criteria/snapshot_check/:version', generateStatus(false)); // Snapshot outdated @@ -95,7 +105,9 @@ describe('E2E test - Switcher offline - Snapshot:', function () { Error, 'Something went wrong: Connection has been refused - ECONNREFUSED'); }); - it('should update snapshot', testSettings, function () { + it('should update snapshot', testSettings, async function () { + await delay(2000); + //given given('POST@/criteria/auth', generateAuth(token, 5)); given('GET@/criteria/snapshot_check/:version', generateStatus(false)); // Snapshot outdated @@ -106,17 +118,20 @@ describe('E2E test - Switcher offline - Snapshot:', function () { snapshotLocation: 'generated-snapshots/' }); - // Ignored test: not working - // await Switcher.loadSnapshot(); - // assertExists(Switcher.snapshot); + await Switcher.loadSnapshot(); + assertExists(Switcher.snapshot); }); it('should not throw when switcher keys provided were configured properly', testSettings, async function () { + await delay(2000); + await Switcher.loadSnapshot(); await Switcher.checkSwitchers(['FF2FOR2030']); }); it('should throw when switcher keys provided were not configured properly', testSettings, async function () { + await delay(2000); + await Switcher.loadSnapshot(); await assertRejects(async () => await Switcher.checkSwitchers(['FEATURE02']), diff --git a/test/switcher-watch-snapshot.test.ts b/test/switcher-watch-snapshot.test.ts index d3819ad..3bb3cd1 100644 --- a/test/switcher-watch-snapshot.test.ts +++ b/test/switcher-watch-snapshot.test.ts @@ -1,6 +1,6 @@ // deno-lint-ignore-file no-explicit-any -import { describe, it, afterAll, beforeEach } from 'https://deno.land/std@0.176.0/testing/bdd.ts'; -import { assertEquals, assertFalse } from 'https://deno.land/std@0.176.0/testing/asserts.ts'; +import { describe, it, afterAll, beforeEach } from 'https://deno.land/std@0.177.0/testing/bdd.ts'; +import { assertEquals, assertFalse } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; import { existsSync } from 'https://deno.land/std@0.110.0/fs/mod.ts'; import { assertTrue } from './helper/utils.ts'; diff --git a/test/timed-match.test.ts b/test/timed-match.test.ts index 1fd8590..5dba844 100644 --- a/test/timed-match.test.ts +++ b/test/timed-match.test.ts @@ -1,5 +1,5 @@ -import { beforeEach, describe, it } from "https://deno.land/std@0.176.0/testing/bdd.ts"; -import { assertFalse } from 'https://deno.land/std@0.176.0/testing/asserts.ts'; +import { beforeEach, describe, it } from "https://deno.land/std@0.177.0/testing/bdd.ts"; +import { assertFalse } from 'https://deno.land/std@0.177.0/testing/asserts.ts'; import { assertTrue } from './helper/utils.ts'; import TimedMatch from "../src/lib/utils/timed-match/index.ts"; From b62bb75ae69dc2bd393ae034cefa41922c2d5fbe Mon Sep 17 00:00:00 2001 From: petruki <31597636+petruki@users.noreply.github.com> Date: Wed, 8 Feb 2023 21:14:51 -0800 Subject: [PATCH 2/2] Fixing flaky snapshot tests #2 --- test/switcher-client.test.ts | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/test/switcher-client.test.ts b/test/switcher-client.test.ts index 9d5e60c..2ba20f5 100644 --- a/test/switcher-client.test.ts +++ b/test/switcher-client.test.ts @@ -188,6 +188,18 @@ describe('E2E test - Switcher offline:', function () { assertTrue(await switcher.isItOn('FF2FOR2020')); }); + it('should be valid - Offline mode', testSettings, async function () { + await delay(2000); + + Switcher.buildContext({ url, apiKey, domain, component, environment }, { + offline: true, + snapshotLocation: 'generated-snapshots/' + }); + + await Switcher.loadSnapshot(); + assertExists(Switcher.snapshot); + }); + it('should be invalid - Offline mode cannot load snapshot from an invalid path', testSettings, async function () { await delay(2000); @@ -202,15 +214,4 @@ describe('E2E test - Switcher offline:', function () { Error, 'Something went wrong: It was not possible to load the file at //somewhere/'); }); - it('should be valid - Offline mode', testSettings, async function () { - await delay(2000); - - Switcher.buildContext({ url, apiKey, domain, component, environment }, { - offline: true, - snapshotLocation: 'generated-snapshots/' - }); - - await Switcher.loadSnapshot(); - assertExists(Switcher.snapshot); - }); }); \ No newline at end of file