✨ Introduce a way to mark equal values as skipped (#1629)

documentation/Runners.md

@@ -110,8 +110,13 @@ export interface Parameters<T = void> {
     interruptAfterTimeLimit?: number; // optional, interrupt test execution after a given time limit
                         // in milliseconds (relies on Date.now): disabled by default
     markInterruptAsFailure?: boolean; // optional, mark interrupted runs as failure: disabled by default
+    skipEqualValues?: boolean; // optional, skip repeated runs: disabled by default
+                        // If a same input is encountered multiple times only the first one will be executed,
+                        // next ones will be skipped. Be aware that skipping runs may lead to property failure
+                        // if the arbitrary does not have enough values. In that case use `ignoreEqualValues` instead.
     ignoreEqualValues?: boolean; // optional, do not repeat runs with already covered cases: disabled by default
-    					// This is useful when arbitrary has a limited number of variants
+                        // Similar to `skipEqualValues` but instead of skipping runs, it just don't rerun them.
+                        // It can be useful when arbitrary has a limited number of variants.
     reporter?: (runDetails: RunDetails<T>) => void; // optional, custom reporter replacing the default one
                         // reporter is responsible for throwing in case of failure, as an example default one throws
                         // whenever `runDetails.failed` is true but it is up to you

src/check/property/IgnoreEqualValuesProperty.ts

@@ -2,19 +2,56 @@ import { IRawProperty } from './IRawProperty';
 import { Random } from '../../random/generator/Random';
 import { Shrinkable } from '../arbitrary/definition/Shrinkable';
 import { stringify } from '../../utils/stringify';
+import { PreconditionFailure } from '../precondition/PreconditionFailure';
+
+/** @internal */
+function fromSyncCached<Ts>(
+  cachedValue: ReturnType<IRawProperty<Ts, false>['run']>
+): ReturnType<IRawProperty<Ts, false>['run']> {
+  return cachedValue === null ? new PreconditionFailure() : cachedValue;
+}
+
+/** @internal */
+function fromCached<Ts>(
+  cachedValue: ReturnType<IRawProperty<Ts, false>['run']>,
+  isAsync: false
+): ReturnType<IRawProperty<Ts, false>['run']>;
+/** @internal */
+function fromCached<Ts>(
+  cachedValue: ReturnType<IRawProperty<Ts, true>['run']>,
+  isAsync: true
+): ReturnType<IRawProperty<Ts, true>['run']>;
+function fromCached<Ts>(
+  ...data: [ReturnType<IRawProperty<Ts, true>['run']>, true] | [ReturnType<IRawProperty<Ts, false>['run']>, false]
+) {
+  if (data[1]) return data[0].then(fromSyncCached);
+  return fromSyncCached(data[0]);
+}
+
+/** @internal */
+function fromCachedUnsafe<Ts, IsAsync extends boolean>(
+  cachedValue: ReturnType<IRawProperty<Ts, IsAsync>['run']>,
+  isAsync: IsAsync
+): ReturnType<IRawProperty<Ts, IsAsync>['run']> {
+  return fromCached(cachedValue as any, isAsync as any) as any;
+}
 
 /** @internal */
 export class IgnoreEqualValuesProperty<Ts, IsAsync extends boolean> implements IRawProperty<Ts, IsAsync> {
   private coveredCases: Map<string, ReturnType<IRawProperty<Ts, IsAsync>['run']>> = new Map();
 
-  constructor(readonly property: IRawProperty<Ts, IsAsync>) {}
+  constructor(readonly property: IRawProperty<Ts, IsAsync>, readonly skipRuns: boolean) {}
 
   isAsync = (): IsAsync => this.property.isAsync();
   generate = (mrng: Random, runId?: number): Shrinkable<Ts> => this.property.generate(mrng, runId);
   run = (v: Ts): ReturnType<IRawProperty<Ts, IsAsync>['run']> => {
     const stringifiedValue = stringify(v);
     if (this.coveredCases.has(stringifiedValue)) {
-      return this.coveredCases.get(stringifiedValue) as ReturnType<IRawProperty<Ts, IsAsync>['run']>;
+      const lastOutput = this.coveredCases.get(stringifiedValue) as ReturnType<IRawProperty<Ts, IsAsync>['run']>;
+      if (!this.skipRuns) {
+        return lastOutput;
+      }
+      return fromCachedUnsafe(lastOutput, this.property.isAsync());
     }
     const out = this.property.run(v);
     this.coveredCases.set(stringifiedValue, out);

src/check/runner/DecorateProperty.ts

@@ -8,7 +8,7 @@ import { IgnoreEqualValuesProperty } from '../property/IgnoreEqualValuesProperty
 /** @internal */
 type MinimalQualifiedParameters<Ts> = Pick<
   QualifiedParameters<Ts>,
-  'unbiased' | 'timeout' | 'skipAllAfterTimeLimit' | 'interruptAfterTimeLimit' | 'ignoreEqualValues'
+  'unbiased' | 'timeout' | 'skipAllAfterTimeLimit' | 'interruptAfterTimeLimit' | 'skipEqualValues' | 'ignoreEqualValues'
 >;
 
 /** @internal */
@@ -20,7 +20,7 @@ export function decorateProperty<Ts>(
   if (rawProperty.isAsync() && qParams.timeout != null) {
     prop = new TimeoutProperty(prop, qParams.timeout);
   }
-  if (qParams.unbiased === true) {
+  if (qParams.unbiased) {
     prop = new UnbiasedProperty(prop);
   }
   if (qParams.skipAllAfterTimeLimit != null) {
@@ -29,8 +29,11 @@ export function decorateProperty<Ts>(
   if (qParams.interruptAfterTimeLimit != null) {
     prop = new SkipAfterProperty(prop, Date.now, qParams.interruptAfterTimeLimit, true);
   }
+  if (qParams.skipEqualValues) {
+    prop = new IgnoreEqualValuesProperty(prop, true);
+  }
   if (qParams.ignoreEqualValues) {
-    prop = new IgnoreEqualValuesProperty(prop);
+    prop = new IgnoreEqualValuesProperty(prop, false);
   }
   return prop;
 }

src/check/runner/configuration/Parameters.ts

@@ -92,10 +92,26 @@ export interface Parameters<T = void> {
    */
   markInterruptAsFailure?: boolean;
   /**
-   * Do not repeat runs with already covered cases.
-   * This is useful when arbitrary has a limited number of variants.
+   * Skip runs corresponding to already tried values.
    *
-   * NOTE: Values are compared by equality of fc.stringify results.
+   * WARNING:
+   * Discarded runs will be retried. Under the hood they are simple calls to `fc.pre`.
+   * In other words, if you ask for 100 runs but your generator can only generate 10 values then the property will fail as 100 runs will never be reached.
+   * Contrary to `ignoreEqualValues` you always have the number of runs you requested.
+   *
+   * NOTE: Relies on `fc.stringify` to check the equality.
+   *
+   * @remarks Since 2.14.0
+   */
+  skipEqualValues?: boolean;
+  /**
+   * Discard runs corresponding to already tried values.
+   *
+   * WARNING:
+   * Discarded runs will not be replaced.
+   * In other words, if you ask for 100 runs and have 2 discarded runs you will only have 98 effective runs.
+   *
+   * NOTE: Relies on `fc.stringify` to check the equality.
    *
    * @remarks Since 2.14.0
    */

src/check/runner/configuration/QualifiedParameters.ts

@@ -25,6 +25,7 @@ export class QualifiedParameters<T> {
   skipAllAfterTimeLimit: number | null;
   interruptAfterTimeLimit: number | null;
   markInterruptAsFailure: boolean;
+  skipEqualValues: boolean;
   ignoreEqualValues: boolean;
   reporter: ((runDetails: RunDetails<T>) => void) | null;
   asyncReporter: ((runDetails: RunDetails<T>) => Promise<void>) | null;
@@ -40,6 +41,7 @@ export class QualifiedParameters<T> {
     this.skipAllAfterTimeLimit = QualifiedParameters.readOrDefault(p, 'skipAllAfterTimeLimit', null);
     this.interruptAfterTimeLimit = QualifiedParameters.readOrDefault(p, 'interruptAfterTimeLimit', null);
     this.markInterruptAsFailure = QualifiedParameters.readBoolean(p, 'markInterruptAsFailure');
+    this.skipEqualValues = QualifiedParameters.readBoolean(p, 'skipEqualValues');
     this.ignoreEqualValues = QualifiedParameters.readBoolean(p, 'ignoreEqualValues');
     this.logger = QualifiedParameters.readOrDefault(p, 'logger', (v: string) => {
       // tslint:disable-next-line:no-console
@@ -64,6 +66,7 @@ export class QualifiedParameters<T> {
       skipAllAfterTimeLimit: orUndefined(this.skipAllAfterTimeLimit),
       interruptAfterTimeLimit: orUndefined(this.interruptAfterTimeLimit),
       markInterruptAsFailure: this.markInterruptAsFailure,
+      skipEqualValues: this.skipEqualValues,
       ignoreEqualValues: this.ignoreEqualValues,
       path: this.path,
       logger: this.logger,

test/e2e/IgnoreEqualValues.spec.ts

@@ -2,20 +2,39 @@ import { seed } from './seed';
 import * as fc from '../../src/fast-check';
 
 describe(`IgnoreEqualValues (seed: ${seed})`, () => {
-  it('should not run more than 4 times', () => {
-    let numRuns = 0;
-    const out = fc.check(
-      fc.property(fc.boolean(), fc.boolean(), () => {
-        ++numRuns;
-        return true;
-      }),
-      { ignoreEqualValues: true }
-    );
-    expect(out.failed).toBe(false);
-    expect(out.interrupted).toBe(false);
-    expect(out.numRuns).toBe(100);
-    expect(out.numShrinks).toBe(0);
-    expect(out.numSkips).toBe(0);
-    expect(numRuns).toBeLessThanOrEqual(4);
+  describe('ignoreEqualValues', () => {
+    it('should not run more than 4 times', () => {
+      let numRuns = 0;
+      const out = fc.check(
+        fc.property(fc.boolean(), fc.boolean(), () => {
+          ++numRuns;
+        }),
+        { ignoreEqualValues: true }
+      );
+      expect(out.failed).toBe(false);
+      expect(out.interrupted).toBe(false);
+      expect(out.numRuns).toBe(100);
+      expect(out.numShrinks).toBe(0);
+      expect(out.numSkips).toBe(0);
+      expect(numRuns).toBeLessThanOrEqual(4);
+    });
+  });
+
+  describe('skipEqualValues', () => {
+    it('should not run more than 4 times but mark run as failed due to too many skipped values', () => {
+      let numRuns = 0;
+      const out = fc.check(
+        fc.property(fc.boolean(), fc.boolean(), () => {
+          ++numRuns;
+        }),
+        { skipEqualValues: true }
+      );
+      expect(out.failed).toBe(true);
+      expect(out.interrupted).toBe(false);
+      expect(out.numRuns).toBe(4);
+      expect(out.numShrinks).toBe(0);
+      expect(out.numSkips).not.toBe(0);
+      expect(numRuns).toBeLessThanOrEqual(4);
+    });
   });
 });

test/unit/check/property/IgnoreEqualValuesProperty.spec.ts

@@ -1,5 +1,7 @@
 import { IRawProperty } from '../../../../src/check/property/IRawProperty';
 import { IgnoreEqualValuesProperty } from '../../../../src/check/property/IgnoreEqualValuesProperty';
+import { PreconditionFailure } from '../../../../src/check/precondition/PreconditionFailure';
+import { Shrinkable } from '../../../../src/check/arbitrary/definition/Shrinkable';
 
 function buildProperty() {
   const mocks = {
@@ -11,18 +13,93 @@ function buildProperty() {
 }
 
 describe('IgnoreEqualValuesProperty', () => {
-  it('should not run decorated property when property is run on the same value', async () => {
+  it.each`
+    skipRuns
+    ${false}
+    ${true}
+  `('should not run decorated property when property is run on the same value', ({ skipRuns }) => {
     const { mocks: propertyMock, property: decoratedProperty } = buildProperty();
-    const property = new IgnoreEqualValuesProperty(decoratedProperty);
+    const property = new IgnoreEqualValuesProperty(decoratedProperty, skipRuns);
     property.run(1);
     property.run(1);
 
     expect(propertyMock.run.mock.calls.length).toBe(1);
   });
 
-  it('should run decorated property when property is run on another value', async () => {
+  it.each`
+    originalValue                           | isAsync
+    ${null /* success */}                   | ${false}
+    ${'error' /* failure */}                | ${false}
+    ${new PreconditionFailure() /* skip */} | ${false}
+    ${null /* success */}                   | ${true}
+    ${'error' /* failure */}                | ${true}
+    ${new PreconditionFailure() /* skip */} | ${true}
+  `(
+    'should always return the cached value for skipRuns=false, originalValue=$originalValue, isAsync=$isAsync',
+    ({ originalValue, isAsync }) => {
+      // success -> success
+      // failure -> failure
+      // skip    -> skip
+      const property = new IgnoreEqualValuesProperty(
+        {
+          isAsync: () => isAsync,
+          run: () => (isAsync ? Promise.resolve(originalValue) : originalValue),
+          generate: () => new Shrinkable(null),
+        },
+        false
+      );
+
+      const initialRunOutput = property.run(null);
+      const secondRunOutput = property.run(null);
+
+      expect(secondRunOutput).toBe(initialRunOutput);
+    }
+  );
+
+  it.each`
+    originalValue                           | isAsync
+    ${null /* success */}                   | ${false}
+    ${'error' /* failure */}                | ${false}
+    ${new PreconditionFailure() /* skip */} | ${false}
+    ${null /* success */}                   | ${true}
+    ${'error' /* failure */}                | ${true}
+    ${new PreconditionFailure() /* skip */} | ${true}
+  `(
+    'should return the cached value but skip success for skipRuns=true, originalValue=$originalValue, isAsync=$isAsync',
+    // success -> skip
+    // failure -> failure
+    // skip    -> skip
+    async ({ originalValue, isAsync }) => {
+      const property = new IgnoreEqualValuesProperty(
+        {
+          isAsync: () => isAsync,
+          run: () => (isAsync ? Promise.resolve(originalValue) : originalValue),
+          generate: () => new Shrinkable(null),
+        },
+        true
+      );
+
+      const initialRunOutput = await property.run(null);
+      const secondRunOutput = await property.run(null);
+
+      if (initialRunOutput === null) {
+        // success
+        expect(secondRunOutput).not.toBe(initialRunOutput);
+        expect(PreconditionFailure.isFailure(secondRunOutput)).toBe(true);
+      } else {
+        // failure or skip
+        expect(secondRunOutput).toBe(initialRunOutput);
+      }
+    }
+  );
+
+  it.each`
+    skipRuns
+    ${false}
+    ${true}
+  `('should run decorated property when property is run on another value', ({ skipRuns }) => {
     const { mocks: propertyMock, property: decoratedProperty } = buildProperty();
-    const property = new IgnoreEqualValuesProperty(decoratedProperty);
+    const property = new IgnoreEqualValuesProperty(decoratedProperty, skipRuns);
     property.run(1);
     property.run(2);