Add --filter-quietly config flag

This makes it possible to use `--grep` and `--only` along with
`--fail-skip`, without making a ton of excessive noise. If you *want*
`--fail-skip` to fail on filtered tests, then you can set
`--no-filter-quietly` to force the skip messages.

Also, it's kind of nicer when filtering tests anyway, because it
provides a way to not have them show a bunch of noisy repetitive skip
messages.

src/filter/README.md

@@ -163,3 +163,22 @@ t.runOnly = false
 t.only('this will run', t => t.end())
 t.test('so will this, even with --only', t => t.end())
 ```
+
+## `--filter-quietly`
+
+By default, when a test is skipped with `--grep` or `--only`, a
+skip message is applied, indicating why it was omitted.
+
+This is often desireable, but can be noisy. The
+`--filter-quietly` config flag will disable this reporting,
+making filtered tests look like empty passing assertions.
+
+Since a skip message will cause failures when `--fail-skip` is
+set, in that case `--filter-quietly` will be enabled by default.
+Presumably, if you tell tap "fail on skipped tests", you don't
+also mean for it to fail on tests that you have told it to skip
+in that very same command with `--grep` or `--only`.
+
+If you _do_ mean to have it fail on intentionally skipped tests,
+then you can set `--no-filter-quietly` (or `filter-quietly:
+false` in a `.taprc` file) along with `--fail-skip`.

src/filter/src/index.ts

@@ -50,6 +50,7 @@ export class Filter {
   #grep: (string | RegExp)[] = []
   #grepInvert: boolean
   #runOnly: boolean
+  #filterQuietly: boolean
 
   get runOnly() {
     return this.#runOnly
@@ -60,6 +61,9 @@ export class Filter {
 
   constructor(t: TestBase, opts: FilterOptions) {
     this.#t = t
+    const eq = process.env.TAP_FILTER_QUIETLY
+    this.#filterQuietly =
+      eq === '1' || (!!t.options.failSkip && eq !== '0')
 
     // don't filter test files when we're the cli test runner
     const { grep, grepInvert, runOnly } =
@@ -116,11 +120,13 @@ export class Filter {
           : pattern.test(name)
       const match = this.#grepInvert ? !m : m
       if (!match) {
-        const p = `filter${
-          this.#grepInvert ? ' out' : ''
-        }: ${pattern}`
-        opts.skip = p
-        return shouldSkipChild(opts)
+        if (!this.#filterQuietly) {
+          opts.skip = `filter${
+            this.#grepInvert ? ' out' : ''
+          }: ${pattern}`
+        }
+        shouldSkipChild(opts)
+        return true
       } else {
         opts.grep = rest
       }
@@ -130,9 +136,11 @@ export class Filter {
       opts.grep = []
     }
     if (this.#runOnly && !opts.only) {
-      const p = 'filter: only'
-      opts.skip = p
-      return shouldSkipChild(opts)
+      if (!this.#filterQuietly) {
+        opts.skip = 'filter: only'
+      }
+      shouldSkipChild(opts)
+      return true
     }
     if (opts.only && !this.#runOnly) {
       this.#t.comment(
@@ -257,4 +265,29 @@ export const config = {
     description:
       'Do not invert the matches to --grep patterns. (default)',
   },
+
+  /**
+   * flag
+   *
+   * Do not apply a skip message to tests filtered using `--grep`
+   * and `--only`.
+   *
+   * Enabled by default if `--fail-skip` is set.
+   */
+  'filter-quietly': {
+    type: 'boolean',
+    description: `
+       Do not apply a skip message to tests filtered using \`--grep\`
+       and \`--only\`.
+
+       Defaults to true if \`--fail-skip\` is set.
+   `,
+  },
+  'no-filter-quietly': {
+    type: 'boolean',
+    description: `
+       Always set a skip message on filtered tests, even if \`--fail-skip\`
+       is enabled.
+   `,
+  },
 }

src/filter/tap-snapshots/test/index.ts.test.cjs

@@ -174,6 +174,48 @@ ok 3 - dog # SKIP filter out: /unicorn|dog/
 
 `
 
+exports[`test/index.ts > TAP > grep invert, failSkip:true > must match snapshot 1`] = `
+TAP version 14
+# Subtest: cat
+    # Subtest: hiss
+        ok 1 - sssss
+        1..1
+    ok 1 - hiss # time={TIME}
+    
+    # Subtest: purr
+        ok 1 - rrrrr
+        1..1
+    ok 2 - purr # time={TIME}
+    
+    ok 3 - meow
+    1..3
+ok 1 - cat # time={TIME}
+
+ok 2 - unicorn
+ok 3 - dog
+1..3
+
+`
+
+exports[`test/index.ts > TAP > grep, failSkip:true > must match snapshot 1`] = `
+TAP version 14
+# Subtest: cat
+    ok 1 - hiss
+    # Subtest: purr
+        ok 1 - rrrrr
+        1..1
+    ok 2 - purr # time={TIME}
+    
+    ok 3 - meow
+    1..3
+ok 1 - cat # time={TIME}
+
+ok 2 - unicorn
+ok 3 - dog
+1..3
+
+`
+
 exports[`test/index.ts > TAP > only > must match snapshot 1`] = `
 TAP version 14
 ok 1 - cat # SKIP filter: only
@@ -200,6 +242,19 @@ ok 3 - dog # SKIP filter: only
 
 `
 
+exports[`test/index.ts > TAP > only, failSkip: true > must match snapshot 1`] = `
+TAP version 14
+ok 1 - cat
+# Subtest: unicorn
+    ok 1 - this is fine
+    1..1
+ok 2 - unicorn # time={TIME}
+
+ok 3 - dog
+1..3
+
+`
+
 exports[`test/index.ts > TAP > warn if using only() unnecessarily > must match snapshot 1`] = `
 TAP version 14
 # Subtest: cat

src/filter/test/index.ts

@@ -53,6 +53,12 @@ t.test('grep', async t =>
   )
 )
 
+t.test('grep, failSkip:true', async t =>
+  t.matchSnapshot(
+    await run({ name: 'grepper', failSkip: true, grep: ['cat', 'purr'] })
+  )
+)
+
 t.test('grep invert', async t =>
   t.matchSnapshot(
     await run({
@@ -63,12 +69,29 @@ t.test('grep invert', async t =>
   )
 )
 
+t.test('grep invert, failSkip:true', async t =>
+  t.matchSnapshot(
+    await run({
+      name: 'grep invert',
+      grep: /unicorn|dog/,
+      grepInvert: true,
+      failSkip: true,
+    })
+  )
+)
+
 t.test('only', async t =>
   t.matchSnapshot(
     await run({ name: 'only the lonely', runOnly: true })
   )
 )
 
+t.test('only, failSkip: true', async t =>
+  t.matchSnapshot(
+    await run({ name: 'only the lonely', runOnly: true, failSkip: true })
+  )
+)
+
 t.test('only, but by setting t.runOnly', async t =>
   t.matchSnapshot(await run({ name: 'only the lonely' }, true))
 )