From f2a7dc5cb30ea8e09c0f410ad8ff2dc667204563 Mon Sep 17 00:00:00 2001 From: Wojtek Majewski Date: Thu, 9 Oct 2025 16:27:54 +0200 Subject: [PATCH] refactor(typecheck): fix tsc type tests by removing --project - tsc runs in strict mode then - Replaces temporary tsconfig creation with direct tsc invocation on each file - Enables detection of unused @ts-expect-error errors during strict type checks - Improves script clarity and reliability by removing unnecessary config file handling --- .github/workflows/type-test-health.yml | 44 +++++ pkgs/client/project.json | 16 +- pkgs/client/src/lib/FlowRun.ts | 6 +- pkgs/client/src/lib/FlowStep.ts | 6 +- pkgs/core/project.json | 16 +- pkgs/dsl/__tests__/types/README.md | 166 ++++++++++++++++++ pkgs/dsl/__tests__/types/__health__.test-d.ts | 76 ++++++++ .../__tests__/types/array-method.test-d.ts | 25 ++- pkgs/dsl/project.json | 20 ++- pkgs/dsl/src/dsl.ts | 27 +-- pkgs/dsl/tsconfig.typecheck.json | 3 + scripts/typecheck-strict.sh | 102 ----------- scripts/typecheck-ts2578.sh | 88 ++++++++++ scripts/verify-type-test-health.sh | 106 +++++++++++ 14 files changed, 574 insertions(+), 127 deletions(-) create mode 100644 .github/workflows/type-test-health.yml create mode 100644 pkgs/dsl/__tests__/types/README.md create mode 100644 pkgs/dsl/__tests__/types/__health__.test-d.ts delete mode 100755 scripts/typecheck-strict.sh create mode 100755 scripts/typecheck-ts2578.sh create mode 100755 scripts/verify-type-test-health.sh diff --git a/.github/workflows/type-test-health.yml b/.github/workflows/type-test-health.yml new file mode 100644 index 000000000..e893d17e1 --- /dev/null +++ b/.github/workflows/type-test-health.yml @@ -0,0 +1,44 @@ +name: Type Testing Health Check + +# Run on every push/PR to ensure type testing infrastructure always works +# Cost: ~1-2 seconds per run +# Benefit: Never ship broken type validation +on: + pull_request: + push: + branches: + - main + - 'release/**' + + # Allow manual trigger + workflow_dispatch: + +jobs: + health-check: + name: Verify Type Testing Infrastructure + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v2 + with: + version: 8 + + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run Type Testing Health Check + run: pnpm nx test:types:health dsl + + - name: Report Results + if: failure() + run: | + echo "::error::Type testing infrastructure is broken!" + echo "This means type tests may not be catching bugs." + echo "Check the health check output above for details." diff --git a/pkgs/client/project.json b/pkgs/client/project.json index f37ac0aca..072700b5a 100644 --- a/pkgs/client/project.json +++ b/pkgs/client/project.json @@ -187,12 +187,24 @@ "parallel": false } }, - "test:types": { + "test:types:vitest": { + "executor": "nx:run-commands", + "dependsOn": ["build"], + "options": { + "cwd": "{projectRoot}", + "command": "pnpm vitest --typecheck.only --run" + } + }, + "test:types:strict": { "executor": "nx:run-commands", "options": { "cwd": "{projectRoot}", - "command": "bash ../../scripts/typecheck-strict.sh" + "command": "bash ../../scripts/typecheck-ts2578.sh" } + }, + "test:types": { + "executor": "nx:noop", + "dependsOn": ["test:types:vitest", "test:types:strict"] } } } diff --git a/pkgs/client/src/lib/FlowRun.ts b/pkgs/client/src/lib/FlowRun.ts index 308bba509..abd56fa41 100644 --- a/pkgs/client/src/lib/FlowRun.ts +++ b/pkgs/client/src/lib/FlowRun.ts @@ -345,10 +345,8 @@ export class FlowRun break; default: { - // Exhaustiveness check - should never happen with proper types - // @ts-expect-error Intentional exhaustiveness check - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const _exhaustivenessCheck: never = event; + // Exhaustiveness check - ensures all event statuses are handled + event satisfies never; return false; } } diff --git a/pkgs/client/src/lib/FlowStep.ts b/pkgs/client/src/lib/FlowStep.ts index 9a68bd28f..382a5aea7 100644 --- a/pkgs/client/src/lib/FlowStep.ts +++ b/pkgs/client/src/lib/FlowStep.ts @@ -235,10 +235,8 @@ export class FlowStep< break; default: { - // Exhaustiveness check - should never happen with proper types - // @ts-expect-error Intentional exhaustiveness check - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const _exhaustivenessCheck: never = event; + // Exhaustiveness check - ensures all event statuses are handled + event satisfies never; return false; } } diff --git a/pkgs/core/project.json b/pkgs/core/project.json index 727af79f9..70ada3b2e 100644 --- a/pkgs/core/project.json +++ b/pkgs/core/project.json @@ -270,12 +270,24 @@ }, "cache": true }, - "test:types": { + "test:types:vitest": { "executor": "nx:run-commands", + "dependsOn": ["build"], "options": { "cwd": "{projectRoot}", - "command": "bash ../../scripts/typecheck-strict.sh" + "command": "pnpm vitest --typecheck.only --run" } + }, + "test:types:strict": { + "executor": "nx:run-commands", + "options": { + "cwd": "{projectRoot}", + "command": "bash ../../scripts/typecheck-ts2578.sh" + } + }, + "test:types": { + "executor": "nx:noop", + "dependsOn": ["test:types:vitest", "test:types:strict"] } } } diff --git a/pkgs/dsl/__tests__/types/README.md b/pkgs/dsl/__tests__/types/README.md new file mode 100644 index 000000000..ced79677c --- /dev/null +++ b/pkgs/dsl/__tests__/types/README.md @@ -0,0 +1,166 @@ +# Type Testing Infrastructure + +This directory contains TypeScript type tests using a **hybrid approach** that combines: +1. **Vitest `expectTypeOf`** for positive type assertions (clear error messages) +2. **`@ts-expect-error`** for negative assertions (enforces code doesn't compile) + +## Running Type Tests + +```bash +# Run all type tests (vitest + TS2578 validation) +pnpm nx test:types dsl + +# Individual targets for faster iteration: +pnpm nx test:types:vitest dsl # Vitest only (fast, ~300ms) +pnpm nx test:types:strict dsl # TS2578 validation only + +# Health check (verifies testing infrastructure works) +pnpm nx test:types:health dsl +``` + +## Development Workflow + +**Fast iteration (during development):** +```bash +pnpm nx test:types:vitest dsl # Fast, nice output +``` + +**Full validation (before commit):** +```bash +pnpm nx test:types dsl # Runs both vitest and TS2578 checks +``` + +## Health Check System + +### Purpose +The health check ensures the type testing infrastructure itself is working correctly. Without this, type tests could silently break and fail to catch bugs. + +### Components + +#### 1. Canary File (`__canary__.test-d.ts`) +- **Intentionally fails** type checking +- Contains known type errors that MUST be detected +- Excluded from normal test runs (see `tsconfig.typecheck.json`) +- Tests both `expectTypeOf` and `@ts-expect-error` mechanisms + +#### 2. Verification Script (`scripts/verify-type-test-health.sh`) +Runs diagnostic tests to verify: +- tsc detects type errors +- `expectTypeOf` produces clear "Expected X, Actual Y" messages +- TS2578 detection works (unused `@ts-expect-error`) +- Expected error patterns appear (TS2344, TS2578) + +#### 3. Nx Target (`test:types:health`) +Run with: `pnpm nx test:types:health dsl` + +### When to Run Health Check + +- **After TypeScript version upgrades** +- **After Vitest updates** +- **After modifying tsconfig files** +- **After changing typecheck-ts2578.sh script** +- **In CI** (recommended: run weekly or on release branches) + +### Expected Output + +✅ **Healthy system:** +``` +✅ SUCCESS: Type testing infrastructure is healthy + +All checks passed: + • Pass 1 (project-wide) catches type errors + • expectTypeOf produces clear error messages + • Pass 2 (individual files) catches TS2578 errors + • Canary file exhibits expected error patterns +``` + +❌ **Broken system:** +``` +❌ FAILURE: Type testing infrastructure is BROKEN + +This means type tests may not be catching bugs! +``` + +## Hybrid Testing Approach + +### Use `expectTypeOf` for Positive Assertions + +```typescript +it('should correctly infer types', () => { + const flow = new Flow<{ userId: string }>({ slug: 'test' }) + .step({ slug: 's1' }, () => 42); + + const step = flow.getStepDefinition('s1'); + + // Clear, explicit type assertion + expectTypeOf(step.handler).returns.toEqualTypeOf(); +}); +``` + +**Benefits:** +- ✅ Clear error messages: `"Expected: number, Actual: string"` +- ✅ Explicit about what types SHOULD be +- ✅ Good for complex type testing + +### Use `@ts-expect-error` for Negative Assertions + +```typescript +it('should reject invalid handlers', () => { + new Flow({ slug: 'test' }) + // @ts-expect-error - should reject null return + .array({ slug: 'bad' }, () => null) + + // @ts-expect-error - should reject undefined return + .array({ slug: 'bad2' }, () => undefined); +}); +``` + +**Benefits:** +- ✅ Actually enforces code doesn't compile +- ✅ Detected by typecheck-ts2578.sh via TS2578 +- ✅ Tests real-world usage patterns + +### Why Both? + +- **`expectTypeOf`**: Tests type relationships and transformations +- **`@ts-expect-error`**: Tests that invalid code is actually rejected + +Using both provides comprehensive coverage and catches different classes of bugs. + +## Troubleshooting + +### Health Check Fails + +1. Check TypeScript version: `pnpm tsc --version` +2. Check Vitest version: `pnpm vitest --version` +3. Review `tsconfig.typecheck.json` +4. Check `typecheck-ts2578.sh` script +5. Look at canary file errors: `pnpm tsc --noEmit __tests__/types/__canary__.test-d.ts` + +### Type Tests Pass But Should Fail + +This usually means: +- Type signatures are too permissive +- Missing `@ts-expect-error` directives +- Run health check to verify infrastructure works + +### TS2578 Not Detected + +- Ensure `typecheck-ts2578.sh` runs correctly +- Check that file is included in typecheck +- Verify `@ts-expect-error` is on its own line above the code + +## Adding New Type Tests + +1. **Choose the right tool**: + - Positive assertion (what type IS) → `expectTypeOf` + - Negative assertion (what shouldn't compile) → `@ts-expect-error` + +2. **Follow existing patterns** in the test files + +3. **Run health check** after major changes: + ```bash + pnpm nx test:types:health dsl + ``` + +4. **Test your tests**: Temporarily break the type signature and ensure the test fails diff --git a/pkgs/dsl/__tests__/types/__health__.test-d.ts b/pkgs/dsl/__tests__/types/__health__.test-d.ts new file mode 100644 index 000000000..27c4e3597 --- /dev/null +++ b/pkgs/dsl/__tests__/types/__health__.test-d.ts @@ -0,0 +1,76 @@ +/** + * TYPE TESTING HEALTH CHECK FILE + * + * This file intentionally contains type errors to verify that our type testing + * infrastructure is working correctly. It should NEVER pass type checking. + * + * ⚠️ DO NOT INCLUDE IN NORMAL TEST RUNS ⚠️ + * ⚠️ EXCLUDED FROM tsconfig.typecheck.json ⚠️ + * + * Purpose: + * - Verifies that expectTypeOf detects type mismatches + * - Verifies that @ts-expect-error unused directive detection works + * - Verifies that typecheck-ts2578.sh catches errors correctly + * + * Verified by: scripts/verify-type-test-health.sh + * Run with: pnpm nx test:types:health dsl + */ + +import { describe, it, expectTypeOf } from 'vitest'; +import { Flow } from '../../src/index.js'; + +describe('Health: expectTypeOf detects type mismatches', () => { + it('MUST FAIL: detects wrong return type with clear error message', () => { + const flow = new Flow<{ count: number }>({ slug: 'health_test' }) + .array({ slug: 'items' }, () => [1, 2, 3]); + + const arrayStep = flow.getStepDefinition('items'); + + // This MUST fail with: "Expected: string, Actual: number" + expectTypeOf(arrayStep.handler).returns.toEqualTypeOf(); + }); + + it('MUST FAIL: detects input type mismatch', () => { + const flow = new Flow<{ userId: string }>({ slug: 'health_test2' }) + .step({ slug: 'step1' }, () => 42); + + const step = flow.getStepDefinition('step1'); + + // This MUST fail - expecting number input but it's { run: { userId: string } } + expectTypeOf(step.handler).parameter(0).toEqualTypeOf<{ run: { userId: number } }>(); + }); +}); + +describe('Health: @ts-expect-error unused directive detection', () => { + it('MUST FAIL: detects unused @ts-expect-error (TS2578)', () => { + const validArray = new Flow({ slug: 'health_test3' }) + .array({ slug: 'numbers' }, () => [1, 2, 3]); + + // These @ts-expect-error directives are UNUSED because the types are actually valid + // They MUST trigger TS2578 errors in Pass 2 of typecheck-ts2578.sh + + // @ts-expect-error - HEALTH: This is actually valid, so TS2578 should fire + const step1 = validArray.getStepDefinition('numbers'); + + // @ts-expect-error - HEALTH: This is actually valid, so TS2578 should fire + const handler = step1.handler; + + // Suppress unused variable warnings + void handler; + }); +}); + +describe('Health: Hybrid approach validation', () => { + it('MUST FAIL: both expectTypeOf and @ts-expect-error work together', () => { + const flow = new Flow<{ value: number }>({ slug: 'health_hybrid' }) + .step({ slug: 's1' }, () => 'result'); + + // expectTypeOf assertion that MUST fail + expectTypeOf(flow.getStepDefinition('s1').handler).returns.toEqualTypeOf(); + + // @ts-expect-error that is unused (no actual error) - MUST trigger TS2578 + // @ts-expect-error - HEALTH: This line is actually valid + const validStep = flow.getStepDefinition('s1'); + void validStep; + }); +}); diff --git a/pkgs/dsl/__tests__/types/array-method.test-d.ts b/pkgs/dsl/__tests__/types/array-method.test-d.ts index 8d4b34333..0d7fa7d74 100644 --- a/pkgs/dsl/__tests__/types/array-method.test-d.ts +++ b/pkgs/dsl/__tests__/types/array-method.test-d.ts @@ -20,10 +20,31 @@ describe('.array() method type constraints', () => { }); it('should reject handlers that return non-arrays', () => { - new Flow>({ slug: 'test' }) + const flow = new Flow>({ slug: 'test' }); + + // Test that these handlers are NOT assignable to array handler type + type ArrayHandler = (input: { run: Record }, context: any) => Array | Promise>; + + const invalidNumber = () => 42; + expectTypeOf(invalidNumber).not.toMatchTypeOf(); + + const invalidString = () => 'not an array'; + expectTypeOf(invalidString).not.toMatchTypeOf(); + + const invalidObject = () => ({ not: 'array' }); + expectTypeOf(invalidObject).not.toMatchTypeOf(); + + const invalidNull = () => null; + expectTypeOf(invalidNull).not.toMatchTypeOf(); + + const invalidUndefined = () => undefined; + expectTypeOf(invalidUndefined).not.toMatchTypeOf(); + + // Keep the runtime tests with @ts-expect-error for actual type enforcement + new Flow>({ slug: 'test2' }) // @ts-expect-error - should reject number return .array({ slug: 'invalid_number' }, () => 42) - // @ts-expect-error - should reject string return + // @ts-expect-error - should reject string return .array({ slug: 'invalid_string' }, () => 'not an array') // @ts-expect-error - should reject object return .array({ slug: 'invalid_object' }, () => ({ not: 'array' })) diff --git a/pkgs/dsl/project.json b/pkgs/dsl/project.json index f93ed7e72..5461ed74f 100644 --- a/pkgs/dsl/project.json +++ b/pkgs/dsl/project.json @@ -34,11 +34,29 @@ "reportsDirectory": "{workspaceRoot}/coverage/{projectRoot}" } }, + "test:types:vitest": { + "executor": "nx:run-commands", + "options": { + "cwd": "{projectRoot}", + "command": "pnpm vitest --typecheck.only --run" + } + }, + "test:types:strict": { + "executor": "nx:run-commands", + "options": { + "cwd": "{projectRoot}", + "command": "bash ../../scripts/typecheck-ts2578.sh" + } + }, "test:types": { + "executor": "nx:noop", + "dependsOn": ["test:types:vitest", "test:types:strict"] + }, + "test:types:health": { "executor": "nx:run-commands", "options": { "cwd": "{projectRoot}", - "command": "bash ../../scripts/typecheck-strict.sh" + "command": "bash ../../scripts/verify-type-test-health.sh" } }, "test": { diff --git a/pkgs/dsl/src/dsl.ts b/pkgs/dsl/src/dsl.ts index 5bd8ecd12..53b7e0ae3 100644 --- a/pkgs/dsl/src/dsl.ts +++ b/pkgs/dsl/src/dsl.ts @@ -509,7 +509,7 @@ export class Flow< * @template THandler - The handler function that must return an array or Promise * @template Deps - The step dependencies (must be existing step slugs) * @param opts - Step configuration including slug, dependencies, and runtime options - * @param handler - Function that processes input and returns an array + * @param handler - Function that processes input and returns an array (null/undefined rejected) * @returns A new Flow instance with the array step added */ array< @@ -523,7 +523,7 @@ export class Flow< } >, context: BaseContext & TContext - ) => Array | Promise>, + ) => readonly any[] | Promise, Deps extends Extract = never >( opts: Simplify<{ slug: Slug extends keyof Steps ? never : Slug; dependsOn?: Deps[] } & StepRuntimeOptions>, @@ -551,11 +551,14 @@ export class Flow< * @returns A new Flow instance with the map step added */ // Overload for root map - map( - opts: Simplify<{ slug: Slug extends keyof Steps ? never : Slug } & StepRuntimeOptions>, - handler: TFlowInput extends readonly (infer Item)[] - ? THandler & ((item: Item, context: BaseContext & TContext) => Json | Promise) + map< + Slug extends string, + THandler extends TFlowInput extends readonly (infer Item)[] + ? (item: Item, context: BaseContext & TContext) => Json | Promise : never + >( + opts: Simplify<{ slug: Slug extends keyof Steps ? never : Slug } & StepRuntimeOptions>, + handler: THandler ): Flow< TFlowInput, TContext & BaseContext, @@ -564,11 +567,15 @@ export class Flow< >; // Overload for dependent map - map, THandler>( - opts: Simplify<{ slug: Slug extends keyof Steps ? never : Slug; array: TArrayDep } & StepRuntimeOptions>, - handler: Steps[TArrayDep] extends readonly (infer Item)[] - ? THandler & ((item: Item, context: BaseContext & TContext) => Json | Promise) + map< + Slug extends string, + TArrayDep extends Extract, + THandler extends Steps[TArrayDep] extends readonly (infer Item)[] + ? (item: Item, context: BaseContext & TContext) => Json | Promise : never + >( + opts: Simplify<{ slug: Slug extends keyof Steps ? never : Slug; array: TArrayDep } & StepRuntimeOptions>, + handler: THandler ): Flow< TFlowInput, TContext & BaseContext, diff --git a/pkgs/dsl/tsconfig.typecheck.json b/pkgs/dsl/tsconfig.typecheck.json index 165d7dba7..7527c5595 100644 --- a/pkgs/dsl/tsconfig.typecheck.json +++ b/pkgs/dsl/tsconfig.typecheck.json @@ -17,5 +17,8 @@ "include": [ "__tests__/**/*.test-d.ts", "__tests__/**/*.spec-d.ts" + ], + "exclude": [ + "__tests__/**/__health__.test-d.ts" ] } diff --git a/scripts/typecheck-strict.sh b/scripts/typecheck-strict.sh deleted file mode 100755 index 39efc5592..000000000 --- a/scripts/typecheck-strict.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash -# Two-pass TypeScript type checking -# Pass 1: Project-wide type check -# Pass 2: Individual file checks to catch unused @ts-expect-error directives -# -# Usage: -# typecheck-strict.sh # Check all *.test-d.ts files -# typecheck-strict.sh path/to/file.ts # Check specific file -# typecheck-strict.sh path/to/dir/ # Check all *.test-d.ts in directory - -set +e # Don't exit on first error, collect all errors - -EXIT_CODE=0 -ERRORS_FILE=$(mktemp) -TARGET_PATH="${1:-.}" # Use first argument or current directory - -echo "=========================================" -echo "Pass 1: Project-wide type check" -echo "=========================================" -pnpm tsc --project tsconfig.typecheck.json --noEmit 2>&1 | tee -a "$ERRORS_FILE" -if [ ${PIPESTATUS[0]} -ne 0 ]; then - EXIT_CODE=1 - echo "❌ Project-wide type check failed" -else - echo "✓ Project-wide type check passed" -fi -echo "" - -echo "=========================================" -echo "Pass 2: Individual file strict checks" -echo "=========================================" -echo "Checking for unused @ts-expect-error directives..." -echo "" - -FILE_ERRORS=0 - -# Determine which files to check -if [ -f "$TARGET_PATH" ]; then - # Single file provided - echo "Targeting specific file: $TARGET_PATH" - FILES_TO_CHECK="$TARGET_PATH" -elif [ -d "$TARGET_PATH" ]; then - # Directory provided - find all test-d.ts files - echo "Targeting directory: $TARGET_PATH" - FILES_TO_CHECK=$(find "$TARGET_PATH" -name "*.test-d.ts" -type f) -else - echo "Error: '$TARGET_PATH' is not a valid file or directory" - exit 1 -fi - -# Check each file -while IFS= read -r file; do - [ -z "$file" ] && continue # Skip empty lines - echo "Checking: $file" - - # Create temporary tsconfig in current directory that extends the main one - TEMP_CONFIG=".tsconfig.typecheck-strict-$(basename "$file").json" - cat > "$TEMP_CONFIG" <&1) - TSC_EXIT=$? - rm -f "$TEMP_CONFIG" - - if [ $TSC_EXIT -ne 0 ]; then - # Filter out node_modules errors, keep only test file errors - FILTERED=$(echo "$OUTPUT" | grep -v "node_modules") - - if [ -n "$FILTERED" ]; then - echo "$FILTERED" | tee -a "$ERRORS_FILE" - FILE_ERRORS=$((FILE_ERRORS + 1)) - EXIT_CODE=1 - echo "" - fi - fi -done <<< "$FILES_TO_CHECK" - -if [ $FILE_ERRORS -eq 0 ]; then - echo "✓ All individual file checks passed" -else - echo "❌ $FILE_ERRORS file(s) had type errors" -fi -echo "" - -echo "=========================================" -echo "Summary" -echo "=========================================" -if [ $EXIT_CODE -eq 0 ]; then - echo "✅ All type checks passed!" -else - echo "❌ Type checking failed" - echo "" - echo "Errors found:" - cat "$ERRORS_FILE" -fi - -rm -f "$ERRORS_FILE" -exit $EXIT_CODE diff --git a/scripts/typecheck-ts2578.sh b/scripts/typecheck-ts2578.sh new file mode 100755 index 000000000..8861859ca --- /dev/null +++ b/scripts/typecheck-ts2578.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# TS2578 validation - checks for unused @ts-expect-error directives +# +# This script validates that @ts-expect-error directives are actually +# suppressing TypeScript errors. If they're unused, it means the type +# constraint they were testing has become too permissive. +# +# Usage: +# typecheck-ts2578.sh # Check all *.test-d.ts files +# typecheck-ts2578.sh path/to/file.ts # Check specific file +# typecheck-ts2578.sh path/to/dir/ # Check all *.test-d.ts in directory + +set +e # Don't exit on first error, collect all errors + +EXIT_CODE=0 +ERRORS_FILE=$(mktemp) +TARGET_PATH="${1:-.}" # Use first argument or current directory + +echo "=========================================" +echo "TS2578 Validation (@ts-expect-error)" +echo "=========================================" +echo "Checking for unused @ts-expect-error directives..." +echo "(This catches when type constraints become too permissive)" +echo "" + +FILE_ERRORS=0 + +# Determine which files to check +if [ -f "$TARGET_PATH" ]; then + # Single file provided + echo "Targeting specific file: $TARGET_PATH" + FILES_TO_CHECK="$TARGET_PATH" +elif [ -d "$TARGET_PATH" ]; then + # Directory provided - find all test-d.ts files (excluding __health__.test-d.ts) + echo "Targeting directory: $TARGET_PATH" + FILES_TO_CHECK=$(find "$TARGET_PATH" -name "*.test-d.ts" -type f ! -name "__health__.test-d.ts") +else + echo "Error: '$TARGET_PATH' is not a valid file or directory" + exit 1 +fi + +# Check each file +while IFS= read -r file; do + [ -z "$file" ] && continue # Skip empty lines + echo "Checking: $file" + + # Run tsc directly on the file without --project flag + # This allows TS2578 errors (unused @ts-expect-error) to be detected + # Use --strict to enable strictNullChecks and other strict mode features + OUTPUT=$(pnpm tsc --noEmit --strict "$file" 2>&1) + TSC_EXIT=$? + + if [ $TSC_EXIT -ne 0 ]; then + # Only check for TS2578 errors (unused @ts-expect-error directives) + # Other errors are caught by vitest typecheck + FILTERED=$(echo "$OUTPUT" | grep "TS2578") + + if [ -n "$FILTERED" ]; then + echo "$FILTERED" | tee -a "$ERRORS_FILE" + FILE_ERRORS=$((FILE_ERRORS + 1)) + EXIT_CODE=1 + echo "" + fi + fi +done <<< "$FILES_TO_CHECK" + +echo "" +if [ $FILE_ERRORS -eq 0 ]; then + echo "✅ All @ts-expect-error directives are valid" + echo "" + echo "This means:" + echo " • Type constraints are working correctly" + echo " • Invalid code is being properly rejected" + echo " • No @ts-expect-error directives have become unused" +else + echo "❌ Found $FILE_ERRORS file(s) with unused @ts-expect-error directives" + echo "" + echo "This means:" + echo " • Type constraints have become too permissive" + echo " • Code that should fail type checking now passes" + echo " • Type validation is broken - fix the type signatures!" + echo "" + echo "Errors found:" + cat "$ERRORS_FILE" +fi + +rm -f "$ERRORS_FILE" +exit $EXIT_CODE diff --git a/scripts/verify-type-test-health.sh b/scripts/verify-type-test-health.sh new file mode 100755 index 000000000..73dc5a56e --- /dev/null +++ b/scripts/verify-type-test-health.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Type Testing Health Check +# +# Verifies that the type testing infrastructure is working correctly by +# running tests on a canary file that MUST fail in expected ways. +# +# This ensures that if type checking breaks, we'll know about it! + +set -e + +HEALTH_FILE="__tests__/types/__health__.test-d.ts" +TEMP_OUTPUT=$(mktemp) +EXIT_CODE=0 + +echo "=========================================" +echo "Type Testing Infrastructure Health Check" +echo "=========================================" +echo "" +echo "Running diagnostics on: $HEALTH_FILE" +echo "" + +# Test 1: Verify tsc catches errors in canary file +echo "[1/4] Testing Pass 1: Type error detection..." +pnpm tsc --noEmit "$HEALTH_FILE" 2>&1 | tee "$TEMP_OUTPUT" > /dev/null +if grep -q "error TS" "$TEMP_OUTPUT"; then + echo " ✓ tsc detects type errors in health file" +else + echo " ✗ FAIL: tsc did not detect errors" + EXIT_CODE=1 +fi +echo "" + +# Test 2: Verify expectTypeOf type mismatch detection +echo "[2/4] Testing expectTypeOf type mismatch detection..." +if grep -q "Expected.*Actual" "$TEMP_OUTPUT"; then + echo " ✓ expectTypeOf produces clear error messages" + # Show example + EXAMPLE_MSG=$(grep -m1 "Expected.*Actual" "$TEMP_OUTPUT" | sed 's/.*\(Expected[^"]*\).*/\1/') + echo " Example: $EXAMPLE_MSG" +else + echo " ✗ FAIL: expectTypeOf type mismatches not detected clearly" + EXIT_CODE=1 +fi +echo "" + +# Test 3: Verify Pass 2 (individual file) catches TS2578 +echo "[3/4] Testing Pass 2: Unused @ts-expect-error detection..." +if pnpm tsc --noEmit "$HEALTH_FILE" 2>&1 | grep -q "TS2578"; then + echo " ✓ Pass 2 detects unused @ts-expect-error directives" + # Count how many + TS2578_COUNT=$(pnpm tsc --noEmit "$HEALTH_FILE" 2>&1 | grep -c "TS2578" || true) + echo " Found $TS2578_COUNT unused @ts-expect-error directive(s)" +else + echo " ✗ FAIL: TS2578 not detected - @ts-expect-error validation broken" + EXIT_CODE=1 +fi +echo "" + +# Test 4: Verify the health file has expected error patterns +echo "[4/4] Testing health file error patterns..." +EXPECTED_PATTERNS=( + "TS2344" # Type constraint violations from expectTypeOf + "TS2578" # Unused @ts-expect-error directives +) + +ALL_ERRORS=$(pnpm tsc --noEmit "$HEALTH_FILE" 2>&1 || true) +for PATTERN in "${EXPECTED_PATTERNS[@]}"; do + if echo "$ALL_ERRORS" | grep -q "$PATTERN"; then + echo " ✓ Found expected error pattern: $PATTERN" + else + echo " ✗ MISSING expected error pattern: $PATTERN" + EXIT_CODE=1 + fi +done +echo "" + +# Cleanup +rm -f "$TEMP_OUTPUT" + +# Final verdict +echo "=========================================" +if [ $EXIT_CODE -eq 0 ]; then + echo "✅ SUCCESS: Type testing infrastructure is healthy" + echo "" + echo "All checks passed:" + echo " • Pass 1 (project-wide) catches type errors" + echo " • expectTypeOf produces clear error messages" + echo " • Pass 2 (individual files) catches TS2578 errors" + echo " • Health file exhibits expected error patterns" + echo "" + echo "Your type tests are working correctly!" +else + echo "❌ FAILURE: Type testing infrastructure is BROKEN" + echo "" + echo "This means type tests may not be catching bugs!" + echo "Investigate the failed checks above and fix the testing setup." + echo "" + echo "Common issues:" + echo " • TypeScript version mismatch" + echo " • tsconfig.typecheck.json misconfigured" + echo " • typecheck-ts2578.sh script broken" + echo " • Vitest expectTypeOf not working" +fi +echo "=========================================" + +exit $EXIT_CODE