Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions .github/workflows/type-test-health.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
name: Type Testing Health Check

# Run on every push/PR to ensure type testing infrastructure always works
# Cost: ~1-2 seconds per run
# Benefit: Never ship broken type validation
on:
pull_request:
push:
branches:
- main
- 'release/**'

# Allow manual trigger
workflow_dispatch:

jobs:
health-check:
name: Verify Type Testing Infrastructure
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4

- uses: pnpm/action-setup@v2
with:
version: 8

- uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'pnpm'

- name: Install dependencies
run: pnpm install --frozen-lockfile

- name: Run Type Testing Health Check
run: pnpm nx test:types:health dsl

- name: Report Results
if: failure()
run: |
echo "::error::Type testing infrastructure is broken!"
echo "This means type tests may not be catching bugs."
echo "Check the health check output above for details."
16 changes: 14 additions & 2 deletions pkgs/client/project.json
Original file line number Diff line number Diff line change
Expand Up @@ -187,12 +187,24 @@
"parallel": false
}
},
"test:types": {
"test:types:vitest": {
"executor": "nx:run-commands",
"dependsOn": ["build"],
"options": {
"cwd": "{projectRoot}",
"command": "pnpm vitest --typecheck.only --run"
}
},
"test:types:strict": {
"executor": "nx:run-commands",
"options": {
"cwd": "{projectRoot}",
"command": "bash ../../scripts/typecheck-strict.sh"
"command": "bash ../../scripts/typecheck-ts2578.sh"
}
},
"test:types": {
"executor": "nx:noop",
"dependsOn": ["test:types:vitest", "test:types:strict"]
}
}
}
6 changes: 2 additions & 4 deletions pkgs/client/src/lib/FlowRun.ts
Original file line number Diff line number Diff line change
Expand Up @@ -345,10 +345,8 @@ export class FlowRun<TFlow extends AnyFlow>
break;

default: {
// Exhaustiveness check - should never happen with proper types
// @ts-expect-error Intentional exhaustiveness check
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const _exhaustivenessCheck: never = event;
// Exhaustiveness check - ensures all event statuses are handled
event satisfies never;
return false;
}
}
Expand Down
6 changes: 2 additions & 4 deletions pkgs/client/src/lib/FlowStep.ts
Original file line number Diff line number Diff line change
Expand Up @@ -235,10 +235,8 @@ export class FlowStep<
break;

default: {
// Exhaustiveness check - should never happen with proper types
// @ts-expect-error Intentional exhaustiveness check
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const _exhaustivenessCheck: never = event;
// Exhaustiveness check - ensures all event statuses are handled
event satisfies never;
return false;
}
}
Expand Down
16 changes: 14 additions & 2 deletions pkgs/core/project.json
Original file line number Diff line number Diff line change
Expand Up @@ -270,12 +270,24 @@
},
"cache": true
},
"test:types": {
"test:types:vitest": {
"executor": "nx:run-commands",
"dependsOn": ["build"],
"options": {
"cwd": "{projectRoot}",
"command": "bash ../../scripts/typecheck-strict.sh"
"command": "pnpm vitest --typecheck.only --run"
}
},
"test:types:strict": {
"executor": "nx:run-commands",
"options": {
"cwd": "{projectRoot}",
"command": "bash ../../scripts/typecheck-ts2578.sh"
}
},
"test:types": {
"executor": "nx:noop",
"dependsOn": ["test:types:vitest", "test:types:strict"]
}
}
}
166 changes: 166 additions & 0 deletions pkgs/dsl/__tests__/types/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
# Type Testing Infrastructure

This directory contains TypeScript type tests using a **hybrid approach** that combines:
1. **Vitest `expectTypeOf`** for positive type assertions (clear error messages)
2. **`@ts-expect-error`** for negative assertions (enforces code doesn't compile)

## Running Type Tests

```bash
# Run all type tests (vitest + TS2578 validation)
pnpm nx test:types dsl

# Individual targets for faster iteration:
pnpm nx test:types:vitest dsl # Vitest only (fast, ~300ms)
pnpm nx test:types:strict dsl # TS2578 validation only

# Health check (verifies testing infrastructure works)
pnpm nx test:types:health dsl
```

## Development Workflow

**Fast iteration (during development):**
```bash
pnpm nx test:types:vitest dsl # Fast, nice output
```

**Full validation (before commit):**
```bash
pnpm nx test:types dsl # Runs both vitest and TS2578 checks
```

## Health Check System

### Purpose
The health check ensures the type testing infrastructure itself is working correctly. Without this, type tests could silently break and fail to catch bugs.

### Components

#### 1. Canary File (`__canary__.test-d.ts`)
- **Intentionally fails** type checking
- Contains known type errors that MUST be detected
- Excluded from normal test runs (see `tsconfig.typecheck.json`)
- Tests both `expectTypeOf` and `@ts-expect-error` mechanisms

#### 2. Verification Script (`scripts/verify-type-test-health.sh`)
Runs diagnostic tests to verify:
- tsc detects type errors
- `expectTypeOf` produces clear "Expected X, Actual Y" messages
- TS2578 detection works (unused `@ts-expect-error`)
- Expected error patterns appear (TS2344, TS2578)

#### 3. Nx Target (`test:types:health`)
Run with: `pnpm nx test:types:health dsl`

### When to Run Health Check

- **After TypeScript version upgrades**
- **After Vitest updates**
- **After modifying tsconfig files**
- **After changing typecheck-ts2578.sh script**
- **In CI** (recommended: run weekly or on release branches)

### Expected Output

✅ **Healthy system:**
```
✅ SUCCESS: Type testing infrastructure is healthy

All checks passed:
• Pass 1 (project-wide) catches type errors
• expectTypeOf produces clear error messages
• Pass 2 (individual files) catches TS2578 errors
• Canary file exhibits expected error patterns
```

❌ **Broken system:**
```
❌ FAILURE: Type testing infrastructure is BROKEN

This means type tests may not be catching bugs!
```

## Hybrid Testing Approach

### Use `expectTypeOf` for Positive Assertions

```typescript
it('should correctly infer types', () => {
const flow = new Flow<{ userId: string }>({ slug: 'test' })
.step({ slug: 's1' }, () => 42);

const step = flow.getStepDefinition('s1');

// Clear, explicit type assertion
expectTypeOf(step.handler).returns.toEqualTypeOf<number>();
});
```

**Benefits:**
- ✅ Clear error messages: `"Expected: number, Actual: string"`
- ✅ Explicit about what types SHOULD be
- ✅ Good for complex type testing

### Use `@ts-expect-error` for Negative Assertions

```typescript
it('should reject invalid handlers', () => {
new Flow({ slug: 'test' })
// @ts-expect-error - should reject null return
.array({ slug: 'bad' }, () => null)

// @ts-expect-error - should reject undefined return
.array({ slug: 'bad2' }, () => undefined);
});
```

**Benefits:**
- ✅ Actually enforces code doesn't compile
- ✅ Detected by typecheck-ts2578.sh via TS2578
- ✅ Tests real-world usage patterns

### Why Both?

- **`expectTypeOf`**: Tests type relationships and transformations
- **`@ts-expect-error`**: Tests that invalid code is actually rejected

Using both provides comprehensive coverage and catches different classes of bugs.

## Troubleshooting

### Health Check Fails

1. Check TypeScript version: `pnpm tsc --version`
2. Check Vitest version: `pnpm vitest --version`
3. Review `tsconfig.typecheck.json`
4. Check `typecheck-ts2578.sh` script
5. Look at canary file errors: `pnpm tsc --noEmit __tests__/types/__canary__.test-d.ts`

### Type Tests Pass But Should Fail

This usually means:
- Type signatures are too permissive
- Missing `@ts-expect-error` directives
- Run health check to verify infrastructure works

### TS2578 Not Detected

- Ensure `typecheck-ts2578.sh` runs correctly
- Check that file is included in typecheck
- Verify `@ts-expect-error` is on its own line above the code

## Adding New Type Tests

1. **Choose the right tool**:
- Positive assertion (what type IS) → `expectTypeOf`
- Negative assertion (what shouldn't compile) → `@ts-expect-error`

2. **Follow existing patterns** in the test files

3. **Run health check** after major changes:
```bash
pnpm nx test:types:health dsl
```

4. **Test your tests**: Temporarily break the type signature and ensure the test fails
76 changes: 76 additions & 0 deletions pkgs/dsl/__tests__/types/__health__.test-d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/**
* TYPE TESTING HEALTH CHECK FILE
*
* This file intentionally contains type errors to verify that our type testing
* infrastructure is working correctly. It should NEVER pass type checking.
*
* ⚠️ DO NOT INCLUDE IN NORMAL TEST RUNS ⚠️
* ⚠️ EXCLUDED FROM tsconfig.typecheck.json ⚠️
*
* Purpose:
* - Verifies that expectTypeOf detects type mismatches
* - Verifies that @ts-expect-error unused directive detection works
* - Verifies that typecheck-ts2578.sh catches errors correctly
*
* Verified by: scripts/verify-type-test-health.sh
* Run with: pnpm nx test:types:health dsl
*/

import { describe, it, expectTypeOf } from 'vitest';
import { Flow } from '../../src/index.js';

describe('Health: expectTypeOf detects type mismatches', () => {
it('MUST FAIL: detects wrong return type with clear error message', () => {
const flow = new Flow<{ count: number }>({ slug: 'health_test' })
.array({ slug: 'items' }, () => [1, 2, 3]);

const arrayStep = flow.getStepDefinition('items');

// This MUST fail with: "Expected: string, Actual: number"
expectTypeOf(arrayStep.handler).returns.toEqualTypeOf<string[]>();
});

it('MUST FAIL: detects input type mismatch', () => {
const flow = new Flow<{ userId: string }>({ slug: 'health_test2' })
.step({ slug: 'step1' }, () => 42);

const step = flow.getStepDefinition('step1');

// This MUST fail - expecting number input but it's { run: { userId: string } }
expectTypeOf(step.handler).parameter(0).toEqualTypeOf<{ run: { userId: number } }>();
});
});

describe('Health: @ts-expect-error unused directive detection', () => {
it('MUST FAIL: detects unused @ts-expect-error (TS2578)', () => {
const validArray = new Flow({ slug: 'health_test3' })
.array({ slug: 'numbers' }, () => [1, 2, 3]);

// These @ts-expect-error directives are UNUSED because the types are actually valid
// They MUST trigger TS2578 errors in Pass 2 of typecheck-ts2578.sh

// @ts-expect-error - HEALTH: This is actually valid, so TS2578 should fire
const step1 = validArray.getStepDefinition('numbers');

// @ts-expect-error - HEALTH: This is actually valid, so TS2578 should fire
const handler = step1.handler;

// Suppress unused variable warnings
void handler;
});
});

describe('Health: Hybrid approach validation', () => {
it('MUST FAIL: both expectTypeOf and @ts-expect-error work together', () => {
const flow = new Flow<{ value: number }>({ slug: 'health_hybrid' })
.step({ slug: 's1' }, () => 'result');

// expectTypeOf assertion that MUST fail
expectTypeOf(flow.getStepDefinition('s1').handler).returns.toEqualTypeOf<number>();

// @ts-expect-error that is unused (no actual error) - MUST trigger TS2578
// @ts-expect-error - HEALTH: This line is actually valid
const validStep = flow.getStepDefinition('s1');
void validStep;
});
});
Loading