diff --git a/src/config/index.ts b/src/config/index.ts index 9697fa5d..cb922e3a 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -129,6 +129,7 @@ const VALID_OUTPUT_FORMATS = ['esm', 'cjs', 'both']; const VALID_COMPRESSION = ['auto', 'gzip', 'brotli', 'none']; const VALID_TYPE_HINTS = ['strict', 'loose', 'ignore']; const VALID_TYPE_PRESETS = new Set([ + // Accepted as an explicit no-op in 0.9; ndarray shape typing depends on #268. 'numpy', 'pandas', 'pydantic', diff --git a/src/core/annotation-parser.ts b/src/core/annotation-parser.ts index 32e4a5a4..5dfc2864 100644 --- a/src/core/annotation-parser.ts +++ b/src/core/annotation-parser.ts @@ -407,8 +407,7 @@ export function parseAnnotationToPythonType( const builtInClassMatch = raw.match(/^$/); if (builtInClassMatch) { const inner = (raw.match(/^$/) ?? [])[1] ?? ''; - const name = (inner.split('.').pop() ?? '').toString(); - return mapSimpleName(name); + return mapSimpleName(inner); } if (raw.includes('|')) { @@ -506,7 +505,11 @@ export function parseAnnotationToPythonType( }; } - return mapSimpleName(raw); + const qualified = splitQualifiedName(raw); + if (qualified.module) { + return { kind: 'custom', name: qualified.name, module: qualified.module }; + } + return mapSimpleName(qualified.name); }; return parse(annotation, 0); diff --git a/src/core/generator.ts b/src/core/generator.ts index 8ef4659d..a135dfef 100644 --- a/src/core/generator.ts +++ b/src/core/generator.ts @@ -29,14 +29,30 @@ interface GenericRenderParam { interface GenericRenderContext { currentModule?: string; + localDeclaredNames: Set; declaration: string; typeArguments: string; emittedNames: Set; emittedParamSpecs: Set; } +export interface CodeGeneratorOptions { + /** Reports a generated annotation that cannot be represented by emitted declarations. */ + onTypeDegrade?: (typeName: string) => void; +} + export class CodeGenerator { private readonly mapper: TypeMapper; + private readonly onTypeDegrade?: (typeName: string) => void; + private readonly builtinGenericNames = new Set([ + 'Array', + 'AsyncIterator', + 'Generator', + 'Iterable', + 'Iterator', + 'Promise', + 'Record', + ]); private readonly reservedTsIdentifiers = new Set([ 'default', 'delete', @@ -66,8 +82,9 @@ export class CodeGenerator { 'false', ]); - constructor(mapper: TypeMapper = new TypeMapper()) { + constructor(mapper: TypeMapper = new TypeMapper(), options: CodeGeneratorOptions = {}) { this.mapper = mapper; + this.onTypeDegrade = options.onTypeDegrade; } /** @@ -140,7 +157,8 @@ export class CodeGenerator { private buildGenericRenderContext( typeParameters: readonly PythonGenericParameter[], types: readonly PythonType[], - currentModule?: string + currentModule?: string, + localDeclaredNames: Set = new Set() ): GenericRenderContext { const callableParamSpecs = new Set(); types.forEach(type => this.collectCallableParamSpecs(type, callableParamSpecs)); @@ -170,6 +188,7 @@ export class CodeGenerator { return { currentModule, + localDeclaredNames, declaration: emitted.length > 0 ? `<${emitted.map(param => param.declaration).join(', ')}>` : '', typeArguments: emitted.length > 0 ? `<${emitted.map(param => param.name).join(', ')}>` : '', @@ -184,6 +203,7 @@ export class CodeGenerator { ): GenericRenderContext { return { currentModule: inner.currentModule ?? outer.currentModule, + localDeclaredNames: inner.localDeclaredNames, declaration: inner.declaration, typeArguments: inner.typeArguments, emittedNames: new Set([...outer.emittedNames, ...inner.emittedNames]), @@ -308,7 +328,27 @@ export class CodeGenerator { ctx: GenericRenderContext, mappingContext: 'value' | 'return' ): string { - return this.typeToTs(this.mapper.mapPythonType(this.sanitizeType(type, ctx), mappingContext)); + return this.typeToTs( + this.mapper.mapPythonType(this.sanitizeType(type, ctx), mappingContext), + ctx, + mappingContext + ); + } + + private isLocalTypeIdentity( + type: { name: string; module?: string }, + ctx: GenericRenderContext + ): boolean { + if (!ctx.localDeclaredNames.has(type.name)) { + return false; + } + return type.module === undefined || type.module === ctx.currentModule; + } + + private degradeType(type: { name: string; module?: string }): string { + const identity = type.module ? `${type.module}.${type.name}` : type.name; + this.onTypeDegrade?.(identity); + return 'unknown'; } private renderLooksLikeKwargsExpr( @@ -360,7 +400,8 @@ export class CodeGenerator { generateFunctionWrapper( func: PythonFunction, moduleName?: string, - annotatedJSDoc = false + annotatedJSDoc = false, + localDeclaredNames: Set = new Set() ): GeneratedCode { const jsdoc = this.generateJsDoc( func.docstring, @@ -379,7 +420,8 @@ export class CodeGenerator { const genericContext = this.buildGenericRenderContext( this.getTypeParameters(func.typeParameters), [func.returnType, ...filteredParams.map(param => param.type)], - moduleName + moduleName, + localDeclaredNames ); const typeParamDecl = genericContext.declaration; @@ -553,8 +595,11 @@ ${callPrelude}${guards} return getRuntimeBridge().call('${moduleId}', '${func.n generateClassWrapper( cls: PythonClass, moduleName?: string, - _annotatedJSDoc = false + _annotatedJSDoc = false, + localDeclaredNames: Set = new Set() ): GeneratedCode { + const moduleDeclaredNames = new Set(localDeclaredNames); + moduleDeclaredNames.add(cls.name); const jsdoc = this.generateJsDoc(cls.docstring); const classGenericContext = this.buildGenericRenderContext( this.getTypeParameters(cls.typeParameters), @@ -566,7 +611,8 @@ ${callPrelude}${guards} return getRuntimeBridge().call('${moduleId}', '${func.n ...method.parameters.map(p => p.type), ]), ], - moduleName + moduleName, + moduleDeclaredNames ); const classTypeParamDecl = classGenericContext.declaration; const cname = this.escapeIdentifier(cls.name); @@ -617,7 +663,8 @@ ${callPrelude}${guards} return getRuntimeBridge().call('${moduleId}', '${func.n const methodOwnGenericContext = this.buildGenericRenderContext( this.getTypeParameters(m.typeParameters), [m.returnType, ...fparams.map(param => param.type)], - moduleName + moduleName, + moduleDeclaredNames ); const methodGenericContext = this.mergeGenericRenderContexts( classGenericContext, @@ -668,7 +715,8 @@ ${callPrelude}${guards} return getRuntimeBridge().call('${moduleId}', '${func.n const methodOwnGenericContext = this.buildGenericRenderContext( this.getTypeParameters(method.typeParameters), [method.returnType, ...fparams.map(param => param.type)], - moduleName + moduleName, + moduleDeclaredNames ); const methodGenericContext = this.mergeGenericRenderContexts( classGenericContext, @@ -816,11 +864,16 @@ ${migrationNote}${declarationMethodsSection} return this.wrap(ts, declaration, [cls.name]); } - generateTypeAlias(alias: PythonTypeAlias, moduleName?: string): GeneratedCode { + generateTypeAlias( + alias: PythonTypeAlias, + moduleName?: string, + localDeclaredNames: Set = new Set() + ): GeneratedCode { const genericContext = this.buildGenericRenderContext( this.getTypeParameters(alias.typeParameters), [alias.type], - moduleName + moduleName, + localDeclaredNames ); const aliasName = this.escapeIdentifier(alias.name, { preserveCase: true }); const body = this.typeToTsFromPython(alias.type, genericContext, 'value'); @@ -854,15 +907,19 @@ ${migrationNote}${declarationMethodsSection} } generateModuleDefinition(module: PythonModule, annotatedJSDoc = false): GeneratedCode { + const localDeclaredNames = new Set([ + ...module.classes.map(cls => cls.name), + ...(module.typeAliases ?? []).map(alias => alias.name), + ]); const functionResults = [...module.functions] .sort((a, b) => a.name.localeCompare(b.name)) - .map(f => this.generateFunctionWrapper(f, module.name, annotatedJSDoc)); + .map(f => this.generateFunctionWrapper(f, module.name, annotatedJSDoc, localDeclaredNames)); const classResults = [...module.classes] .sort((a, b) => a.name.localeCompare(b.name)) - .map(c => this.generateClassWrapper(c, module.name, annotatedJSDoc)); + .map(c => this.generateClassWrapper(c, module.name, annotatedJSDoc, localDeclaredNames)); const typeAliasResults = [...(module.typeAliases ?? [])] .sort((a, b) => a.name.localeCompare(b.name)) - .map(alias => this.generateTypeAlias(alias, module.name)); + .map(alias => this.generateTypeAlias(alias, module.name, localDeclaredNames)); const functionCodes = functionResults.map(result => result.typescript).join('\n'); const classCodes = classResults.map(result => result.typescript).join('\n'); @@ -917,23 +974,27 @@ ${migrationNote}${declarationMethodsSection} }; } - private typeToTs(type: TypescriptType): string { + private typeToTs( + type: TypescriptType, + ctx?: GenericRenderContext, + mappingContext: 'value' | 'return' = 'value' + ): string { switch (type.kind) { case 'primitive': return type.name; case 'array': - return `${this.typeToTs(type.elementType)}[]`; + return `${this.typeToTs(type.elementType, ctx, mappingContext)}[]`; case 'tuple': { const t = type as { kind: 'tuple'; elementTypes: TypescriptType[] }; - const parts = t.elementTypes.map(e => this.typeToTs(e)).join(', '); + const parts = t.elementTypes.map(e => this.typeToTs(e, ctx, mappingContext)).join(', '); return `[${parts}]`; } case 'object': { const t = type; // If it's a simple Record pattern, use that syntax if (t.properties.length === 0 && t.indexSignature) { - const keyType = this.typeToTs(t.indexSignature.keyType); - const valueType = this.typeToTs(t.indexSignature.valueType); + const keyType = this.typeToTs(t.indexSignature.keyType, ctx, mappingContext); + const valueType = this.typeToTs(t.indexSignature.valueType, ctx, mappingContext); if (keyType === 'string') { return `Record`; } @@ -942,34 +1003,53 @@ ${migrationNote}${declarationMethodsSection} const props = t.properties .map( p => - `${p.readonly ? 'readonly ' : ''}${p.name}${p.optional ? '?' : ''}: ${this.typeToTs(p.type)};` + `${p.readonly ? 'readonly ' : ''}${p.name}${p.optional ? '?' : ''}: ${this.typeToTs(p.type, ctx, mappingContext)};` ) .join(' '); const indexSig = t.indexSignature - ? `[key: ${this.typeToTs(t.indexSignature.keyType)}]: ${this.typeToTs(t.indexSignature.valueType)};` + ? `[key: ${this.typeToTs(t.indexSignature.keyType, ctx, mappingContext)}]: ${this.typeToTs(t.indexSignature.valueType, ctx, mappingContext)};` : ''; return `{ ${props} ${indexSig} }`; } case 'union': - return type.types.map(t => this.typeToTs(t)).join(' | '); + return type.types.map(t => this.typeToTs(t, ctx, mappingContext)).join(' | '); case 'function': { const ft = type; const params = ft.parameters .map( - p => `${p.rest ? '...' : ''}${p.name}${p.optional ? '?' : ''}: ${this.typeToTs(p.type)}` + p => + `${p.rest ? '...' : ''}${p.name}${p.optional ? '?' : ''}: ${this.typeToTs(p.type, ctx, mappingContext)}` ) .join(', '); - return `(${params}) => ${this.typeToTs(ft.returnType)}`; + return `(${params}) => ${this.typeToTs(ft.returnType, ctx, mappingContext)}`; } case 'generic': { - const g = type as { kind: 'generic'; name: string; typeArgs: TypescriptType[] }; - const args = g.typeArgs.map(a => this.typeToTs(a)).join(', '); + const g = type as { + kind: 'generic'; + name: string; + module?: string; + typeArgs: TypescriptType[]; + }; + if (!/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(g.name)) { + return this.degradeType(g); + } + if (ctx && !this.builtinGenericNames.has(g.name) && !this.isLocalTypeIdentity(g, ctx)) { + return this.degradeType(g); + } + const args = g.typeArgs.map(a => this.typeToTs(a, ctx, mappingContext)).join(', '); return `${g.name}<${args}>`; } case 'custom': { - const c = type as { kind: 'custom'; name: string }; + const c = type as { kind: 'custom'; name: string; module?: string }; if (!/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(c.name)) { - return 'unknown'; + return this.degradeType(c); + } + if ( + ctx && + !this.isLocalTypeIdentity(c, ctx) && + !(c.module === 'typing' && ctx.emittedNames.has(c.name)) + ) { + return this.degradeType(c); } return c.name; } diff --git a/src/core/mapper.ts b/src/core/mapper.ts index 37f4efc9..af0bf0c3 100644 --- a/src/core/mapper.ts +++ b/src/core/mapper.ts @@ -94,7 +94,7 @@ export class TypeMapper { : type.name === 'bool' ? 'boolean' : type.name === 'bytes' - ? 'string' + ? 'Uint8Array' : // None context === 'return' ? 'void' @@ -137,16 +137,15 @@ export class TypeMapper { return { kind: 'tuple', elementTypes } satisfies TSTupleType; } - // set[T] -> Set + // Python sets cross the JSON bridge as arrays in both directions. if (type.name === 'set' || type.name === 'frozenset') { const elementType = this.mapPythonType( type.itemTypes[0] ?? { kind: 'custom', name: 'Any', module: 'typing' } ); return { - kind: 'generic', - name: 'Set', - typeArgs: [elementType], - } satisfies TSGenericType; + kind: 'array', + elementType, + } satisfies TSArrayType; } // dict[K, V] -> { [key: K]: V } @@ -204,6 +203,7 @@ export class TypeMapper { return { kind: 'generic', name: normalized.name, + module: normalized.module, typeArgs, }; } diff --git a/src/types/index.ts b/src/types/index.ts index 02aaba5f..d19c4417 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -261,7 +261,8 @@ export interface TSPrimitiveType { | 'void' | 'unknown' | 'never' - | 'object'; + | 'object' + | 'Uint8Array'; } export interface TSArrayType { @@ -314,6 +315,7 @@ export interface TSParameter { export interface TSGenericType { kind: 'generic'; name: string; + module?: string; typeArgs: TypescriptType[]; } @@ -404,6 +406,8 @@ export interface PerformanceConfig { compression: 'auto' | 'gzip' | 'brotli' | 'none'; } +// `numpy` is intentionally accepted as a no-op in 0.9: decoded ndarrays are JS +// arrays, and a faithful static shape awaits #268 return validation. export type TypePreset = 'numpy' | 'pandas' | 'pydantic' | 'stdlib' | 'scipy' | 'torch' | 'sklearn'; export interface TypeMappingConfig { diff --git a/src/tywrap.ts b/src/tywrap.ts index 94dbaf6b..24d1aa0b 100644 --- a/src/tywrap.ts +++ b/src/tywrap.ts @@ -21,6 +21,7 @@ import { fsUtils, pathUtils, processUtils, isWindows } from './utils/runtime.js' import { globalCache } from './utils/cache.js'; import { resolvePythonExecutable } from './utils/python.js'; import { computeIrCacheFilename } from './utils/ir-cache.js'; +import { logger } from './utils/logger.js'; const TYWRAP_IR_VERSION = '0.3.0'; @@ -42,7 +43,9 @@ interface TywrapInstance { export async function tywrap(options: Partial = {}): Promise { const mapper = new TypeMapper({ presets: options.types?.presets }); - const generator = new CodeGenerator(mapper); + const generator = new CodeGenerator(mapper, { + onTypeDegrade: typeName => recordUnknown(`unresolvable generated type ${typeName}`), + }); globalCache.setDebug(options.debug ?? false); @@ -339,6 +342,24 @@ export async function generate( continue; } + const irWarnings = Array.isArray((ir as Record).warnings) + ? ((ir as Record).warnings as unknown[]) + : []; + for (const warning of irWarnings) { + if (typeof warning !== 'string') { + continue; + } + // Only type-honesty degrades feed --fail-on-warn. Environment capability + // notices (e.g. typing.get_overloads missing on Python < 3.11) describe + // the analyzing interpreter, not the generated types, and must not fail + // an otherwise clean build. + if (warning.startsWith('Return annotation for ')) { + recordUnknown(`Python IR warning: ${warning}`); + } else { + logger.info(`Python IR notice: ${warning}`, { component: 'Generate' }); + } + } + const moduleModel = transformIrToTsModel(ir); // Apply module-level export filtering (functions/classes + excludes). diff --git a/test/annotation-parser.test.ts b/test/annotation-parser.test.ts index bbc024bf..9d87863e 100644 --- a/test/annotation-parser.test.ts +++ b/test/annotation-parser.test.ts @@ -65,6 +65,7 @@ describe('annotation parser', () => { it('does not treat arbitrary dotted names as ParamSpec packs', () => { const t = parseAnnotationToPythonType('Request.args'); expect(t.kind).toBe('custom'); - expect((t as any).name).toBe('Request.args'); + expect((t as any).name).toBe('args'); + expect((t as any).module).toBe('Request'); }); }); diff --git a/test/e2e_smoke.test.ts b/test/e2e_smoke.test.ts index 69193032..adc3330c 100644 --- a/test/e2e_smoke.test.ts +++ b/test/e2e_smoke.test.ts @@ -75,7 +75,7 @@ describeNodeOnly('E2E Smoke - CLI generate + runtime bridge', () => { modulePath, [ 'class Greeter:', - ' def __init__(self, name: str):', + ' def __init__(self, name: str) -> None:', ' self.name = name', '', ' def greet(self, suffix: str = "!") -> str:', diff --git a/test/generator.test.ts b/test/generator.test.ts index 0750c9a2..7333d573 100644 --- a/test/generator.test.ts +++ b/test/generator.test.ts @@ -94,6 +94,85 @@ describe('CodeGenerator', () => { expect(code.typescript).toMatch(/Promise<\[number, string\]>/); }); + it('degrades undeclared generated type leaves to unknown and reports them', () => { + const degraded: string[] = []; + const honestGenerator = new CodeGenerator(undefined, { + onTypeDegrade: typeName => degraded.push(typeName), + }); + const code = honestGenerator.generateModuleDefinition({ + name: 'local_module', + functions: [ + { + name: 'external', + signature: { + parameters: [], + returnType: { kind: 'custom', name: 'Remote', module: 'other' }, + isAsync: false, + isGenerator: false, + }, + decorators: [], + isAsync: false, + isGenerator: false, + returnType: { kind: 'custom', name: 'Remote', module: 'other' }, + parameters: [], + }, + ], + classes: [], + typeAliases: [], + imports: [], + exports: [], + }); + + expect(code.typescript).toContain('Promise'); + expect(code.typescript).not.toContain('Promise'); + expect(degraded).toEqual(['other.Remote']); + }); + + it('degrades qualified generic heads and module-colliding leaves', () => { + const degraded: string[] = []; + const honestGenerator = new CodeGenerator(undefined, { + onTypeDegrade: typeName => degraded.push(typeName), + }); + const makeFunction = (name: string, returnType: any): any => ({ + name, + signature: { parameters: [], returnType, isAsync: false, isGenerator: false }, + decorators: [], + isAsync: false, + isGenerator: false, + returnType, + parameters: [], + }); + const code = honestGenerator.generateModuleDefinition({ + name: 'local_module', + functions: [ + makeFunction('generic', { + kind: 'generic', + name: 'Remote', + module: 'other', + typeArgs: [{ kind: 'primitive', name: 'int' }], + }), + makeFunction('collision', { kind: 'custom', name: 'Point', module: 'external' }), + makeFunction('invalid', { kind: 'custom', name: 'bad-name', module: 'external' }), + ], + classes: [ + { + name: 'Point', + bases: [], + methods: [], + properties: [], + decorators: [], + }, + ], + typeAliases: [], + imports: [], + exports: [], + }); + + expect(code.typescript).not.toContain('Remote'); + expect(code.typescript).not.toContain('Promise'); + expect(degraded).toEqual(['external.Point', 'other.Remote', 'external.bad-name']); + }); + it('emits JSDoc with Annotated metadata when enabled', () => { const code = gen.generateFunctionWrapper( { diff --git a/test/menagerie/__snapshots__/gen.test.ts.snap b/test/menagerie/__snapshots__/gen.test.ts.snap index 9d62bc3a..12f39a8f 100644 --- a/test/menagerie/__snapshots__/gen.test.ts.snap +++ b/test/menagerie/__snapshots__/gen.test.ts.snap @@ -18,12 +18,12 @@ export async function asyncText(): Promise { return getRuntimeBridge().call('fixtures.typing_torture', 'async_text', __args); } -export async function bytesEcho(value: string): Promise { +export async function bytesEcho(value: Uint8Array): Promise { const __args: unknown[] = [value]; return getRuntimeBridge().call('fixtures.typing_torture', 'bytes_echo', __args); } -export async function frozenValues(value: Set): Promise> { +export async function frozenValues(value: string[]): Promise { const __args: unknown[] = [value]; return getRuntimeBridge().call('fixtures.typing_torture', 'frozen_values', __args); } @@ -95,7 +95,7 @@ export async function protocolIdentity(value: SupportsClose): Promise): Promise> { +export async function setValues(value: number[]): Promise { const __args: unknown[] = [value]; return getRuntimeBridge().call('fixtures.typing_torture', 'set_values', __args); } @@ -129,6 +129,8 @@ export class Box { export type Movie = { title: string; year: number; } export type SupportsClose = { close: () => void; } + +export type UserId = number ", "// Generated by tywrap // Type Declarations @@ -138,9 +140,9 @@ export function annotatedEcho(value: number): Promise; export function asyncText(): Promise; -export function bytesEcho(value: string): Promise; +export function bytesEcho(value: Uint8Array): Promise; -export function frozenValues(value: Set): Promise>; +export function frozenValues(value: string[]): Promise; export function generatorValues(): Promise>; @@ -164,7 +166,7 @@ export function paramspecApply

(callback: (...args: P) => export function protocolIdentity(value: SupportsClose): Promise; -export function setValues(value: Set): Promise>; +export function setValues(value: number[]): Promise; export function tupleKeyDict(value: Record): Promise>; @@ -183,6 +185,8 @@ export class Box { export type Movie = { title: string; year: number; } export type SupportsClose = { close: () => void; } + +export type UserId = number ", ] `; @@ -200,12 +204,12 @@ export async function boolsAndInts(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'bools_and_ints', __args); } -export async function bytesEcho(value: string): Promise { +export async function bytesEcho(value: Uint8Array): Promise { const __args: unknown[] = [value]; return getRuntimeBridge().call('fixtures.values_torture', 'bytes_echo', __args); } -export async function complexValue(): Promise { +export async function complexValue(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'complex_value', __args); } @@ -220,7 +224,7 @@ export async function dataclassInstance(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'dataclass_instance', __args); } -export async function decimalValues(): Promise { +export async function decimalValues(): Promise { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'decimal_values', __args); } @@ -235,7 +239,7 @@ export async function echoInt(value: number): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'echo_int', __args); } -export async function emptyValues(): Promise<[[unknown], object[], Record, Set]> { +export async function emptyValues(): Promise<[[unknown], object[], Record, object[]]> { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'empty_values', __args); } @@ -275,7 +279,7 @@ export async function megabyteText(): Promise { return getRuntimeBridge().call('fixtures.values_torture', 'megabyte_text', __args); } -export async function setAndFrozenset(): Promise<[Set, Set]> { +export async function setAndFrozenset(): Promise<[number[], string[]]> { const __args: unknown[] = []; return getRuntimeBridge().call('fixtures.values_torture', 'set_and_frozenset', __args); } @@ -329,21 +333,21 @@ export class TrafficLight { export function boolsAndInts(): Promise; -export function bytesEcho(value: string): Promise; +export function bytesEcho(value: Uint8Array): Promise; -export function complexValue(): Promise; +export function complexValue(): Promise; export function coroutineValue(): Promise; export function dataclassInstance(): Promise; -export function decimalValues(): Promise; +export function decimalValues(): Promise; export function deeplyNested(): Promise; export function echoInt(value: number): Promise; -export function emptyValues(): Promise<[[unknown], object[], Record, Set]>; +export function emptyValues(): Promise<[[unknown], object[], Record, object[]]>; export function enumMember(): Promise; @@ -359,7 +363,7 @@ export function loneSurrogate(): Promise; export function megabyteText(): Promise; -export function setAndFrozenset(): Promise<[Set, Set]>; +export function setAndFrozenset(): Promise<[number[], string[]]>; export function specialFloats(): Promise; export function specialFloats(include?: boolean): Promise; diff --git a/test/menagerie/gen.test.ts b/test/menagerie/gen.test.ts index 066cd9a8..204495be 100644 --- a/test/menagerie/gen.test.ts +++ b/test/menagerie/gen.test.ts @@ -73,51 +73,42 @@ describe('menagerie generation gate', () => { 30_000 ); - // FIXME(#267): map Decimal and complex to declared TypeScript types. - it.fails( - 'typechecks the generated plain-values fixture', - async () => { - const tempDir = await mkdtemp(join(process.cwd(), '.tmp-menagerie-values-')); - try { - const outputDir = join(tempDir, 'generated'); - const result = await generate({ - pythonModules: { 'fixtures.values_torture': { runtime: 'node', typeHints: 'strict' } }, - pythonImportPath: [fixtureImportPath], - output: { dir: outputDir, format: 'esm', declaration: true, sourceMap: false }, - runtime: { node: { pythonPath: defaultPythonPath } }, - performance: { caching: false, batching: false, compression: 'none' }, - } as never); - const generatedTs = result.written.find(path => path.endsWith('.generated.ts')); - expect(generatedTs).toBeDefined(); - await compileGeneratedFile(generatedTs as string); - } finally { - await rm(tempDir, { recursive: true, force: true }); - } - }, - 30_000 - ); + it('typechecks the generated plain-values fixture', async () => { + const tempDir = await mkdtemp(join(process.cwd(), '.tmp-menagerie-values-')); + try { + const outputDir = join(tempDir, 'generated'); + const result = await generate({ + pythonModules: { 'fixtures.values_torture': { runtime: 'node', typeHints: 'strict' } }, + pythonImportPath: [fixtureImportPath], + output: { dir: outputDir, format: 'esm', declaration: true, sourceMap: false }, + runtime: { node: { pythonPath: defaultPythonPath } }, + performance: { caching: false, batching: false, compression: 'none' }, + } as never); + expect(result.warnings.some(warning => warning.includes('Python IR warning:'))).toBe(true); + const generatedTs = result.written.find(path => path.endsWith('.generated.ts')); + expect(generatedTs).toBeDefined(); + await compileGeneratedFile(generatedTs as string); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }, 30_000); - // FIXME(#267): preserve NewType declarations or map their use-sites to a declared TS alias. - it.fails( - 'typechecks the generated NewType fixture', - async () => { - const tempDir = await mkdtemp(join(process.cwd(), '.tmp-menagerie-newtype-')); - try { - const outputDir = join(tempDir, 'generated'); - const result = await generate({ - pythonModules: { 'fixtures.typing_torture': { runtime: 'node', typeHints: 'strict' } }, - pythonImportPath: [fixtureImportPath], - output: { dir: outputDir, format: 'esm', declaration: true, sourceMap: false }, - runtime: { node: { pythonPath: defaultPythonPath } }, - performance: { caching: false, batching: false, compression: 'none' }, - } as never); - const generatedTs = result.written.find(path => path.endsWith('.generated.ts')); - expect(generatedTs).toBeDefined(); - await compileGeneratedFile(generatedTs as string); - } finally { - await rm(tempDir, { recursive: true, force: true }); - } - }, - 30_000 - ); + it('typechecks the generated NewType fixture', async () => { + const tempDir = await mkdtemp(join(process.cwd(), '.tmp-menagerie-newtype-')); + try { + const outputDir = join(tempDir, 'generated'); + const result = await generate({ + pythonModules: { 'fixtures.typing_torture': { runtime: 'node', typeHints: 'strict' } }, + pythonImportPath: [fixtureImportPath], + output: { dir: outputDir, format: 'esm', declaration: true, sourceMap: false }, + runtime: { node: { pythonPath: defaultPythonPath } }, + performance: { caching: false, batching: false, compression: 'none' }, + } as never); + const generatedTs = result.written.find(path => path.endsWith('.generated.ts')); + expect(generatedTs).toBeDefined(); + await compileGeneratedFile(generatedTs as string); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }, 30_000); }); diff --git a/test/menagerie/manifest.ts b/test/menagerie/manifest.ts index 7cf0c7f2..b9a9f0c0 100644 --- a/test/menagerie/manifest.ts +++ b/test/menagerie/manifest.ts @@ -28,10 +28,17 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [ }, { fixture: 'values_torture', - call: 'empty_values(), set_and_frozenset()', + call: 'empty_values()', status: 'KNOWN_LIE', - currentBehavior: 'Python tuples, sets, and frozensets decode as JavaScript arrays.', - expectedFix: 'Future tagged tuple/set transport.', + currentBehavior: + 'The empty Python tuple is not represented exactly in the generated tuple type.', + expectedFix: 'Future exact empty-tuple typing.', + }, + { + fixture: 'values_torture', + call: 'set_and_frozenset()', + status: 'EXPECTED_OK', + currentBehavior: 'Python sets and frozensets are declared and delivered as JavaScript arrays.', }, { fixture: 'values_torture', @@ -43,17 +50,15 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [ { fixture: 'values_torture', call: 'bytes_echo()', - status: 'KNOWN_LIE', - currentBehavior: - 'The bridge delivers Uint8Array while generated wrappers currently declare string.', - expectedFix: 'Future bytes mapper alignment.', + status: 'EXPECTED_OK', + currentBehavior: 'Python bytes are declared and delivered as Uint8Array.', }, { fixture: 'values_torture', call: 'temporal_values(), decimal_values(), uuid_and_path()', status: 'KNOWN_LIE', currentBehavior: - 'Python temporal values, Decimal, UUID, and Path decode as strings or seconds.', + 'Python temporal values, Decimal, UUID, and Path decode as strings or seconds; Decimal is now generated as unknown rather than an undeclared TypeScript leaf.', expectedFix: 'Future tagged stdlib-value transport.', }, { @@ -82,7 +87,7 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [ call: 'coroutine_value(), dataclass_instance(), complex_value(), generator_value()', status: 'LOUD_FAIL', currentBehavior: - 'Coroutines and unsupported Python objects reject instead of silently coercing.', + 'Coroutines and unsupported Python objects reject instead of silently coercing; unsupported return leaves (including complex) are generated as unknown. Local Point stays typed to document intent, while its dataclass instance fails loudly at serialization as the catalogued Class-C residual.', }, { fixture: 'library_torture', diff --git a/test/menagerie/roundtrip.test.ts b/test/menagerie/roundtrip.test.ts index 54079837..e252b31a 100644 --- a/test/menagerie/roundtrip.test.ts +++ b/test/menagerie/roundtrip.test.ts @@ -37,9 +37,10 @@ async function callValues(functionName: string, args: unknown[] = []): Promis * | --- | --- | --- | --- | * | values_torture | echo_int(), bools_and_ints(), finite_float_edges() | EXPECTED_OK | safe integers, booleans, -0, and subnormal float survive | * | values_torture | unicode_text(), lone_surrogate(), megabyte_text(), deeply_nested() | EXPECTED_OK | text and 100-level nesting survive | - * | values_torture | bytes_echo() | KNOWN_LIE | bridge delivers Uint8Array; generated wrapper declares string | + * | values_torture | bytes_echo() | EXPECTED_OK | wrapper and bridge both use Uint8Array | * | values_torture | integer_boundaries() | KNOWN_LIE | unsafe integers lose precision | - * | values_torture | empty_values(), set_and_frozenset() | KNOWN_LIE | tuple/set/frozenset become arrays | + * | values_torture | empty_values() | KNOWN_LIE | empty tuple typing remains inexact | + * | values_torture | set_and_frozenset() | EXPECTED_OK | set/frozenset are declared and delivered as arrays | * | values_torture | int_key_dict() | KNOWN_LIE | integer object keys stringify | * | values_torture | temporal_values(), decimal_values(), uuid_and_path() | KNOWN_LIE | values become strings or seconds | * | values_torture | special_floats(true) | LOUD_FAIL | non-finite numbers reject | @@ -115,7 +116,7 @@ describe.skipIf(!bridgeAvailable)('menagerie runtime gate', () => { expect(sets[0]).toEqual(expect.arrayContaining([1, 2])); expect(sets[1]).toEqual(expect.arrayContaining(['a', 'b'])); await expect(callValues('int_key_dict')).resolves.toEqual({ 1: 'one', 2: 'two' }); - expect(RUNTIME_CATALOGUE.filter(row => row.status === 'KNOWN_LIE')).toHaveLength(5); + expect(RUNTIME_CATALOGUE.filter(row => row.status === 'KNOWN_LIE')).toHaveLength(4); }, timeoutMs ); diff --git a/test/type_mapping_advanced.test.ts b/test/type_mapping_advanced.test.ts index 68578994..43fb5a0d 100644 --- a/test/type_mapping_advanced.test.ts +++ b/test/type_mapping_advanced.test.ts @@ -27,7 +27,7 @@ describe('TypeMapper - Advanced Type Mapping Validation', () => { { python: 'float', expected: 'number' }, { python: 'str', expected: 'string' }, { python: 'bool', expected: 'boolean' }, - { python: 'bytes', expected: 'string' }, + { python: 'bytes', expected: 'Uint8Array' }, { python: 'None', expected: 'null', context: 'value' }, { python: 'None', expected: 'void', context: 'return' }, ]; @@ -106,32 +106,30 @@ describe('TypeMapper - Advanced Type Mapping Validation', () => { expect(result.elementTypes).toEqual([{ kind: 'primitive', name: 'undefined' }]); }); - test('maps set[T] to Set', () => { + test('maps set[T] to T[]', () => { const pythonType: PythonType = { kind: 'collection', name: 'set', itemTypes: [{ kind: 'primitive', name: 'str' }], }; - const result = mapper.mapPythonType(pythonType) as TSGenericType; + const result = mapper.mapPythonType(pythonType) as TSArrayType; - expect(result.kind).toBe('generic'); - expect(result.name).toBe('Set'); - expect(result.typeArgs).toEqual([{ kind: 'primitive', name: 'string' }]); + expect(result.kind).toBe('array'); + expect(result.elementType).toEqual({ kind: 'primitive', name: 'string' }); }); - test('maps frozenset[T] to Set', () => { + test('maps frozenset[T] to T[]', () => { const pythonType: PythonType = { kind: 'collection', name: 'frozenset', itemTypes: [{ kind: 'primitive', name: 'int' }], }; - const result = mapper.mapPythonType(pythonType) as TSGenericType; + const result = mapper.mapPythonType(pythonType) as TSArrayType; - expect(result.kind).toBe('generic'); - expect(result.name).toBe('Set'); - expect(result.typeArgs).toEqual([{ kind: 'primitive', name: 'number' }]); + expect(result.kind).toBe('array'); + expect(result.elementType).toEqual({ kind: 'primitive', name: 'number' }); }); test('maps dict[K, V] to index signature object', () => { diff --git a/test/type_mapping_property.test.ts b/test/type_mapping_property.test.ts index b5a8437d..297c7f65 100644 --- a/test/type_mapping_property.test.ts +++ b/test/type_mapping_property.test.ts @@ -178,7 +178,7 @@ describe('TypeMapper - Property-Based Tests', () => { ); case 'set': case 'frozenset': - return result.kind === 'generic' && (result as any).name === 'Set'; + return result.kind === 'array'; default: return false; } @@ -304,6 +304,8 @@ describe('TypeMapper - Property-Based Tests', () => { 'void', 'unknown', 'never', + 'object', + 'Uint8Array', ]; return primitiveNames.includes((result as any).name); diff --git a/tywrap_ir/tests/test_ir_return_honesty.py b/tywrap_ir/tests/test_ir_return_honesty.py new file mode 100644 index 00000000..d213c5c7 --- /dev/null +++ b/tywrap_ir/tests/test_ir_return_honesty.py @@ -0,0 +1,47 @@ +import decimal +import sys +import types +import unittest +from pathlib import Path + +from tywrap_ir.ir import extract_module_ir + + +class IRReturnHonestyTests(unittest.TestCase): + def test_preserves_external_return_path_and_warns(self) -> None: + module_name = "tywrap_ir_test_external_return" + module = types.ModuleType(module_name) + + def external() -> decimal.Decimal: + return decimal.Decimal("1") + + external.__module__ = module_name + module.external = external + sys.modules[module_name] = module + try: + ir = extract_module_ir(module_name) + finally: + del sys.modules[module_name] + + function = next(item for item in ir["functions"] if item["name"] == "external") + self.assertEqual(function["returns"], "decimal.Decimal") + self.assertTrue( + any("resolves outside analyzed module: decimal.Decimal" in warning for warning in ir["warnings"]) + ) + + def test_extracts_local_newtype_as_alias_to_its_base(self) -> None: + fixture_root = Path(__file__).resolve().parents[2] / "test" / "menagerie" + sys.path.insert(0, str(fixture_root)) + try: + ir = extract_module_ir("fixtures.typing_torture") + finally: + sys.path.pop(0) + sys.modules.pop("fixtures.typing_torture", None) + sys.modules.pop("fixtures", None) + + alias = next(item for item in ir["type_aliases"] if item["name"] == "UserId") + self.assertEqual(alias["definition"], "int") + + +if __name__ == "__main__": + unittest.main() diff --git a/tywrap_ir/tywrap_ir/ir.py b/tywrap_ir/tywrap_ir/ir.py index 6eb5f242..2bbd0cb1 100644 --- a/tywrap_ir/tywrap_ir/ir.py +++ b/tywrap_ir/tywrap_ir/ir.py @@ -7,6 +7,7 @@ import inspect import json import platform +import re import sys import types import typing @@ -137,8 +138,6 @@ def _stringify_annotation(annotation: Any) -> Optional[str]: str_repr = str(annotation) if str_repr.startswith(""): class_path = str_repr[8:-2] - if "." in class_path: - return class_path.split(".")[-1] return class_path return str_repr except Exception: @@ -310,6 +309,8 @@ def _collect_scoped_type_params( def _unwrap_type_alias_value(value: Any) -> Any: + if hasattr(value, "__supertype__"): + return value.__supertype__ type_alias_type = getattr(typing, "TypeAliasType", None) if type_alias_type is None or not isinstance(value, type_alias_type): return value @@ -347,6 +348,8 @@ def _top_level_assigned_names(module: Any) -> set[str]: def _is_type_alias_value(value: Any) -> bool: + if hasattr(value, "__supertype__"): + return True if inspect.isfunction(value) or inspect.isbuiltin(value) or inspect.isclass(value) or inspect.ismodule(value): return False type_alias_type = getattr(typing, "TypeAliasType", None) @@ -918,6 +921,36 @@ def extract_module_ir( warnings.extend(_pending_overload_warnings) _pending_overload_warnings.clear() + # Class annotations were historically reduced to their leaf name (for + # example ``decimal.Decimal`` became ``Decimal``), making it impossible for + # the generator to tell a local declaration from an imported object. Keep the + # full path above and surface return annotations that cannot be resolved from + # this module as structured diagnostics. The TS generator turns these leaves + # into ``unknown`` unless it emits a declaration for them. + external_return_pattern = re.compile(r"(? None: + if fn.returns is None: + warnings.append(f"Return annotation for {fn.qualname} is missing.") + return + for candidate in external_return_pattern.findall(fn.returns): + if candidate.startswith(exempt_return_prefixes) or candidate.startswith(f"{module_name}."): + continue + warnings.append( + f"Return annotation for {fn.qualname} resolves outside analyzed module: {candidate}." + ) + break + + for fn in functions: + record_return_warning(fn) + for cls in classes: + for method in cls.methods: + record_return_warning(method) + for accessor in cls.accessors: + if accessor.returns is None: + warnings.append(f"Return annotation for {cls.qualname}.{accessor.name} is missing.") + ir = IRModule( ir_version=ir_version, module=module_name,