Fix stack overflow caused by simplification of recursive type (#4260)

Author: ahejlsberg

internal/checker/checker.go

@@ -27716,6 +27716,16 @@ func (c *Checker) getSimplifiedIndexedAccessType(t *Type, writing bool) *Type {
 		return core.IfElse(cached == c.circularConstraintType, t, cached)
 	}
 	c.cachedTypes[key] = t
+	result := c.getSimplifiedIndexedAccessTypeWorker(t, writing)
+	if result != t {
+		// If the simplification is a union type that includes t, remove t from the type.
+		result = c.removeType(result, t)
+		c.cachedTypes[key] = result
+	}
+	return result
+}
+
+func (c *Checker) getSimplifiedIndexedAccessTypeWorker(t *Type, writing bool) *Type {
 	// We recursively simplify the object type as it may in turn be an indexed access type. For example, with
 	// '{ [P in T]: { [Q in U]: number } }[T][U]' we want to first simplify the inner indexed access type.
 	objectType := c.getSimplifiedType(t.AsIndexedAccessType().objectType, writing)
@@ -27724,7 +27734,6 @@ func (c *Checker) getSimplifiedIndexedAccessType(t *Type, writing bool) *Type {
 	// T[A | B] -> T[A] & T[B] (writing)
 	distributedOverIndex := c.distributeObjectOverIndexType(objectType, indexType, writing)
 	if distributedOverIndex != nil {
-		c.cachedTypes[key] = distributedOverIndex
 		return distributedOverIndex
 	}
 	// Only do the inner distributions if the index can no longer be instantiated to cause index distribution again
@@ -27734,7 +27743,6 @@ func (c *Checker) getSimplifiedIndexedAccessType(t *Type, writing bool) *Type {
 		// (T & U)[K] -> T[K] & U[K]
 		distributedOverObject := c.distributeIndexOverObjectType(objectType, indexType, writing)
 		if distributedOverObject != nil {
-			c.cachedTypes[key] = distributedOverObject
 			return distributedOverObject
 		}
 	}
@@ -27746,7 +27754,6 @@ func (c *Checker) getSimplifiedIndexedAccessType(t *Type, writing bool) *Type {
 	if c.isGenericTupleType(objectType) && indexType.flags&TypeFlagsNumberLike != 0 {
 		elementType := c.getElementTypeOfSliceOfTupleType(objectType, core.IfElse(indexType.flags&TypeFlagsNumber != 0, 0, objectType.TargetTupleType().fixedLength), 0 /*endSkipCount*/, writing, false)
 		if elementType != nil {
-			c.cachedTypes[key] = elementType
 			return elementType
 		}
 	}
@@ -27755,11 +27762,9 @@ func (c *Checker) getSimplifiedIndexedAccessType(t *Type, writing bool) *Type {
 	// For example, for an index access { [P in K]: Box<T[P]> }[X], we construct the type Box<T[X]>.
 	if c.isGenericMappedType(objectType) {
 		if c.getMappedTypeNameTypeKind(objectType) != MappedTypeNameTypeKindRemapping {
-			result := c.mapType(c.substituteIndexedMappedType(objectType, t.AsIndexedAccessType().indexType), func(t *Type) *Type {
+			return c.mapType(c.substituteIndexedMappedType(objectType, t.AsIndexedAccessType().indexType), func(t *Type) *Type {
 				return c.getSimplifiedType(t, writing)
 			})
-			c.cachedTypes[key] = result
-			return result
 		}
 	}
 	return t

testdata/baselines/reference/compiler/recursiveIndexedAccessSimplification.errors.txt

@@ -0,0 +1,42 @@
+recursiveIndexedAccessSimplification.ts(5,5): error TS4109: Type arguments for 'Array' circularly reference themselves.
+recursiveIndexedAccessSimplification.ts(5,9): error TS2536: Type 'number' cannot be used to index type 'Recur<T>'.
+recursiveIndexedAccessSimplification.ts(8,5): error TS2322: Type '[string, ...Recur<T>[]]' is not assignable to type 'Recur<T>'.
+  Type '[string, ...Recur<T>[]]' is not assignable to type 'Recur<T>[number][]'.
+    Type 'string | Recur<T>' is not assignable to type 'Recur<T>[number] & (T extends unknown[] ? {} : { [K in keyof T]?: Recur<T[K]> | undefined; })[number]'.
+      Type 'string' is not assignable to type 'Recur<T>[number] & (T extends unknown[] ? {} : { [K in keyof T]?: Recur<T[K]> | undefined; })[number]'.
+        Type 'string' is not assignable to type 'Recur<T>[number] & (T extends unknown[] ? {} : { [K in keyof T]?: Recur<T[K]> | undefined; })[number]'.
+          Type 'string' is not assignable to type '(T extends unknown[] ? {} : { [K in keyof T]?: Recur<T[K]> | undefined; })[number]'.
+recursiveIndexedAccessSimplification.ts(8,12): error TS2589: Type instantiation is excessively deep and possibly infinite.
+recursiveIndexedAccessSimplification.ts(12,37): error TS2589: Type instantiation is excessively deep and possibly infinite.
+
+
+==== recursiveIndexedAccessSimplification.ts (5 errors) ====
+    // https://github.com/microsoft/TypeScript/issues/63270
+    
+    type Recur<T> =
+        (T extends  (unknown[]) ? {} : { [K in keyof T]?: Recur<T[K]>}) |
+        [...Recur<T>[number][]];
+        ~~~~~~~~~~~~~~~~~~~~~~~
+!!! error TS4109: Type arguments for 'Array' circularly reference themselves.
+            ~~~~~~~~~~~~~~~~
+!!! error TS2536: Type 'number' cannot be used to index type 'Recur<T>'.
+    
+    function join<T>(l: Recur<T>[]): Recur<T> {
+        return ['marker', ...l];
+        ~~~~~~
+!!! error TS2322: Type '[string, ...Recur<T>[]]' is not assignable to type 'Recur<T>'.
+!!! error TS2322:   Type '[string, ...Recur<T>[]]' is not assignable to type 'Recur<T>[number][]'.
+!!! error TS2322:     Type 'string | Recur<T>' is not assignable to type 'Recur<T>[number] & (T extends unknown[] ? {} : { [K in keyof T]?: Recur<T[K]> | undefined; })[number]'.
+!!! error TS2322:       Type 'string' is not assignable to type 'Recur<T>[number] & (T extends unknown[] ? {} : { [K in keyof T]?: Recur<T[K]> | undefined; })[number]'.
+!!! error TS2322:         Type 'string' is not assignable to type 'Recur<T>[number] & (T extends unknown[] ? {} : { [K in keyof T]?: Recur<T[K]> | undefined; })[number]'.
+!!! error TS2322:           Type 'string' is not assignable to type '(T extends unknown[] ? {} : { [K in keyof T]?: Recur<T[K]> | undefined; })[number]'.
+               ~~~~~~~~~~~~~~~~
+!!! error TS2589: Type instantiation is excessively deep and possibly infinite.
+    }
+    
+    function a<T>(l: Recur<T>[]): void {
+        const x: Recur<T> | undefined = join(l);
+                                        ~~~~~~~
+!!! error TS2589: Type instantiation is excessively deep and possibly infinite.
+    }
+    

testdata/baselines/reference/compiler/recursiveIndexedAccessSimplification.symbols

@@ -0,0 +1,49 @@
+//// [tests/cases/compiler/recursiveIndexedAccessSimplification.ts] ////
+
+=== recursiveIndexedAccessSimplification.ts ===
+// https://github.com/microsoft/TypeScript/issues/63270
+
+type Recur<T> =
+>Recur : Symbol(Recur, Decl(recursiveIndexedAccessSimplification.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveIndexedAccessSimplification.ts, 2, 11))
+
+    (T extends  (unknown[]) ? {} : { [K in keyof T]?: Recur<T[K]>}) |
+>T : Symbol(T, Decl(recursiveIndexedAccessSimplification.ts, 2, 11))
+>K : Symbol(K, Decl(recursiveIndexedAccessSimplification.ts, 3, 38))
+>T : Symbol(T, Decl(recursiveIndexedAccessSimplification.ts, 2, 11))
+>Recur : Symbol(Recur, Decl(recursiveIndexedAccessSimplification.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveIndexedAccessSimplification.ts, 2, 11))
+>K : Symbol(K, Decl(recursiveIndexedAccessSimplification.ts, 3, 38))
+
+    [...Recur<T>[number][]];
+>Recur : Symbol(Recur, Decl(recursiveIndexedAccessSimplification.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveIndexedAccessSimplification.ts, 2, 11))
+
+function join<T>(l: Recur<T>[]): Recur<T> {
+>join : Symbol(join, Decl(recursiveIndexedAccessSimplification.ts, 4, 28))
+>T : Symbol(T, Decl(recursiveIndexedAccessSimplification.ts, 6, 14))
+>l : Symbol(l, Decl(recursiveIndexedAccessSimplification.ts, 6, 17))
+>Recur : Symbol(Recur, Decl(recursiveIndexedAccessSimplification.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveIndexedAccessSimplification.ts, 6, 14))
+>Recur : Symbol(Recur, Decl(recursiveIndexedAccessSimplification.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveIndexedAccessSimplification.ts, 6, 14))
+
+    return ['marker', ...l];
+>l : Symbol(l, Decl(recursiveIndexedAccessSimplification.ts, 6, 17))
+}
+
+function a<T>(l: Recur<T>[]): void {
+>a : Symbol(a, Decl(recursiveIndexedAccessSimplification.ts, 8, 1))
+>T : Symbol(T, Decl(recursiveIndexedAccessSimplification.ts, 10, 11))
+>l : Symbol(l, Decl(recursiveIndexedAccessSimplification.ts, 10, 14))
+>Recur : Symbol(Recur, Decl(recursiveIndexedAccessSimplification.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveIndexedAccessSimplification.ts, 10, 11))
+
+    const x: Recur<T> | undefined = join(l);
+>x : Symbol(x, Decl(recursiveIndexedAccessSimplification.ts, 11, 9))
+>Recur : Symbol(Recur, Decl(recursiveIndexedAccessSimplification.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveIndexedAccessSimplification.ts, 10, 11))
+>join : Symbol(join, Decl(recursiveIndexedAccessSimplification.ts, 4, 28))
+>l : Symbol(l, Decl(recursiveIndexedAccessSimplification.ts, 10, 14))
+}
+

testdata/baselines/reference/compiler/recursiveIndexedAccessSimplification.types

@@ -0,0 +1,33 @@
+//// [tests/cases/compiler/recursiveIndexedAccessSimplification.ts] ////
+
+=== recursiveIndexedAccessSimplification.ts ===
+// https://github.com/microsoft/TypeScript/issues/63270
+
+type Recur<T> =
+>Recur : Recur<T>
+
+    (T extends  (unknown[]) ? {} : { [K in keyof T]?: Recur<T[K]>}) |
+    [...Recur<T>[number][]];
+
+function join<T>(l: Recur<T>[]): Recur<T> {
+>join : <T>(l: Recur<T>[]) => Recur<T>
+>l : Recur<T>[]
+
+    return ['marker', ...l];
+>['marker', ...l] : [string, ...Recur<T>[]]
+>'marker' : "marker"
+>...l : Recur<T>
+>l : Recur<T>[]
+}
+
+function a<T>(l: Recur<T>[]): void {
+>a : <T>(l: Recur<T>[]) => void
+>l : Recur<T>[]
+
+    const x: Recur<T> | undefined = join(l);
+>x : Recur<T> | undefined
+>join(l) : Recur<T>
+>join : <T_1>(l: Recur<T_1>[]) => Recur<T_1>
+>l : Recur<T>[]
+}
+

testdata/tests/cases/compiler/recursiveIndexedAccessSimplification.ts

@@ -0,0 +1,15 @@
+// @noEmit: true
+
+// https://github.com/microsoft/TypeScript/issues/63270
+
+type Recur<T> =
+    (T extends  (unknown[]) ? {} : { [K in keyof T]?: Recur<T[K]>}) |
+    [...Recur<T>[number][]];
+
+function join<T>(l: Recur<T>[]): Recur<T> {
+    return ['marker', ...l];
+}
+
+function a<T>(l: Recur<T>[]): void {
+    const x: Recur<T> | undefined = join(l);
+}