Don't use symbol.lock / unlock in checkNonCyclic

When checking type params / abstract types for cyclic subtyping,
don't use `symbol.lock` to detect cycles.

The same symbol may occur multiple times and expand to a different
type, for example `a.Id[a.Id[Int]]`.

Seen types need to be compared with `==` (not `=:=`), otherwise
`a.Id[Int]` is matches `a.Id[a.Id[Int]]`.

I tried using `symbol.lock` in a first step and only adding types
to the stack from the second iteration, but some of the new test
cases then still fail to compile because the same symbol lock is
also used for type completers (detecting cyclic references).

An issue was that diverging types were no longer reported as cyclic,
as in neg/t510.scala. I added a recursion depth limit for this case.

```
abstract class C { type T }
abstract class D[S <: C](val c: S) extends C {
  type T <: c.T
}
abstract class E(e: E) extends D[E](e)
object Test {
  def f(e: E): Unit = {
    def g(t: e.T): Unit = ()
    ()
  }
}
```

`e.T` keeps expanding to `e.c.T` to `e.c.c.T` and so on.

Note that removing `object Test` from the example and compiling
only the definitions runs into a stack overflow in a later phase,
so the compiler is vulnerable to diverging types in other places.

For the record, in Scala 3 catches stack overflows and reports
"Recursion limit exceeded" errors.

src/compiler/scala/tools/nsc/typechecker/Typers.scala

@@ -401,28 +401,45 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
       )
     }
 
+    class NonCyclicStack {
+      // for diverging types, neg/t510.scala
+      private val maxRecursion = 42
+
+      // For each abstract type symbol (type member, type parameter), keep track of seen types represented by that symbol
+      private lazy val map = collection.mutable.HashMap[Symbol, mutable.ListBuffer[Type]]()
+
+      def lockSymbol[T](sym: Symbol, tp: Type)(body: => T): T = {
+        val stk = map.getOrElseUpdate(sym, ListBuffer.empty)
+        stk.prepend(tp)
+        try body
+        finally stk.remove(0)
+      }
+
+      def isUnlocked(sym: Symbol, tp: Type): Boolean =
+        !sym.isNonClassType || !map.get(sym).exists(tps => tps.length > maxRecursion || tps.contains(tp))
+    }
+
     /** Check that type `tp` is not a subtype of itself
      */
-    def checkNonCyclic(pos: Position, tp: Type): Boolean = {
+    def checkNonCyclic(pos: Position, tp: Type, stack: NonCyclicStack = new NonCyclicStack): Boolean = {
       def checkNotLocked(sym: Symbol) =
-        sym.initialize.lockOK || { CyclicAliasingOrSubtypingError(pos, sym); false }
+        stack.isUnlocked(sym, tp) || { CyclicAliasingOrSubtypingError(pos, sym); false }
 
       tp match {
         case TypeRef(pre, sym, args) =>
           checkNotLocked(sym) && {
-            !sym.isNonClassType || {
-              sym.lock(())
-              try checkNonCyclic(pos, appliedType(pre.memberInfo(sym), args))
-              finally sym.unlock()
-            }
+            !sym.isNonClassType ||
+              stack.lockSymbol(sym, tp) {
+                checkNonCyclic(pos, appliedType(pre.memberInfo(sym), args), stack)
+              }
           }
 
         case SingleType(_, sym) =>
           checkNotLocked(sym)
         case st: SubType =>
-          checkNonCyclic(pos, st.supertype)
+          checkNonCyclic(pos, st.supertype, stack)
         case ct: CompoundType =>
-          ct.parents forall (x => checkNonCyclic(pos, x))
+          ct.parents forall (x => checkNonCyclic(pos, x, stack))
         case _ =>
           true
       }

test/files/pos/t10669.scala

@@ -0,0 +1,10 @@
+abstract class CharSet[P] {
+  type Type <: P
+}
+
+object LetterOrDigit extends CharSet[Char]
+object Digit extends CharSet[LetterOrDigit.Type]
+
+object t {
+  type D = Digit.Type
+}

test/files/pos/t12622.scala

@@ -0,0 +1,43 @@
+trait ScenarioParam {
+  type Builder <: Type
+}
+
+trait ScenarioParamBuilder
+
+trait Type {
+  type Builder <: ScenarioParamBuilder
+}
+
+trait Types[H <: ScenarioParam, T <: Type] extends Type {
+  type Builder = H#Builder with T#Builder
+}
+
+trait Nil extends Type {
+  type Builder = ScenarioParamBuilder
+}
+
+trait ScenarioTarget {
+  type FilterParam <: Type
+}
+
+class P1 extends ScenarioParam
+class P2 extends ScenarioParam
+
+object someTarget extends ScenarioTarget {
+  type FilterParam = Types[P1, Types[P2, Nil]]
+}
+
+class WhereClauseBuilder1[T <: ScenarioTarget] {
+  type FilterBuilderType = T#FilterParam#Builder
+  def m1(f: FilterBuilderType => Any): Any = null
+  def m2(f: T#FilterParam#Builder => Any): Any = null
+}
+
+object t {
+  (null: WhereClauseBuilder1[someTarget.type]).m1(x => null)
+
+  val stabilizer: WhereClauseBuilder1[someTarget.type] = null
+  stabilizer.m1(x => null)
+
+  (null: WhereClauseBuilder1[someTarget.type]).m2(x => null)
+}

test/files/pos/t12814.scala

@@ -0,0 +1,98 @@
+// https://github.com/scala/bug/issues/12814#issuecomment-1822770100
+object t1 {
+  trait A[X] { type T = X }
+  object B extends A[String]
+  object C extends A[B.T] {
+    def f: C.T = "hai"
+  }
+}
+
+// https://github.com/scala/bug/issues/12814
+object t2 {
+  sealed trait Common
+  sealed trait One extends Common
+  sealed trait Two extends Common
+
+
+  trait Module[C <: Common] {
+    val name: String
+    type Narrow = C
+    def narrow: PartialFunction[Common, C]
+  }
+
+  object ModuleA extends Module[One] {
+    val name = "A"
+    val narrow: PartialFunction[Common, Narrow] = {
+      case cc: Narrow => cc
+    }
+  }
+
+  object ModuleB extends Module[ModuleA.Narrow] {
+    val name = "B"
+    val narrow: PartialFunction[Common, Narrow] = {
+      case cc: Narrow => cc
+    }
+  }
+
+  object ModuleC extends Module[Two] {
+    val name = "C"
+    val narrow: PartialFunction[Common, Narrow] = {
+      case cc: Narrow => cc
+    }
+  }
+
+  object ModuleD extends Module[One with Two] {
+    val name = "D"
+    val narrow: PartialFunction[Common, Narrow] = {
+      case cc: Narrow => cc
+    }
+  }
+
+  val one    = new One {}
+  val two    = new Two {}
+  val oneTwo = new One with Two {}
+
+  Seq(ModuleA, ModuleB, ModuleC, ModuleD).foreach { module =>
+    println(s"${module.name} at One    = ${module.narrow.isDefinedAt(one)}")
+    println(s"${module.name} at Two    = ${module.narrow.isDefinedAt(two)}")
+    println(s"${module.name} at OneTwo = ${module.narrow.isDefinedAt(oneTwo)}")
+    println("-" * 10)
+  }
+}
+
+// https://github.com/scala/scala/pull/10457/files
+object t3 {
+  sealed trait A
+
+  sealed trait B extends A
+
+  trait F[C] {
+    type T = C
+  }
+
+  object O extends F[B]
+
+  object P1 extends F[O.T] {
+    val f: PartialFunction[A, P1.T] = {
+      case x: P1.T => x
+    }
+  }
+
+  object P2 extends F[O.T] {
+    val f: PartialFunction[A, P2.T] = x => x match {
+      case x: P2.T => x
+    }
+  }
+
+  object P3 extends F[O.T] {
+    val f: Function1[A, P3.T] = {
+      case x: P3.T => x
+    }
+  }
+
+  object P4 extends F[O.T] {
+    val f: Function1[A, P4.T] = x => x match {
+      case x: P4.T => x
+    }
+  }
+}

test/files/pos/t8252.scala

@@ -0,0 +1,129 @@
+// https://github.com/scala/bug/issues/8252
+object t1 {
+  import scala.language.higherKinds
+  type Id[A] = A
+  def foo[F[_], G[_], A](a: F[G[A]]): F[G[A]] = { println("what?! " + a); a }
+  def oops(): Unit = {
+    foo[Id, Id, Int](1)
+  }
+  oops()
+  def expected(): Unit = {
+    val kaboom = foo[Id, Id, Int](1)
+  }
+}
+
+// https://github.com/scala/bug/issues/8252#issuecomment-534822175
+object t2 {
+  trait Foo { type X }
+  trait HL extends Foo { override type X }
+  trait HC[H <: Foo, T <: HL] extends HL { override type X = H#X with T#X }
+  trait HN extends HL { override type X = Any }
+  class A extends Foo { trait X } ; class B extends Foo { trait X }
+  class Test {
+      def test: Unit = {
+          val bad = new HC[A, HC[B, HN]] {}
+          val xx: bad.X = ???
+      }
+  }
+}
+
+// https://github.com/scala/bug/issues/8252#issuecomment-347417206
+object t3 {
+  import scala.language.reflectiveCalls
+
+  trait Schema
+
+  class Thing[S] {
+    def apply[T](col: S => T): T = ???
+  }
+
+  type :+:[H, T <: Schema] = H with T
+  type End = Schema
+
+  class Test {
+
+    def test = {
+      new Thing[{val foo: String} :+: End].apply(_.foo)
+
+      new Thing[{val foo: String} :+: {val bar: Int} :+: End].apply(x => x.foo)
+    }
+  }
+
+  // https://github.com/scala/bug/issues/8252#issuecomment-347456209
+  trait Example[T] {
+    type Out
+
+    def apply[A](fn: Out => A): A = ???
+  }
+
+  object Example {
+    def apply[A](implicit inst: Example[A]): Aux[A, inst.Out] = inst
+
+    type Aux[T, Out0] = Example[T] {type Out = Out0}
+
+    implicit def forall[T]: Aux[T, T] = new Example[T] {
+      type Out = T
+    }
+  }
+
+  Example[ {val foo: Int} :+: {val bar: String} :+: {val baz: Boolean} :+: {val buzz: Double} :+: {val booze: Float} :+: End].apply(_.foo)
+
+  // https://github.com/scala/bug/issues/8252#issuecomment-347562144
+  new Example[Unit] {
+    type Out = {val foo: Int} :+: {val bar: String} :+: {val baz: Boolean} :+: {val buzz: Double} :+: {val booze: Float} :+: End
+  }.apply(_.foo)
+
+  // https://github.com/scala/bug/issues/8252#issuecomment-347562502
+  new Example[Unit] {
+    type Out = {val foo: Int} :+: End
+  }.apply(_.foo)
+}
+
+// https://github.com/scala/bug/issues/8252#issuecomment-347565398
+object t4 {
+  import scala.language.reflectiveCalls
+
+  object Example1 {
+
+    trait Schema
+
+    type :+:[H, T <: Schema] = H with T
+    type End = Schema
+
+    class Thing[S] {
+      type Out = S
+
+      def apply[T](fn: Out => T): T = ???
+    }
+
+    new Thing[ {val foo: String} :+: {val bar: Int} :+: {val baz: Boolean} :+: {val booze: Double} :+: End
+    ].apply(x => x.foo)
+
+    val foo = new Thing[ {val foo: String} :+: {val bar: Int} :+: {val baz: Boolean} :+: {val booze: Double} :+: End
+    ]
+
+    foo.apply(x => x.foo)
+  }
+
+  object Example2 {
+
+    trait Schema
+
+    type :+:[H, T <: Schema] = H with T
+    type End = Schema
+
+    class Thing[S]
+
+    implicit class ThingOps[S](self: Thing[S]) {
+      type Out = S
+
+      def apply[T](fn: Out => T): T = ???
+    }
+
+    val foo = new Thing[ {val foo: String} :+: {val bar: Int} :+: {val baz: Boolean} :+: {val booze: Double} :+: End
+    ]
+
+    foo.apply(x => x.foo)
+
+  }
+}

test/files/pos/t9568.scala

@@ -0,0 +1,20 @@
+object Cyclic {
+  class Node[T]() {
+    type Self = T
+  }
+
+  val nodeA = new Node[Int]()
+  val nodeB = new NodeB(a => a)
+  val nodeC = new NodeC(a => a)
+  /*
+  val nodeB = new NodeB(a => a + 1)
+  val nodeC = new NodeC(a => a + 1)
+  */
+  val nodeD = new NodeD((b, c) => b + c)
+
+  class NodeB[T](fun: Function[nodeA.Self, T]) extends Node[T]
+
+  class NodeC[T](fun: Function[nodeA.Self, T]) extends Node[T]
+
+  class NodeD[T](fun: Function2[nodeB.Self, nodeC.Self, T]) extends Node[T]
+}

test/tasty/neg-isolated/src-2/TestChildren.check

@@ -1,7 +1,4 @@
 TestChildren_fail.scala:5: error: can't find type in parents of class tastytest.Child: tastytest.Parent; perhaps it is missing from the classpath.
   def test1 = new Child
                   ^
-TestChildren_fail.scala:6: error: can't find type in parents of object tastytest.Module: tastytest.Parent; perhaps it is missing from the classpath.
-  def test2 = Module
-      ^
-2 errors
+1 error