Fix 2.3.0 Scala 3 regression "T is not a type lambda, it cannot be pa…

…rameterized" (#385)

Fix 2.3.0 Scala 3 regression: Convert abstract higher-kinded type members into lambdas on Scala 3, fix inheritance of abstract proper type members on Scala 3

Add test for problem with higher-kinded type members, on Scala 3 form TestModel::HigherKindedTypeMember::T|<λ %0,%1 → Unit..λ %0,%1 → AnyVal> prevents lambda application

Fix inheritance comparison failing if type constructor inherited a proper type, but basesdb had no data for fully applied type inheriting a proper type

README.md

@@ -53,6 +53,7 @@ Known limitations are:
 4. This-Types such as `X.this.type` are ignored and identical to `X`
 5. `izumi-reflect` is less powerful than `scala-reflect`: it does not preserve fields and methods when it's not necessary for equality and subtype checks, it does not preserve code trees, internal compiler data structures, etc.
 6. There are some optimizations in place which reduce correctness, namely: subtype check for `scala.Matchable` will always return true, no distinction is made between `scala.Any` and `scala.AnyRef`.
+7. Lower bounds are not preserved in abstract higher-kinded type members which may produce false comparisons.
 
 ## Debugging
 

docs/index.md

@@ -38,6 +38,7 @@ Known limitations are:
 4. This-Types such as `X.this.type` are ignored and identical to `X`
 5. `izumi-reflect` is less powerful than `scala-reflect`: it does not preserve fields and methods when it's not necessary for equality and subtype checks, it does not preserve code trees, internal compiler data structures, etc.
 6. There are some optimizations in place which reduce correctness, namely: subtype check for `scala.Matchable` will always return true, no distinction is made between `scala.Any` and `scala.AnyRef`.
+7. Lower bounds are not preserved in abstract higher-kinded type members which may produce false comparisons.
 
 ## Installation
 

izumi-reflect/izumi-reflect/src/main/scala-2/izumi/reflect/macrortti/LightTypeTagImpl.scala

@@ -146,7 +146,7 @@ final class LightTypeTagImpl[U <: Universe with Singleton](val u: U, withCache:
       // we need to use tpe.etaExpand but 2.13 has a bug: https://github.com/scala/bug/issues/11673#
       // tpe.etaExpand.resultType.dealias.typeArgs.flatMap(_.dealias.resultType.typeSymbol.typeSignature match {
 
-      def doExtract(t: Type) = {
+      def doExtract(t: Type): List[Type] = {
         val tpePolyTypeResultType = Dealias.fullNormDealiasSquashHKTToPolyTypeResultType(t)
 
         tpePolyTypeResultType.typeArgs.flatMap {
@@ -190,11 +190,11 @@ final class LightTypeTagImpl[U <: Universe with Singleton](val u: U, withCache:
       out
     }
 
-    val inh = mutable.HashSet[Type]()
-    extractComponents(mainTpe, inh)
-    logger.log(s"Extracted all type references for mainTpe=$mainTpe inh=${inh.iterator.map(t => (t, t.getClass.asInstanceOf[Class[Any]])).toMap.niceList()}")
+    val parts = mutable.HashSet[Type]()
+    extractComponents(mainTpe, parts)
+    logger.log(s"Extracted all type references for mainTpe=$mainTpe parts=${parts.iterator.map(t => (t, t.getClass.asInstanceOf[Class[Any]])).toMap.niceList()}")
 
-    inh.toSet
+    parts.toSet
   }
 
   private[this] def makeAppliedBases(mainTpe: Type, allReferenceComponents: Set[Type]): Set[(AbstractReference, AbstractReference)] = {
@@ -299,7 +299,7 @@ final class LightTypeTagImpl[U <: Universe with Singleton](val u: U, withCache:
     }
 
     val unappliedBases = allReferenceComponents.flatMap(processLambdasReturningRefinements)
-    logger.log(s"Computed unapplied lambda bases for tpe=$mainTpe unappliedBases=${unappliedBases.toMultimap.niceList()}")
+    logger.log(s"Computed lambda only bases for tpe=$mainTpe lambdaBases=${unappliedBases.toMultimap.niceList()}")
     unappliedBases
   }
 
@@ -345,7 +345,19 @@ final class LightTypeTagImpl[U <: Universe with Singleton](val u: U, withCache:
     // val tpef = Dealias.fullNormDealiasResultType(t0, squashHKTRefToPolyTypeResultType = false)
     // no PolyTypes passed to here [but actually we should preserve polyTypes]
     val tpe = Dealias.fullNormDealias(t0)
-    tpe
+    val upperBound = {
+      val tpeSig = tpe.typeSymbol.typeSignature
+      tpeSig.finalResultType match {
+        // handle abstract higher-kinded type members specially,
+        // move their upper bound into inheritance db, because they
+        // will lose it after application. (Unlike proper type members)
+        case b: TypeBoundsApi if tpeSig.takesTypeArgs =>
+          List(b.hi)
+        case _ =>
+          Nil
+      }
+    }
+    upperBound ++ tpe
       .baseClasses
       .iterator
       .map(tpe.baseType)
@@ -390,7 +402,9 @@ final class LightTypeTagImpl[U <: Universe with Singleton](val u: U, withCache:
       }
       tOrTypeSymBounds match {
         case b: TypeBoundsApi =>
-          if ((b.lo =:= nothing && b.hi =:= any) || (nestedIn.contains(b.lo) || nestedIn.contains(b.hi))) {
+          if ((b.lo =:= nothing && b.hi =:= any) ||
+            // prevent recursion
+            (nestedIn.contains(b.lo) || nestedIn.contains(b.hi))) {
             Boundaries.Empty
           } else {
             val lo = makeRefSub(b.lo, Map.empty, Set.empty)

izumi-reflect/izumi-reflect/src/main/scala-3/izumi/reflect/dottyreflection/FullDbInspector.scala

@@ -43,7 +43,7 @@ abstract class FullDbInspector(protected val shift: Int) extends InspectorBase {
           extractBase(a, selfRef, recurseIntoBases = false)
 
         case typeLambda: TypeLambda =>
-          val selfL = i.inspectTypeRepr(typeLambda).asInstanceOf[LightTypeTagRef.Lambda]
+          val selfL = selfRef.asInstanceOf[LightTypeTagRef.Lambda]
 
           val parents = new Run(i.nextLam(typeLambda)).inspectTypeBoundsToFull(typeLambda.resType)
 
@@ -92,7 +92,7 @@ abstract class FullDbInspector(protected val shift: Int) extends InspectorBase {
           extractBase(termRef, selfRef, false)
 
         case b: TypeBounds =>
-          inspectTypeReprToFullBases(b.hi) ++ inspectTypeReprToFullBases(b.low)
+          processTypeBounds(b)
 
         case c: ConstantType =>
           extractBase(c, selfRef, false)
@@ -116,7 +116,7 @@ abstract class FullDbInspector(protected val shift: Int) extends InspectorBase {
           extractBase(r, selfRef, recurseIntoBases = true)
 
         case s if s.isTypeDef =>
-          inspectTypeReprToFullBases(r._underlying)
+          processTypeMemberWithTypeLambdaBounds(r)
 
         case o =>
           throw new RuntimeException(s"Shit tree: ${o.getClass} $o $r ${o.tree}")
@@ -164,11 +164,56 @@ abstract class FullDbInspector(protected val shift: Int) extends InspectorBase {
     private def inspectTypeBoundsToFull(tpe: TypeRepr): List[(AbstractReference, AbstractReference)] = {
       tpe.dealias match {
         case t: TypeBounds =>
-          inspectTypeReprToFullBases(t.hi) ++ inspectTypeReprToFullBases(t.low)
+          processTypeBounds(t)
         case t: TypeRepr =>
           inspectTypeReprToFullBases(t)
       }
     }
+
+    private def processTypeBounds(tb: TypeBounds): List[(AbstractReference, AbstractReference)] = {
+      inspectTypeReprToFullBases(tb.hi) ++ inspectTypeReprToFullBases(tb.low)
+    }
+
+    private def processTypeMemberWithTypeLambdaBounds(t: TypeRef | ParamRef): List[(AbstractReference, AbstractReference)] = {
+      def replaceUpperBoundWithSelfInUpperBoundBases(selfRef: AbstractReference, upperBound: AbstractReference, upperBoundTpe: TypeRepr)
+        : List[(AbstractReference, AbstractReference)] = {
+        val basesOfUpperBound = inspectTypeReprToFullBases(upperBoundTpe)
+        basesOfUpperBound.map {
+          case (k, v) if k == upperBound =>
+            // bases of upper bound are also bases of the abstract type
+            selfRef -> v
+          case kv =>
+            kv
+        }
+      }
+
+      val underlying = t._underlying
+      underlying match {
+        // handle abstract higher-kinded type members specially,
+        // move their upper bound into inheritance db, because they
+        // will lose it after application. (Unlike proper type members)
+        case TypeBounds(_, tl0: TypeLambda) =>
+          val selfRef = i.inspectTypeRepr(t)
+          // include only upper bound: we discard the lower bound for abstract higher-kinded type members
+          val tl = tl0.dealias.simplified
+          val hiTypeLambda = i.inspectTypeRepr(tl)
+
+          (selfRef, hiTypeLambda) :: replaceUpperBoundWithSelfInUpperBoundBases(selfRef, hiTypeLambda, tl)
+
+        // for abstract proper type members, we do not include the upper bound itself into db
+        // (because it's already in the type bound and unlike for type lambda members, the type bound is not lost.
+        case TypeBounds(_, hi0) =>
+          val selfRef = i.inspectTypeRepr(t)
+          val hi = hi0.dealias.simplified
+          val upperBound = i.inspectTypeRepr(hi)
+
+          replaceUpperBoundWithSelfInUpperBoundBases(selfRef, upperBound, hi)
+
+        case _ =>
+          inspectTypeReprToFullBases(underlying)
+      }
+    }
+
   }
 
 }

izumi-reflect/izumi-reflect/src/main/scala-3/izumi/reflect/dottyreflection/InheritanceDbInspector.scala

@@ -90,12 +90,27 @@ abstract class InheritanceDbInspector(protected val shift: Int) extends Inspecto
     val onlyParameterizedBases =
       typeRepr
         .baseClasses
-        .filter(s => s.isType)
-        .map(s => typeRepr.baseType(s))
+        .filter(_.isType)
+        .map(typeRepr.baseType)
 
     val allbases = onlyParameterizedBases.filterNot(_ =:= typeRepr)
 
-    allbases
+    val upperBoundBases = typeRepr match {
+      case t: TypeRef =>
+        t._underlying match {
+          // handle abstract higher-kinded type members specially,
+          // move their upper bound into inheritance db, because they
+          // will lose it after application. (Unlike proper type members)
+          case TypeBounds(_, tl: TypeLambda) =>
+            List(tl.resType.dealias.simplified)
+          case _ =>
+            Nil
+        }
+      case _ =>
+        Nil
+    }
+
+    upperBoundBases ++ allbases
   }
 
   extension (t: TypeRepr) {

izumi-reflect/izumi-reflect/src/main/scala-3/izumi/reflect/dottyreflection/Inspector.scala

@@ -2,6 +2,7 @@ package izumi.reflect.dottyreflection
 
 import izumi.reflect.macrortti.{LightTypeTagInheritance, LightTypeTagRef}
 import izumi.reflect.macrortti.LightTypeTagRef.*
+import izumi.reflect.macrortti.LightTypeTagRef.SymName.SymTypeName
 
 import scala.annotation.{tailrec, targetName}
 import scala.collection.immutable.Queue
@@ -43,10 +44,9 @@ abstract class Inspector(protected val shift: Int, val context: Queue[Inspector.
   }
 
   def buildTypeRef[T <: AnyKind: Type]: AbstractReference = {
-    val tpeTree = TypeTree.of[T]
-    log(s" -------- about to inspect ${tpeTree.show} --------")
+    log(s" -------- about to inspect ${TypeTree.of[T].show} (${TypeRepr.of[T]}) --------")
     val res = inspectTypeRepr(TypeRepr.of[T])
-    log(s" -------- done inspecting ${tpeTree.show} --------")
+    log(s" -------- done inspecting ${TypeTree.of[T].show} (${TypeRepr.of[T]}) --------")
     res
   }
 
@@ -103,9 +103,6 @@ abstract class Inspector(protected val shift: Int, val context: Queue[Inspector.
         val paramNames = inspector.context.last.params.map(_.asParam)
         LightTypeTagRef.Lambda(paramNames, resType)
 
-      case p: ParamRef =>
-        makeNameReferenceFromType(p)
-
       case t: ThisType =>
         next().inspectTypeRepr(t.tref)
 
@@ -125,6 +122,9 @@ abstract class Inspector(protected val shift: Int, val context: Queue[Inspector.
           UnionReference(elements)
         }
 
+      case p: ParamRef =>
+        makeNameReferenceFromType(p)
+
       case term: TermRef =>
         makeNameReferenceFromType(term)
 
@@ -137,14 +137,13 @@ abstract class Inspector(protected val shift: Int, val context: Queue[Inspector.
       case constant: ConstantType =>
         makeNameReferenceFromType(constant)
 
+      case ref: Refinement =>
+        next().inspectRefinements(ref)
+
       case lazyref if lazyref.getClass.getName.contains("LazyRef") => // upstream bug seems like
         log(s"TYPEREPR UNSUPPORTED: LazyRef occured $lazyref")
         NameReference(SymName.SymTypeName("???"))
 
-      // Matches CachedRefinedType for this ZIO issue https://github.com/zio/zio/issues/6071
-      case ref: Refinement =>
-        next().inspectRefinements(ref)
-
       case o =>
         log(s"TYPEREPR UNSUPPORTED: $o")
         throw new RuntimeException(s"TYPEREPR, UNSUPPORTED: ${o.getClass} - $o")
@@ -204,7 +203,8 @@ abstract class Inspector(protected val shift: Int, val context: Queue[Inspector.
   }
 
   private def inspectBounds(outerTypeRef: Option[TypeRef], tb: TypeBounds, isParamWildcard: Boolean): AbstractReference = {
-    inspectBoundsImpl(tb) match {
+    log(s"inspectBounds: found TypeBounds $tb outer=$outerTypeRef isParamWildcard=$isParamWildcard")
+    val res = inspectBoundsImpl(tb) match {
       case Left(hi) =>
         hi
       case Right(boundaries) =>
@@ -219,6 +219,16 @@ abstract class Inspector(protected val shift: Int, val context: Queue[Inspector.
           symref
         }
     }
+    invertTypeMemberWithTypeLambdaBounds(res)
+  }
+
+  private def invertTypeMemberWithTypeLambdaBounds(abstractReference: AbstractReference): AbstractReference = abstractReference match {
+    case NameReference(symName, Boundaries.Defined(bottom @ _, LightTypeTagRef.Lambda(input, realTop @ _)), prefix) =>
+      // We throw away both upper and lower boundaries
+      // Upper boundaries we'll recover later in fulldb and inheritancedb
+      // But lower boundaries we don't recover
+      LightTypeTagRef.Lambda(input, FullReference(symName, input.map(p => TypeParam(NameReference(p), Variance.Invariant)), prefix))
+    case other => other
   }
 
   private def inspectBoundsImpl(tb: TypeBounds): Either[AbstractReference, Boundaries] = {
@@ -239,6 +249,7 @@ abstract class Inspector(protected val shift: Int, val context: Queue[Inspector.
   private[dottyreflection] def inspectSymbol(symbol: Symbol, outerTypeRef: Option[TypeRef], prefixSource: Option[NamedType]): AbstractReference = {
     symbol match {
       case s if s.isClassDef || s.isValDef || s.isBind =>
+        log(s"inspectSymbol: Found Cls=${s.isClassDef} Val=${s.isValDef} Bind=${s.isBind} symbol $s")
         makeNameReferenceFromSymbol(symbol, prefixSource)
 
       case s if s.isTypeDef =>
@@ -300,6 +311,7 @@ abstract class Inspector(protected val shift: Int, val context: Queue[Inspector.
         log(s"make name reference from term $term")
         makeNameReferenceFromSymbol(term.termSymbol, Some(term))
       case t: ParamRef =>
+        log(s"make name reference from paramRef $t")
         val isInContext = context.flatMap(_.params.map(_.tpe)).contains(t)
         if (isInContext) {
           // assert(isInContext, s"${context.flatMap(_.params.map(_.tpe)).map(t => t)} must contain $t")
@@ -316,12 +328,13 @@ abstract class Inspector(protected val shift: Int, val context: Queue[Inspector.
 
         } else {
           val lt = t.binder.asInstanceOf[LambdaType]
-          val paramName = lt.paramNames(t.paramNum).toString
           NameReference(SymName.LambdaParamName(t.paramNum, -1, lt.paramNames.size), Boundaries.Empty, None)
         }
 
       case constant: ConstantType =>
+        log(s"make name reference from ConstantType $constant")
         NameReference(SymName.SymLiteral(constant.constant.value), Boundaries.Empty, prefix = None)
+
       case ref =>
         log(s"make name reference from what? $ref ${ref.getClass} ${ref.termSymbol}")
         makeNameReferenceFromSymbol(ref.typeSymbol, None)
@@ -331,12 +344,12 @@ abstract class Inspector(protected val shift: Int, val context: Queue[Inspector.
   @tailrec
   private[dottyreflection] final def makeNameReferenceFromSymbol(symbol: Symbol, prefixSource1: Option[NamedType]): NameReference = {
     def default: NameReference = {
-      val (symName, s, prefixSource2) = if (symbol.isTerm || symbol.isBind || symbol.isValDef) {
-        (SymName.SymTermName(symbol.fullName), symbol, symbol.termRef)
+      val (symName, prefixSource2) = if (symbol.isTerm || symbol.isBind || symbol.isValDef) {
+        (SymName.SymTermName(symbol.fullName), symbol.termRef)
       } else if (symbol.flags.is(Flags.Module)) { // Handle ModuleClasses (can creep in from ThisType)
-        (SymName.SymTermName(symbol.companionModule.fullName), symbol.companionModule, symbol.companionModule.termRef)
+        (SymName.SymTermName(symbol.companionModule.fullName), symbol.companionModule.termRef)
       } else {
-        (SymName.SymTypeName(symbol.fullName), symbol, symbol.termRef)
+        (SymName.SymTypeName(symbol.fullName), symbol.termRef)
       }
       val prefix = getPrefixFromQualifier(prefixSource1.getOrElse(prefixSource2))
       NameReference(symName, Boundaries.Empty, prefix)

izumi-reflect/izumi-reflect/src/main/scala/izumi/reflect/macrortti/LightTypeTagInheritance.scala

@@ -116,6 +116,7 @@ final class LightTypeTagInheritance(self: LightTypeTag, other: LightTypeTag) {
           all(
             compareBounds(ctx)(s, t.boundaries),
             any(
+              oneOfUnparameterizedParentsIsInheritedFrom(ctx)(s.asName, t), // constructor inherits from rhs, where rhs is an unparameterized type
 //              outerLambdaParams.map(_.name).contains(t.ref.name), // lambda parameter may accept anything within bounds      // UNSOUND-LAMBDA-COMPARISON
               t.ref.maybeName.exists(outerDecls.map(_.name).contains) // refinement type decl may accept anything within bounds
             )
@@ -129,10 +130,12 @@ final class LightTypeTagInheritance(self: LightTypeTag, other: LightTypeTag) {
       case (s: NameReference, t: NameReference) =>
         val boundIsOk = compareBounds(ctx)(s, t.boundaries)
         any(
-          all(boundIsOk, parameterizedParentsOf(s).exists(ctx.isChild(_, t))),
-          all(boundIsOk, unparameterizedParentsOf(s).exists(ctx.isChild(_, t))),
-//          all(boundIsOk, outerLambdaParams.map(_.name).contains(t.ref.name)), // lambda parameter may accept anything within bounds       // UNSOUND-LAMBDA-COMPARISON
-          all(boundIsOk, t.ref.maybeName.exists(outerDecls.map(_.name).contains)), // refinement decl may accept anything within bounds
+          boundIsOk && any(
+            oneOfParameterizedParentsIsInheritedFrom(ctx)(s, t),
+            oneOfUnparameterizedParentsIsInheritedFrom(ctx)(s, t),
+            // outerLambdaParams.map(_.name).contains(t.ref.name), // lambda parameter may accept anything within bounds       // UNSOUND-LAMBDA-COMPARISON
+            t.ref.maybeName.exists(outerDecls.map(_.name).contains) // refinement decl may accept anything within bounds
+          ),
           s.boundaries match {
             case Boundaries.Defined(_, sUp) =>
               ctx.isChild(sUp, t)
@@ -339,6 +342,12 @@ final class LightTypeTagInheritance(self: LightTypeTag, other: LightTypeTag) {
     parents.exists(ctx.isChild(_, parent))
   }
 
+  private def oneOfUnparameterizedParentsIsInheritedFrom(ctx: Ctx)(child: NameReference, parent: NameReference): Boolean = {
+    ctx.logger.log(s"Looking up unparameterized parents of $child => ${unparameterizedParentsOf(child)}")
+    val parents = unparameterizedParentsOf(child)
+    parents.exists(ctx.isChild(_, parent))
+  }
+
   private def unparameterizedParentsOf(t: NameReference): mutable.HashSet[NameReference] = {
     def parentsOf(t: NameReference, out: mutable.HashSet[NameReference], tested: mutable.HashSet[NameReference]): Unit = {
       val direct = idb.get(t).toSet.flatten