Fix regression #17245: Overloaded methods with ClassTags (#18286)

The problem lied with slightly adjusted unapply of FunctionOf in a
previous PR, which caused different behavior in `resolveOverloaded`,
where due to a pattern match into a FunctionOf
`resolveOverloaded1` would return no candidates, causing more issues
later on.

To keep the new behavior of FunctionOf unapply (which as a side-effect
ended up fixing few issues represented with added tests), with the previous
behavior of overloaded functions, we allow the method candidate
filtering to fallback from the FunctionOf candidate filtering into the
previous behavior in case no candidates are kept. This also fixes an
additional case, which is not part of the regression, but produces
an incorrect error in similar manner.

Fixes #17245

compiler/src/dotty/tools/dotc/typer/Applications.scala

@@ -2059,31 +2059,35 @@ trait Applications extends Compatibility {
           if isDetermined(alts2) then alts2
           else resolveMapped(alts1, _.widen.appliedTo(targs1.tpes), pt1)
 
-      case defn.FunctionOf(args, resultType, _) =>
-        narrowByTypes(alts, args, resultType)
-
       case pt =>
-        val compat = alts.filterConserve(normalizedCompatible(_, pt, keepConstraint = false))
-        if (compat.isEmpty)
-          /*
-           * the case should not be moved to the enclosing match
-           * since SAM type must be considered only if there are no candidates
-           * For example, the second f should be chosen for the following code:
-           *   def f(x: String): Unit = ???
-           *   def f: java.io.OutputStream = ???
-           *   new java.io.ObjectOutputStream(f)
-           */
-          pt match {
-            case SAMType(mtp, _) =>
-              narrowByTypes(alts, mtp.paramInfos, mtp.resultType)
-            case _ =>
-              // pick any alternatives that are not methods since these might be convertible
-              // to the expected type, or be used as extension method arguments.
-              val convertible = alts.filterNot(alt =>
-                  normalize(alt, IgnoredProto(pt)).widenSingleton.isInstanceOf[MethodType])
-              if convertible.length == 1 then convertible else compat
-          }
-        else compat
+        val compat0 = pt match
+          case defn.FunctionOf(args, resType, _) =>
+            narrowByTypes(alts, args, resType)
+          case _ =>
+            Nil
+        if (compat0.isEmpty) then
+          val compat = alts.filterConserve(normalizedCompatible(_, pt, keepConstraint = false))
+          if (compat.isEmpty)
+            /*
+            * the case should not be moved to the enclosing match
+            * since SAM type must be considered only if there are no candidates
+            * For example, the second f should be chosen for the following code:
+            *   def f(x: String): Unit = ???
+            *   def f: java.io.OutputStream = ???
+            *   new java.io.ObjectOutputStream(f)
+            */
+            pt match {
+              case SAMType(mtp, _) =>
+                narrowByTypes(alts, mtp.paramInfos, mtp.resultType)
+              case _ =>
+                // pick any alternatives that are not methods since these might be convertible
+                // to the expected type, or be used as extension method arguments.
+                val convertible = alts.filterNot(alt =>
+                    normalize(alt, IgnoredProto(pt)).widenSingleton.isInstanceOf[MethodType])
+                if convertible.length == 1 then convertible else compat
+            }
+          else compat
+        else compat0
     }
 
     /** The type of alternative `alt` after instantiating its first parameter

tests/pos/i17245.scala

@@ -0,0 +1,20 @@
+import scala.reflect.ClassTag
+
+trait MockSettings
+
+object Mockito {
+  def mock[T : ClassTag]: T = ???
+  def mock[T : ClassTag](settings: MockSettings): T = ???
+}
+
+trait Channel
+type OnChannel = Channel => Any
+
+@main def Test =
+  val case1: OnChannel = Mockito.mock[OnChannel]
+  val case2: OnChannel = Mockito.mock
+  val case3 = Mockito.mock[OnChannel]
+  val case4: OnChannel =  Mockito.mock[OnChannel](summon[ClassTag[OnChannel]])
+
+  // not a regressive case, but an added improvement with the fix for the above
+  val case5: Channel => Any = Mockito.mock[Channel => Any]