Runner Discovered Suites

js/core/src/main/scala/org/scalatest/tools/MasterRunner.scala

@@ -19,6 +19,8 @@ import org.scalatest.Tracker
 import org.scalatest.events.Summary
 import sbt.testing.{Framework => BaseFramework, Event => SbtEvent, Status => SbtStatus, _}
 
+import scala.scalajs.reflect.Reflect
+
 import scala.compat.Platform
 import ArgsParser._
 import org.scalatest.prop.Seed
@@ -155,6 +157,12 @@ class MasterRunner(theArgs: Array[String], theRemoteArgs: Array[String], testCla
     theArgs
   }
 
+  private def runnerInstance() = {
+    val runnerCompanionClass = Reflect.lookupLoadableModuleClass("org.scalatest.tools.Runner$").getOrElse(throw new RuntimeException("Cannot load org.scalatest.tools.Runner$ class."))
+    val obj = runnerCompanionClass.loadModule()
+    obj.asInstanceOf[Runner.type]
+  }
+
   def tasks(taskDefs: Array[TaskDef]): Array[Task] = {
     def filterWildcard(paths: List[String], taskDefs: Array[TaskDef]): Array[TaskDef] =
       taskDefs.filter(td => paths.exists(td.fullyQualifiedName.startsWith(_)))
@@ -168,7 +176,12 @@ class MasterRunner(theArgs: Array[String], theRemoteArgs: Array[String], testCla
       new TaskRunner(t, testClassLoader, tracker, tagsToInclude, tagsToExclude, t.selectors ++ autoSelectors, false, presentAllDurations, presentInColor, presentShortStackTraces, presentFullStackTraces, presentUnformatted, presentReminder,
         presentReminderWithShortStackTraces, presentReminderWithFullStackTraces, presentReminderWithoutCanceledTests, presentFilePathname, presentJson, Some(send))
 
-    (if (wildcard.isEmpty && membersOnly.isEmpty) taskDefs else (filterWildcard(wildcard, taskDefs) ++ filterMembersOnly(membersOnly, taskDefs)).distinct).map(createTask)
+    val tasks = (if (wildcard.isEmpty && membersOnly.isEmpty) taskDefs else (filterWildcard(wildcard, taskDefs) ++ filterMembersOnly(membersOnly, taskDefs)).distinct).map(createTask)
+
+    val discoveredSuites = tasks.map(_.taskDef.fullyQualifiedName).toSet
+    runnerInstance().internalDiscoveredSuites.set(Some(discoveredSuites))
+
+    tasks
   }
 
   private def send(msg: String): Unit = {

js/core/src/main/scala/org/scalatest/tools/Runner.scala

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2001-2024 Artima, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.scalatest.tools
+
+import java.util.concurrent.atomic.AtomicReference
+import scala.scalajs.reflect.annotation.EnableReflectiveInstantiation
+
+@EnableReflectiveInstantiation
+object Runner {
+
+  private[scalatest] val internalDiscoveredSuites: AtomicReference[Option[Set[String]]] = new AtomicReference(None)
+
+  def discoveredSuites: Option[Set[String]] = internalDiscoveredSuites.get
+
+}

js/core/src/main/scala/org/scalatest/tools/SlaveRunner.scala

@@ -20,6 +20,8 @@ import org.scalatest.events.Summary
 import sbt.testing.{Framework => BaseFramework, Event => SbtEvent, Status => SbtStatus, _}
 import ArgsParser._
 
+import scala.scalajs.reflect.Reflect
+
 import scala.compat.Platform
 import org.scalatest.prop.Seed
 
@@ -122,6 +124,12 @@ class SlaveRunner(theArgs: Array[String], theRemoteArgs: Array[String], testClas
     theArgs
   }
 
+  private def runnerInstance() = {
+    val runnerCompanionClass = Reflect.lookupLoadableModuleClass("org.scalatest.tools.Runner$").getOrElse(throw new RuntimeException("Cannot load org.scalatest.tools.Runner$ class."))
+    val obj = runnerCompanionClass.loadModule()
+    obj.asInstanceOf[Runner.type]
+  }
+
   def tasks(taskDefs: Array[TaskDef]): Array[Task] = {
 
     def filterWildcard(paths: List[String], taskDefs: Array[TaskDef]): Array[TaskDef] =
@@ -136,7 +144,12 @@ class SlaveRunner(theArgs: Array[String], theRemoteArgs: Array[String], testClas
       new TaskRunner(t, testClassLoader, tracker, tagsToInclude, tagsToExclude, t.selectors ++ autoSelectors, false, presentAllDurations, presentInColor, presentShortStackTraces, presentFullStackTraces, presentUnformatted, presentReminder,
         presentReminderWithShortStackTraces, presentReminderWithFullStackTraces, presentReminderWithoutCanceledTests, presentFilePathname, presentJson, Some(notifyServer))
 
-    (if (wildcard.isEmpty && membersOnly.isEmpty) taskDefs else (filterWildcard(wildcard, taskDefs) ++ filterMembersOnly(membersOnly, taskDefs)).distinct).map(createTask)
+    val tasks = (if (wildcard.isEmpty && membersOnly.isEmpty) taskDefs else (filterWildcard(wildcard, taskDefs) ++ filterMembersOnly(membersOnly, taskDefs)).distinct).map(createTask)
+
+    val discoveredSuites = tasks.map(_.taskDef.fullyQualifiedName).toSet
+    runnerInstance().internalDiscoveredSuites.set(Some(discoveredSuites))
+
+    tasks
   }
 
   def receiveMessage(msg: String): Option[String] =

jvm/core/src/main/scala/org/scalatest/tools/Framework.scala

@@ -745,12 +745,24 @@ class Framework extends SbtFramework {
         paths.exists(path => td.fullyQualifiedName.startsWith(path) && td.fullyQualifiedName.substring(path.length).lastIndexOf('.') <= 0)
       }
 
-    def tasks(taskDefs: Array[TaskDef]): Array[Task] =
-      for {
-        taskDef <- if (wildcard.isEmpty && membersOnly.isEmpty) taskDefs else (filterWildcard(wildcard, taskDefs) ++ filterMembersOnly(membersOnly, taskDefs)).distinct
-        task = createTask(taskDef)
-        if task.shouldDiscover
-      } yield task
+    def tasks(taskDefs: Array[TaskDef]): Array[Task] = {
+      val filteredTaskDefs = 
+        (if (wildcard.isEmpty && membersOnly.isEmpty) 
+          taskDefs 
+        else 
+          (filterWildcard(wildcard, taskDefs) ++ filterMembersOnly(membersOnly, taskDefs)).distinct)
+      val tasks: Array[Task] = 
+        for {
+          taskDef <- filteredTaskDefs
+          task = createTask(taskDef)
+          if task.shouldDiscover
+        } yield task
+
+      val discoveredSuites = tasks.map(_.taskDef.fullyQualifiedName).toSet
+      runnerInstance(loader).internalDiscoveredSuites.set(Some(discoveredSuites))
+
+      tasks 
+    }
 
     def done = {
       if (!isDone.getAndSet(true)) {
@@ -949,6 +961,13 @@ class Framework extends SbtFramework {
     wildcards.toList
   }
 
+  private def runnerInstance(loader: ClassLoader) = {
+    val runnerCompanionClass = loader.loadClass("org.scalatest.tools.Runner$")
+    val module = runnerCompanionClass.getField("MODULE$")
+    val obj = module.get(runnerCompanionClass)
+    obj.asInstanceOf[Runner.type]
+  }
+
   /**
    *
    * Initiates a ScalaTest run.
@@ -1011,12 +1030,7 @@ class Framework extends SbtFramework {
         case _ => (false, 60000L, 60000L)
       }
 
-    val runnerCompanionClass = testClassLoader.loadClass("org.scalatest.tools.Runner$")
-    val module = runnerCompanionClass.getField("MODULE$")
-    val obj = module.get(runnerCompanionClass)
-    val runnerInstance = obj.asInstanceOf[Runner.type]
-
-    runnerInstance.spanScaleFactor = parseDoubleArgument(spanScaleFactors, "-F", 1.0)
+    runnerInstance(testClassLoader).spanScaleFactor = parseDoubleArgument(spanScaleFactors, "-F", 1.0)
 
     parseLongArgument(seedArgs, "-S") match {
       case Some(seed) => Seed.configuredRef.getAndSet(Some(seed))

jvm/core/src/main/scala/org/scalatest/tools/Runner.scala

@@ -29,6 +29,7 @@ import org.scalatest.events._
 import java.util.concurrent.Executors
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.ThreadFactory
+import java.util.concurrent.atomic.AtomicReference
 import SuiteDiscoveryHelper._
 import org.scalatest.time.Span
 import org.scalatest.time.Seconds
@@ -691,6 +692,10 @@ object Runner {
   @volatile private[scalatest] var spanScaleFactor: Double = 1.0
 
   private final val DefaultNumFilesToArchive = 2
+
+  private[scalatest] val internalDiscoveredSuites: AtomicReference[Option[Set[String]]] = new AtomicReference(None)
+
+  def discoveredSuites: Option[Set[String]] = internalDiscoveredSuites.get
 
   //                     TO
   // We always include a PassFailReporter on runs in order to determine
@@ -1144,6 +1149,8 @@ object Runner {
         dispatch(
           DiscoveryCompleted(tracker.nextOrdinal(), Some(discoveryDuration)))
 
+        internalDiscoveredSuites.set(Some(discoSuites.map(_.suite.getClass.getName).toSet))  
+
         discoSuites
       }
     }

native/core/src/main/scala/org/scalatest/tools/Framework.scala

@@ -33,10 +33,9 @@ class Framework extends BaseFramework {
         def requireNoArgConstructor(): Boolean = true
       })
 
-
-  def slaveRunner(args: Array[String], remoteArgs: Array[String], testClassLoader: ClassLoader, send: (String) => Unit): Runner =
+  def slaveRunner(args: Array[String], remoteArgs: Array[String], testClassLoader: ClassLoader, send: (String) => Unit): Runner = 
     new SlaveRunner(args, remoteArgs, testClassLoader, send)
 
-  def runner(args: Array[String], remoteArgs: Array[String], testClassLoader: ClassLoader): Runner =
+  def runner(args: Array[String], remoteArgs: Array[String], testClassLoader: ClassLoader): Runner = 
     new MasterRunner(args, remoteArgs, testClassLoader)
 }

native/core/src/main/scala/org/scalatest/tools/MasterRunner.scala

@@ -19,10 +19,14 @@ import org.scalatest.Tracker
 import org.scalatest.events.Summary
 import sbt.testing.{Framework => BaseFramework, Event => SbtEvent, Status => SbtStatus, _}
 
+import scala.scalanative.reflect.Reflect
+
 import scala.compat.Platform
 import ArgsParser._
 import org.scalatest.prop.Seed
 
+import java.io.PrintWriter
+
 class MasterRunner(theArgs: Array[String], theRemoteArgs: Array[String], testClassLoader: ClassLoader) extends Runner {
 
   // TODO: To take these from test arguments
@@ -147,13 +151,11 @@ class MasterRunner(theArgs: Array[String], theRemoteArgs: Array[String], testCla
     fragments.map(_.toPossiblyColoredText(presentInColor)).mkString("\n")
   }
 
-  def remoteArgs(): Array[String] = {
+  def remoteArgs(): Array[String] = 
     theRemoteArgs
-  }
 
-  def args: Array[String] = {
+  def args: Array[String] = 
     theArgs
-  }
 
   def tasks(taskDefs: Array[TaskDef]): Array[Task] = {
     def filterWildcard(paths: List[String], taskDefs: Array[TaskDef]): Array[TaskDef] =
@@ -164,11 +166,18 @@ class MasterRunner(theArgs: Array[String], theRemoteArgs: Array[String], testCla
         paths.exists(path => td.fullyQualifiedName().startsWith(path) && td.fullyQualifiedName().substring(path.length).lastIndexOf('.') <= 0)
       }
 
+    val discoveredSuites = taskDefs.map(_.fullyQualifiedName()).toSet
+    Runner.setDiscoveredSuites(discoveredSuites)  
+
     def createTask(t: TaskDef): Task =
       new TaskRunner(t, testClassLoader, tracker, tagsToInclude, tagsToExclude, t.selectors() ++ autoSelectors, false, presentAllDurations, presentInColor, presentShortStackTraces, presentFullStackTraces, presentUnformatted, presentReminder,
         presentReminderWithShortStackTraces, presentReminderWithFullStackTraces, presentReminderWithoutCanceledTests, presentFilePathname, presentJson, Some(send))
 
-    (if (wildcard.isEmpty && membersOnly.isEmpty) taskDefs else (filterWildcard(wildcard, taskDefs) ++ filterMembersOnly(membersOnly, taskDefs)).distinct).map(createTask)
+    val tasks = (if (wildcard.isEmpty && membersOnly.isEmpty) taskDefs else (filterWildcard(wildcard, taskDefs) ++ filterMembersOnly(membersOnly, taskDefs)).distinct).map(createTask)
+
+    //runnerInstance().internalDiscoveredSuites.set(Some(discoveredSuites))
+
+    tasks
   }
 
   private def send(msg: String): Unit = {
@@ -194,13 +203,23 @@ class MasterRunner(theArgs: Array[String], theRemoteArgs: Array[String], testCla
         summaryCounter.incrementSuitesAbortedCount()
       case "org.scalatest.events.ScopePending" =>
         summaryCounter.incrementScopesPendingCount()
+      case s"getDiscoveredSuites-$filePath" =>
+        val tempFile = new java.io.File(filePath)
+        val writer = new PrintWriter(filePath)
+        try {
+          writer.write(Runner.discoveredSuites.get.mkString(","))
+          writer.flush()
+        } finally {
+          writer.close()
+        }  
       case _ =>
     }
     None
   }
 
-  def serializeTask(task: Task, serializer: (TaskDef) => String): String =
+  def serializeTask(task: Task, serializer: (TaskDef) => String): String = {
     serializer(task.taskDef())
+  }
 
   def deserializeTask(task: String, deserializer: (String) => TaskDef): Task = {
     val taskDef = deserializer(task)

native/core/src/main/scala/org/scalatest/tools/Runner.scala

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2001-2024 Artima, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.scalatest.tools
+
+import java.util.concurrent.atomic.AtomicReference
+import scala.scalanative.reflect.annotation.EnableReflectiveInstantiation
+import java.io.{File, PrintWriter}
+import org.scalatest.{Retries, Succeeded, Failed}
+import org.scalatest.time.Span
+import org.scalatest.time.Second
+
+@EnableReflectiveInstantiation
+object Runner {
+
+  private[scalatest] val masterFun: AtomicReference[Option[String => Unit]] = new AtomicReference(None)
+
+  def setMasterFun(fun: String => Unit): Unit = {
+    masterFun.set(Some(fun))
+  }
+
+  def setDiscoveredSuites(suites: Set[String]): Unit = {
+    System.setProperty("scalatest.DiscoveredSuites", suites.mkString(","))
+  }
+
+  def discoveredSuites: Option[Set[String]] = {
+    masterFun.get match {
+      case Some(fun) => 
+        val tempFile = File.createTempFile("scalatest-", "-getDiscoveredSuites")
+        // Tell the master runner to write the discovered suites to the temp file.
+        fun(s"getDiscoveredSuites-${tempFile.getAbsolutePath}")
+        // The above is asynchronous, so we need to wait until the temp file is written.
+        // Bad idea to block, but I think that's the only way to do it here, and we do not expect discoveredSuites to be called often.
+        while (tempFile.length() == 0)
+          Thread.sleep(500)  
+        val source = scala.io.Source.fromFile(tempFile)
+        try {
+          Some(source.mkString.split(",").toSet)
+        } finally { 
+          source.close()
+          tempFile.delete()
+        }
+      case None => 
+        val discoveredSuites = System.getProperty("scalatest.DiscoveredSuites")
+        if (discoveredSuites == null || discoveredSuites.trim.isEmpty) None
+        else Some(discoveredSuites.split(",").toSet)
+    }
+  }
+
+}

native/core/src/main/scala/org/scalatest/tools/SlaveRunner.scala

@@ -21,10 +21,11 @@ import sbt.testing.{Framework => BaseFramework, Event => SbtEvent, Status => Sbt
 import ArgsParser._
 import org.scalatest.prop.Seed
 
+import scala.scalanative.reflect.Reflect
+
 import scala.compat.Platform
 
 class SlaveRunner(theArgs: Array[String], theRemoteArgs: Array[String], testClassLoader: ClassLoader, notifyServer: String => Unit) extends Runner {
-
   val sbtNoFormat = false   // System property not supported in scala-js
   val ParsedArgs(
     reporterArgs,
@@ -36,6 +37,8 @@ class SlaveRunner(theArgs: Array[String], theRemoteArgs: Array[String], testClas
     seedArgs
   ) = parseArgs(args)
 
+  Runner.setMasterFun(notifyServer)
+
   val (
   presentAllDurations,
   presentInColor,
@@ -114,13 +117,11 @@ class SlaveRunner(theArgs: Array[String], theRemoteArgs: Array[String], testClas
 
   def done(): String = ""
 
-  def remoteArgs(): Array[String] = {
+  def remoteArgs(): Array[String] = 
     theRemoteArgs
-  }
 
-  def args: Array[String] = {
+  def args: Array[String] = 
     theArgs
-  }
 
   def tasks(taskDefs: Array[TaskDef]): Array[Task] = {
 
@@ -132,11 +133,16 @@ class SlaveRunner(theArgs: Array[String], theRemoteArgs: Array[String], testClas
         paths.exists(path => td.fullyQualifiedName().startsWith(path) && td.fullyQualifiedName().substring(path.length).lastIndexOf('.') <= 0)
       }
 
+    val discoveredSuites = taskDefs.map(_.fullyQualifiedName()).toSet
+    Runner.setDiscoveredSuites(discoveredSuites)  
+
     def createTask(t: TaskDef): Task =
       new TaskRunner(t, testClassLoader, tracker, tagsToInclude, tagsToExclude, t.selectors() ++ autoSelectors, false, presentAllDurations, presentInColor, presentShortStackTraces, presentFullStackTraces, presentUnformatted, presentReminder,
         presentReminderWithShortStackTraces, presentReminderWithFullStackTraces, presentReminderWithoutCanceledTests, presentFilePathname, presentJson, Some(notifyServer))
 
-    (if (wildcard.isEmpty && membersOnly.isEmpty) taskDefs else (filterWildcard(wildcard, taskDefs) ++ filterMembersOnly(membersOnly, taskDefs)).distinct).map(createTask)
+    val tasks = (if (wildcard.isEmpty && membersOnly.isEmpty) taskDefs else (filterWildcard(wildcard, taskDefs) ++ filterMembersOnly(membersOnly, taskDefs)).distinct).map(createTask)
+
+    tasks
   }
 
   def receiveMessage(msg: String): Option[String] =