Merge pull request #11349 from romanowski/scaladoc/remote-dottydoc

remove dottydoc

.github/workflows/ci.yaml

@@ -100,7 +100,7 @@ jobs:
       - name: Add SBT proxy repositories
         run: cp -vf .github/workflows/repositories /root/.sbt/ ; true
 
-      - name: Test
+      - name: Cmd Tests
         run: |
           ./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test;sjsCompilerTests/test ;sbt-dotty/scripted scala2-compat/* ;configureIDE ;stdlib-bootstrapped/test:run ;stdlib-bootstrapped-tasty-tests/test"
           ./project/scripts/bootstrapCmdTests

.github/workflows/scaladoc.yaml

@@ -69,7 +69,7 @@ jobs:
             DOC_DEST=pr-${PR_NUMBER:-${GITHUB_REF##*/}}
             echo uplading docs to https://scala3doc.virtuslab.com/$DOC_DEST
             az storage container create --name $DOC_DEST --account-name scala3docstorage --public-access container
-            az storage blob sync -s scaladoc/output -c $DOC_DEST --account-name scala3docstorage
+            az storage blob upload-batch -s scaladoc/output -d $DOC_DEST --account-name scala3docstorage
 
   community-docs:
     env:

bin/scalad → bin/scaladoc

@@ -2,4 +2,4 @@
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/.."
 
-"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scalad" "$@"
+"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scaladoc" "$@"

build.sbt

@@ -1,8 +1,6 @@
 val scala3 = Build.scala3
 val `scala3-bootstrapped` = Build.`scala3-bootstrapped`
 val `scala3-interfaces` = Build.`scala3-interfaces`
-val `scala3-doc` = Build.`scala3-doc`
-val `scala3-doc-bootstrapped` = Build.`scala3-doc-bootstrapped`
 val `scala3-compiler` = Build.`scala3-compiler`
 val `scala3-compiler-bootstrapped` = Build.`scala3-compiler-bootstrapped`
 val `scala3-library` = Build.`scala3-library`
@@ -12,6 +10,7 @@ val `scala3-sbt-bridge` = Build.`scala3-sbt-bridge`
 val `scala3-sbt-bridge-tests` = Build.`scala3-sbt-bridge-tests`
 val `scala3-staging` = Build.`scala3-staging`
 val `scala3-tasty-inspector` = Build.`scala3-tasty-inspector`
+val `scala3-tasty-inspector-nonbootstrapped` = Build.`scala3-tasty-inspector-nonbootstrapped`
 val `scala3-language-server` = Build.`scala3-language-server`
 val `scala3-bench` = Build.`scala3-bench`
 val `scala3-bench-bootstrapped` = Build.`scala3-bench-bootstrapped`
@@ -21,6 +20,7 @@ val `tasty-core` = Build.`tasty-core`
 val `tasty-core-bootstrapped` = Build.`tasty-core-bootstrapped`
 val `tasty-core-scala2` = Build.`tasty-core-scala2`
 val scaladoc = Build.scaladoc
+val `scaladoc-nonBootstrapped` = Build.`scaladoc-nonBootstrapped`
 val `scaladoc-testcases` = Build.`scaladoc-testcases`
 val `scaladoc-js` = Build.`scaladoc-js`
 val `scala3-bench-run` = Build.`scala3-bench-run`

community-build/src/scala/dotty/communitybuild/Main.scala

@@ -19,10 +19,36 @@ object Main:
         e.printStackTrace()
         Nil
 
+  def withProjects[T](names: Seq[String], opName: String)(op: CommunityProject => T): Seq[T] =
+    val missing = names.filterNot(projectMap.contains)
+    if missing.nonEmpty then
+      val allNames = allProjects.map(_.project).mkString(", ")
+      println(s"Missing projects: ${missing.mkString(", ")}. All projects: $allNames")
+      sys.exit(1)
+
+    val (failed, completed) = names.flatMap(projectMap.apply).partitionMap( o =>
+      try
+        Right(op(o))
+      catch case e: Throwable =>
+        e.printStackTrace()
+        Left(o)
+    )
+
+    if failed.nonEmpty then
+      println(s"$opName failed for ${failed.mkString(", ")}")
+      sys.exit(1)
+
+    completed
+
   /** Allows running various commands on community build projects. */
   def main(args: Array[String]): Unit =
     args.toList match
-      case "publish" :: name :: Nil =>
+      case "publish" :: names if names.nonEmpty =>
+        withProjects(names, "Publishing")(_.publish())
+
+      case "build" :: names if names.nonEmpty =>
+        withProjects(names, "Build")(_.build())
+
       case "doc" :: "all" :: destStr :: Nil =>
         val dest = Paths.get(destStr)
         Seq("rm", "-rf", destStr).!
@@ -66,26 +92,23 @@ object Main:
           sys.exit(1)
 
       case "doc" :: names if names.nonEmpty =>
-        val missing = names.filterNot(projectMap.contains)
-        if missing.nonEmpty then
-          println(s"Missing projects: ${missing.mkString(", ")}. All projects: ${allProjects.mkString(", ")}")
-          sys.exit(1)
-
-        val failed = names.filter{ p =>
-          val docsRoots = generateDocs(projectMap(p))
+        val failed = withProjects(names, "Documenting"){ p =>
+          val docsRoots = generateDocs(p)
+          println(docsRoots)
           if docsRoots.nonEmpty then println(s"Docs for $p generated in $docsRoots")
-          docsRoots.isEmpty
-        }
+          if docsRoots.isEmpty then Some(p.project) else None
+        }.flatten
+
         if failed.nonEmpty then
           println(s"Documentation not found for ${failed.mkString(", ")}")
           sys.exit(1)
 
       case args =>
         println("USAGE: <COMMAND> <PROJECT NAME>")
-        println("COMMAND is one of: publish doc")
+        println("COMMAND is one of: publish doc run")
         println("Available projects are:")
         allProjects.foreach { k =>
-          println(s"\t$k")
+          println(s"\t${k.project}")
         }
         sys.exit(1)
 

community-build/src/scala/dotty/communitybuild/projects.scala

@@ -84,16 +84,24 @@ sealed trait CommunityProject:
     if exitCode != 0 then
       throw RuntimeException(s"Doc command exited with code $exitCode for project $project. Project details:\n$this")
 
+  final def build(): Int = exec(projectDir, binaryName, buildCommands: _*)
+
+  final def buildCommands = runCommandsArgs :+ testCommand
+
 end CommunityProject
 
 final case class MillCommunityProject(
     project: String,
     baseCommand: String,
-    dependencies: List[CommunityProject] = Nil) extends CommunityProject:
+    dependencies: List[CommunityProject] = Nil,
+    ignoreDocs: Boolean = false,
+    ) extends CommunityProject:
   override val binaryName: String = "./mill"
   override val testCommand = s"$baseCommand.test"
   override val publishCommand = s"$baseCommand.publishLocal"
   override val docCommand = null
+    // uncomment once mill is released
+    // if ignoreDocs then null else s"$baseCommand.docJar"
   override val runCommandsArgs = List("-i", "-D", s"dottyVersion=$compilerVersion")
 
 final case class SbtCommunityProject(
@@ -160,10 +168,9 @@ final case class SbtCommunityProject(
     """set testOptions in Global += Tests.Argument(TestFramework("munit.Framework"), "+l"); """
     ++ s"$baseCommand$sbtTestCommand"
 
-  override val publishCommand = if sbtPublishCommand eq null then null else
-    val disableDocCommand =
-      if sbtDocCommand eq null then "" else "set every useScaladoc := false;"
-    s"$baseCommand$disableDocCommand$sbtPublishCommand"
+  override val publishCommand =
+    if sbtPublishCommand eq null then null else s"$baseCommand$sbtPublishCommand"
+
   override val docCommand =
     if sbtDocCommand eq null then null else
       val cmd = if sbtDocCommand.startsWith(";") then sbtDocCommand else s";$sbtDocCommand"
@@ -195,11 +202,13 @@ object projects:
   lazy val utest = MillCommunityProject(
     project = "utest",
     baseCommand = s"utest.jvm[$compilerVersion]",
+    ignoreDocs = true
   )
 
   lazy val sourcecode = MillCommunityProject(
     project = "sourcecode",
     baseCommand = s"sourcecode.jvm[$compilerVersion]",
+    ignoreDocs = true
   )
 
   lazy val oslib = MillCommunityProject(
@@ -211,7 +220,8 @@ object projects:
   lazy val oslibWatch = MillCommunityProject(
     project = "os-lib",
     baseCommand = s"os.watch[$compilerVersion]",
-    dependencies = List(utest, sourcecode)
+    dependencies = List(utest, sourcecode),
+    ignoreDocs = true
   )
 
   lazy val ujson = MillCommunityProject(
@@ -241,13 +251,15 @@ object projects:
   lazy val fansi = MillCommunityProject(
     project = "fansi",
     baseCommand = s"fansi.jvm[$compilerVersion]",
-    dependencies = List(utest, sourcecode)
+    dependencies = List(utest, sourcecode),
+    ignoreDocs = true
   )
 
   lazy val pprint = MillCommunityProject(
     project = "PPrint",
     baseCommand = s"pprint.jvm[$compilerVersion]",
-    dependencies = List(fansi)
+    dependencies = List(fansi),
+    ignoreDocs = true
   )
 
   lazy val requests = MillCommunityProject(
@@ -409,7 +421,7 @@ object projects:
     sbtTestCommand  = "testsJVM/test;testsJS/test;",
     // Hardcode the version to avoid having to deal with something set by sbt-dynver
     sbtPublishCommand   = s"""set every version := "${Versions.munit}"; munitJVM/publishLocal; munitJS/publishLocal; munitScalacheckJVM/publishLocal; munitScalacheckJS/publishLocal; junit/publishLocal""",
-    sbtDocCommand   = "munitJVM/doc",
+    sbtDocCommand   = "junit/doc; munitJVM/doc",
     dependencies = List(scalacheck)
   )
 
@@ -627,7 +639,7 @@ object projects:
     sbtPublishCommand = "publishLocal",
     dependencies = List(scalatest)
   )
-  
+
   lazy val perspective = SbtCommunityProject(
     project = "perspective",
     // No library with easy typeclasses to verify data against exist for Dotty, so no tests yet
@@ -702,4 +714,4 @@ def allProjects = List(
   projects.perspective,
 )
 
-lazy val projectMap = allProjects.map(p => p.project -> p).toMap
+lazy val projectMap = allProjects.groupBy(_.project)

community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala

@@ -22,7 +22,7 @@ abstract class CommunityBuildTest:
    */
   extension (self: CommunityProject) def run()(using suite: CommunityBuildTest) =
     self.dependencies.foreach(_.publish())
-    suite.test(self.project, self.binaryName, self.runCommandsArgs :+ self.testCommand)
+    suite.test(self)
 
   /** Build the given project with the published local compiler and sbt plugin.
    *
@@ -34,10 +34,14 @@ abstract class CommunityBuildTest:
    *                    a build tool like SBT or Mill
    *  @param arguments  Arguments to pass to the testing program
    */
-  def test(project: String, command: String, arguments: Seq[String]): Unit = {
+  def test(projectDef: CommunityProject): Unit = {
+    val project = projectDef.project
+    val command = projectDef.binaryName
+    val arguments = projectDef.buildCommands
+
     @annotation.tailrec
-    def execTimes(task: => Int, timesToRerun: Int): Boolean =
-      val exitCode = task
+    def execTimes(task: () => Int, timesToRerun: Int): Boolean =
+      val exitCode = task()
       if exitCode == 0
       then true
       else if timesToRerun == 0
@@ -60,7 +64,7 @@ abstract class CommunityBuildTest:
         |""".stripMargin)
     }
 
-    val testsCompletedSuccessfully = execTimes(exec(projectDir, command, arguments: _*), 3)
+    val testsCompletedSuccessfully = execTimes(projectDef.build, 3)
 
     if (!testsCompletedSuccessfully) {
       fail(s"""