Adding tests in CI

.github/workflows/ci.yml

@@ -235,7 +235,7 @@ jobs:
       - name: Submit Dependencies
         uses: scalacenter/sbt-dependency-submission@v2
         with:
-          modules-ignore: rootjs_2.13 rootjs_3 docs_3 rootjvm_2.13 rootjvm_3 rootnative_2.13 rootnative_3
+          modules-ignore: rootjs_2.13 rootjs_3 docs_3 tests_sjs1_2.13 tests_sjs1_3 rootjvm_2.13 rootjvm_3 rootnative_2.13 rootnative_3 tests_2.13 tests_3 tests_native0.4_2.13 tests_native0.4_3
           configs-ignore: test scala-tool scala-doc-tool test-internal
 
   site:

.mergify.yml

@@ -26,6 +26,14 @@ pull_request_rules:
       add:
       - site
       remove: []
+- name: Label tests PRs
+  conditions:
+  - files~=^tests/
+  actions:
+    label:
+      add:
+      - tests
+      remove: []
 - name: Label toolkit PRs
   conditions:
   - files~=^toolkit/

build.sbt

@@ -1,5 +1,6 @@
 import laika.helium.config._
 import laika.rewrite.nav.{ChoiceConfig, Selections, SelectionConfig}
+import java.io.File
 
 ThisBuild / tlBaseVersion := "0.1"
 ThisBuild / startYear := Some(2023)
@@ -11,7 +12,8 @@ ThisBuild / mergifyStewardConfig ~= {
 
 ThisBuild / crossScalaVersions := Seq("2.13.12", "3.3.1")
 
-lazy val root = tlCrossRootProject.aggregate(toolkit, toolkitTest)
+lazy val root = tlCrossRootProject
+  .aggregate(toolkit, toolkitTest, tests)
 
 lazy val toolkit = crossProject(JVMPlatform, JSPlatform, NativePlatform)
   .in(file("toolkit"))
@@ -44,6 +46,71 @@ lazy val toolkitTest = crossProject(JVMPlatform, JSPlatform, NativePlatform)
     mimaPreviousArtifacts := Set()
   )
 
+lazy val tests = crossProject(JVMPlatform, JSPlatform, NativePlatform)
+  .in(file("tests"))
+  .settings(
+    name := "tests",
+    scalacOptions ++= {
+      if (scalaBinaryVersion.value == "2.13") Seq("-Ytasty-reader") else Nil
+    },
+    libraryDependencies ++= Seq(
+      "org.typelevel" %%% "munit-cats-effect" % "2.0.0-M3" % Test,
+      "co.fs2" %%% "fs2-io" % "3.9.2" % Test,
+      // https://github.com/VirtusLab/scala-cli/issues/2421
+      "org.virtuslab.scala-cli" %% "cli" % "1.0.4" cross (CrossVersion.for2_13Use3) excludeAll (
+        ExclusionRule("com.lihaoyi:geny_2.13"),
+        ExclusionRule(
+          "org.scala-lang.modules",
+          "scala-collection-compat_2.13"
+        ),
+        ExclusionRule(
+          "com.github.plokhotnyuk.jsoniter-scala",
+          "jsoniter-scala-core_2.13"
+        ),
+        ExclusionRule("com.lihaoyi", "sourcecode_2.13"),
+        ExclusionRule("ai.kien", "python-native-libs_2.13"),
+        ExclusionRule("com.lihaoyi", "os-lib_2.13")
+      )
+    ),
+    buildInfoKeys += BuildInfoKey.map(Compile / dependencyClasspath) {
+      case (_, v) =>
+        "classPath" -> v.seq
+          .map(_.data.getAbsolutePath)
+          .mkString(File.pathSeparator)
+    },
+    buildInfoKeys += BuildInfoKey.action("javaHome") {
+      val path = sys.env.get("JAVA_HOME").orElse(sys.props.get("java.home")).get
+      if (path.endsWith("/jre")) {
+        // handle JDK 8 installations
+        path.replace("/jre", "")
+      } else path
+    },
+    buildInfoKeys += "scala3" -> (scalaVersion.value.head == '3')
+  )
+  .jvmSettings(
+    Test / test := (Test / test)
+      .dependsOn(toolkit.jvm / publishLocal, toolkitTest.jvm / publishLocal)
+      .value,
+    buildInfoKeys += "platform" -> "jvm"
+  )
+  .jsSettings(
+    Test / test := (Test / test)
+      .dependsOn(toolkit.js / publishLocal, toolkitTest.js / publishLocal)
+      .value,
+    buildInfoKeys += "platform" -> "js",
+    scalaJSLinkerConfig ~= { _.withModuleKind(ModuleKind.CommonJSModule) }
+  )
+  .nativeSettings(
+    Test / test := (Test / test)
+      .dependsOn(
+        toolkit.native / publishLocal,
+        toolkitTest.native / publishLocal
+      )
+      .value,
+    buildInfoKeys += "platform" -> "native"
+  )
+  .enablePlugins(BuildInfoPlugin, NoPublishPlugin)
+
 lazy val docs = project
   .in(file("site"))
   .enablePlugins(TypelevelSitePlugin)

project/plugins.sbt

@@ -4,3 +4,4 @@ addSbtPlugin("org.typelevel" % "sbt-typelevel-mergify" % sbtTlVersion)
 addSbtPlugin("org.typelevel" % "sbt-typelevel-site" % sbtTlVersion)
 addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.13.2")
 addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.15")
+addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0")

tests/shared/src/test/scala/org/typelevel/toolkit/ScalaCliProcess.scala

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2023 Typelevel
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.typelevel.toolkit
+
+import cats.effect.kernel.Resource
+import cats.effect.std.Console
+import cats.effect.IO
+import cats.syntax.parallel._
+import buildinfo.BuildInfo
+import fs2.Stream
+import fs2.io.file.Files
+import fs2.io.process.ProcessBuilder
+import munit.Assertions.fail
+
+object ScalaCliProcess {
+
+  private val ClassPath: String = BuildInfo.classPath
+  private val JavaHome: String = BuildInfo.javaHome
+
+  private def scalaCli(args: List[String]): IO[Unit] = ProcessBuilder(
+    s"$JavaHome/bin/java",
+    args.prependedAll(List("-cp", ClassPath, "scala.cli.ScalaCli"))
+  ).spawn[IO]
+    .use(process =>
+      (
+        process.exitValue,
+        process.stdout.through(fs2.text.utf8.decode).compile.string,
+        process.stderr.through(fs2.text.utf8.decode).compile.string
+      ).parFlatMapN {
+        case (0, _, _) => IO.unit
+        case (exitCode, stdout, stdErr) =>
+          IO.println(stdout) >> Console[IO].errorln(stdErr) >> IO.delay(
+            fail(s"Non zero exit code ($exitCode) for ${args.mkString(" ")}")
+          )
+      }
+    )
+
+  private def writeToFile(
+      scriptBody: String
+  )(isTest: Boolean): Resource[IO, String] =
+    Files[IO]
+      .tempFile(
+        None,
+        "",
+        if (isTest) "-toolkit.test.scala" else "-toolkit.scala",
+        None
+      )
+      .evalTap { path =>
+        val header = List(
+          s"//> using scala ${BuildInfo.scalaVersion}",
+          s"//> using toolkit typelevel:${BuildInfo.version}",
+          s"//> using platform ${BuildInfo.platform}"
+        ).mkString("", "\n", "\n")
+        Stream(header, scriptBody.stripMargin)
+          .through(Files[IO].writeUtf8(path))
+          .compile
+          .drain
+      }
+      .map(_.toString)
+
+  def command(args: List[String]): IO[Unit] = scalaCli(args)
+
+  def run(body: String): IO[Unit] =
+    writeToFile(body)(false).use(f =>
+      scalaCli("run" :: "--native-version" :: "0.4.15" :: f :: Nil)
+    )
+
+  def test(body: String): IO[Unit] =
+    writeToFile(body)(true).use(f =>
+      scalaCli("test" :: "--native-version" :: "0.4.15" :: f :: Nil)
+    )
+
+}

tests/shared/src/test/scala/org/typelevel/toolkit/ToolkitCompilationTest.scala

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2023 Typelevel
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.typelevel.toolkit
+
+import munit.{CatsEffectSuite, TestOptions}
+import buildinfo.BuildInfo.scala3
+import scala.concurrent.duration._
+
+class ToolkitCompilationTest extends CatsEffectSuite {
+
+  // 2 minutes may seem a lot, but consider that the first test for
+  // each (scalaVersion, platform) will have to download the compiler
+  // (if it's not the default), compile (that for native takes awhile)
+  // and then finally run the code.
+  override def munitIOTimeout: Duration = 2.minute
+
+  testRun("Toolkit should run a simple Hello Cats Effect") {
+    if (scala3)
+      """|import cats.effect.*
+         |
+         |object Hello extends IOApp.Simple:
+         |  def run = IO.println("Hello toolkit!")"""
+    else
+      """|import cats.effect._
+         |
+         |object Hello extends IOApp.Simple {
+         |  def run = IO.println("Hello toolkit!")
+         |}"""
+  }
+
+  testRun("Toolkit should run a script with every dependency") {
+    if (scala3)
+      """|import cats.syntax.all.*
+         |import cats.effect.*
+         |import com.monovore.decline.effect.*
+         |import fs2.data.csv.generic.semiauto.*
+         |import fs2.io.file.*
+         |import org.http4s.ember.client.*
+         |
+         |object Hello extends IOApp.Simple:
+         |  def run = IO.println("Hello toolkit!")"""
+    else
+      """|import cats.syntax.all._
+         |import cats.effect._
+         |import com.monovore.decline.effect._
+         |import fs2.data.csv.generic.semiauto._
+         |import fs2.io.file._
+         |import io.circe._
+         |import org.http4s.ember.client._
+         |
+         |object Hello extends IOApp.Simple {
+         |  def run = IO.println("Hello toolkit!")
+         |}"""
+  }
+
+  testTest("Toolkit should execute a simple munit suite") {
+    if (scala3)
+      """|import cats.effect.*
+         |import munit.*
+         |
+         |class Test extends CatsEffectSuite:
+         |  test("test")(IO.unit)"""
+    else
+      """|import cats.effect._
+         |import munit._
+         |
+         |class Test extends CatsEffectSuite {
+         |  test("test")(IO.unit)
+         |}"""
+  }
+
+  def testRun(testName: TestOptions)(scriptBody: String): Unit =
+    test(testName)(ScalaCliProcess.run(scriptBody))
+
+  def testTest(testName: TestOptions)(scriptBody: String): Unit =
+    test(testName)(ScalaCliProcess.test(scriptBody))
+
+}