CI: Tweaks for backend tests

build/build/src/engine/context.rs

@@ -394,15 +394,23 @@ impl RunContext {
 
         // === Unit tests and Enso tests ===
         debug!("Running unit tests and Enso tests.");
-        if self.config.test_scala {
+        // We store Scala test result but not immediately fail on it, as we want to run all the
+        // tests (including standard library ones) even if Scala tests fail.
+        let scala_test_result = if self.config.test_scala {
             // Make sure that `sbt buildEngineDistributionNoIndex` is run before
             // `project-manager/test`. Note that we do not have to run
             // `buildEngineDistribution` (with indexing), because it is unnecessary.
             sbt.call_arg("buildEngineDistributionNoIndex").await?;
 
             // Run unit tests
-            sbt.call_arg("set Global / parallelExecution := false; test").await?;
-        }
+            sbt.call_arg("set Global / parallelExecution := false; test").await.inspect_err(|e| {
+                ide_ci::actions::workflow::error(format!("Scala Tests failed: {e:?}"))
+            })
+        } else {
+            // No tests - no fail.
+            Ok(())
+        };
+
         if self.config.test_standard_library {
             enso.run_tests(IrCaches::No, &sbt, PARALLEL_ENSO_TESTS).await?;
         }
@@ -540,13 +548,6 @@ impl RunContext {
         // Verify License Packages in Distributions
         // FIXME apparently this does not work on Windows due to some CRLF issues?
         if self.config.verify_packages && TARGET_OS != OS::Windows {
-            /*  refversion=${{ env.ENSO_VERSION }}
-                binversion=${{ env.DIST_VERSION }}
-                engineversion=$(${{ env.ENGINE_DIST_DIR }}/bin/enso --version --json | jq -r '.version')
-                test $binversion = $refversion || (echo "Tag version $refversion and the launcher version $binversion do not match" && false)
-                test $engineversion = $refversion || (echo "Tag version $refversion and the engine version $engineversion do not match" && false)
-            */
-
             for package in ret.packages() {
                 package.verify_package_sbt(&sbt).await?;
             }
@@ -588,6 +589,8 @@ impl RunContext {
             bundle.create(&self.repo_root, &graal_version).await?;
         }
 
+        scala_test_result?;
+
         Ok(ret)
     }
 

build/build/src/enso.rs

@@ -80,13 +80,12 @@ impl BuiltEnso {
             .await
     }
 
-    pub fn run_test(&self, test: impl AsRef<Path>, ir_caches: IrCaches) -> Result<Command> {
-        let test_path = self.paths.repo_root.test.join(test);
+    pub fn run_test(&self, test_path: impl AsRef<Path>, ir_caches: IrCaches) -> Result<Command> {
         let mut command = self.cmd()?;
         command
             .arg(ir_caches)
             .arg("--run")
-            .arg(test_path)
+            .arg(test_path.as_ref())
             // This flag enables assertions in the JVM. Some of our stdlib tests had in the past
             // failed on Graal/Truffle assertions, so we want to have them triggered.
             .set_env(JAVA_OPTS, &ide_ci::programs::java::Option::EnableAssertions.as_ref())?;
@@ -157,8 +156,9 @@ impl BuiltEnso {
             _ => None,
         };
 
-        let futures = crate::paths::LIBRARIES_TO_TEST.map(ToString::to_string).map(|test| {
-            let command = self.run_test(test, ir_caches);
+        let std_tests = crate::paths::discover_standard_library_tests(&paths.repo_root)?;
+        let futures = std_tests.into_iter().map(|test_path| {
+            let command = self.run_test(test_path, ir_caches);
             async move { command?.run_ok().await }
         });
 

build/build/src/paths.rs

@@ -42,17 +42,16 @@ ide_ci::define_env_var! {
 
 pub const EDITION_FILE_ARTIFACT_NAME: &str = "Edition File";
 
-pub const LIBRARIES_TO_TEST: [&str; 7] = [
-    "Examples_Tests",
-    "Geo_Tests",
-    "Image_Tests",
-    // Temporarily disabled due to https://www.pivotaltracker.com/story/show/184042416
-    // "Meta_Test_Suite_Tests",
-    "Table_Tests",
-    "Base_Tests",
-    "AWS_Tests",
-    "Visualization_Tests",
-];
+/// Get paths of directories containing standard library test project.
+pub fn discover_standard_library_tests(repo_root: &generated::RepoRoot) -> Result<Vec<PathBuf>> {
+    // The glob pattern will discover paths like "H:\NBO\enso\test\AWS_Tests\package.yaml".
+    let glob_pattern = "**/*_Tests/package.yaml";
+    let glob_pattern = repo_root.test.join(glob_pattern);
+    glob::glob(glob_pattern.as_str())?
+        // Package manifest path -> Parent directory.
+        .map(|package_path_result| Result::Ok(package_path_result?.try_parent()?.to_path_buf()))
+        .try_collect_vec()
+}
 
 pub fn new_repo_root(repo_root: impl Into<PathBuf>, triple: &TargetTriple) -> generated::RepoRoot {
     generated::RepoRoot::new_root(

build/ci_utils/src/actions/workflow.rs

@@ -109,6 +109,10 @@ pub fn message(level: MessageLevel, text: impl AsRef<str>) {
     Message { level, text: text.as_ref().into() }.send()
 }
 
+pub fn error(text: impl AsRef<str>) {
+    message(MessageLevel::Error, text)
+}
+
 pub fn warn(text: impl AsRef<str>) {
     message(MessageLevel::Warning, text)
 }