[MMF-2861] Embed Node.js runtime in analyzer

.cirrus.yml

@@ -20,9 +20,6 @@
   CIRRUS_SHELL: bash
 
 container_definition: &CONTAINER_DEFINITION
-  dockerfile: .cirrus/nodejs.Dockerfile
-  docker_arguments:
-    CIRRUS_AWS_ACCOUNT: ${CIRRUS_AWS_ACCOUNT}
   cluster_name: ${CIRRUS_CLUSTER_NAME}
   builder_role: cirrus-builder
   builder_image: docker-builder-v*
@@ -37,13 +34,18 @@
     folder: ${CIRRUS_WORKING_DIR}/.m2/repository
     fingerprint_script: cat **/pom.xml
 
+nodejs_runtimes_cache_definition: &RUNTIME_CACHE
+  runtime_cache:
+    folder: tools/fetch-node/downloads/
+    fingerprint_script: cat tools/fetch-node/node-distros.mjs
+
 win_vm_definition: &WINDOWS_VM_DEFINITION
   ec2_instance:
     experimental: true # see https://github.com/cirruslabs/cirrus-ci-docs/issues/1051
     image: base-windows-jdk17-v*
     platform: windows
     region: eu-central-1
-    type: c6id.2xlarge
+    type: c6i.4xlarge
     subnet_id: ${CIRRUS_AWS_SUBNET}
     preemptible: false
     use_ssd: true
@@ -63,17 +65,18 @@
     CIRRUS_CLONE_DEPTH: 10
     SONARSOURCE_QA: true
   <<: *MAVEN_CACHE
-  node_version_script:
-    - node -v
   qa_script:
     - source cirrus-env QA
     - source set_maven_build_version $BUILD_NUMBER
-    - mvn -f its/plugin/pom.xml -Dsonar.runtimeVersion=${SQ_VERSION} -B -e -V verify surefire-report:report
+    - mvn -f its/plugin/pom.xml -Dsonar.runtimeVersion=${SQ_VERSION} ${MVN_TEST} -B -e -V verify surefire-report:report
   cleanup_before_cache_script: cleanup_maven_repository
 
 build_task:
   eks_container:
     <<: *CONTAINER_DEFINITION
+    dockerfile: .cirrus/nodejs.Dockerfile
+    docker_arguments:
+      CIRRUS_AWS_ACCOUNT: ${CIRRUS_AWS_ACCOUNT}
     cpu: 15
     memory: 30G
   env:
@@ -86,6 +89,7 @@
     SIGN_KEY: VAULT[development/kv/data/sign data.key]
     PGP_PASSPHRASE: VAULT[development/kv/data/sign data.passphrase]
   <<: *MAVEN_CACHE
+  <<: *RUNTIME_CACHE
   sonar_cache:
     folder: ${HOME}/.sonar/cache
     fingerprint_script:
@@ -104,9 +108,12 @@
   <<: *WINDOWS_VM_DEFINITION
   <<: *ONLY_SONARSOURCE_QA
   <<: *MAVEN_CACHE
+  <<: *RUNTIME_CACHE
   build_script:
     - source cirrus-env CI
     - npm config set registry "${ARTIFACTORY_URL}/api/npm/npm"
+    #  TODO should be as close to build_task as possible, ie. remove incompatible bash scripts
+    - npm run build && npm run test
     - mvn test
   cleanup_before_cache_script: cleanup_maven_repository
 
@@ -115,6 +122,9 @@
     - build
   eks_container:
     <<: *CONTAINER_DEFINITION
+    dockerfile: .cirrus/nodejs.Dockerfile
+    docker_arguments:
+      CIRRUS_AWS_ACCOUNT: ${CIRRUS_AWS_ACCOUNT}
     cpu: 4
     memory: 8G
   # run only on master and long-term branches
@@ -133,25 +143,31 @@
     ws_artifacts:
       path: 'whitesource/**/*'
 
-plugin_qa_task:
+plugin_qa_with_node_task:
   <<: *PLUGIN_QA_BODY
   eks_container:
     dockerfile: .cirrus/nodejs.jdk17.Dockerfile
-    docker_arguments:
-      matrix:
-        - NODE_VERSION: 14
-        - NODE_VERSION: 16
-        - NODE_VERSION: 18
-        - NODE_VERSION: 20
   env:
     SQ_VERSION: LATEST_RELEASE
+    MVN_TEST: ''
+
+plugin_qa_no_node_task:
+  <<: *PLUGIN_QA_BODY
+  eks_container:
+    image: ${CIRRUS_AWS_ACCOUNT}.dkr.ecr.eu-central-1.amazonaws.com/base:j17-latest
+  env:
+    SQ_VERSION: LATEST_RELEASE
+    SONARJS_ARTIFACT: multi
+    MVN_TEST: '-Dtest=!EslintCustomRulesTest,!SonarJsIntegrationTest --projects !org.sonarsource.javascript:eslint-custom-rules-plugin'
 
 plugin_qa_sq_dev_task:
   <<: *PLUGIN_QA_BODY
   eks_container:
-    dockerfile: .cirrus/nodejs.jdk17.Dockerfile
+    image: ${CIRRUS_AWS_ACCOUNT}.dkr.ecr.eu-central-1.amazonaws.com/base:j17-latest
   env:
     SQ_VERSION: DEV
+    SONARJS_ARTIFACT: multi
+    MVN_TEST: '-Dtest=!EslintCustomRulesTest,!SonarJsIntegrationTest --projects !org.sonarsource.javascript:eslint-custom-rules-plugin'
 
 # Plugin QA for Windows is splint into 2 parts to make it faster
 plugin_qa_win_task:
@@ -180,7 +196,7 @@
   <<: *ONLY_SONARSOURCE_QA
   eks_container:
     <<: *CONTAINER_DEFINITION
-    dockerfile: .cirrus/nodejs.jdk17.Dockerfile
+    image: ${CIRRUS_AWS_ACCOUNT}.dkr.ecr.eu-central-1.amazonaws.com/base:j17-latest
     cpu: 15
     memory: 24G
   env:
@@ -203,19 +219,20 @@
     diff_artifacts:
       path: '**/target/actual/**/*'
 
 promote_task:
   depends_on:
     - ws_scan
     - build_win
-    - plugin_qa
+    - plugin_qa_with_node
+    - plugin_qa_no_node
     - plugin_qa_sq_dev
     - plugin_qa_win
     - ruling
   <<: *ONLY_SONARSOURCE_QA
   eks_container:
     <<: *CONTAINER_DEFINITION
-    cpu: 1
-    memory: 1G
+    cpu: 2
+    memory: 4G
   env:
     #promotion cloud function
     GCF_ACCESS_TOKEN: VAULT[development/kv/data/promote data.token]

.prettierignore

@@ -30,3 +30,5 @@ sonar-plugin/sonar-javascript-plugin/src/test/resources
 
 typedoc/site
 typedoc/models
+
+tools/fetch-node/downloads

its/plugin/sonarlint-tests/pom.xml

@@ -88,6 +88,7 @@
                     <artifactItem>
                       <groupId>${project.groupId}</groupId>
                       <artifactId>sonar-javascript-plugin</artifactId>
+                      <classifier>multi</classifier>
                       <type>sonar-plugin</type>
                       <overWrite>true</overWrite>
                     </artifactItem>

its/plugin/sonarlint-tests/src/test/java/SonarLintTest.java

@@ -103,14 +103,20 @@ void should_raise_issues() throws IOException {
         tuple("javascript:S3504", 4, filePath, "CRITICAL")
       );
 
-    assertThat(
-      logs
-        .stream()
-        .anyMatch(s ->
-          s.matches("Using Node\\.js executable .* from property sonar\\.nodejs\\.executable\\.")
-        )
-    )
-      .isTrue();
+    if (!usingEmbeddedNode()) {
+      assertThat(
+        logs
+          .stream()
+          .anyMatch(s ->
+            s.matches("Using Node\\.js executable .* from property sonar\\.nodejs\\.executable\\.")
+          )
+      )
+        .isTrue();
+    }
+  }
+
+  private static boolean usingEmbeddedNode() {
+    return TestUtils.JAVASCRIPT_PLUGIN_LOCATION.toString().contains("multi");
   }
 
   @Test

its/plugin/sonarlint-tests/src/test/java/TestUtils.java

@@ -30,28 +30,35 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.util.regex.Pattern;
 import org.sonarsource.sonarlint.core.analysis.api.ClientInputFile;
 
 public class TestUtils {
 
-  static final Path JAVASCRIPT_PLUGIN_LOCATION;
-
-  static {
-    var start = homeDir().resolve("../sonar-plugin/sonar-javascript-plugin/target");
-    try (var walk = Files.walk(start)) {
-      JAVASCRIPT_PLUGIN_LOCATION =
-        walk
-          .filter(p -> {
-            var filename = p.getFileName().toString();
-            return filename.startsWith("sonar-javascript-plugin-") && filename.endsWith(".jar");
-          })
-          .findAny()
-          .orElseThrow();
+  static final Path JAVASCRIPT_PLUGIN_LOCATION = artifact();
+
+  /**
+   * This is used to test artifact with and without embedded runtime during plugin QA integration tests
+   *
+   */
+  private static Path artifact() {
+    var target = homeDir().resolve("../sonar-plugin/sonar-javascript-plugin/target");
+    try (var stream = Files.walk(target, 1)) {
+      return stream
+        .filter(p -> pluginFilenameMatcher().matcher(p.getFileName().toString()).matches())
+        .findAny()
+        .orElseThrow();
     } catch (IOException e) {
-      throw new IllegalStateException(e);
+      throw new UncheckedIOException(e);
     }
   }
 
+  private static Pattern pluginFilenameMatcher() {
+    return "multi".equals(System.getenv("SONARJS_ARTIFACT"))
+      ? Pattern.compile("sonar-javascript-plugin-.*-multi\\.jar")
+      : Pattern.compile("sonar-javascript-plugin-[0-9.]*(?:-SNAPSHOT)?\\.jar");
+  }
+
   public static Path homeDir() {
     return Path.of("../../");
   }

its/plugin/tests/pom.xml

@@ -109,6 +109,27 @@
                       <type>sonar-plugin</type>
                       <overWrite>true</overWrite>
                     </artifactItem>
+                    <artifactItem>
+                      <groupId>${project.groupId}</groupId>
+                      <artifactId>sonar-javascript-plugin</artifactId>
+                      <classifier>win-x64</classifier>
+                      <type>sonar-plugin</type>
+                      <overWrite>true</overWrite>
+                    </artifactItem>
+                    <artifactItem>
+                      <groupId>${project.groupId}</groupId>
+                      <artifactId>sonar-javascript-plugin</artifactId>
+                      <classifier>linux-x64</classifier>
+                      <type>sonar-plugin</type>
+                      <overWrite>true</overWrite>
+                    </artifactItem>
+                    <artifactItem>
+                      <groupId>${project.groupId}</groupId>
+                      <artifactId>sonar-javascript-plugin</artifactId>
+                      <classifier>multi</classifier>
+                      <type>sonar-plugin</type>
+                      <overWrite>true</overWrite>
+                    </artifactItem>
                   </artifactItems>
                   <outputDirectory>../../../sonar-plugin/sonar-javascript-plugin/target</outputDirectory>
                   <overWriteReleases>true</overWriteReleases>

its/plugin/tests/src/test/java/com/sonar/javascript/it/plugin/EmbeddedNodeTest.java

@@ -0,0 +1,84 @@
+/*
+ * SonarQube JavaScript Plugin
+ * Copyright (C) 2012-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package com.sonar.javascript.it.plugin;
+
+import static com.sonar.javascript.it.plugin.OrchestratorStarter.getSonarScanner;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.sonar.orchestrator.junit5.OrchestratorExtension;
+import com.sonar.orchestrator.locator.FileLocation;
+import java.io.File;
+import java.util.Locale;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.parallel.Isolated;
+
+@Isolated
+class EmbeddedNodeTest {
+
+  @Test
+  void embedded_node() {
+    var plugin = FileLocation.byWildcardMavenFilename(
+      new File("../../../sonar-plugin/sonar-javascript-plugin/target"),
+      "sonar-javascript-plugin-*" + classifier() + ".jar"
+    );
+
+    var orchestrator = OrchestratorExtension
+      .builderEnv()
+      .useDefaultAdminCredentialsForBuilds(true)
+      .setSonarVersion(System.getProperty("sonar.runtimeVersion", "LATEST_RELEASE"))
+      .restoreProfileAtStartup(FileLocation.ofClasspath("/eslint-based-rules.xml"))
+      .addPlugin(plugin)
+      .build();
+
+    synchronized (OrchestratorStarter.class) {
+      orchestrator.start();
+    }
+
+    var projectKey = "eslint_based_rules";
+    var server = orchestrator.getServer();
+    server.provisionProject(projectKey, projectKey);
+    server.associateProjectToQualityProfile(projectKey, "js", "eslint-based-rules-profile");
+
+    var projectDir = TestUtils.projectDir(projectKey);
+    var build = getSonarScanner()
+      .setProjectKey(projectKey)
+      .setSourceEncoding("UTF-8")
+      .setSourceDirs(".")
+      .setProjectDir(projectDir);
+
+    var buildResult = orchestrator.executeBuild(build);
+    assertThat(buildResult.isSuccess()).isTrue();
+    assertThat(buildResult.getLogs()).contains("INFO: Using embedded Node.js runtime");
+    assertThat(buildResult.getLogsLines(l -> l.startsWith("ERROR"))).isEmpty();
+    orchestrator.stop();
+  }
+
+  private static String classifier() {
+    var os = System.getProperty("os.name").toLowerCase(Locale.ROOT);
+    var arch = System.getProperty("os.arch");
+    if (os.contains("linux") && arch.contains("64")) {
+      return "-linux-x64";
+    } else if (os.contains("windows")) {
+      return "-win-x64";
+    } else {
+      return "-multi";
+    }
+  }
+}

its/plugin/tests/src/test/java/com/sonar/javascript/it/plugin/EslintBasedRulesTest.java

@@ -164,6 +164,7 @@ void should_not_use_node_in_cwd() throws Exception {
       .setSourceEncoding("UTF-8")
       .setSourceDirs(".")
       .setDebugLogs(true)
+      .setProperty("sonar.nodejs.forceHost", "true")
       .setProjectDir(projectDir);
 
     // copy ping.exe to node.exe and place it in the project directory