Add npm ci support (#413)

Author: barbelity

Diff for: build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java

@@ -47,6 +47,14 @@ public String install(File workingDirectory, List<String> extraArgs, Log logger)
         }
     }
 
+    public String ci(File workingDirectory, List<String> extraArgs, Log logger) throws IOException {
+        try {
+            return runCommand(workingDirectory, new String[]{"ci"}, extraArgs, logger);
+        } catch (IOException | InterruptedException e) {
+            throw new IOException("npm ci failed: " + e.getMessage(), e);
+        }
+    }
+
     public String pack(File workingDirectory, List<String> extraArgs) throws IOException {
         try {
             return runCommand(workingDirectory, new String[]{"pack"}, extraArgs);

Diff for: build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java

@@ -60,14 +60,17 @@ public class NpmBuildInfoExtractor implements BuildInfoExtractor<NpmProject> {
     @Override
     public Build extract(NpmProject npmProject) throws Exception {
         String resolutionRepository = npmProject.getResolutionRepository();
-        List<String> installationArgs = npmProject.getInstallationArgs();
+        List<String> commandArgs = npmProject.getCommandArgs();
         Path workingDir = npmProject.getWorkingDir();
 
         preparePrerequisites(resolutionRepository, workingDir);
-        createTempNpmrc(workingDir, installationArgs);
-        runInstall(workingDir, installationArgs);
+        createTempNpmrc(workingDir, commandArgs);
+        if (npmProject.isCiCommand()) {
+            runCi(workingDir, commandArgs);
+        } else {
+            runInstall(workingDir, commandArgs);
+        }
         restoreNpmrc(workingDir);
-
         List<Dependency> dependencies = collectDependencies(workingDir);
         String moduleId = StringUtils.isNotBlank(module) ? module : npmPackageInfo.toString();
         return createBuild(dependencies, moduleId);
@@ -128,10 +131,10 @@ private void backupProjectNpmrc(Path workingDir) throws IOException {
         Files.copy(npmrcPath, npmrcBackupPath, StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING);
     }
 
-    private void createTempNpmrc(Path workingDir, List<String> installationArgs) throws IOException, InterruptedException {
+    private void createTempNpmrc(Path workingDir, List<String> commandArgs) throws IOException, InterruptedException {
         Path npmrcPath = workingDir.resolve(NPMRC_FILE_NAME);
         Files.deleteIfExists(npmrcPath); // Delete old npmrc file
-        final String configList = npmDriver.configList(workingDir.toFile(), installationArgs);
+        final String configList = npmDriver.configList(workingDir.toFile(), commandArgs);
 
         Properties npmrcProperties = new Properties();
 
@@ -140,8 +143,8 @@ private void createTempNpmrc(Path workingDir, List<String> installationArgs) thr
         JsonNode manifestTree = mapper.readTree(configList);
         manifestTree.fields().forEachRemaining(entry -> npmrcProperties.setProperty(entry.getKey(), entry.getValue().asText()));
         // Since we run the get config cmd with "--json" flag, we don't want to force the json output on the new npmrc we write.
-        // We will get json output only if it was explicitly required in the installation arguments.
-        npmrcProperties.setProperty("json", String.valueOf(isJsonOutputRequired(installationArgs)));
+        // We will get json output only if it was explicitly required in the command arguments.
+        npmrcProperties.setProperty("json", String.valueOf(isJsonOutputRequired(commandArgs)));
 
         // Save npm auth
         npmrcProperties.putAll(npmAuth);
@@ -172,12 +175,12 @@ private void createTempNpmrc(Path workingDir, List<String> installationArgs) thr
      * 2. --arg=value (true/false)
      * 3. --arg value (true/false)
      */
-    static boolean isJsonOutputRequired(List<String> installationArgs) {
-        int jsonIndex = installationArgs.indexOf("--json");
+    static boolean isJsonOutputRequired(List<String> commandArgs) {
+        int jsonIndex = commandArgs.indexOf("--json");
         if (jsonIndex > -1) {
-            return jsonIndex == installationArgs.size() - 1 || !installationArgs.get(jsonIndex + 1).equals("false");
+            return jsonIndex == commandArgs.size() - 1 || !commandArgs.get(jsonIndex + 1).equals("false");
         }
-        return installationArgs.contains("--json=true");
+        return commandArgs.contains("--json=true");
     }
 
     /**
@@ -198,6 +201,10 @@ private void runInstall(Path workingDir, List<String> installationArgs) throws I
         logger.info(npmDriver.install(workingDir.toFile(), installationArgs, logger));
     }
 
+    private void runCi(Path workingDir, List<String> installationArgs) throws IOException {
+        logger.info(npmDriver.ci(workingDir.toFile(), installationArgs, logger));
+    }
+
     private void restoreNpmrc(Path workingDir) throws IOException {
         Path npmrcPath = workingDir.resolve(NPMRC_FILE_NAME);
         Path npmrcBackupPath = workingDir.resolve(NPMRC_BACKUP_FILE_NAME);

Diff for: build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmCommand.java

@@ -16,7 +16,7 @@
 import java.util.Map;
 
 /**
- * Base class for npm install and npm publish commands.
+ * Base class for npm commands.
  *
  * @author Yahav Itzhak
  */

Diff for: build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmInstall.java renamed to build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmInstallCi.java

@@ -22,25 +22,27 @@
  * @author Yahav Itzhak
  */
 @SuppressWarnings({"unused", "WeakerAccess"})
-public class NpmInstall extends NpmCommand {
+public class NpmInstallCi extends NpmCommand {
 
     NpmBuildInfoExtractor buildInfoExtractor;
-    List<String> installArgs;
+    List<String> commandArgs;
+    boolean isCiCommand;
 
     /**
-     * Install npm package.
+     * Run npm install or npm ci commands.
      *
      * @param clientBuilder        - Build Info client builder.
      * @param resolutionRepository - The repository it'll resolve from.
-     * @param installArgs          - Npm install args.
+     * @param commandArgs          - Npm command args.
      * @param logger               - The logger.
      * @param path                 - Path to directory contains package.json or path to '.tgz' file.
      * @param env                  - Environment variables to use during npm execution.
      */
-    public NpmInstall(ArtifactoryDependenciesClientBuilder clientBuilder, String resolutionRepository, String installArgs, Log logger, Path path, Map<String, String> env, String module) {
+    public NpmInstallCi(ArtifactoryDependenciesClientBuilder clientBuilder, String resolutionRepository, String commandArgs, Log logger, Path path, Map<String, String> env, String module, boolean isCiCommand) {
         super(clientBuilder, resolutionRepository, logger, path, env);
         buildInfoExtractor = new NpmBuildInfoExtractor(clientBuilder, npmDriver, logger, module);
-        this.installArgs = StringUtils.isBlank(installArgs) ? new ArrayList<>() : Arrays.asList(installArgs.trim().split("\\s+"));
+        this.commandArgs = StringUtils.isBlank(commandArgs) ? new ArrayList<>() : Arrays.asList(commandArgs.trim().split("\\s+"));
+        this.isCiCommand = isCiCommand;
     }
 
     @Override
@@ -52,7 +54,7 @@ public Build execute() {
             validateNpmVersion();
             validateRepoExists(client, repo, "Source repo must be specified");
 
-            NpmProject npmProject = new NpmProject(installArgs, repo, workingDir);
+            NpmProject npmProject = new NpmProject(commandArgs, repo, workingDir, isCiCommand);
             return buildInfoExtractor.extract(npmProject);
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
@@ -69,13 +71,14 @@ public static void main(String[] ignored) {
             ArtifactoryClientConfiguration clientConfiguration = createArtifactoryClientConfiguration();
             ArtifactoryDependenciesClientBuilder clientBuilder = new ArtifactoryDependenciesClientBuilder().setClientConfiguration(clientConfiguration, clientConfiguration.resolver);
             ArtifactoryClientConfiguration.PackageManagerHandler npmHandler = clientConfiguration.packageManagerHandler;
-            NpmInstall npmInstall = new NpmInstall(clientBuilder,
+            NpmInstallCi npmInstall = new NpmInstallCi(clientBuilder,
                     clientConfiguration.resolver.getRepoKey(),
                     npmHandler.getArgs(),
                     clientConfiguration.getLog(),
                     Paths.get(npmHandler.getPath() != null ? npmHandler.getPath() : "."),
                     clientConfiguration.getAllProperties(),
-                    npmHandler.getModule());
+                    npmHandler.getModule(),
+                    clientConfiguration.npmHandler.isCiCommand());
             npmInstall.executeAndSaveBuildInfo(clientConfiguration);
         } catch (RuntimeException e) {
             ExceptionUtils.printRootCauseStackTrace(e, System.out);

Diff for: build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/types/NpmProject.java

@@ -11,14 +11,16 @@
  */
 public class NpmProject {
 
-    private List<String> installationArgs;
+    private List<String> commandArgs;
     private String resolutionRepository;
     private Path workingDir;
+    private boolean ciCommand;
 
-    public NpmProject(List<String> installationArgs, String resolutionRepository, Path workingDir) {
-        this.installationArgs = installationArgs;
+    public NpmProject(List<String> commandArgs, String resolutionRepository, Path workingDir, boolean ciCommand) {
+        this.commandArgs = commandArgs;
         this.resolutionRepository = resolutionRepository;
         this.workingDir = workingDir;
+        this.ciCommand = ciCommand;
     }
 
     public String getResolutionRepository() {
@@ -29,7 +31,11 @@ public Path getWorkingDir() {
         return workingDir;
     }
 
-    public List<String> getInstallationArgs() {
-        return installationArgs;
+    public List<String> getCommandArgs() {
+        return commandArgs;
+    }
+
+    public boolean isCiCommand() {
+        return ciCommand;
     }
 }

Diff for: build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmExtractorTest.java

@@ -151,16 +151,50 @@ private Object[][] npmInstallProvider() {
     @SuppressWarnings("unused")
     @Test(dataProvider = "npmInstallProvider")
     public void npmInstallTest(Project project, Set<String> expectedDependencies, String args, boolean packageJsonPath) {
+        runNpmTest(project, expectedDependencies, args, packageJsonPath, false);
+    }
+
+    @DataProvider
+    private Object[][] npmCiProvider() {
+        return new Object[][]{
+                {Project.A, Project.A.dependencies, "", true},
+                {Project.A, Collections.emptySet(), "--only=dev", false},
+                {Project.B, Project.B.dependencies, "", true},
+                {Project.B, Collections.emptySet(), "--production", false},
+                {Project.C, Project.C.dependencies, "", true},
+                {Project.C, Project.A.dependencies, "--only=production", true}
+        };
+    }
+
+    @SuppressWarnings("unused")
+    @Test(dataProvider = "npmCiProvider")
+    public void npmCiTest(Project project, Set<String> expectedDependencies, String args, boolean packageJsonPath) {
+        runNpmTest(project, expectedDependencies, args, packageJsonPath, true);
+    }
+
+    private void runNpmTest(Project project, Set<String> expectedDependencies, String args, boolean packageJsonPath, boolean isNpmCi) {
         Path projectDir = null;
         try {
-            // Run npm install
+            // Prepare.
             projectDir = createProjectDir(project);
             Path path = packageJsonPath ? projectDir.resolve("package.json") : projectDir;
-            NpmInstall npmInstall = new NpmInstall(dependenciesClientBuilder, virtualRepo, args, log, path, null, null);
-            Build build = npmInstall.execute();
+            if (isNpmCi) {
+                // Run npm install to generate package-lock.json file.
+                new NpmInstallCi(dependenciesClientBuilder, virtualRepo, args, log, path, null, null, false).execute();
+            }
+
+            // Execute command.
+            NpmInstallCi buildExecutor;
+            if (isNpmCi) {
+                buildExecutor = new NpmInstallCi(dependenciesClientBuilder, virtualRepo, args, log, path, null, null, true);
+            } else {
+                buildExecutor = new NpmInstallCi(dependenciesClientBuilder, virtualRepo, args, log, path, null, null, false);
+            }
+            Build build = buildExecutor.execute();
+
+            // Validate.
             assertEquals(build.getModules().size(), 1);
             Module module = build.getModules().get(0);
-            // Check correctness of the module and dependencies
             assertEquals(module.getType(), "npm");
             assertEquals(module.getId(), project.getModuleId());
             Set<String> moduleDependencies = module.getDependencies().stream().map(Dependency::getId).collect(Collectors.toSet());

Diff for: build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/ArtifactoryClientConfiguration.java

@@ -55,6 +55,7 @@ public class ArtifactoryClientConfiguration {
     public final BuildInfoHandler info;
     public final ProxyHandler proxy;
     public final PackageManagerHandler packageManagerHandler;
+    public final NpmHandler npmHandler;
     public final PipHandler pipHandler;
     public final DotnetHandler dotnetHandler;
     public final DockerHandler dockerHandler;
@@ -72,6 +73,7 @@ public ArtifactoryClientConfiguration(Log log) {
         this.info = new BuildInfoHandler();
         this.proxy = new ProxyHandler();
         this.packageManagerHandler = new PackageManagerHandler();
+        this.npmHandler = new NpmHandler();
         this.pipHandler = new PipHandler();
         this.dotnetHandler = new DotnetHandler();
         this.dockerHandler = new DockerHandler();
@@ -502,8 +504,21 @@ public void setModule(String packageManagerModule) {
         }
     }
 
-    public class PipHandler extends PrefixPropertyHandler {
+    public class NpmHandler extends PrefixPropertyHandler {
+        public NpmHandler() {
+            super(root, PROP_NPM_PREFIX);
+        }
+
+        public boolean isCiCommand() {
+            return rootConfig.getBooleanValue(NPM_CI_COMMAND, false);
+        }
 
+        public void setCiCommand(boolean ciCommand) {
+            rootConfig.setBooleanValue(NPM_CI_COMMAND, ciCommand);
+        }
+    }
+
+    public class PipHandler extends PrefixPropertyHandler {
         public PipHandler() {
             super(root, PROP_PIP_PREFIX);
         }

Diff for: build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/ClientConfigurationFields.java

@@ -33,6 +33,7 @@ public interface ClientConfigurationFields {
     String PACKAGE_MANAGER_ARGS = "package.manager.args";
     String PACKAGE_MANAGER_PATH = "package.manager.path"; // Path to package-manager execution dir
     String PACKAGE_MANAGER_MODULE = "package.manager.module"; // Custom module name for the build-info
+    String NPM_CI_COMMAND = "npm.ci.command"; // Determines whether the npm build is 'npm install' or 'npm ci' command.
     String PIP_ENV_ACTIVATION = "pip.env.activation";
     String DOTNET_USE_DOTNET_CORE_CLI = "dotnet.use.dotnet.core.cli";
     String DOCKER_IMAGE_TAG = "docker.image.tag";

Diff for: build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/ClientProperties.java

@@ -44,6 +44,8 @@ public interface ClientProperties {
 
     String PROP_PACKAGE_MANAGER_PREFIX = ARTIFACTORY_PREFIX + "package.manager.";
 
+    String PROP_NPM_PREFIX = ARTIFACTORY_PREFIX + "npm.";
+
     String PROP_PIP_PREFIX = ARTIFACTORY_PREFIX + "pip.";
 
     String PROP_DOTNET_PREFIX = ARTIFACTORY_PREFIX + "dotnet.";