curseforge: support parallel downloads and force reinstall

.github/dependabot.yml

@@ -5,3 +5,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+  - package-ecosystem: gradle
+    directory: "/"
+    schedule:
+      interval: "weekly"

build.gradle

@@ -97,7 +97,7 @@ dependencies {
     implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
     implementation 'com.jayway.jsonpath:json-path:2.7.0'
     implementation 'org.apache.httpcomponents.client5:httpclient5:5.2.1'
-    implementation 'io.projectreactor.netty:reactor-netty-http:1.0.24'
+    implementation 'io.projectreactor.netty:reactor-netty-http:1.1.1'
     implementation 'org.apache.maven:maven-artifact:3.8.5'
     implementation 'commons-codec:commons-codec:1.15'
 

src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java

@@ -9,7 +9,6 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -35,6 +34,9 @@
 import me.itzg.helpers.http.SharedFetch;
 import me.itzg.helpers.http.UriBuilder;
 import me.itzg.helpers.json.ObjectMappers;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Schedulers;
 
 @RequiredArgsConstructor
 @Slf4j
@@ -49,6 +51,13 @@ public class CurseForgeInstaller {
     @Getter
     @Setter
     private String apiBaseUrl = "https://api.curse.tools/v1/cf";
+    @Getter
+    @Setter
+    private int parallelism = 4;
+
+    @Getter
+    @Setter
+    private boolean forceSynchronize;
 
     public void install(String slug, String fileMatcher, Integer fileId, Set<Integer> excludedModIds) throws IOException {
         requireNonNull(outputDir, "outputDir is required");
@@ -77,12 +86,12 @@ else if (searchResponse.getData().size() > 1) {
         }
     }
 
-    private void processMod(SharedFetch preparedFetch, UriBuilder uriBuilder, CurseForgeMod mod, Integer fileId, String fileMatcher,
+    private void processMod(SharedFetch preparedFetch, UriBuilder uriBuilder, CurseForgeMod mod, Integer fileId,
+        String fileMatcher,
         Set<Integer> excludedModIds
     )
         throws IOException {
 
-
         final CurseForgeFile modFile;
 
         if (fileId == null) {
@@ -98,13 +107,18 @@ private void processMod(SharedFetch preparedFetch, UriBuilder uriBuilder, CurseF
             && manifest.getFileId() == modFile.getId()
             && manifest.getModId() == modFile.getModId()
         ) {
-            if (Manifests.allFilesPresent(outputDir, manifest)) {
+            if (forceSynchronize) {
+                log.info("Requested force synchronize of {}", modFile.getDisplayName());
+            }
+            else if (Manifests.allFilesPresent(outputDir, manifest)) {
                 log.info("Requested CurseForge modpack {} is already installed for {}",
                     modFile.getDisplayName(), mod.getName()
                 );
                 return;
             }
-            log.warn("Some files from modpack file {} were missing. Proceeding with a re-install", modFile.getFileName());
+            else {
+                log.warn("Some files from modpack file {} were missing. Proceeding with a re-install", modFile.getFileName());
+            }
         }
 
         log.info("Processing modpack {} @ {}:{}", modFile.getDisplayName(), modFile.getModId(), modFile.getId());
@@ -130,10 +144,10 @@ private static CurseForgeFile resolveModpackFile(SharedFetch preparedFetch, UriB
     ) throws IOException {
         // NOTE latestFiles in mod is only one or two files, so retrieve the full list instead
         final GetModFilesResponse resp = preparedFetch.fetch(
-                    uriBuilder.resolve("/mods/{modId}/files", mod.getId())
-                )
-                .toObject(GetModFilesResponse.class)
-                .execute();
+                uriBuilder.resolve("/mods/{modId}/files", mod.getId())
+            )
+            .toObject(GetModFilesResponse.class)
+            .execute();
 
         return resp.getData().stream()
             .filter(file ->
@@ -154,12 +168,12 @@ private static URI normalizeDownloadUrl(String downloadUrl) {
 
         final String filename = downloadUrl.substring(nameStart + 1);
         return URI.create(
-            downloadUrl.substring(0, nameStart+1) +
+            downloadUrl.substring(0, nameStart + 1) +
                 filename
                     .replace(" ", "%20")
                     .replace("[", "%5B")
                     .replace("]", "%5D")
-            );
+        );
     }
 
     private List<Path> processModpackFile(SharedFetch preparedFetch, UriBuilder uriBuilder, URI downloadUrl,
@@ -171,6 +185,9 @@ private List<Path> processModpackFile(SharedFetch preparedFetch, UriBuilder uriB
         try {
             preparedFetch.fetch(downloadUrl)
                 .toFile(downloaded)
+                .handleStatus((status, uri, file) ->
+                    log.debug("Modpack file retrieval: status={} uri={} file={}", status, uri, file)
+                )
                 .execute();
 
             final MinecraftModpackManifest modpackManifest = extractModpackManifest(downloaded);
@@ -182,36 +199,40 @@ private List<Path> processModpackFile(SharedFetch preparedFetch, UriBuilder uriB
 
             final Path modsDir = Files.createDirectories(outputDir.resolve("mods"));
 
-            final List<Path> modFiles = modpackManifest.getFiles().stream()
+            final List<Path> modFiles = Flux.fromIterable(modpackManifest.getFiles())
+                .parallel(parallelism)
+                .runOn(Schedulers.newParallel("downloader"))
                 .filter(ManifestFileRef::isRequired)
                 .filter(manifestFileRef -> !excludedModIds.contains(manifestFileRef.getProjectID()))
-                .map(fileRef ->
+                .flatMap(fileRef ->
                     downloadModFile(preparedFetch, uriBuilder, modsDir, fileRef.getProjectID(), fileRef.getFileID())
                 )
-                .filter(Objects::nonNull)
-                .collect(Collectors.toList());
+                .sequential()
+                .collectList()
+                .block();
 
-            final List<Path> overrides = applyOverrides(downloaded);
+            final List<Path> overrides = applyOverrides(downloaded, modpackManifest.getOverrides());
 
             prepareModLoader(modLoader.getId(), modpackManifest.getMinecraft().getVersion());
 
-            return Stream.concat(modFiles.stream(), overrides.stream())
+            return Stream.concat(
+                    modFiles != null ? modFiles.stream() : Stream.empty(),
+                    overrides.stream()
+                )
                 .collect(Collectors.toList());
-        }
-        finally {
+        } finally {
             Files.delete(downloaded);
         }
     }
 
-    private List<Path> applyOverrides(Path modpackZip) throws IOException {
+    private List<Path> applyOverrides(Path modpackZip, String overridesDir) throws IOException {
         final ArrayList<Path> overrides = new ArrayList<>();
         try (ZipInputStream zip = new ZipInputStream(Files.newInputStream(modpackZip))) {
             ZipEntry entry;
             while ((entry = zip.getNextEntry()) != null) {
                 if (!entry.isDirectory()) {
-                    // TODO lookup "overrides" from file model
-                    if (entry.getName().startsWith("overrides/")) {
-                        final String subpath = entry.getName().substring("overrides/".length());
+                    if (entry.getName().startsWith(overridesDir + "/")) {
+                        final String subpath = entry.getName().substring(overridesDir.length() + 1/*for slash*/);
                         final Path outPath = outputDir.resolve(subpath);
                         log.debug("Applying override {}", subpath);
 
@@ -227,26 +248,36 @@ private List<Path> applyOverrides(Path modpackZip) throws IOException {
         return overrides;
     }
 
-    private Path downloadModFile(SharedFetch preparedFetch, UriBuilder uriBuilder, Path modsDir, int projectID, int fileID) {
+    private Mono<Path> downloadModFile(SharedFetch preparedFetch, UriBuilder uriBuilder, Path modsDir, int projectID, int fileID
+    ) {
         try {
             final CurseForgeFile file = getModFileInfo(preparedFetch, uriBuilder, projectID, fileID);
-            if (!isServerMod(file)) {
-                log.debug("Skipping {} since it is a client mod", file.getFileName());
-                return null;
-            }
-
-            log.info("Download/confirm mod {} @ {}:{}",
+            log.debug("Download/confirm mod {} @ {}:{}",
                 // several mods have non-descriptive display names, like "v1.0.0", so filename tends to be better
                 file.getFileName(),
                 projectID, fileID
             );
+            if (!isServerMod(file)) {
+                log.debug("Skipping {} since it is a client mod", file.getFileName());
+                return Mono.empty();
+            }
 
             return preparedFetch.fetch(
                     normalizeDownloadUrl(file.getDownloadUrl())
                 )
-                .toDirectory(modsDir)
+                .toFile(modsDir.resolve(file.getFileName()))
                 .skipExisting(true)
-                .execute();
+                .handleStatus((status, uri, f) -> {
+                    switch (status) {
+                        case SKIP_FILE_EXISTS:
+                            log.info("Mod file {} already exists", outputDir.relativize(f));
+                            break;
+                        case DOWNLOADED:
+                            log.info("Downloaded mod file {}", outputDir.relativize(f));
+                            break;
+                    }
+                })
+                .assemble();
         } catch (IOException e) {
             throw new GenericException(String.format("Failed to locate mod file modId=%s fileId=%d", projectID, fileID), e);
         }

src/main/java/me/itzg/helpers/curseforge/InstallCurseForgeCommand.java

@@ -43,6 +43,14 @@ public class InstallCurseForgeCommand implements Callable<Integer> {
         description = "Substring to select specific modpack filename")
     String filenameMatcher;
 
+    @Option(names = "--force-synchronize")
+    boolean forceSynchronize;
+
+    @Option(names = "--parallel-downloads", defaultValue = "4",
+        description = "Default: ${DEFAULT-VALUE}"
+    )
+    int parallelDownloads;
+
     private static final Pattern PAGE_URL_PATTERN = Pattern.compile(
         "https://www.curseforge.com/minecraft/modpacks/(?<slug>.+?)(/files(/(?<fileId>\\d+)?)?)?");
 
@@ -71,7 +79,9 @@ public Integer call() throws Exception {
             return ExitCode.USAGE;
         }
 
-        final CurseForgeInstaller installer = new CurseForgeInstaller(outputDirectory, resultsFile);
+        final CurseForgeInstaller installer = new CurseForgeInstaller(outputDirectory, resultsFile)
+            .setForceSynchronize(forceSynchronize)
+            .setParallelism(parallelDownloads);
         installer.install(slug, filenameMatcher, fileId, excludedModIds);
 
         return ExitCode.OK;

src/main/java/me/itzg/helpers/get/GetCommand.java

@@ -211,7 +211,11 @@ null, new JsonPathOutputHandler(
                         .toFile(outputFile)
                         .skipUpToDate(skipUpToDate)
                         .acceptContentTypes(acceptContentTypes)
-                        .logProgressEach(logProgressEach)
+                        .handleDownloaded((uri, f, contentSizeBytes) -> {
+                            if (logProgressEach) {
+                                log.info("Downloaded {}", f);
+                            }
+                        })
                         .execute();
                     if (this.outputFilename) {
                         stdout.println(file);

src/main/java/me/itzg/helpers/http/FetchBuilderBase.java

@@ -80,6 +80,10 @@ protected HttpHead head(boolean withConfigure) throws IOException {
         return request;
     }
 
+    protected URI uri() {
+        return state.uri;
+    }
+
     public List<String> getAcceptContentTypes() {
         return state.acceptContentTypes;
     }

src/main/java/me/itzg/helpers/http/FileDownloadStatus.java

@@ -0,0 +1,8 @@
+package me.itzg.helpers.http;
+
+public enum FileDownloadStatus {
+    SKIP_FILE_EXISTS,
+    SKIP_FILE_UP_TO_DATE,
+    DOWNLOADING,
+    DOWNLOADED
+}

src/main/java/me/itzg/helpers/http/FileDownloadStatusHandler.java

@@ -0,0 +1,10 @@
+package me.itzg.helpers.http;
+
+import java.net.URI;
+import java.nio.file.Path;
+
+@FunctionalInterface
+public interface FileDownloadStatusHandler {
+
+    void call(FileDownloadStatus status, URI uri, Path file);
+}

src/main/java/me/itzg/helpers/http/FileDownloadedHandler.java

@@ -0,0 +1,10 @@
+package me.itzg.helpers.http;
+
+import java.net.URI;
+import java.nio.file.Path;
+
+@FunctionalInterface
+public interface FileDownloadedHandler {
+
+    void call(URI uri, Path file, long contentSizeBytes);
+}

src/main/java/me/itzg/helpers/http/FilenameExtractor.java

@@ -2,6 +2,8 @@
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.hc.client5.http.HttpResponseException;
 import org.apache.hc.core5.http.ClassicHttpResponse;
@@ -13,12 +15,29 @@
 
 @Slf4j
 public class FilenameExtractor {
+  private static final Pattern HTTP_CONTENT_DISPOSITION =
+      Pattern.compile("(?<type>inline|attachment)(\\s*;\\s+filename=\"(?<filename>.+?)\")?");
+
   private final LatchingUrisInterceptor interceptor;
 
   public FilenameExtractor(LatchingUrisInterceptor interceptor) {
     this.interceptor = Objects.requireNonNull(interceptor, "interceptor is required");
   }
 
+  static String filenameFromContentDisposition(String headerValue) {
+      if (headerValue == null) {
+          return null;
+      }
+
+      final Matcher m = HTTP_CONTENT_DISPOSITION.matcher(headerValue);
+      if (m.matches()) {
+          if (m.group("type").equals("attachment")) {
+              return m.group("filename");
+          }
+      }
+      return null;
+  }
+
   public String extract(ClassicHttpResponse response) throws IOException, ProtocolException {
     // Same as AbstractHttpClientResponseHandler
     if (response.getCode() >= HttpStatus.SC_REDIRECTION) {