modrinth: expand downloaded zip files

src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java

@@ -211,7 +211,7 @@ else if (Manifests.allFilesPresent(outputDir, manifest)) {
             .levelName(results.getLevelName())
             .build();
 
-        Manifests.cleanup(outputDir, manifest, newManifest, f -> log.info("Removing old file {}", f));
+        Manifests.cleanup(outputDir, manifest, newManifest, log);
 
         Manifests.save(outputDir, CURSEFORGE_ID, newManifest);
 

src/main/java/me/itzg/helpers/fabric/FabricLauncherInstaller.java

@@ -136,7 +136,7 @@ private Path processInstallUsingVersions(String minecraftVersion, String loaderV
             .launcherPath(launcherPath.toString())
             .build();
 
-        Manifests.cleanup(outputDir, manifest, newManifest, f -> log.debug("Removing {}", f));
+        Manifests.cleanup(outputDir, manifest, newManifest, log);
 
         Manifests.save(outputDir, MANIFEST_ID, newManifest);
 
@@ -243,7 +243,7 @@ public void installGivenLauncherFile(Path launcher) throws IOException {
                 }
             }
             else if (manifest != null) {
-                Manifests.cleanup(outputDir, manifest, newManifest, f -> log.debug("Removing {}", f));
+                Manifests.cleanup(outputDir, manifest, newManifest, log);
 
                 if (manifest.getOrigin() != null) {
                     log.info("Switching from {} to provided launcher", manifest.getOrigin());
@@ -293,7 +293,7 @@ public Path installUsingUri(URI loaderUri) throws IOException {
             .build();
         Manifests.save(outputDir, MANIFEST_ID, newManifest);
 
-        Manifests.cleanup(outputDir, oldManifest, newManifest, f -> log.debug("Removing {}", f));
+        Manifests.cleanup(outputDir, oldManifest, newManifest, log);
 
         if (resultsFile != null) {
             try (ResultsFileWriter results = new ResultsFileWriter(resultsFile)) {

src/main/java/me/itzg/helpers/files/BaseManifest.java

@@ -2,6 +2,7 @@
 
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.fasterxml.jackson.annotation.JsonTypeInfo.Id;
+import java.nio.file.Path;
 import java.time.Instant;
 import java.util.List;
 import lombok.Builder.Default;
@@ -25,5 +26,8 @@ public abstract class BaseManifest {
     @Default
     Instant timestamp = Instant.now();
 
+    /**
+     * NOTE: use {@link Manifests#relativizeAll(Path, List)} to remap regular paths into relative paths
+     */
     List<String> files;
 }

src/main/java/me/itzg/helpers/files/Manifests.java

@@ -9,6 +9,7 @@
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import me.itzg.helpers.json.ObjectMappers;
+import org.slf4j.Logger;
 
 public class Manifests {
 
@@ -37,6 +38,15 @@ public static void cleanup(Path baseDir, Collection<String> oldFiles, Collection
         }
     }
 
+    /**
+     * @param oldManifest can be null
+     */
+    public static void cleanup(Path baseDir, BaseManifest oldManifest,
+        BaseManifest newManifest, Logger log
+    ) throws IOException {
+        cleanup(baseDir, oldManifest, newManifest, s -> log.info("Removing old file {}", s));
+    }
+
     /**
      * @param oldManifest can be null
      * @param removeListener passed the path of a file being removed. Useful for debug logging as removed.

src/main/java/me/itzg/helpers/http/SpecificFileFetchBuilder.java

@@ -65,7 +65,7 @@ public Mono<Path> assemble() {
         final URI uri = uri();
 
         if (skipExisting && Files.exists(file)) {
-            log.debug("File already exists and skip requested");
+            log.debug("Skipping file={} that already exists due to request", file);
             statusHandler.call(FileDownloadStatus.SKIP_FILE_EXISTS, uri, file);
             return Mono.just(file);
         }

src/main/java/me/itzg/helpers/modrinth/Manifest.java → src/main/java/me/itzg/helpers/modrinth/LegacyModrinthManifest.java

@@ -9,7 +9,7 @@
 @Data
 @Builder
 @Jacksonized
-public class Manifest {
+public class LegacyModrinthManifest {
 
     public static final String FILENAME = ".modrinth-files.manifest";
 

src/main/java/me/itzg/helpers/modrinth/ModrinthCommand.java

@@ -8,14 +8,16 @@
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.time.Instant;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 import lombok.extern.slf4j.Slf4j;
 import me.itzg.helpers.errors.GenericException;
 import me.itzg.helpers.files.Manifests;
@@ -69,46 +71,47 @@ public ModrinthCommand(String baseUrl) {
 
     @Override
     public Integer call() throws Exception {
-        final ObjectMapper objectMapper = ObjectMappers.defaultMapper();
-
         Files.createDirectories(outputDirectory);
 
-        final Path manifestPath = outputDirectory.resolve(Manifest.FILENAME);
-
-        final Manifest oldManifest;
-        if (Files.exists(manifestPath)) {
-            oldManifest = objectMapper.readValue(manifestPath.toFile(), Manifest.class);
-            log.debug("Loaded existing manifest={}", oldManifest);
-        } else {
-            oldManifest = null;
-        }
+        final ModrinthManifest prevManifest = loadManifest();
 
         final List<Path> outputFiles = projects.stream()
             .flatMap(this::processProject)
             .collect(Collectors.toList());
 
-        final Manifest newManifest = Manifest.builder()
-            .timestamp(Instant.now())
-            .files(
-                outputFiles.stream()
-                    .map(path -> outputDirectory.relativize(path))
-                    .map(Path::toString)
-                    .collect(Collectors.toSet())
-            )
+        final ModrinthManifest newManifest = ModrinthManifest.builder()
+            .files(Manifests.relativizeAll(outputDirectory, outputFiles))
+            .projects(projects)
             .build();
 
-        if (oldManifest != null) {
-            Manifests.cleanup(outputDirectory, oldManifest.getFiles(), newManifest.getFiles(),
-                file -> log.debug("Deleting old file={}", file)
-            );
-        }
+        Manifests.cleanup(outputDirectory, prevManifest, newManifest, log);
 
-        objectMapper.writerWithDefaultPrettyPrinter()
-            .writeValue(manifestPath.toFile(), newManifest);
+        Manifests.save(outputDirectory, ModrinthManifest.ID, newManifest);
 
         return ExitCode.OK;
     }
 
+    private ModrinthManifest loadManifest() throws IOException {
+        final Path legacyManifestPath = outputDirectory.resolve(LegacyModrinthManifest.FILENAME);
+
+        if (Files.exists(legacyManifestPath)) {
+            final ObjectMapper objectMapper = ObjectMappers.defaultMapper();
+
+            final LegacyModrinthManifest legacyManifest = objectMapper.readValue(legacyManifestPath.toFile(),
+                LegacyModrinthManifest.class
+            );
+
+            Files.delete(legacyManifestPath);
+
+            return ModrinthManifest.builder()
+                .timestamp(legacyManifest.getTimestamp())
+                .files(new ArrayList<>(legacyManifest.getFiles()))
+                .build();
+        }
+
+        return Manifests.load(outputDirectory, ModrinthManifest.ID, ModrinthManifest.class);
+    }
+
     private Stream<Version> expandDependencies(Version version) {
         log.debug("Expanding dependencies of version={}", version);
         return version.getDependencies().stream()
@@ -279,9 +282,58 @@ private Stream<? extends Path> processProject(String projectRef) {
                         expandDependencies(version)
                     )
                     .map(this::pickVersionFile)
-                    .map(versionFile -> download(project.getProjectType(), versionFile));
+                    .map(versionFile -> download(project.getProjectType(), versionFile))
+                    .flatMap(this::expandIfZip);
             }
         }
         return Stream.empty();
     }
+
+    /**
+     * If downloadedFile ends in .zip, then expand it, return its files and given file.
+     * @return a stream of at least the given file along with unzipped contents
+     */
+    private Stream<Path> expandIfZip(Path downloadedFile) {
+        if (downloadedFile.getFileName().toString().endsWith(".zip")) {
+            return Stream.concat(
+                Stream.of(downloadedFile),
+                expandZip(downloadedFile)
+            );
+        }
+        else {
+            return Stream.of(downloadedFile);
+        }
+    }
+
+    private Stream<Path> expandZip(Path zipFile) {
+        log.debug("Unzipping downloaded file={}", zipFile);
+        final Path outDir = zipFile.getParent();
+
+        final ArrayList<Path> contents = new ArrayList<>();
+
+        try (ZipInputStream zipIn = new ZipInputStream(Files.newInputStream(zipFile))) {
+            ZipEntry entry;
+            while ((entry = zipIn.getNextEntry()) != null) {
+                if (!entry.isDirectory()) {
+                    final String name = entry.getName();
+                    final Path resolved = outDir.resolve(name);
+                    if (!Files.exists(resolved)) {
+                        log.debug("Expanding from zip to={}", resolved);
+                        if (name.contains("/")) {
+                            Files.createDirectories(resolved.getParent());
+                        }
+                        Files.copy(zipIn, resolved);
+                    }
+                    else {
+                        log.debug("File={} from zip already exists", resolved);
+                    }
+                    contents.add(resolved);
+                }
+            }
+        } catch (IOException e) {
+            throw new GenericException("Unable to unzip downloaded file", e);
+        }
+
+        return contents.stream();
+    }
 }

src/main/java/me/itzg/helpers/modrinth/ModrinthManifest.java

@@ -0,0 +1,17 @@
+package me.itzg.helpers.modrinth;
+
+import java.util.List;
+import lombok.Getter;
+import lombok.experimental.SuperBuilder;
+import lombok.extern.jackson.Jacksonized;
+import me.itzg.helpers.files.BaseManifest;
+
+@SuperBuilder
+@Getter
+@Jacksonized
+public class ModrinthManifest extends BaseManifest {
+
+    public static final String ID = "modrinth";
+
+    List<String> projects;
+}