From 31423fd255ec3160855cf3d5a22421f5950740ea Mon Sep 17 00:00:00 2001 From: Geoff Bourne Date: Sun, 5 Mar 2023 15:13:35 -0600 Subject: [PATCH 1/2] modrinth: expand downloaded zip files --- .../curseforge/CurseForgeInstaller.java | 2 +- .../fabric/FabricLauncherInstaller.java | 6 +- .../me/itzg/helpers/files/BaseManifest.java | 4 + .../java/me/itzg/helpers/files/Manifests.java | 10 ++ ...ifest.java => LegacyModrinthManifest.java} | 2 +- .../helpers/modrinth/ModrinthCommand.java | 107 +++++++++++++----- .../helpers/modrinth/ModrinthManifest.java | 14 +++ 7 files changed, 112 insertions(+), 33 deletions(-) rename src/main/java/me/itzg/helpers/modrinth/{Manifest.java => LegacyModrinthManifest.java} (89%) create mode 100644 src/main/java/me/itzg/helpers/modrinth/ModrinthManifest.java diff --git a/src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java b/src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java index b0599e9a..88e087d3 100644 --- a/src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java +++ b/src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java @@ -211,7 +211,7 @@ else if (Manifests.allFilesPresent(outputDir, manifest)) { .levelName(results.getLevelName()) .build(); - Manifests.cleanup(outputDir, manifest, newManifest, f -> log.info("Removing old file {}", f)); + Manifests.cleanup(outputDir, manifest, newManifest, log); Manifests.save(outputDir, CURSEFORGE_ID, newManifest); diff --git a/src/main/java/me/itzg/helpers/fabric/FabricLauncherInstaller.java b/src/main/java/me/itzg/helpers/fabric/FabricLauncherInstaller.java index 7320d622..c2aec6c6 100644 --- a/src/main/java/me/itzg/helpers/fabric/FabricLauncherInstaller.java +++ b/src/main/java/me/itzg/helpers/fabric/FabricLauncherInstaller.java @@ -136,7 +136,7 @@ private Path processInstallUsingVersions(String minecraftVersion, String loaderV .launcherPath(launcherPath.toString()) .build(); - Manifests.cleanup(outputDir, manifest, newManifest, f -> log.debug("Removing {}", f)); + Manifests.cleanup(outputDir, manifest, newManifest, log); Manifests.save(outputDir, MANIFEST_ID, newManifest); @@ -243,7 +243,7 @@ public void installGivenLauncherFile(Path launcher) throws IOException { } } else if (manifest != null) { - Manifests.cleanup(outputDir, manifest, newManifest, f -> log.debug("Removing {}", f)); + Manifests.cleanup(outputDir, manifest, newManifest, log); if (manifest.getOrigin() != null) { log.info("Switching from {} to provided launcher", manifest.getOrigin()); @@ -293,7 +293,7 @@ public Path installUsingUri(URI loaderUri) throws IOException { .build(); Manifests.save(outputDir, MANIFEST_ID, newManifest); - Manifests.cleanup(outputDir, oldManifest, newManifest, f -> log.debug("Removing {}", f)); + Manifests.cleanup(outputDir, oldManifest, newManifest, log); if (resultsFile != null) { try (ResultsFileWriter results = new ResultsFileWriter(resultsFile)) { diff --git a/src/main/java/me/itzg/helpers/files/BaseManifest.java b/src/main/java/me/itzg/helpers/files/BaseManifest.java index 3c26a719..09b99948 100644 --- a/src/main/java/me/itzg/helpers/files/BaseManifest.java +++ b/src/main/java/me/itzg/helpers/files/BaseManifest.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; +import java.nio.file.Path; import java.time.Instant; import java.util.List; import lombok.Builder.Default; @@ -25,5 +26,8 @@ public abstract class BaseManifest { @Default Instant timestamp = Instant.now(); + /** + * NOTE: use {@link Manifests#relativizeAll(Path, List)} to remap regular paths into relative paths + */ List files; } diff --git a/src/main/java/me/itzg/helpers/files/Manifests.java b/src/main/java/me/itzg/helpers/files/Manifests.java index 3e518521..600dcc3d 100644 --- a/src/main/java/me/itzg/helpers/files/Manifests.java +++ b/src/main/java/me/itzg/helpers/files/Manifests.java @@ -9,6 +9,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import me.itzg.helpers.json.ObjectMappers; +import org.slf4j.Logger; public class Manifests { @@ -37,6 +38,15 @@ public static void cleanup(Path baseDir, Collection oldFiles, Collection } } + /** + * @param oldManifest can be null + */ + public static void cleanup(Path baseDir, BaseManifest oldManifest, + BaseManifest newManifest, Logger log + ) throws IOException { + cleanup(baseDir, oldManifest, newManifest, s -> log.info("Removing old file {}", s)); + } + /** * @param oldManifest can be null * @param removeListener passed the path of a file being removed. Useful for debug logging as removed. diff --git a/src/main/java/me/itzg/helpers/modrinth/Manifest.java b/src/main/java/me/itzg/helpers/modrinth/LegacyModrinthManifest.java similarity index 89% rename from src/main/java/me/itzg/helpers/modrinth/Manifest.java rename to src/main/java/me/itzg/helpers/modrinth/LegacyModrinthManifest.java index bee2d013..a5b4afe8 100644 --- a/src/main/java/me/itzg/helpers/modrinth/Manifest.java +++ b/src/main/java/me/itzg/helpers/modrinth/LegacyModrinthManifest.java @@ -9,7 +9,7 @@ @Data @Builder @Jacksonized -public class Manifest { +public class LegacyModrinthManifest { public static final String FILENAME = ".modrinth-files.manifest"; diff --git a/src/main/java/me/itzg/helpers/modrinth/ModrinthCommand.java b/src/main/java/me/itzg/helpers/modrinth/ModrinthCommand.java index 40f67145..6fefc597 100644 --- a/src/main/java/me/itzg/helpers/modrinth/ModrinthCommand.java +++ b/src/main/java/me/itzg/helpers/modrinth/ModrinthCommand.java @@ -8,7 +8,7 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Instant; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -16,6 +16,8 @@ import java.util.concurrent.Callable; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import lombok.extern.slf4j.Slf4j; import me.itzg.helpers.errors.GenericException; import me.itzg.helpers.files.Manifests; @@ -69,46 +71,46 @@ public ModrinthCommand(String baseUrl) { @Override public Integer call() throws Exception { - final ObjectMapper objectMapper = ObjectMappers.defaultMapper(); - Files.createDirectories(outputDirectory); - final Path manifestPath = outputDirectory.resolve(Manifest.FILENAME); - - final Manifest oldManifest; - if (Files.exists(manifestPath)) { - oldManifest = objectMapper.readValue(manifestPath.toFile(), Manifest.class); - log.debug("Loaded existing manifest={}", oldManifest); - } else { - oldManifest = null; - } + final ModrinthManifest prevManifest = loadManifest(); final List outputFiles = projects.stream() .flatMap(this::processProject) .collect(Collectors.toList()); - final Manifest newManifest = Manifest.builder() - .timestamp(Instant.now()) - .files( - outputFiles.stream() - .map(path -> outputDirectory.relativize(path)) - .map(Path::toString) - .collect(Collectors.toSet()) - ) + final ModrinthManifest newManifest = ModrinthManifest.builder() + .files(Manifests.relativizeAll(outputDirectory, outputFiles)) .build(); - if (oldManifest != null) { - Manifests.cleanup(outputDirectory, oldManifest.getFiles(), newManifest.getFiles(), - file -> log.debug("Deleting old file={}", file) - ); - } + Manifests.cleanup(outputDirectory, prevManifest, newManifest, log); - objectMapper.writerWithDefaultPrettyPrinter() - .writeValue(manifestPath.toFile(), newManifest); + Manifests.save(outputDirectory, ModrinthManifest.ID, newManifest); return ExitCode.OK; } + private ModrinthManifest loadManifest() throws IOException { + final Path legacyManifestPath = outputDirectory.resolve(LegacyModrinthManifest.FILENAME); + + if (Files.exists(legacyManifestPath)) { + final ObjectMapper objectMapper = ObjectMappers.defaultMapper(); + + final LegacyModrinthManifest legacyManifest = objectMapper.readValue(legacyManifestPath.toFile(), + LegacyModrinthManifest.class + ); + + Files.delete(legacyManifestPath); + + return ModrinthManifest.builder() + .timestamp(legacyManifest.getTimestamp()) + .files(new ArrayList<>(legacyManifest.getFiles())) + .build(); + } + + return Manifests.load(outputDirectory, ModrinthManifest.ID, ModrinthManifest.class); + } + private Stream expandDependencies(Version version) { log.debug("Expanding dependencies of version={}", version); return version.getDependencies().stream() @@ -279,9 +281,58 @@ private Stream processProject(String projectRef) { expandDependencies(version) ) .map(this::pickVersionFile) - .map(versionFile -> download(project.getProjectType(), versionFile)); + .map(versionFile -> download(project.getProjectType(), versionFile)) + .flatMap(this::expandIfZip); } } return Stream.empty(); } + + /** + * If downloadedFile ends in .zip, then expand it, return its files and given file. + * @return a stream of at least the given file along with unzipped contents + */ + private Stream expandIfZip(Path downloadedFile) { + if (downloadedFile.getFileName().toString().endsWith(".zip")) { + return Stream.concat( + Stream.of(downloadedFile), + expandZip(downloadedFile) + ); + } + else { + return Stream.of(downloadedFile); + } + } + + private Stream expandZip(Path zipFile) { + log.debug("Unzipping downloaded file={}", zipFile); + final Path outDir = zipFile.getParent(); + + final ArrayList contents = new ArrayList<>(); + + try (ZipInputStream zipIn = new ZipInputStream(Files.newInputStream(zipFile))) { + ZipEntry entry; + while ((entry = zipIn.getNextEntry()) != null) { + if (!entry.isDirectory()) { + final String name = entry.getName(); + final Path resolved = outDir.resolve(name); + if (!Files.exists(resolved)) { + log.debug("Expanding from zip to={}", resolved); + if (name.contains("/")) { + Files.createDirectories(resolved.getParent()); + } + Files.copy(zipIn, resolved); + } + else { + log.debug("File={} from zip already exists", resolved); + } + contents.add(resolved); + } + } + } catch (IOException e) { + throw new GenericException("Unable to unzip downloaded file", e); + } + + return contents.stream(); + } } diff --git a/src/main/java/me/itzg/helpers/modrinth/ModrinthManifest.java b/src/main/java/me/itzg/helpers/modrinth/ModrinthManifest.java new file mode 100644 index 00000000..14f57cf7 --- /dev/null +++ b/src/main/java/me/itzg/helpers/modrinth/ModrinthManifest.java @@ -0,0 +1,14 @@ +package me.itzg.helpers.modrinth; + +import lombok.Getter; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; +import me.itzg.helpers.files.BaseManifest; + +@SuperBuilder +@Getter +@Jacksonized +public class ModrinthManifest extends BaseManifest { + + public static final String ID = "modrinth"; +} From 148c7a692a9fe778cfa569e183cc0d65d38e78e7 Mon Sep 17 00:00:00 2001 From: Geoff Bourne Date: Sun, 5 Mar 2023 15:18:58 -0600 Subject: [PATCH 2/2] Include projects in manifest --- .../java/me/itzg/helpers/http/SpecificFileFetchBuilder.java | 2 +- src/main/java/me/itzg/helpers/modrinth/ModrinthCommand.java | 1 + src/main/java/me/itzg/helpers/modrinth/ModrinthManifest.java | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/me/itzg/helpers/http/SpecificFileFetchBuilder.java b/src/main/java/me/itzg/helpers/http/SpecificFileFetchBuilder.java index 5867ba07..0c8ae9f4 100644 --- a/src/main/java/me/itzg/helpers/http/SpecificFileFetchBuilder.java +++ b/src/main/java/me/itzg/helpers/http/SpecificFileFetchBuilder.java @@ -65,7 +65,7 @@ public Mono assemble() { final URI uri = uri(); if (skipExisting && Files.exists(file)) { - log.debug("File already exists and skip requested"); + log.debug("Skipping file={} that already exists due to request", file); statusHandler.call(FileDownloadStatus.SKIP_FILE_EXISTS, uri, file); return Mono.just(file); } diff --git a/src/main/java/me/itzg/helpers/modrinth/ModrinthCommand.java b/src/main/java/me/itzg/helpers/modrinth/ModrinthCommand.java index 6fefc597..5e5690c8 100644 --- a/src/main/java/me/itzg/helpers/modrinth/ModrinthCommand.java +++ b/src/main/java/me/itzg/helpers/modrinth/ModrinthCommand.java @@ -81,6 +81,7 @@ public Integer call() throws Exception { final ModrinthManifest newManifest = ModrinthManifest.builder() .files(Manifests.relativizeAll(outputDirectory, outputFiles)) + .projects(projects) .build(); Manifests.cleanup(outputDirectory, prevManifest, newManifest, log); diff --git a/src/main/java/me/itzg/helpers/modrinth/ModrinthManifest.java b/src/main/java/me/itzg/helpers/modrinth/ModrinthManifest.java index 14f57cf7..59cdfd87 100644 --- a/src/main/java/me/itzg/helpers/modrinth/ModrinthManifest.java +++ b/src/main/java/me/itzg/helpers/modrinth/ModrinthManifest.java @@ -1,5 +1,6 @@ package me.itzg.helpers.modrinth; +import java.util.List; import lombok.Getter; import lombok.experimental.SuperBuilder; import lombok.extern.jackson.Jacksonized; @@ -11,4 +12,6 @@ public class ModrinthManifest extends BaseManifest { public static final String ID = "modrinth"; + + List projects; }