From f6aec2f5aa802dd09fb7ebf18912e82e828348f2 Mon Sep 17 00:00:00 2001 From: Geoff Bourne Date: Sat, 9 Aug 2025 15:57:48 -0500 Subject: [PATCH 1/4] fabric: use Fabric installer to retrieve server launcher Also - validate fabric launcher jar and retry --- src/main/java/me/itzg/helpers/McImageHelper.java | 2 ++ .../helpers/errors/InvalidContentException.java | 13 +++++++++++++ .../http/OutputToDirectoryFetchBuilder.java | 4 ++++ .../java/me/itzg/helpers/http/SharedFetch.java | 15 +++++++++++---- 4 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 src/main/java/me/itzg/helpers/errors/InvalidContentException.java diff --git a/src/main/java/me/itzg/helpers/McImageHelper.java b/src/main/java/me/itzg/helpers/McImageHelper.java index 30d39d53..51502975 100644 --- a/src/main/java/me/itzg/helpers/McImageHelper.java +++ b/src/main/java/me/itzg/helpers/McImageHelper.java @@ -150,6 +150,8 @@ private static void setLevel(boolean enabled, Level level) { if (Level.TRACE.isGreaterOrEqual(level)) { ((Logger) LoggerFactory.getLogger("org.apache.hc.client5.http")).setLevel( enabled ? level : Level.INFO); + ((Logger) LoggerFactory.getLogger("reactor.netty.http.client.HttpClient")).setLevel( + enabled ? level : Level.INFO); } } diff --git a/src/main/java/me/itzg/helpers/errors/InvalidContentException.java b/src/main/java/me/itzg/helpers/errors/InvalidContentException.java new file mode 100644 index 00000000..d71eb109 --- /dev/null +++ b/src/main/java/me/itzg/helpers/errors/InvalidContentException.java @@ -0,0 +1,13 @@ +package me.itzg.helpers.errors; + +import java.io.IOException; + +public class InvalidContentException extends IOException { + + public InvalidContentException(String message) { + super(message); + } + public InvalidContentException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/me/itzg/helpers/http/OutputToDirectoryFetchBuilder.java b/src/main/java/me/itzg/helpers/http/OutputToDirectoryFetchBuilder.java index 76886d28..00cf67af 100644 --- a/src/main/java/me/itzg/helpers/http/OutputToDirectoryFetchBuilder.java +++ b/src/main/java/me/itzg/helpers/http/OutputToDirectoryFetchBuilder.java @@ -231,6 +231,10 @@ else if (skipUpToDate) { return failedRequestMono(resp, byteBufFlux.aggregate(), "Downloading file"); } + if (log.isTraceEnabled()) { + resp.responseHeaders().forEach(header -> log.trace("Response header: {}={}", header.getKey(), header.getValue())); + } + return copyBodyInputStreamToFile(byteBufFlux, outputFile); }) .last() diff --git a/src/main/java/me/itzg/helpers/http/SharedFetch.java b/src/main/java/me/itzg/helpers/http/SharedFetch.java index 2ab46258..8ea107bc 100644 --- a/src/main/java/me/itzg/helpers/http/SharedFetch.java +++ b/src/main/java/me/itzg/helpers/http/SharedFetch.java @@ -12,9 +12,11 @@ import lombok.extern.slf4j.Slf4j; import me.itzg.helpers.McImageHelper; import me.itzg.helpers.errors.GenericException; -import reactor.netty.http.Http11SslContextSpec; +import reactor.netty.http.Http2SslContextSpec; +import reactor.netty.http.HttpProtocol; import reactor.netty.http.client.HttpClient; import reactor.netty.resources.ConnectionProvider; +import reactor.netty.tcp.SslProvider.GenericSslContextSpec; /** * Provides an efficient way to make multiple web requests since a single client @@ -58,6 +60,8 @@ public SharedFetch(String forCommand, Options options) { reactiveClient = HttpClient.create(connectionProvider) .proxyWithSystemProperties() + // https://projectreactor.io/docs/netty/release/reference/http-client.html#HTTP2 + .protocol(HttpProtocol.HTTP11, HttpProtocol.H2) .headers(headers -> { headers .set(HttpHeaderNames.USER_AGENT.toString(), userAgent) @@ -69,11 +73,14 @@ public SharedFetch(String forCommand, Options options) { ) // Reference https://projectreactor.io/docs/netty/release/reference/index.html#response-timeout .responseTimeout(options.getResponseTimeout()) - // Reference https://projectreactor.io/docs/netty/release/reference/index.html#ssl-tls-timeout .secure(spec -> - spec.sslContext(Http11SslContextSpec.forClient()) + // Http2 SSL supports both HTTP/2 and HTTP/1.1 + spec.sslContext((GenericSslContextSpec) Http2SslContextSpec.forClient()) + // Reference https://projectreactor.io/docs/netty/release/reference/index.html#ssl-tls-timeout .handshakeTimeout(options.getTlsHandshakeTimeout()) - ); + ) + + ; headers.put("x-fetch-session", fetchSessionId); From 2025819efeb2a6fabf8ad3967894f0a6a61a46cd Mon Sep 17 00:00:00 2001 From: Geoff Bourne Date: Sat, 9 Aug 2025 15:58:48 -0500 Subject: [PATCH 2/4] fabric: use Fabric installer to retrieve server launcher Also - validate fabric launcher jar and retry --- .../itzg/helpers/fabric/FabricMetaClient.java | 56 ++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/src/main/java/me/itzg/helpers/fabric/FabricMetaClient.java b/src/main/java/me/itzg/helpers/fabric/FabricMetaClient.java index 3fc930e3..ad042c2e 100644 --- a/src/main/java/me/itzg/helpers/fabric/FabricMetaClient.java +++ b/src/main/java/me/itzg/helpers/fabric/FabricMetaClient.java @@ -1,20 +1,28 @@ package me.itzg.helpers.fabric; +import java.io.BufferedReader; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.util.List; +import java.util.Properties; import java.util.function.Predicate; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import me.itzg.helpers.errors.GenericException; +import me.itzg.helpers.errors.InvalidContentException; +import me.itzg.helpers.files.IoStreams; import me.itzg.helpers.http.FileDownloadStatusHandler; import me.itzg.helpers.http.SharedFetch; import me.itzg.helpers.http.UriBuilder; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; import reactor.util.retry.Retry; +@Slf4j public class FabricMetaClient { private final SharedFetch sharedFetch; @@ -32,7 +40,7 @@ public class FabricMetaClient { private Duration retryMinBackoff = Duration.ofMillis(500); @Setter - private int downloadRetryMaxAttempts = 5; + private int downloadRetryMaxAttempts = 10; @Setter private Duration downloadRetryMinBackoff = Duration.ofMillis(500); @@ -146,9 +154,55 @@ public Mono downloadLauncher( .handleStatus(statusHandler) .assemble() .retryWhen(Retry.backoff(downloadRetryMaxAttempts, downloadRetryMinBackoff).filter(IOException.class::isInstance)) + .flatMap(this::validateLauncherJar) + .retryWhen(Retry.backoff(downloadRetryMaxAttempts, downloadRetryMinBackoff).filter(InvalidContentException.class::isInstance)) .checkpoint("downloadLauncher"); } + private Mono validateLauncherJar(Path path) { + return Mono.fromCallable(() -> { + log.debug("Validating Fabric launcher file {}", path); + + if (!path.toFile().isFile()) { + throw new InvalidContentException("Downloaded launcher jar is not a file"); + } + try { + final Properties installProperties = IoStreams.readFileFromZip(path, "install.properties", in -> { + Properties p = new Properties(); + p.load(in); + return p; + } + ); + if (installProperties == null) { + debugDownloadedContent(path); + throw new InvalidContentException("Downloaded launcher jar does not contain an install.properties"); + } + if (!installProperties.containsKey("game-version")) { + debugDownloadedContent(path); + throw new InvalidContentException("Downloaded launcher jar does not contain a valid install.properties"); + } + } catch (IOException e) { + debugDownloadedContent(path); + throw new InvalidContentException("Downloaded launcher jar could not be read as a jar/zip", e); + } + + return path; + }) + .subscribeOn(Schedulers.boundedElastic()); + } + + private static void debugDownloadedContent(Path path) { + if (log.isDebugEnabled()) { + try (BufferedReader reader = Files.newBufferedReader(path)) { + final char[] buf = new char[100]; + final int amount = reader.read(buf); + log.debug("Downloaded launcher jar content starts with: {}", new String(buf, 0, amount)); + } catch (IOException e) { + throw new GenericException("Failed to read downloaded launcher jar for debugging", e); + } + } + } + @NotNull private static Mono findFirst(List versionEntries, Predicate condition ) { From cccbb7dade5035ea10397cf59eeb55d716b33ed3 Mon Sep 17 00:00:00 2001 From: Geoff Bourne Date: Sat, 9 Aug 2025 16:16:59 -0500 Subject: [PATCH 3/4] Allow for skipping validation --- .../helpers/fabric/FabricLauncherInstaller.java | 10 +++++++++- .../me/itzg/helpers/fabric/FabricMetaClient.java | 6 ++++-- .../helpers/fabric/FabricLauncherInstallerTest.java | 13 ++++++++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/main/java/me/itzg/helpers/fabric/FabricLauncherInstaller.java b/src/main/java/me/itzg/helpers/fabric/FabricLauncherInstaller.java index 00112a10..34489121 100644 --- a/src/main/java/me/itzg/helpers/fabric/FabricLauncherInstaller.java +++ b/src/main/java/me/itzg/helpers/fabric/FabricLauncherInstaller.java @@ -8,6 +8,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Properties; +import lombok.AccessLevel; import lombok.Getter; import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -42,6 +43,12 @@ public class FabricLauncherInstaller { @Getter @Setter private boolean forceReinstall; + /** + * For testing purposes. + */ + @Getter @Setter(AccessLevel.PACKAGE) + private boolean skipValidation; + public void installUsingVersions( Options sharedFetchOptions, @NonNull String minecraftVersion, @Nullable String loaderVersion, @@ -97,7 +104,8 @@ private Mono downloadResolvedLauncher(FabricMetaClient fabricMet if (needsInstall) { return fabricMetaClient.downloadLauncher( outputDir, minecraftVersion, loaderVersion, installerVersion, - Fetch.loggingDownloadStatusHandler(log) + Fetch.loggingDownloadStatusHandler(log), + skipValidation ) .publishOn(Schedulers.boundedElastic()) .flatMap(launcherPath -> diff --git a/src/main/java/me/itzg/helpers/fabric/FabricMetaClient.java b/src/main/java/me/itzg/helpers/fabric/FabricMetaClient.java index ad042c2e..1ecc23fb 100644 --- a/src/main/java/me/itzg/helpers/fabric/FabricMetaClient.java +++ b/src/main/java/me/itzg/helpers/fabric/FabricMetaClient.java @@ -142,7 +142,8 @@ public Mono resolveInstallerVersion(String installerVersion) { public Mono downloadLauncher( Path outputDir, String minecraftVersion, String loaderVersion, String installerVersion, - FileDownloadStatusHandler statusHandler + FileDownloadStatusHandler statusHandler, + boolean skipValidation ) { return sharedFetch.fetch( uriBuilder.resolve( @@ -154,7 +155,8 @@ public Mono downloadLauncher( .handleStatus(statusHandler) .assemble() .retryWhen(Retry.backoff(downloadRetryMaxAttempts, downloadRetryMinBackoff).filter(IOException.class::isInstance)) - .flatMap(this::validateLauncherJar) + .flatMap(path -> skipValidation ? Mono.just(path) : validateLauncherJar(path)) + .doOnError(InvalidContentException.class, e -> log.warn("Invalid launcher jar, will try again", e)) .retryWhen(Retry.backoff(downloadRetryMaxAttempts, downloadRetryMinBackoff).filter(InvalidContentException.class::isInstance)) .checkpoint("downloadLauncher"); } diff --git a/src/test/java/me/itzg/helpers/fabric/FabricLauncherInstallerTest.java b/src/test/java/me/itzg/helpers/fabric/FabricLauncherInstallerTest.java index 3faf3e0c..2eec63a2 100644 --- a/src/test/java/me/itzg/helpers/fabric/FabricLauncherInstallerTest.java +++ b/src/test/java/me/itzg/helpers/fabric/FabricLauncherInstallerTest.java @@ -56,6 +56,7 @@ void testInstallUsingVersions_onlyGameVersion(WireMockRuntimeInfo wmRuntimeInfo) final FabricLauncherInstaller installer = new FabricLauncherInstaller( tempDir ) + .setSkipValidation(true) .setResultsFile(resultsFile); installer.setFabricMetaBaseUrl(wmRuntimeInfo.getHttpBaseUrl()); @@ -106,6 +107,7 @@ void testWithProvidedUri(WireMockRuntimeInfo wmRuntimeInfo) throws IOException { final Path expectedResultsPath = tempDir.resolve("results.env"); final FabricLauncherInstaller installer = new FabricLauncherInstaller(tempDir) + .setSkipValidation(true) .setResultsFile(expectedResultsPath); final URI loaderUri = URI.create(wmRuntimeInfo.getHttpBaseUrl() + "/fabric-launcher.jar"); @@ -142,7 +144,8 @@ void testWithProvidedUri_contentDisposition(WireMockRuntimeInfo wmRuntimeInfo) t ) ); - final FabricLauncherInstaller installer = new FabricLauncherInstaller(tempDir); + final FabricLauncherInstaller installer = new FabricLauncherInstaller(tempDir) + .setSkipValidation(true); installer.installUsingUri( Options.builder().build(), URI.create(wmRuntimeInfo.getHttpBaseUrl() + "/server") ); @@ -157,6 +160,7 @@ void testWithProvidedUri_contentDisposition(WireMockRuntimeInfo wmRuntimeInfo) t void testWithLocalLauncherFile() throws IOException { final Path expectedResultsPath = tempDir.resolve("results.env"); final FabricLauncherInstaller installer = new FabricLauncherInstaller(tempDir) + .setSkipValidation(true) .setResultsFile(expectedResultsPath); final Path launcherFile = Paths.get("src/test/resources/__files/fabric-empty-launcher.jar"); @@ -179,7 +183,8 @@ void testUpgradeFromVersionToVersion(WireMockRuntimeInfo wmRuntimeInfo) { final FabricLauncherInstaller installer = new FabricLauncherInstaller( tempDir - ); + ) + .setSkipValidation(true); installer.setFabricMetaBaseUrl(wmRuntimeInfo.getHttpBaseUrl()); installer.installUsingVersions( @@ -213,7 +218,8 @@ void testNoNetworkUsageWhenVersionMatches(WireMockRuntimeInfo wmRuntimeInfo) { final FabricLauncherInstaller installer = new FabricLauncherInstaller( tempDir - ); + ) + .setSkipValidation(true); installer.setFabricMetaBaseUrl(wmRuntimeInfo.getHttpBaseUrl()); installer.installUsingVersions( @@ -255,6 +261,7 @@ void testNoNetworkUsageWhenVersionMatches(WireMockRuntimeInfo wmRuntimeInfo) { void forRecordingVersionDiscovery() { final Path resultsFile = tempDir.resolve("results.env"); final FabricLauncherInstaller installer = new FabricLauncherInstaller(tempDir) + .setSkipValidation(true) .setResultsFile(resultsFile); installer.setFabricMetaBaseUrl("http://localhost:8080"); From 8acd78831e461a27a8a5aedd8903c593eaaeaaac Mon Sep 17 00:00:00 2001 From: Geoff Bourne Date: Sat, 9 Aug 2025 16:29:46 -0500 Subject: [PATCH 4/4] Added a bit more Modrinth modpack logging --- .../java/me/itzg/helpers/modrinth/ModrinthApiClient.java | 5 ++--- .../me/itzg/helpers/modrinth/ModrinthHttpPackFetcher.java | 4 +--- .../me/itzg/helpers/modrinth/ModrinthPackInstaller.java | 6 +++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/main/java/me/itzg/helpers/modrinth/ModrinthApiClient.java b/src/main/java/me/itzg/helpers/modrinth/ModrinthApiClient.java index f455bd04..124a88bf 100644 --- a/src/main/java/me/itzg/helpers/modrinth/ModrinthApiClient.java +++ b/src/main/java/me/itzg/helpers/modrinth/ModrinthApiClient.java @@ -20,7 +20,6 @@ import me.itzg.helpers.errors.InvalidParameterException; import me.itzg.helpers.http.FailedRequestException; import me.itzg.helpers.http.Fetch; -import me.itzg.helpers.http.FileDownloadedHandler; import me.itzg.helpers.http.SharedFetch; import me.itzg.helpers.http.SharedFetch.Options; import me.itzg.helpers.http.UriBuilder; @@ -266,10 +265,10 @@ public void close() { sharedFetch.close(); } - public Mono downloadFileFromUrl(Path outputFile, URI uri, FileDownloadedHandler fileDownloadedHandler) { + public Mono downloadFileFromUrl(Path outputFile, URI uri) { return sharedFetch.fetch(uri) .toFile(outputFile) - .handleDownloaded(fileDownloadedHandler) + .handleStatus(Fetch.loggingDownloadStatusHandler(log)) .skipExisting(true) .assemble(); } diff --git a/src/main/java/me/itzg/helpers/modrinth/ModrinthHttpPackFetcher.java b/src/main/java/me/itzg/helpers/modrinth/ModrinthHttpPackFetcher.java index 347b0376..2a4706b0 100644 --- a/src/main/java/me/itzg/helpers/modrinth/ModrinthHttpPackFetcher.java +++ b/src/main/java/me/itzg/helpers/modrinth/ModrinthHttpPackFetcher.java @@ -22,9 +22,7 @@ public class ModrinthHttpPackFetcher implements ModrinthPackFetcher { @Override public Mono fetchModpack(ModrinthModpackManifest prevManifest) { return apiClient.downloadFileFromUrl( - destFilePath, modpackUri, - (uri, file, contentSizeBytes) -> - log.info("Downloaded {}", destFilePath) + destFilePath, modpackUri ) .map(mrPackFile -> new FetchedPack(mrPackFile, "custom", deriveVersionId(), deriveVersionName())); } diff --git a/src/main/java/me/itzg/helpers/modrinth/ModrinthPackInstaller.java b/src/main/java/me/itzg/helpers/modrinth/ModrinthPackInstaller.java index 0159d014..68ea880d 100644 --- a/src/main/java/me/itzg/helpers/modrinth/ModrinthPackInstaller.java +++ b/src/main/java/me/itzg/helpers/modrinth/ModrinthPackInstaller.java @@ -118,6 +118,8 @@ public Mono processModpack(SharedFetch sharedFetch) { )); } + log.info("Processing modpack files for {} {}", modpackIndex.getName(), modpackIndex.getVersionId()); + return processModFiles(modpackIndex) .collectList() .map(modFiles -> @@ -168,9 +170,7 @@ private Flux processModFiles(ModpackIndex modpackIndex) { return this.apiClient.downloadFileFromUrl( outFilePath, - modpackFile.getDownloads().get(0), - (uri, file, contentSizeBytes) -> - log.info("Downloaded {}", modpackFilePath) + modpackFile.getDownloads().get(0) ); }, maxConcurrentDownloads); }