From f9a5e4a3715f1ab3608631de4d5f4a3533d0debb Mon Sep 17 00:00:00 2001 From: Geoff Bourne Date: Mon, 14 Oct 2024 17:50:20 -0500 Subject: [PATCH 1/2] cf: API cache duration default is 2 days and can be configured --- .../helpers/{files => cache}/ApiCaching.java | 2 +- .../ApiCachingDisabled.java} | 4 +-- .../{files => cache}/ApiCachingImpl.java | 27 +++++++++++++++---- .../java/me/itzg/helpers/cache/CacheArgs.java | 15 +++++++++++ .../helpers/{files => cache}/CacheIndex.java | 2 +- .../curseforge/CurseForgeApiClient.java | 2 +- .../curseforge/CurseForgeFilesCommand.java | 13 ++++++--- .../curseforge/CurseForgeInstaller.java | 13 ++++++--- .../curseforge/InstallCurseForgeCommand.java | 7 ++++- .../helpers/modrinth/ModrinthApiClient.java | 4 +-- .../me/itzg/helpers/modrinth/ProjectRef.java | 12 ++++----- .../curseforge/CurseForgeApiClientTest.java | 4 +-- .../curseforge/ModFileRefResolverTest.java | 4 +-- .../itzg/helpers/modrinth/ProjectRefTest.java | 10 +++---- 14 files changed, 83 insertions(+), 36 deletions(-) rename src/main/java/me/itzg/helpers/{files => cache}/ApiCaching.java (89%) rename src/main/java/me/itzg/helpers/{files/DisabledApiCaching.java => cache/ApiCachingDisabled.java} (76%) rename src/main/java/me/itzg/helpers/{files => cache}/ApiCachingImpl.java (86%) create mode 100644 src/main/java/me/itzg/helpers/cache/CacheArgs.java rename src/main/java/me/itzg/helpers/{files => cache}/CacheIndex.java (91%) diff --git a/src/main/java/me/itzg/helpers/files/ApiCaching.java b/src/main/java/me/itzg/helpers/cache/ApiCaching.java similarity index 89% rename from src/main/java/me/itzg/helpers/files/ApiCaching.java rename to src/main/java/me/itzg/helpers/cache/ApiCaching.java index 40a229ae..08341302 100644 --- a/src/main/java/me/itzg/helpers/files/ApiCaching.java +++ b/src/main/java/me/itzg/helpers/cache/ApiCaching.java @@ -1,4 +1,4 @@ -package me.itzg.helpers.files; +package me.itzg.helpers.cache; import java.io.IOException; import reactor.core.publisher.Mono; diff --git a/src/main/java/me/itzg/helpers/files/DisabledApiCaching.java b/src/main/java/me/itzg/helpers/cache/ApiCachingDisabled.java similarity index 76% rename from src/main/java/me/itzg/helpers/files/DisabledApiCaching.java rename to src/main/java/me/itzg/helpers/cache/ApiCachingDisabled.java index ecc439d5..d126a170 100644 --- a/src/main/java/me/itzg/helpers/files/DisabledApiCaching.java +++ b/src/main/java/me/itzg/helpers/cache/ApiCachingDisabled.java @@ -1,9 +1,9 @@ -package me.itzg.helpers.files; +package me.itzg.helpers.cache; import java.io.IOException; import reactor.core.publisher.Mono; -public class DisabledApiCaching implements ApiCaching { +public class ApiCachingDisabled implements ApiCaching { @Override public Mono cache(String operation, Class returnType, Mono resolver, Object... keys) { diff --git a/src/main/java/me/itzg/helpers/files/ApiCachingImpl.java b/src/main/java/me/itzg/helpers/cache/ApiCachingImpl.java similarity index 86% rename from src/main/java/me/itzg/helpers/files/ApiCachingImpl.java rename to src/main/java/me/itzg/helpers/cache/ApiCachingImpl.java index 7de82221..d3674d02 100644 --- a/src/main/java/me/itzg/helpers/files/ApiCachingImpl.java +++ b/src/main/java/me/itzg/helpers/cache/ApiCachingImpl.java @@ -1,4 +1,4 @@ -package me.itzg.helpers.files; +package me.itzg.helpers.cache; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; @@ -6,6 +6,7 @@ import java.nio.file.Path; import java.time.Duration; import java.time.Instant; +import java.time.temporal.TemporalAmount; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -14,9 +15,10 @@ import java.util.stream.Stream; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import me.itzg.helpers.files.CacheIndex.CacheEntry; +import me.itzg.helpers.cache.CacheIndex.CacheEntry; import me.itzg.helpers.json.ObjectMappers; import org.jetbrains.annotations.Blocking; +import org.jetbrains.annotations.Nullable; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @@ -30,10 +32,17 @@ public class ApiCachingImpl implements ApiCaching { private final Path cacheNamespaceDir; @Setter - private Duration cacheDuration = Duration.ofHours(24); + private Duration defaultCacheDuration = Duration.ofHours(48); + + @Setter + private Map cacheDurations = new HashMap<>(); @Blocking - public ApiCachingImpl(Path outputDirectory, String namespace) throws IOException { + public ApiCachingImpl(Path outputDirectory, String namespace, @Nullable CacheArgs cacheArgs) throws IOException { + if (cacheArgs != null) { + defaultCacheDuration = cacheArgs.getDefaultCacheDuration(); + cacheDurations = cacheArgs.getCacheDurations(); + } objectMapper = ObjectMappers.defaultMapper(); cacheNamespaceDir = outputDirectory.resolve(CACHE_SUBIDR).resolve(namespace); cacheIndex = loadCacheIndex(); @@ -110,7 +119,9 @@ private Mono saveToCache(String operation, String keys, R value) { synchronized (cacheIndex) { cacheIndex.getOperations().computeIfAbsent(operation, s -> new HashMap<>()) .put(keys, new CacheEntry() - .setExpiresAt(Instant.now().plus(cacheDuration)) + .setExpiresAt(Instant.now().plus( + lookupCacheDuration(operation) + )) .setFilename(filename) ); } @@ -126,6 +137,12 @@ private Mono saveToCache(String operation, String keys, R value) { .subscribeOn(Schedulers.boundedElastic()); } + private TemporalAmount lookupCacheDuration(String operation) { + return cacheDurations != null ? + cacheDurations.getOrDefault(operation, defaultCacheDuration) + : defaultCacheDuration; + } + private Mono loadFromCache(String operation, String keys, CacheEntry entry, Class returnType) { return Mono.fromCallable(() -> { diff --git a/src/main/java/me/itzg/helpers/cache/CacheArgs.java b/src/main/java/me/itzg/helpers/cache/CacheArgs.java new file mode 100644 index 00000000..b1db8347 --- /dev/null +++ b/src/main/java/me/itzg/helpers/cache/CacheArgs.java @@ -0,0 +1,15 @@ +package me.itzg.helpers.cache; + +import java.time.Duration; +import java.util.Map; +import lombok.Data; +import picocli.CommandLine.Option; + +@Data +public class CacheArgs { + @Option(names = "--api-cache-ttl") + Map cacheDurations; + + @Option(names = "--api-cache-default-ttl", defaultValue = "P2D") + Duration defaultCacheDuration; +} diff --git a/src/main/java/me/itzg/helpers/files/CacheIndex.java b/src/main/java/me/itzg/helpers/cache/CacheIndex.java similarity index 91% rename from src/main/java/me/itzg/helpers/files/CacheIndex.java rename to src/main/java/me/itzg/helpers/cache/CacheIndex.java index 4414cc2a..161ca3c8 100644 --- a/src/main/java/me/itzg/helpers/files/CacheIndex.java +++ b/src/main/java/me/itzg/helpers/cache/CacheIndex.java @@ -1,4 +1,4 @@ -package me.itzg.helpers.files; +package me.itzg.helpers.cache; import java.time.Instant; import java.util.HashMap; diff --git a/src/main/java/me/itzg/helpers/curseforge/CurseForgeApiClient.java b/src/main/java/me/itzg/helpers/curseforge/CurseForgeApiClient.java index e7233427..c0434e66 100644 --- a/src/main/java/me/itzg/helpers/curseforge/CurseForgeApiClient.java +++ b/src/main/java/me/itzg/helpers/curseforge/CurseForgeApiClient.java @@ -10,6 +10,7 @@ import java.util.HashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import me.itzg.helpers.cache.ApiCaching; import me.itzg.helpers.curseforge.model.Category; import me.itzg.helpers.curseforge.model.CurseForgeFile; import me.itzg.helpers.curseforge.model.CurseForgeMod; @@ -21,7 +22,6 @@ import me.itzg.helpers.curseforge.model.ModsSearchResponse; import me.itzg.helpers.errors.GenericException; import me.itzg.helpers.errors.InvalidParameterException; -import me.itzg.helpers.files.ApiCaching; import me.itzg.helpers.http.FailedRequestException; import me.itzg.helpers.http.Fetch; import me.itzg.helpers.http.FileDownloadStatusHandler; diff --git a/src/main/java/me/itzg/helpers/curseforge/CurseForgeFilesCommand.java b/src/main/java/me/itzg/helpers/curseforge/CurseForgeFilesCommand.java index 0e120559..a6179dad 100644 --- a/src/main/java/me/itzg/helpers/curseforge/CurseForgeFilesCommand.java +++ b/src/main/java/me/itzg/helpers/curseforge/CurseForgeFilesCommand.java @@ -15,6 +15,10 @@ import java.util.concurrent.Callable; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import me.itzg.helpers.cache.ApiCaching; +import me.itzg.helpers.cache.ApiCachingDisabled; +import me.itzg.helpers.cache.ApiCachingImpl; +import me.itzg.helpers.cache.CacheArgs; import me.itzg.helpers.curseforge.CurseForgeFilesManifest.FileEntry; import me.itzg.helpers.curseforge.model.Category; import me.itzg.helpers.curseforge.model.CurseForgeFile; @@ -24,9 +28,6 @@ import me.itzg.helpers.curseforge.model.ModLoaderType; import me.itzg.helpers.errors.GenericException; import me.itzg.helpers.errors.InvalidParameterException; -import me.itzg.helpers.files.ApiCaching; -import me.itzg.helpers.files.ApiCachingImpl; -import me.itzg.helpers.files.DisabledApiCaching; import me.itzg.helpers.files.Manifests; import me.itzg.helpers.http.SharedFetchArgs; import org.jetbrains.annotations.NotNull; @@ -99,6 +100,9 @@ public void setSlugCategory(String defaultCategory) { @Option(names = "--disable-api-caching", defaultValue = "${env:CF_DISABLE_API_CACHING:-false}") boolean disableApiCaching; + @ArgGroup(exclusive = false) + CacheArgs cacheArgs; + @ArgGroup(exclusive = false) SharedFetchArgs sharedFetchArgs = new SharedFetchArgs(); @@ -123,7 +127,8 @@ public Integer call() throws Exception { if (modFileRefs != null && !modFileRefs.isEmpty()) { try ( - final ApiCaching apiCaching = disableApiCaching ? new DisabledApiCaching() : new ApiCachingImpl(outputDir, CACHING_NAMESPACE); + final ApiCaching apiCaching = disableApiCaching ? new ApiCachingDisabled() + : new ApiCachingImpl(outputDir, CACHING_NAMESPACE, cacheArgs); final CurseForgeApiClient apiClient = new CurseForgeApiClient( apiBaseUrl, apiKey, sharedFetchArgs.options(), CurseForgeApiClient.MINECRAFT_GAME_ID, diff --git a/src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java b/src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java index ba8a396b..54b60618 100644 --- a/src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java +++ b/src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java @@ -32,6 +32,10 @@ import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import me.itzg.helpers.cache.ApiCaching; +import me.itzg.helpers.cache.ApiCachingDisabled; +import me.itzg.helpers.cache.ApiCachingImpl; +import me.itzg.helpers.cache.CacheArgs; import me.itzg.helpers.curseforge.ExcludeIncludesContent.ExcludeIncludes; import me.itzg.helpers.curseforge.OverridesApplier.Result; import me.itzg.helpers.curseforge.model.Category; @@ -45,9 +49,6 @@ import me.itzg.helpers.errors.InvalidParameterException; import me.itzg.helpers.errors.RateLimitException; import me.itzg.helpers.fabric.FabricLauncherInstaller; -import me.itzg.helpers.files.ApiCaching; -import me.itzg.helpers.files.ApiCachingImpl; -import me.itzg.helpers.files.DisabledApiCaching; import me.itzg.helpers.files.Manifests; import me.itzg.helpers.files.ResultsFileWriter; import me.itzg.helpers.forge.ForgeInstaller; @@ -134,6 +135,9 @@ public class CurseForgeInstaller { @Getter @Setter private boolean disableApiCaching; + @Getter @Setter + private CacheArgs cacheArgs; + /** */ public void installFromModpackZip(Path modpackZip, String slug) { @@ -214,7 +218,8 @@ void install(String slug, InstallationEntryPoint entryPoint) { } try ( - final ApiCaching apiCaching = disableApiCaching ? new DisabledApiCaching() : new ApiCachingImpl(outputDir, CACHING_NAMESPACE); + final ApiCaching apiCaching = disableApiCaching ? new ApiCachingDisabled() + : new ApiCachingImpl(outputDir, CACHING_NAMESPACE, cacheArgs); final CurseForgeApiClient cfApi = new CurseForgeApiClient( apiBaseUrl, apiKey, sharedFetchOptions, CurseForgeApiClient.MINECRAFT_GAME_ID, diff --git a/src/main/java/me/itzg/helpers/curseforge/InstallCurseForgeCommand.java b/src/main/java/me/itzg/helpers/curseforge/InstallCurseForgeCommand.java index 0170b477..dc43e9e5 100644 --- a/src/main/java/me/itzg/helpers/curseforge/InstallCurseForgeCommand.java +++ b/src/main/java/me/itzg/helpers/curseforge/InstallCurseForgeCommand.java @@ -13,6 +13,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import me.itzg.helpers.McImageHelper; +import me.itzg.helpers.cache.CacheArgs; import me.itzg.helpers.curseforge.ModpacksPageUrlParser.Parsed; import me.itzg.helpers.files.ResultsFileWriter; import me.itzg.helpers.files.TabularOutput; @@ -163,6 +164,9 @@ static class Listed { @Option(names = "--disable-api-caching", defaultValue = "${env:CF_DISABLE_API_CACHING:-false}") boolean disableApiCaching; + @ArgGroup(exclusive = false) + CacheArgs cacheArgs; + @Override public Integer call() throws Exception { // https://www.curseforge.com/minecraft/modpacks/all-the-mods-8/files @@ -198,7 +202,8 @@ public Integer call() throws Exception { .setSharedFetchOptions(sharedFetchArgs.options()) .setApiKey(apiKey) .setDownloadsRepo(downloadsRepo) - .setDisableApiCaching(disableApiCaching); + .setDisableApiCaching(disableApiCaching) + .setCacheArgs(cacheArgs); if (apiBaseUrl != null) { installer.setApiBaseUrl(apiBaseUrl); diff --git a/src/main/java/me/itzg/helpers/modrinth/ModrinthApiClient.java b/src/main/java/me/itzg/helpers/modrinth/ModrinthApiClient.java index 63985399..2904e35d 100644 --- a/src/main/java/me/itzg/helpers/modrinth/ModrinthApiClient.java +++ b/src/main/java/me/itzg/helpers/modrinth/ModrinthApiClient.java @@ -133,8 +133,8 @@ public Mono resolveProjectVersion(Project project, ProjectRef projectRe .flatMap(versions -> Mono.justOrEmpty(versions.stream() .filter(version -> - version.getVersionNumber().equals(projectRef.getVersionName()) - || version.getName().equals(projectRef.getVersionName()) + version.getVersionNumber().equals(projectRef.getVersionNumber()) + || version.getName().equals(projectRef.getVersionNumber()) ) .findFirst() )); diff --git a/src/main/java/me/itzg/helpers/modrinth/ProjectRef.java b/src/main/java/me/itzg/helpers/modrinth/ProjectRef.java index 1c40d1c1..7dfb780d 100644 --- a/src/main/java/me/itzg/helpers/modrinth/ProjectRef.java +++ b/src/main/java/me/itzg/helpers/modrinth/ProjectRef.java @@ -33,7 +33,7 @@ public class ProjectRef { private final URI projectUri; private final VersionType versionType; private final String versionId; - private final String versionName; + private final String versionNumber; public static ProjectRef parse(String projectRef) { final Matcher m = PROJECT_REF.matcher(projectRef); @@ -67,16 +67,16 @@ public ProjectRef(String projectSlug, @Nullable String version, boolean datapack if (this.versionType == null) { if (isVersionId(version)) { this.versionId = version; - this.versionName = null; + this.versionNumber = null; } else { this.versionId = null; - this.versionName = version; + this.versionNumber = version; } } else { this.versionId = null; - this.versionName = null; + this.versionNumber = null; } } @@ -88,7 +88,7 @@ public ProjectRef(URI projectUri, String versionId) { this.idOrSlug = filename.endsWith(".mrpack") ? filename.substring(0, filename.length() - ".mrpack".length()) : filename; - this.versionName = null; + this.versionNumber = null; this.versionType = null; this.versionId = versionId; } @@ -138,7 +138,7 @@ public static ProjectRef fromPossibleUrl( } public boolean hasVersionName() { - return versionName != null; + return versionNumber != null; } public boolean hasVersionType() { diff --git a/src/test/java/me/itzg/helpers/curseforge/CurseForgeApiClientTest.java b/src/test/java/me/itzg/helpers/curseforge/CurseForgeApiClientTest.java index 8ac6b392..702bc8d1 100644 --- a/src/test/java/me/itzg/helpers/curseforge/CurseForgeApiClientTest.java +++ b/src/test/java/me/itzg/helpers/curseforge/CurseForgeApiClientTest.java @@ -6,7 +6,7 @@ import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; import java.util.Collections; -import me.itzg.helpers.files.DisabledApiCaching; +import me.itzg.helpers.cache.ApiCachingDisabled; import me.itzg.helpers.http.SharedFetch.Options; import org.intellij.lang.annotations.Language; import org.junit.jupiter.api.Test; @@ -26,7 +26,7 @@ void apiKeyHeaderIsTrimmed(WireMockRuntimeInfo wmInfo) { final CategoryInfo result; try (CurseForgeApiClient client = new CurseForgeApiClient(wmInfo.getHttpBaseUrl(), "key\n", Options.builder().build(), - "test", new DisabledApiCaching() + "test", new ApiCachingDisabled() )) { result = client.loadCategoryInfo(Collections.singleton("mc-mods")) .block(); diff --git a/src/test/java/me/itzg/helpers/curseforge/ModFileRefResolverTest.java b/src/test/java/me/itzg/helpers/curseforge/ModFileRefResolverTest.java index 1a656109..5a08a007 100644 --- a/src/test/java/me/itzg/helpers/curseforge/ModFileRefResolverTest.java +++ b/src/test/java/me/itzg/helpers/curseforge/ModFileRefResolverTest.java @@ -10,9 +10,9 @@ import java.util.Collections; import java.util.List; import java.util.stream.Stream; +import me.itzg.helpers.cache.ApiCachingDisabled; import me.itzg.helpers.curseforge.model.CurseForgeFile; import me.itzg.helpers.curseforge.model.ModLoaderType; -import me.itzg.helpers.files.DisabledApiCaching; import me.itzg.helpers.http.SharedFetch.Options; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; @@ -53,7 +53,7 @@ public static Stream testMcModsRefsArgs() { void testMcModsRefs(String ref, String defaultCategory, String gameVersion, ModLoaderType modLoader, ModFileIds expected) { final CurseForgeApiClient apiClient = new CurseForgeApiClient(wm.baseUrl(), "testing", Options.builder().build(), - CurseForgeApiClient.MINECRAFT_GAME_ID, new DisabledApiCaching() + CurseForgeApiClient.MINECRAFT_GAME_ID, new ApiCachingDisabled() ); final CategoryInfo categoryInfo = apiClient.loadCategoryInfo( diff --git a/src/test/java/me/itzg/helpers/modrinth/ProjectRefTest.java b/src/test/java/me/itzg/helpers/modrinth/ProjectRefTest.java index f5e698b4..d40e45cb 100644 --- a/src/test/java/me/itzg/helpers/modrinth/ProjectRefTest.java +++ b/src/test/java/me/itzg/helpers/modrinth/ProjectRefTest.java @@ -61,7 +61,7 @@ void testProjectRefHasVersionNameForOtherValues() { projectRefUT = new ProjectRef( this.expectedSlug, this.expectedVersionName); - assertThat(projectRefUT.getVersionName()) + assertThat(projectRefUT.getVersionNumber()) .isEqualTo(expectedVersionName); assertThat(projectRefUT.hasVersionId()).isFalse(); assertThat(projectRefUT.hasVersionType()).isFalse(); @@ -74,7 +74,7 @@ void fromPossibleUrlDefaultsToGeneratingRefWithPassedValues() { assertThat(projectRefUT.getIdOrSlug()) .isEqualTo(this.expectedSlug); - assertThat(projectRefUT.getVersionName()) + assertThat(projectRefUT.getVersionNumber()) .isEqualTo(expectedVersionName); } @@ -88,7 +88,7 @@ void fromPossibleUrlExtractsProjectSlugFromUrl() { assertThat(projectRefUT.getIdOrSlug()) .isEqualTo(this.expectedSlug); - assertThat(projectRefUT.getVersionName()) + assertThat(projectRefUT.getVersionNumber()) .isEqualTo(expectedVersionName); } @@ -102,7 +102,7 @@ void fromPossibleUrlExtractsProjectVersionFromUrlWhenPresent() { assertThat(projectRefUT.getIdOrSlug()) .isEqualTo(this.expectedSlug); - assertThat(projectRefUT.getVersionName()) + assertThat(projectRefUT.getVersionNumber()) .isEqualTo(expectedVersionName); } @@ -144,7 +144,7 @@ void parseProjectRef(String input, String slugId, VersionType versionType, Strin assertThat(result.getIdOrSlug()).isEqualTo(slugId); assertThat(result.getVersionType()).isEqualTo(versionType); assertThat(result.getVersionId()).isEqualTo(versionId); - assertThat(result.getVersionName()).isEqualTo(versionName); + assertThat(result.getVersionNumber()).isEqualTo(versionName); assertThat(result.isDatapack()).isEqualTo(datapack); } From dd26d6aa42e79f195e68f617845723d993bbc878 Mon Sep 17 00:00:00 2001 From: Geoff Bourne Date: Mon, 14 Oct 2024 18:16:01 -0500 Subject: [PATCH 2/2] Add option descriptions --- src/main/java/me/itzg/helpers/cache/CacheArgs.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/me/itzg/helpers/cache/CacheArgs.java b/src/main/java/me/itzg/helpers/cache/CacheArgs.java index b1db8347..63ae6d7b 100644 --- a/src/main/java/me/itzg/helpers/cache/CacheArgs.java +++ b/src/main/java/me/itzg/helpers/cache/CacheArgs.java @@ -7,9 +7,13 @@ @Data public class CacheArgs { - @Option(names = "--api-cache-ttl") + @Option(names = "--api-cache-ttl", paramLabel = "OPERATION=DURATION", + description = "Set individual operation TTLs" + ) Map cacheDurations; - @Option(names = "--api-cache-default-ttl", defaultValue = "P2D") + @Option(names = "--api-cache-default-ttl", defaultValue = "P2D", paramLabel = "DURATION", + description = "Set default/fallback TTL in ISO-8601 duration format.\nDefault: ${DEFAULT-VALUE}" + ) Duration defaultCacheDuration; }