manami-project · manami-project · Jan 9, 2025 · Jan 9, 2025 · Jan 9, 2025 · Jan 9, 2025
diff --git a/README.md b/README.md
@@ -32,3 +32,11 @@ Manami creates an index file for the anime that you already watched and stored o
 * Requires JDK 21 and the possibility to run JavaFX
 * Download the *.jar file of the latest release
 * No installation or additional setup needed. Just Download the `*.jar` and start it by double click or via console `java -jar manami.jar`.
+
+## Configuration
+
+See ["Configuration Management"](https://github.com/manami-project/modb-core/tree/master#configuration-management)
+
+| parameter                    | type      | default | description                                                                                                  |
+|------------------------------|-----------|---------|--------------------------------------------------------------------------------------------------------------|
+| `manami.cache.useLocalFiles` | `Boolean` | `true`  | Downloads anime-offline-database files once and stores them next to the *.jar file. Redownload after 24 hrs. |
diff --git a/manami-app/src/main/kotlin/io/github/manamiproject/manami/app/Manami.kt b/manami-app/src/main/kotlin/io/github/manamiproject/manami/app/Manami.kt
@@ -21,9 +21,13 @@ import io.github.manamiproject.manami.app.search.SearchHandler
 import io.github.manamiproject.manami.app.versioning.DefaultLatestVersionChecker
 import io.github.manamiproject.modb.anidb.AnidbConfig
 import io.github.manamiproject.modb.anilist.AnilistConfig
+import io.github.manamiproject.modb.core.coroutines.ModbDispatchers
 import io.github.manamiproject.modb.core.logging.LoggerDelegate
 import io.github.manamiproject.modb.kitsu.KitsuConfig
 import io.github.manamiproject.modb.myanimelist.MyanimelistConfig
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
 import java.net.URI
 import java.util.concurrent.Executors
 import java.util.concurrent.atomic.AtomicReference
@@ -50,12 +54,14 @@ class Manami(
         log.info {"Starting manami" }
         SimpleEventBus.subscribe(this)
         runInBackground {
-            DefaultLatestVersionChecker().checkLatestVersion()
-            AnimeCachePopulator().populate(DefaultAnimeCache.instance)
-            DeadEntriesCachePopulator(config = AnidbConfig, url = URI("$DEAD_ENTRIES_BASE_URL/anidb.zip").toURL()).populate(DefaultAnimeCache.instance)
-            DeadEntriesCachePopulator(config = AnilistConfig, url = URI("$DEAD_ENTRIES_BASE_URL/anilist.zip").toURL()).populate(DefaultAnimeCache.instance)
-            DeadEntriesCachePopulator(config = KitsuConfig, url = URI("$DEAD_ENTRIES_BASE_URL/kitsu.zip").toURL()).populate(DefaultAnimeCache.instance)
-            DeadEntriesCachePopulator(config = MyanimelistConfig, url = URI("$DEAD_ENTRIES_BASE_URL/myanimelist.zip").toURL()).populate(DefaultAnimeCache.instance)
+            withContext(ModbDispatchers.LIMITED_CPU) {
+                launch { DefaultLatestVersionChecker().checkLatestVersion() }
+                launch { AnimeCachePopulator().populate(DefaultAnimeCache.instance) }
+                launch { DeadEntriesCachePopulator(config = AnidbConfig, url = URI("$DEAD_ENTRIES_BASE_URL/anidb.zip").toURL()).populate(DefaultAnimeCache.instance) }
+                launch { DeadEntriesCachePopulator(config = AnilistConfig, url = URI("$DEAD_ENTRIES_BASE_URL/anilist.zip").toURL()).populate(DefaultAnimeCache.instance) }
+                launch { DeadEntriesCachePopulator(config = KitsuConfig, url = URI("$DEAD_ENTRIES_BASE_URL/kitsu.zip").toURL()).populate(DefaultAnimeCache.instance) }
+                launch { DeadEntriesCachePopulator(config = MyanimelistConfig, url = URI("$DEAD_ENTRIES_BASE_URL/myanimelist.zip").toURL()).populate(DefaultAnimeCache.instance) }
+            }
         }
     }
 
@@ -85,8 +91,10 @@ class Manami(
 
 private val backgroundTasks = Executors.newCachedThreadPool()
 
-internal fun runInBackground(action: () -> Unit) {
+internal fun runInBackground(action: suspend () -> Unit) {
     backgroundTasks.submit {
-        action.invoke()
+        runBlocking {
+            action.invoke()
+        }
     }
 }
diff --git a/...src/main/kotlin/io/github/manamiproject/manami/app/cache/populator/AnimeCachePopulator.kt b/...src/main/kotlin/io/github/manamiproject/manami/app/cache/populator/AnimeCachePopulator.kt
@@ -5,25 +5,74 @@ import io.github.manamiproject.manami.app.cache.CacheEntry
 import io.github.manamiproject.manami.app.cache.PresentValue
 import io.github.manamiproject.manami.app.events.EventBus
 import io.github.manamiproject.manami.app.events.SimpleEventBus
+import io.github.manamiproject.modb.core.config.BooleanPropertyDelegate
+import io.github.manamiproject.modb.core.config.ConfigRegistry
+import io.github.manamiproject.modb.core.config.DefaultConfigRegistry
+import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_FS
+import io.github.manamiproject.modb.core.extensions.regularFileExists
+import io.github.manamiproject.modb.core.extensions.writeToFile
+import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient
+import io.github.manamiproject.modb.core.httpclient.HttpClient
 import io.github.manamiproject.modb.core.logging.LoggerDelegate
 import io.github.manamiproject.modb.core.models.Anime
 import io.github.manamiproject.modb.serde.json.AnimeListJsonStringDeserializer
 import io.github.manamiproject.modb.serde.json.DefaultExternalResourceJsonDeserializer
 import io.github.manamiproject.modb.serde.json.ExternalResourceJsonDeserializer
 import io.github.manamiproject.modb.serde.json.models.Dataset
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
 import java.net.URI
+import java.nio.file.Files
+import java.nio.file.attribute.BasicFileAttributes
+import java.time.LocalDate
+import java.time.ZoneId
+import java.time.temporal.ChronoUnit
+import kotlin.io.path.Path
 
 internal class AnimeCachePopulator(
-    private val uri: URI = URI("https://raw.githubusercontent.com/manami-project/anime-offline-database/master/anime-offline-database.zip"),
+    private val fileName: String = "anime-offline-database.zip",
+    private val uri: URI = URI("https://raw.githubusercontent.com/manami-project/anime-offline-database/master/$fileName"),
     private val parser: ExternalResourceJsonDeserializer<Dataset> = DefaultExternalResourceJsonDeserializer(deserializer = AnimeListJsonStringDeserializer.instance),
     private val eventBus: EventBus = SimpleEventBus,
+    private val httpClient: HttpClient = DefaultHttpClient.instance,
+    configRegistry: ConfigRegistry = DefaultConfigRegistry.instance,
 ) : CachePopulator<URI, CacheEntry<Anime>> {
 
-    override fun populate(cache: Cache<URI, CacheEntry<Anime>>) {
-        log.info {"Populating cache with anime from [$uri]." }
+    private val isUseLocalFiles by BooleanPropertyDelegate(
+        namespace = "manami.cache.useLocalFiles",
+        configRegistry = configRegistry,
+        default = true
+    )
 
-        val parsedAnime = runBlocking { parser.deserialize(uri.toURL()).data }
+    override suspend fun populate(cache: Cache<URI, CacheEntry<Anime>>) {
+        val parsedAnime = if (isUseLocalFiles) {
+            val file = Path(fileName)
+
+            val isDownloadFile = if (!file.regularFileExists()) {
+                true
+            } else {
+                val attrs = withContext(LIMITED_FS) {
+                    Files.readAttributes(file, BasicFileAttributes::class.java)
+                }
+                val creationTime = attrs.creationTime()
+                    .toInstant()
+                    .atZone(ZoneId.systemDefault())
+                    .toLocalDate()
+
+                ChronoUnit.DAYS.between(creationTime, LocalDate.now()) >= 1L
+            }
+
+            if (isDownloadFile) {
+                log.info {"Downloading dataset from [$uri], because a local file doesn't exist." }
+                httpClient.get(uri.toURL()).body.writeToFile(file)
+            }
+
+            log.info {"Populating cache with anime." }
+
+            parser.deserialize(file).data
+        } else {
+            log.info {"Populating cache with anime from [$uri]." }
+            parser.deserialize(uri.toURL()).data
+        }
 
         parsedAnime.forEach { anime ->
             anime.sources.forEach { source ->
@@ -40,7 +89,7 @@ internal class AnimeCachePopulator(
         eventBus.post(NumberOfEntriesPerMetaDataProviderEvent(numberOfEntriesPerMetaDataProvider))
 
         eventBus.post(CachePopulatorFinishedEvent)
-        log.info { "Finished populating cache with anime from [$uri]." }
+        log.info { "Finished populating cache with anime." }
     }
 
     private companion object {

diff --git a/...-app/src/main/kotlin/io/github/manamiproject/manami/app/cache/populator/CachePopulator.kt b/...-app/src/main/kotlin/io/github/manamiproject/manami/app/cache/populator/CachePopulator.kt
@@ -4,5 +4,5 @@ import io.github.manamiproject.manami.app.cache.Cache
 import io.github.manamiproject.manami.app.cache.CacheEntry
 
 internal interface CachePopulator<KEY, VALUE: CacheEntry<*>> {
-    fun populate(cache: Cache<KEY, VALUE>)
+    suspend fun populate(cache: Cache<KEY, VALUE>)
 }
diff --git a/...in/kotlin/io/github/manamiproject/manami/app/cache/populator/DeadEntriesCachePopulator.kt b/...in/kotlin/io/github/manamiproject/manami/app/cache/populator/DeadEntriesCachePopulator.kt
@@ -3,31 +3,78 @@ package io.github.manamiproject.manami.app.cache.populator
 import io.github.manamiproject.manami.app.cache.Cache
 import io.github.manamiproject.manami.app.cache.CacheEntry
 import io.github.manamiproject.manami.app.cache.DeadEntry
+import io.github.manamiproject.modb.core.config.BooleanPropertyDelegate
+import io.github.manamiproject.modb.core.config.ConfigRegistry
+import io.github.manamiproject.modb.core.config.DefaultConfigRegistry
 import io.github.manamiproject.modb.core.config.MetaDataProviderConfig
+import io.github.manamiproject.modb.core.coroutines.ModbDispatchers.LIMITED_FS
+import io.github.manamiproject.modb.core.extensions.regularFileExists
+import io.github.manamiproject.modb.core.extensions.writeToFile
+import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient
+import io.github.manamiproject.modb.core.httpclient.HttpClient
 import io.github.manamiproject.modb.core.logging.LoggerDelegate
 import io.github.manamiproject.modb.core.models.Anime
 import io.github.manamiproject.modb.serde.json.DeadEntriesJsonStringDeserializer
 import io.github.manamiproject.modb.serde.json.DefaultExternalResourceJsonDeserializer
 import io.github.manamiproject.modb.serde.json.ExternalResourceJsonDeserializer
 import io.github.manamiproject.modb.serde.json.models.DeadEntries
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
 import java.net.URI
 import java.net.URL
+import java.nio.file.Files
+import java.nio.file.attribute.BasicFileAttributes
+import java.time.LocalDate
+import java.time.ZoneId
+import java.time.temporal.ChronoUnit
+import kotlin.io.path.Path
 
 internal class DeadEntriesCachePopulator(
     private val config: MetaDataProviderConfig,
     private val url: URL,
     private val parser: ExternalResourceJsonDeserializer<DeadEntries> = DefaultExternalResourceJsonDeserializer(deserializer = DeadEntriesJsonStringDeserializer.instance),
+    private val httpClient: HttpClient = DefaultHttpClient.instance,
+    configRegistry: ConfigRegistry = DefaultConfigRegistry.instance,
 ) : CachePopulator<URI, CacheEntry<Anime>> {
 
-    override fun populate(cache: Cache<URI, CacheEntry<Anime>>) {
+    private val isUseLocalFiles by BooleanPropertyDelegate(
+        namespace = "manami.cache.useLocalFiles",
+        configRegistry = configRegistry,
+        default = true
+    )
+
+    override suspend fun populate(cache: Cache<URI, CacheEntry<Anime>>) {
         log.info { "Populating cache with dead entries from [${config.hostname()}]" }
 
-        runBlocking {
-            parser.deserialize(url).deadEntries.forEach { animeId ->
-                val source = config.buildAnimeLink(animeId)
-                cache.populate(source, DeadEntry())
+        val parsedData = if (isUseLocalFiles) {
+            val file = Path("${config.hostname()}.zip")
+
+            val isDownloadFile = if (!file.regularFileExists()) {
+                true
+            } else {
+                val attrs = withContext(LIMITED_FS) {
+                    Files.readAttributes(file, BasicFileAttributes::class.java)
+                }
+                val creationTime = attrs.creationTime()
+                    .toInstant()
+                    .atZone(ZoneId.systemDefault())
+                    .toLocalDate()
+
+                ChronoUnit.DAYS.between(creationTime, LocalDate.now()) >= 1L
+            }
+
+            if (isDownloadFile) {
+                log.info {"Downloading dead entries file from [$url], because a local file doesn't exist." }
+                httpClient.get(url).body.writeToFile(file)
             }
+
+            parser.deserialize(file)
+        } else {
+            parser.deserialize(url)
+        }
+
+        parsedData.deadEntries.forEach { animeId ->
+            val source = config.buildAnimeLink(animeId)
+            cache.populate(source, DeadEntry())
         }
 
         log.info { "Finished populating cache with dead entries from [${config.hostname()}]" }

diff --git a/...pp/src/main/kotlin/io/github/manamiproject/manami/app/versioning/GithubVersionProvider.kt b/...pp/src/main/kotlin/io/github/manamiproject/manami/app/versioning/GithubVersionProvider.kt
@@ -1,22 +1,28 @@
 package io.github.manamiproject.manami.app.versioning
 
-import io.github.manamiproject.modb.core.extensions.EMPTY
 import io.github.manamiproject.modb.core.httpclient.DefaultHttpClient
 import io.github.manamiproject.modb.core.httpclient.HttpClient
+import io.github.manamiproject.modb.core.json.Json
+import io.github.manamiproject.modb.core.logging.LoggerDelegate
 import kotlinx.coroutines.runBlocking
 import java.net.URI
 
 internal class GithubVersionProvider(
-    private val uri: URI = URI("https://github.com/manami-project/manami/releases/latest"),
+    private val uri: URI = URI("https://api.github.com/repos/manami-project/manami/releases/latest"),
     private val httpClient: HttpClient = DefaultHttpClient.instance,
 ): VersionProvider {
 
     override fun version(): SemanticVersion {
-        val response = runBlocking { httpClient.get(uri.toURL()) }
-        check(response.isOk()) { "Unable to check latest version, because response code wasn't 200." }
+        return runBlocking {
+            val response = httpClient.get(uri.toURL())
+            check(response.isOk()) { "Unable to check latest version, because response code wasn't 200." }
 
-        val title = Regex("<title>.*?</title>").find(response.bodyAsText)?.value ?: EMPTY
-        val rawVersion = Regex("([0-9]+\\.?){3}").find(title)?.value ?: EMPTY
-        return  SemanticVersion(rawVersion)
+            val version =  Json.parseJson<GithubResponse>(response.bodyAsText)!!.name
+            SemanticVersion(version)
+        }
     }
-}
+}
+
+private data class GithubResponse(
+    val name: String,
+)
diff --git a/manami-app/src/test/kotlin/io/github/manamiproject/manami/app/cache/TestingAssets.kt b/manami-app/src/test/kotlin/io/github/manamiproject/manami/app/cache/TestingAssets.kt
@@ -1,10 +1,7 @@
 package io.github.manamiproject.manami.app.cache
 
 import io.github.manamiproject.manami.app.cache.loader.CacheLoader
-import io.github.manamiproject.modb.core.config.AnimeId
-import io.github.manamiproject.modb.core.config.FileSuffix
-import io.github.manamiproject.modb.core.config.Hostname
-import io.github.manamiproject.modb.core.config.MetaDataProviderConfig
+import io.github.manamiproject.modb.core.config.*
 import io.github.manamiproject.modb.core.converter.AnimeConverter
 import io.github.manamiproject.modb.core.downloader.Downloader
 import io.github.manamiproject.modb.core.httpclient.HttpClient
@@ -15,6 +12,9 @@ import io.github.manamiproject.modb.core.models.Tag
 import io.github.manamiproject.modb.test.shouldNotBeInvoked
 import java.net.URI
 import java.net.URL
+import java.time.LocalDate
+import java.time.LocalDateTime
+import java.time.OffsetDateTime
 
 internal object TestCacheLoader : CacheLoader {
     override fun hostname(): Hostname = shouldNotBeInvoked()
@@ -53,4 +53,17 @@ internal object TestAnimeCache: AnimeCache {
     override fun fetch(key: URI): CacheEntry<Anime> = shouldNotBeInvoked()
     override fun populate(key: URI, value: CacheEntry<Anime>) = shouldNotBeInvoked()
     override fun clear() = shouldNotBeInvoked()
+}
+
+internal object TestConfigRegistry: ConfigRegistry {
+    override fun boolean(key: String): Boolean = shouldNotBeInvoked()
+    override fun double(key: String): Double = shouldNotBeInvoked()
+    override fun int(key: String): Int = shouldNotBeInvoked()
+    override fun <T : Any> list(key: String): List<T> = shouldNotBeInvoked()
+    override fun localDate(key: String): LocalDate = shouldNotBeInvoked()
+    override fun localDateTime(key: String): LocalDateTime = shouldNotBeInvoked()
+    override fun long(key: String): Long = shouldNotBeInvoked()
+    override fun <T : Any> map(key: String): Map<String, T> = shouldNotBeInvoked()
+    override fun offsetDateTime(key: String): OffsetDateTime = shouldNotBeInvoked()
+    override fun string(key: String): String? = shouldNotBeInvoked()
 }
diff --git a/...test/kotlin/io/github/manamiproject/manami/app/cache/populator/AnimeCachePopulatorTest.kt b/...test/kotlin/io/github/manamiproject/manami/app/cache/populator/AnimeCachePopulatorTest.kt
@@ -5,9 +5,11 @@ import com.github.tomakehurst.wiremock.client.WireMock.*
 import io.github.manamiproject.manami.app.cache.DefaultAnimeCache
 import io.github.manamiproject.manami.app.cache.PresentValue
 import io.github.manamiproject.manami.app.cache.TestCacheLoader
+import io.github.manamiproject.manami.app.cache.TestConfigRegistry
 import io.github.manamiproject.manami.app.events.Event
 import io.github.manamiproject.manami.app.events.EventBus
 import io.github.manamiproject.manami.app.events.TestEventBus
+import io.github.manamiproject.modb.core.config.ConfigRegistry
 import io.github.manamiproject.modb.core.models.Anime
 import io.github.manamiproject.modb.core.models.Anime.Status.FINISHED
 import io.github.manamiproject.modb.core.models.Anime.Type.TV
@@ -16,6 +18,7 @@ import io.github.manamiproject.modb.core.models.AnimeSeason.Season.FALL
 import io.github.manamiproject.modb.test.MockServerTestCase
 import io.github.manamiproject.modb.test.WireMockServerCreator
 import io.github.manamiproject.modb.test.loadTestResource
+import kotlinx.coroutines.runBlocking
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import java.net.URI
@@ -113,9 +116,14 @@ internal class AnimeCachePopulatorTest: MockServerTestCase<WireMockServer> by Wi
             }
         }
 
+        val testConfigRegistry = object: ConfigRegistry by TestConfigRegistry {
+            override fun boolean(key: String): Boolean = false
+        }
+
         val animeCachePopulator = AnimeCachePopulator(
-                uri = URI("http://localhost:$port/anime/1535"),
-                eventBus = testEventBus,
+            uri = URI("http://localhost:$port/anime/1535"),
+            eventBus = testEventBus,
+            configRegistry = testConfigRegistry,
         )
 
         serverInstance.stubFor(
@@ -128,7 +136,9 @@ internal class AnimeCachePopulatorTest: MockServerTestCase<WireMockServer> by Wi
         )
 
         // when
-        animeCachePopulator.populate(testCache)
+        runBlocking {
+            animeCachePopulator.populate(testCache)
+        }
 
         // then
         val expectedAnidbEntry = expectedAnime.copy(