add support for Extensions Lib 1.4 (#496)

* Support extensions lib 1.4

* Fix build

* Support UpdateStrategy

* Update extension lib min/max to match Tachiyomi

* Use HttpSource.getMangaUrl and add Chapter.realUrl

gradle/libs.versions.toml

@@ -1,8 +1,8 @@
 [versions]
-kotlin = "1.7.20"
+kotlin = "1.8.0"
 coroutines = "1.6.4"
 serialization = "1.4.1"
-okhttp = "4.10.0" # Major version is locked by Tachiyomi extensions
+okhttp = "5.0.0-alpha.11" # Major version is locked by Tachiyomi extensions
 javalin = "4.6.6" # Javalin 5.0.0+ requires Java 11
 jackson = "2.13.3" # jackson version locked by javalin, ref: `io.javalin.core.util.OptionalDependency`
 exposed = "0.40.1"
@@ -36,7 +36,7 @@ kotlinlogging = "io.github.microutils:kotlin-logging:3.0.5"
 okhttp-core = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" }
 okhttp-logging = { module = "com.squareup.okhttp3:logging-interceptor", version.ref = "okhttp" }
 okhttp-dnsoverhttps = { module = "com.squareup.okhttp3:okhttp-dnsoverhttps", version.ref = "okhttp" }
-okio = "com.squareup.okio:okio:3.2.0"
+okio = "com.squareup.okio:okio:3.3.0"
 
 # Javalin api
 javalin-core = { module = "io.javalin:javalin", version.ref = "javalin" }

server/src/main/kotlin/eu/kanade/tachiyomi/AppModule.kt

@@ -16,6 +16,7 @@ package eu.kanade.tachiyomi
 // import eu.kanade.tachiyomi.data.track.TrackManager
 // import eu.kanade.tachiyomi.extension.ExtensionManager
 import android.app.Application
+import eu.kanade.tachiyomi.network.JavaScriptEngine
 import eu.kanade.tachiyomi.network.NetworkHelper
 import kotlinx.serialization.json.Json
 import rx.Observable
@@ -41,6 +42,8 @@ class AppModule(val app: Application) : InjektModule {
 
         addSingletonFactory { NetworkHelper(app) }
 
+        addSingletonFactory { JavaScriptEngine(app) }
+
 //        addSingletonFactory { SourceManager(app).also { get<ExtensionManager>().init(it) } }
 //
 //        addSingletonFactory { ExtensionManager(app) }

server/src/main/kotlin/eu/kanade/tachiyomi/network/JavaScriptEngine.kt

@@ -0,0 +1,27 @@
+package eu.kanade.tachiyomi.network
+
+import android.content.Context
+import app.cash.quickjs.QuickJs
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+
+/**
+ * Util for evaluating JavaScript in sources.
+ */
+class JavaScriptEngine(context: Context) {
+
+    /**
+     * Evaluate arbitrary JavaScript code and get the result as a primitive type
+     * (e.g., String, Int).
+     *
+     * @since extensions-lib 1.4
+     * @param script JavaScript to execute.
+     * @return Result of JavaScript code as a primitive type.
+     */
+    @Suppress("UNUSED", "UNCHECKED_CAST")
+    suspend fun <T> evaluate(script: String): T = withContext(Dispatchers.IO) {
+        QuickJs.create().use {
+            it.evaluate(script) as T
+        }
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/network/Requests.kt

@@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.network
 import okhttp3.CacheControl
 import okhttp3.FormBody
 import okhttp3.Headers
+import okhttp3.HttpUrl
 import okhttp3.Request
 import okhttp3.RequestBody
 import java.util.concurrent.TimeUnit.MINUTES
@@ -23,6 +24,21 @@ fun GET(
         .build()
 }
 
+/**
+ * @since extensions-lib 1.4
+ */
+fun GET(
+    url: HttpUrl,
+    headers: Headers = DEFAULT_HEADERS,
+    cache: CacheControl = DEFAULT_CACHE_CONTROL
+): Request {
+    return Request.Builder()
+        .url(url)
+        .headers(headers)
+        .cacheControl(cache)
+        .build()
+}
+
 fun POST(
     url: String,
     headers: Headers = DEFAULT_HEADERS,

server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SManga.kt

@@ -20,6 +20,8 @@ interface SManga : Serializable {
 
     var thumbnail_url: String?
 
+    var update_strategy: UpdateStrategy
+
     var initialized: Boolean
 
     fun copyFrom(other: SManga) {

server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SMangaImpl.kt

@@ -18,5 +18,7 @@ class SMangaImpl : SManga {
 
     override var thumbnail_url: String? = null
 
+    override var update_strategy: UpdateStrategy = UpdateStrategy.ALWAYS_UPDATE
+
     override var initialized: Boolean = false
 }

server/src/main/kotlin/eu/kanade/tachiyomi/source/model/UpdateStrategy.kt

@@ -0,0 +1,6 @@
+package eu.kanade.tachiyomi.source.model
+
+enum class UpdateStrategy {
+    ALWAYS_UPDATE,
+    ONLY_FETCH_ONCE
+}

server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt

@@ -357,15 +357,36 @@ abstract class HttpSource : CatalogueSource {
         }
     }
 
+    /**
+     * Returns the url of the provided manga
+     *
+     * @since extensions-lib 1.4
+     * @param manga the manga
+     * @return url of the manga
+     */
+    open fun getMangaUrl(manga: SManga): String {
+        return mangaDetailsRequest(manga).url.toString()
+    }
+
+    /**
+     * Returns the url of the provided chapter
+     *
+     * @since extensions-lib 1.4
+     * @param chapter the chapter
+     * @return url of the chapter
+     */
+    open fun getChapterUrl(chapter: SChapter): String {
+        return pageListRequest(chapter).url.toString()
+    }
+
     /**
      * Called before inserting a new chapter into database. Use it if you need to override chapter
      * fields, like the title or the chapter number. Do not change anything to [manga].
      *
      * @param chapter the chapter to be added.
      * @param manga the manga of the chapter.
      */
-    open fun prepareNewChapter(chapter: SChapter, manga: SManga) {
-    }
+    open fun prepareNewChapter(chapter: SChapter, manga: SManga) {}
 
     /**
      * Returns the list of filters for the source.

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt

@@ -1,5 +1,6 @@
 package suwayomi.tachidesk.manga.controller
 
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import io.javalin.http.HttpCode
 import io.javalin.websocket.WsConfig
 import mu.KotlinLogging
@@ -96,6 +97,7 @@ object UpdateController {
             .flatMap { CategoryManga.getCategoryMangaList(it.id) }
             .distinctBy { it.id }
             .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title))
+            .filter { it.updateStrategy == UpdateStrategy.ALWAYS_UPDATE }
             .forEach { manga ->
                 updater.addMangaToQueue(manga)
             }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt

@@ -12,11 +12,16 @@ import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.util.chapter.ChapterRecognition
 import kotlinx.serialization.Serializable
 import org.jetbrains.exposed.dao.id.EntityID
-import org.jetbrains.exposed.sql.*
+import org.jetbrains.exposed.sql.Op
+import org.jetbrains.exposed.sql.SortOrder
 import org.jetbrains.exposed.sql.SortOrder.ASC
 import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
-import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList
+import org.jetbrains.exposed.sql.and
+import org.jetbrains.exposed.sql.deleteWhere
+import org.jetbrains.exposed.sql.insert
+import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
+import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.Manga.getManga
 import suwayomi.tachidesk.manga.impl.util.getChapterDir
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
@@ -27,6 +32,7 @@ import suwayomi.tachidesk.manga.model.dataclass.PaginatedList
 import suwayomi.tachidesk.manga.model.dataclass.paginatedFrom
 import suwayomi.tachidesk.manga.model.table.ChapterMetaTable
 import suwayomi.tachidesk.manga.model.table.ChapterTable
+import suwayomi.tachidesk.manga.model.table.ChapterTable.scanlator
 import suwayomi.tachidesk.manga.model.table.MangaTable
 import suwayomi.tachidesk.manga.model.table.PageTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
@@ -85,6 +91,10 @@ object Chapter {
                         it[sourceOrder] = index + 1
                         it[fetchedAt] = now++
                         it[ChapterTable.manga] = mangaId
+
+                        it[realUrl] = runCatching {
+                            (source as? HttpSource)?.getChapterUrl(fetchedChapter)
+                        }.getOrNull()
                     }
                 } else {
                     ChapterTable.update({ ChapterTable.url eq fetchedChapter.url }) {
@@ -95,6 +105,10 @@ object Chapter {
 
                         it[sourceOrder] = index + 1
                         it[ChapterTable.manga] = mangaId
+
+                        it[realUrl] = runCatching {
+                            (source as? HttpSource)?.getChapterUrl(fetchedChapter)
+                        }.getOrNull()
                     }
                 }
             }
@@ -138,26 +152,27 @@ object Chapter {
             val dbChapter = dbChapterMap.getValue(it.url)
 
             ChapterDataClass(
-                dbChapter[ChapterTable.id].value,
-                it.url,
-                it.name,
-                it.date_upload,
-                it.chapter_number,
-                it.scanlator,
-                mangaId,
-
-                dbChapter[ChapterTable.isRead],
-                dbChapter[ChapterTable.isBookmarked],
-                dbChapter[ChapterTable.lastPageRead],
-                dbChapter[ChapterTable.lastReadAt],
-
-                chapterCount - index,
-                dbChapter[ChapterTable.fetchedAt],
-                dbChapter[ChapterTable.isDownloaded],
-
-                dbChapter[ChapterTable.pageCount],
-
-                chapterList.size,
+                id = dbChapter[ChapterTable.id].value,
+                url = it.url,
+                name = it.name,
+                uploadDate = it.date_upload,
+                chapterNumber = it.chapter_number,
+                scanlator = it.scanlator,
+                mangaId = mangaId,
+
+                read = dbChapter[ChapterTable.isRead],
+                bookmarked = dbChapter[ChapterTable.isBookmarked],
+                lastPageRead = dbChapter[ChapterTable.lastPageRead],
+                lastReadAt = dbChapter[ChapterTable.lastReadAt],
+
+                index = chapterCount - index,
+                fetchedAt = dbChapter[ChapterTable.fetchedAt],
+                realUrl = dbChapter[ChapterTable.realUrl],
+                downloaded = dbChapter[ChapterTable.isDownloaded],
+
+                pageCount = dbChapter[ChapterTable.pageCount],
+
+                chapterCount = chapterList.size,
                 meta = chapterMetas.getValue(dbChapter[ChapterTable.id])
             )
         }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt

@@ -11,6 +11,7 @@ import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.NetworkHelper
 import eu.kanade.tachiyomi.source.local.LocalSource
 import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import eu.kanade.tachiyomi.source.online.HttpSource
 import org.jetbrains.exposed.sql.ResultRow
 import org.jetbrains.exposed.sql.SortOrder
@@ -94,39 +95,42 @@ object Manga {
                     }
 
                     it[MangaTable.realUrl] = runCatching {
-                        (source as? HttpSource)?.mangaDetailsRequest(sManga)?.url?.toString()
+                        (source as? HttpSource)?.getMangaUrl(sManga)
                     }.getOrNull()
 
                     it[MangaTable.lastFetchedAt] = Instant.now().epochSecond
+
+                    it[MangaTable.updateStrategy] = sManga.update_strategy.name
                 }
             }
 
             mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
 
             MangaDataClass(
-                mangaId,
-                mangaEntry[MangaTable.sourceReference].toString(),
-
-                mangaEntry[MangaTable.url],
-                mangaEntry[MangaTable.title],
-                proxyThumbnailUrl(mangaId),
-                mangaEntry[MangaTable.thumbnailUrlLastFetched],
-
-                true,
-
-                sManga.artist,
-                sManga.author,
-                sManga.description,
-                sManga.genre.toGenreList(),
-                MangaStatus.valueOf(sManga.status).name,
-                mangaEntry[MangaTable.inLibrary],
-                mangaEntry[MangaTable.inLibraryAt],
-                getSource(mangaEntry[MangaTable.sourceReference]),
-                getMangaMetaMap(mangaId),
-                mangaEntry[MangaTable.realUrl],
-                mangaEntry[MangaTable.lastFetchedAt],
-                mangaEntry[MangaTable.chaptersLastFetchedAt],
-                true
+                id = mangaId,
+                sourceId = mangaEntry[MangaTable.sourceReference].toString(),
+
+                url = mangaEntry[MangaTable.url],
+                title = mangaEntry[MangaTable.title],
+                thumbnailUrl = proxyThumbnailUrl(mangaId),
+                thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched],
+
+                initialized = true,
+
+                artist = sManga.artist,
+                author = sManga.author,
+                description = sManga.description,
+                genre = sManga.genre.toGenreList(),
+                status = MangaStatus.valueOf(sManga.status).name,
+                inLibrary = mangaEntry[MangaTable.inLibrary],
+                inLibraryAt = mangaEntry[MangaTable.inLibraryAt],
+                source = getSource(mangaEntry[MangaTable.sourceReference]),
+                meta = getMangaMetaMap(mangaId),
+                realUrl = mangaEntry[MangaTable.realUrl],
+                lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt],
+                chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt],
+                updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
+                freshData = true
             )
         }
     }
@@ -166,29 +170,30 @@ object Manga {
     }
 
     private fun getMangaDataClass(mangaId: Int, mangaEntry: ResultRow) = MangaDataClass(
-        mangaId,
-        mangaEntry[MangaTable.sourceReference].toString(),
-
-        mangaEntry[MangaTable.url],
-        mangaEntry[MangaTable.title],
-        proxyThumbnailUrl(mangaId),
-        mangaEntry[MangaTable.thumbnailUrlLastFetched],
-
-        true,
-
-        mangaEntry[MangaTable.artist],
-        mangaEntry[MangaTable.author],
-        mangaEntry[MangaTable.description],
-        mangaEntry[MangaTable.genre].toGenreList(),
-        MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
-        mangaEntry[MangaTable.inLibrary],
-        mangaEntry[MangaTable.inLibraryAt],
-        getSource(mangaEntry[MangaTable.sourceReference]),
-        getMangaMetaMap(mangaId),
-        mangaEntry[MangaTable.realUrl],
-        mangaEntry[MangaTable.lastFetchedAt],
-        mangaEntry[MangaTable.chaptersLastFetchedAt],
-        false
+        id = mangaId,
+        sourceId = mangaEntry[MangaTable.sourceReference].toString(),
+
+        url = mangaEntry[MangaTable.url],
+        title = mangaEntry[MangaTable.title],
+        thumbnailUrl = proxyThumbnailUrl(mangaId),
+        thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched],
+
+        initialized = true,
+
+        artist = mangaEntry[MangaTable.artist],
+        author = mangaEntry[MangaTable.author],
+        description = mangaEntry[MangaTable.description],
+        genre = mangaEntry[MangaTable.genre].toGenreList(),
+        status = MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
+        inLibrary = mangaEntry[MangaTable.inLibrary],
+        inLibraryAt = mangaEntry[MangaTable.inLibraryAt],
+        source = getSource(mangaEntry[MangaTable.sourceReference]),
+        meta = getMangaMetaMap(mangaId),
+        realUrl = mangaEntry[MangaTable.realUrl],
+        lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt],
+        chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt],
+        updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
+        freshData = false
     )
 
     fun getMangaMetaMap(mangaId: Int): Map<String, String> {