Skip to content

Commit

Permalink
Feature/auto download of new chapters improve handling of unhandable …
Browse files Browse the repository at this point in the history 
…reuploads (#921)

* Update test/server-reference file

* Properly handle re-uploaded chapters in auto download of new chapters

In case of unhandable re-uploaded chapters (different chapter numbers) they potentially would have prevented auto downloads due being considered as unread.

Additionally, they would not have been considered to get downloaded due to not having a higher chapter number than the previous latest existing chapter before the chapter list fetch.

* Add option to ignore re-uploads for auto downloads

* Extract check for manga category download inclusion

* Extract logic to get new chapter ids to download

* Simplify manga category download inclusion check

In case the DEFAULT category does not exist, someone messed with the database and it is basically corrupted
  • Loading branch information
@schroda
schroda committed Apr 7, 2024
1 parent 89dd570 commit 48e19f7
Show file tree
Hide file tree
Showing 7 changed files with 144 additions and 96 deletions.
1 change: 1 addition & 0 deletions server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ class SettingsMutation {
updateSetting(settings.excludeEntryWithUnreadChapters, serverConfig.excludeEntryWithUnreadChapters)
updateSetting(settings.autoDownloadAheadLimit, serverConfig.autoDownloadNewChaptersLimit) // deprecated
updateSetting(settings.autoDownloadNewChaptersLimit, serverConfig.autoDownloadNewChaptersLimit)
updateSetting(settings.autoDownloadIgnoreReUploads, serverConfig.autoDownloadIgnoreReUploads)

// extension
updateSetting(settings.extensionRepos, serverConfig.extensionRepos)
Expand Down
4 changes: 4 additions & 0 deletions server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ interface Settings : Node {
)
val autoDownloadAheadLimit: Int?
val autoDownloadNewChaptersLimit: Int?
val autoDownloadIgnoreReUploads: Boolean?

// extension
val extensionRepos: List<String>?
Expand Down Expand Up @@ -118,6 +119,7 @@ data class PartialSettingsType(
)
override val autoDownloadAheadLimit: Int?,
override val autoDownloadNewChaptersLimit: Int?,
override val autoDownloadIgnoreReUploads: Boolean?,
// extension
override val extensionRepos: List<String>?,
// requests
Expand Down Expand Up @@ -179,6 +181,7 @@ class SettingsType(
)
override val autoDownloadAheadLimit: Int,
override val autoDownloadNewChaptersLimit: Int,
override val autoDownloadIgnoreReUploads: Boolean?,
// extension
override val extensionRepos: List<String>,
// requests
Expand Down Expand Up @@ -235,6 +238,7 @@ class SettingsType(
config.excludeEntryWithUnreadChapters.value,
config.autoDownloadNewChaptersLimit.value, // deprecated
config.autoDownloadNewChaptersLimit.value,
config.autoDownloadIgnoreReUploads.value,
// extension
config.extensionRepos.value,
// requests
Expand Down
118 changes: 42 additions & 76 deletions server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import kotlinx.serialization.Serializable
import mu.KotlinLogging
import org.jetbrains.exposed.dao.id.EntityID
import org.jetbrains.exposed.sql.Op
import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList
import org.jetbrains.exposed.sql.and
Expand All @@ -37,7 +36,6 @@ import suwayomi.tachidesk.manga.impl.download.DownloadManager.EnqueueInput
import suwayomi.tachidesk.manga.impl.track.Track
import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
import suwayomi.tachidesk.manga.model.dataclass.IncludeOrExclude
import suwayomi.tachidesk.manga.model.dataclass.MangaChapterDataClass
import suwayomi.tachidesk.manga.model.dataclass.PaginatedList
import suwayomi.tachidesk.manga.model.dataclass.paginatedFrom
Expand Down Expand Up @@ -136,6 +134,7 @@ object Chapter {
url = manga.url
}

val currentLatestChapterNumber = Manga.getLatestChapter(mangaId)?.chapterNumber ?: 0f
val numberOfCurrentChapters = getCountOfMangaChapters(mangaId)
val chapterList = source.getChapterList(sManga)

Expand Down Expand Up @@ -164,7 +163,10 @@ object Chapter {
.toList()
}

val chaptersToInsert = mutableListOf<ChapterDataClass>()
// new chapters after they have been added to the database for auto downloads
val insertedChapters = mutableListOf<ChapterDataClass>()

val chaptersToInsert = mutableListOf<ChapterDataClass>() // do not yet have an ID from the database
val chaptersToUpdate = mutableListOf<ChapterDataClass>()

chapterList.reversed().forEachIndexed { index, fetchedChapter ->
Expand Down Expand Up @@ -260,7 +262,7 @@ object Chapter {
this[ChapterTable.fetchedAt] = it
}
}
}
}.forEach { insertedChapters.add(ChapterTable.toDataClass(it)) }
}

if (chaptersToUpdate.isNotEmpty()) {
Expand All @@ -283,14 +285,8 @@ object Chapter {
}
}

val newChapters =
transaction {
ChapterTable.select { ChapterTable.manga eq mangaId }
.orderBy(ChapterTable.sourceOrder to SortOrder.DESC).toList()
}

if (manga.inLibrary) {
downloadNewChapters(mangaId, numberOfCurrentChapters, newChapters)
downloadNewChapters(mangaId, currentLatestChapterNumber, numberOfCurrentChapters, insertedChapters)
}

chapterList
Expand All @@ -301,16 +297,19 @@ object Chapter {

private fun downloadNewChapters(
mangaId: Int,
prevLatestChapterNumber: Float,
prevNumberOfChapters: Int,
updatedChapterList: List<ResultRow>,
newChapters: List<ChapterDataClass>,
) {
val log =
KotlinLogging.logger(
"${logger.name}::downloadNewChapters(" +
"mangaId= $mangaId, " +
"prevLatestChapterNumber= $prevLatestChapterNumber, " +
"prevNumberOfChapters= $prevNumberOfChapters, " +
"updatedChapterList= ${updatedChapterList.size}, " +
"autoDownloadNewChaptersLimit= ${serverConfig.autoDownloadNewChaptersLimit.value}" +
"newChapters= ${newChapters.size}, " +
"autoDownloadNewChaptersLimit= ${serverConfig.autoDownloadNewChaptersLimit.value}, " +
"autoDownloadIgnoreReUploads= ${serverConfig.autoDownloadIgnoreReUploads.value}" +
")",
)

Expand All @@ -319,86 +318,30 @@ object Chapter {
return
}

// Only download if there are new chapters, or if this is the first fetch
val newNumberOfChapters = updatedChapterList.size
val numberOfNewChapters = newNumberOfChapters - prevNumberOfChapters

val areNewChaptersAvailable = numberOfNewChapters > 0
val wasInitialFetch = prevNumberOfChapters == 0

if (!areNewChaptersAvailable) {
if (newChapters.isEmpty()) {
log.debug { "no new chapters available" }
return
}

val wasInitialFetch = prevNumberOfChapters == 0
if (wasInitialFetch) {
log.debug { "skipping download on initial fetch" }
return
}

// Verify the manga is configured to be downloaded based on it's categories.
var mangaCategories = CategoryManga.getMangaCategories(mangaId).toSet()
// if the manga has no categories, then it's implicitly in the default category
if (mangaCategories.isEmpty()) {
val defaultCategory = Category.getCategoryById(Category.DEFAULT_CATEGORY_ID)
if (defaultCategory != null) {
mangaCategories = setOf(defaultCategory)
} else {
log.warn { "missing default category" }
}
}

if (mangaCategories.isNotEmpty()) {
val downloadCategoriesMap = Category.getCategoryList().groupBy { it.includeInDownload }
val unsetCategories = downloadCategoriesMap[IncludeOrExclude.UNSET].orEmpty()
// We only download if it's in the include list, and not in the exclude list.
// Use the unset categories as the included categories if the included categories is
// empty
val includedCategories = downloadCategoriesMap[IncludeOrExclude.INCLUDE].orEmpty().ifEmpty { unsetCategories }
val excludedCategories = downloadCategoriesMap[IncludeOrExclude.EXCLUDE].orEmpty()
// Only download manga that aren't in any excluded categories
val mangaExcludeCategories = mangaCategories.intersect(excludedCategories.toSet())
if (mangaExcludeCategories.isNotEmpty()) {
log.debug { "download excluded by categories: '${mangaExcludeCategories.joinToString("', '") { it.name }}'" }
return
}
val mangaDownloadCategories = mangaCategories.intersect(includedCategories.toSet())
if (mangaDownloadCategories.isNotEmpty()) {
log.debug { "download inluded by categories: '${mangaDownloadCategories.joinToString("', '") { it.name }}'" }
} else {
log.debug { "skipping download due to download categories configuration" }
return
}
} else {
log.debug { "no categories configured, skipping check for category download include/excludes" }
if (!Manga.isInIncludedDownloadCategory(log, mangaId)) {
return
}

val newChapters = updatedChapterList.subList(0, numberOfNewChapters)

// make sure to only consider the latest chapters. e.g. old unread chapters should be ignored
val latestReadChapterIndex =
updatedChapterList.indexOfFirst { it[ChapterTable.isRead] }.takeIf { it > -1 } ?: (updatedChapterList.size)
val unreadChapters =
updatedChapterList.subList(numberOfNewChapters, latestReadChapterIndex)
.filter { !it[ChapterTable.isRead] }
val unreadChapters = Manga.getUnreadChapters(mangaId).subtract(newChapters.toSet())

val skipDueToUnreadChapters = serverConfig.excludeEntryWithUnreadChapters.value && unreadChapters.isNotEmpty()
if (skipDueToUnreadChapters) {
log.debug { "ignore due to unread chapters" }
return
}

val firstChapterToDownloadIndex =
if (serverConfig.autoDownloadNewChaptersLimit.value > 0) {
(numberOfNewChapters - serverConfig.autoDownloadNewChaptersLimit.value).coerceAtLeast(0)
} else {
0
}

val chapterIdsToDownload =
newChapters.subList(firstChapterToDownloadIndex, numberOfNewChapters)
.filter { !it[ChapterTable.isRead] && !it[ChapterTable.isDownloaded] }
.map { it[ChapterTable.id].value }
val chapterIdsToDownload = getNewChapterIdsToDownload(newChapters, prevLatestChapterNumber)

if (chapterIdsToDownload.isEmpty()) {
log.debug { "no chapters available for download" }
Expand All @@ -410,6 +353,29 @@ object Chapter {
DownloadManager.enqueue(EnqueueInput(chapterIdsToDownload))
}

private fun getNewChapterIdsToDownload(
newChapters: List<ChapterDataClass>,
prevLatestChapterNumber: Float,
): List<Int> {
val reUploadedChapters = newChapters.filter { it.chapterNumber < prevLatestChapterNumber }
val actualNewChapters = newChapters.subtract(reUploadedChapters.toSet())
val chaptersToConsiderForDownloadLimit =
if (serverConfig.autoDownloadIgnoreReUploads.value) {
actualNewChapters
} else {
newChapters
}.sortedBy { it.index }

val latestChapterToDownloadIndex =
if (serverConfig.autoDownloadNewChaptersLimit.value == 0) {
chaptersToConsiderForDownloadLimit.size
} else {
serverConfig.autoDownloadNewChaptersLimit.value.coerceAtMost(chaptersToConsiderForDownloadLimit.size)
}

return chaptersToConsiderForDownloadLimit.subList(0, latestChapterToDownloadIndex).map { it.id }
}

fun modifyChapter(
mangaId: Int,
chapterIndex: Int,
Expand Down
55 changes: 55 additions & 0 deletions server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.model.UpdateStrategy
import eu.kanade.tachiyomi.source.online.HttpSource
import io.javalin.http.HttpCode
import mu.KLogger
import mu.KotlinLogging
import okhttp3.CacheControl
import okhttp3.Response
Expand All @@ -41,6 +42,8 @@ import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.clearCachedImage
import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
import suwayomi.tachidesk.manga.impl.util.updateMangaDownloadDir
import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
import suwayomi.tachidesk.manga.model.dataclass.IncludeOrExclude
import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
import suwayomi.tachidesk.manga.model.dataclass.toGenreList
import suwayomi.tachidesk.manga.model.table.ChapterTable
Expand Down Expand Up @@ -366,4 +369,56 @@ object Manga {
clearCachedImage(applicationDirs.tempThumbnailCacheRoot, fileName)
clearCachedImage(applicationDirs.thumbnailDownloadsRoot, fileName)
}

fun getLatestChapter(mangaId: Int): ChapterDataClass? {
return transaction {
ChapterTable.select { ChapterTable.manga eq mangaId }.maxByOrNull { it[ChapterTable.sourceOrder] }
}?.let { ChapterTable.toDataClass(it) }
}

fun getUnreadChapters(mangaId: Int): List<ChapterDataClass> {
return transaction {
ChapterTable.select { (ChapterTable.manga eq mangaId) and (ChapterTable.isRead eq false) }
.orderBy(ChapterTable.sourceOrder to SortOrder.DESC)
.map { ChapterTable.toDataClass(it) }
}
}

fun isInIncludedDownloadCategory(
logContext: KLogger = logger,
mangaId: Int,
): Boolean {
val log = KotlinLogging.logger { "${logContext.name}::isInExcludedDownloadCategory($mangaId)" }

// Verify the manga is configured to be downloaded based on it's categories.
var mangaCategories = CategoryManga.getMangaCategories(mangaId).toSet()
// if the manga has no categories, then it's implicitly in the default category
if (mangaCategories.isEmpty()) {
val defaultCategory = Category.getCategoryById(Category.DEFAULT_CATEGORY_ID)!!
mangaCategories = setOf(defaultCategory)
}

val downloadCategoriesMap = Category.getCategoryList().groupBy { it.includeInDownload }
val unsetCategories = downloadCategoriesMap[IncludeOrExclude.UNSET].orEmpty()
// We only download if it's in the include list, and not in the exclude list.
// Use the unset categories as the included categories if the included categories is
// empty
val includedCategories = downloadCategoriesMap[IncludeOrExclude.INCLUDE].orEmpty().ifEmpty { unsetCategories }
val excludedCategories = downloadCategoriesMap[IncludeOrExclude.EXCLUDE].orEmpty()
// Only download manga that aren't in any excluded categories
val mangaExcludeCategories = mangaCategories.intersect(excludedCategories.toSet())
if (mangaExcludeCategories.isNotEmpty()) {
log.debug { "download excluded by categories: '${mangaExcludeCategories.joinToString("', '") { it.name }}'" }
return false
}
val mangaDownloadCategories = mangaCategories.intersect(includedCategories.toSet())
if (mangaDownloadCategories.isNotEmpty()) {
log.debug { "download inluded by categories: '${mangaDownloadCategories.joinToString("', '") { it.name }}'" }
} else {
log.debug { "skipping download due to download categories configuration" }
return false
}

return true
}
}
1 change: 1 addition & 0 deletions server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ class ServerConfig(getConfig: () -> Config, val moduleName: String = SERVER_CONF
val autoDownloadNewChapters: MutableStateFlow<Boolean> by OverrideConfigValue(BooleanConfigAdapter)
val excludeEntryWithUnreadChapters: MutableStateFlow<Boolean> by OverrideConfigValue(BooleanConfigAdapter)
val autoDownloadNewChaptersLimit: MutableStateFlow<Int> by OverrideConfigValue(IntConfigAdapter)
val autoDownloadIgnoreReUploads: MutableStateFlow<Boolean> by OverrideConfigValue(BooleanConfigAdapter)

// extensions
val extensionRepos: MutableStateFlow<List<String>> by OverrideConfigValues(StringConfigAdapter)
Expand Down
1 change: 1 addition & 0 deletions server/src/main/resources/server-reference.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ server.downloadsPath = ""
server.autoDownloadNewChapters = false # if new chapters that have been retrieved should get automatically downloaded
server.excludeEntryWithUnreadChapters = true # ignore automatic chapter downloads of entries with unread chapters
server.autoDownloadNewChaptersLimit = 0 # 0 to disable it - how many unread downloaded chapters should be available - if the limit is reached, new chapters won't be downloaded automatically. this limit will also be applied to the auto download of new chapters on an update
server.autoDownloadIgnoreReUploads = false # decides if re-uploads should be ignored during auto download of new chapters

# extension repos
server.extensionRepos = [
Expand Down

0 comments on commit 48e19f7

Please sign in to comment.