Skip to content

Commit

Permalink
perf: lazy sql joins
Browse files Browse the repository at this point in the history
  • Loading branch information
@gotson
gotson committed Aug 27, 2020
1 parent 7b90244 commit 6eb7669
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 36 deletions.
38 changes: 24 additions & 14 deletions komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookDtoDao.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,31 +62,31 @@ class BookDtoDao(
override fun findAll(search: BookSearchWithReadProgress, userId: String, pageable: Pageable): Page<BookDto> {
val conditions = search.toCondition()

return findAll(conditions, userId, pageable)
return findAll(conditions, userId, pageable, search.toJoinConditions())
}

override fun findByReadListId(readListId: String, userId: String, pageable: Pageable): Page<BookDto> {
val conditions = rlb.READLIST_ID.eq(readListId)

return findAll(conditions, userId, pageable, selectReadListNumber = true)
return findAll(conditions, userId, pageable, JoinConditions(selectReadListNumber = true))
}

private fun findAll(conditions: Condition, userId: String, pageable: Pageable, selectReadListNumber: Boolean = false): Page<BookDto> {
private fun findAll(conditions: Condition, userId: String, pageable: Pageable, joinConditions: JoinConditions = JoinConditions()): Page<BookDto> {
val count = dsl.selectDistinct(b.ID)
.from(b)
.leftJoin(m).on(b.ID.eq(m.BOOK_ID))
.leftJoin(d).on(b.ID.eq(d.BOOK_ID))
.leftJoin(r).on(b.ID.eq(r.BOOK_ID)).and(readProgressCondition(userId))
.leftJoin(bt).on(b.ID.eq(bt.BOOK_ID))
.leftJoin(rlb).on(b.ID.eq(rlb.BOOK_ID))
.apply { if (joinConditions.tag) leftJoin(bt).on(b.ID.eq(bt.BOOK_ID)) }
.apply { if (joinConditions.selectReadListNumber) leftJoin(rlb).on(b.ID.eq(rlb.BOOK_ID)) }
.where(conditions)
.groupBy(b.ID)
.fetch()
.size

val orderBy = pageable.sort.toOrderBy(sorts)

val dtos = selectBase(userId, selectReadListNumber)
val dtos = selectBase(userId, joinConditions)
.where(conditions)
.orderBy(orderBy)
.apply { if (pageable.isPaged) limit(pageable.pageSize).offset(pageable.offset) }
Expand Down Expand Up @@ -167,19 +167,19 @@ class BookDtoDao(
.firstOrNull()
}

private fun selectBase(userId: String, selectReadListNumber: Boolean = false) =
private fun selectBase(userId: String, joinConditions: JoinConditions = JoinConditions()) =
dsl.selectDistinct(
*b.fields(),
*m.fields(),
*d.fields(),
*r.fields()
).apply { if (selectReadListNumber) select(rlb.NUMBER) }
).apply { if (joinConditions.selectReadListNumber) select(rlb.NUMBER) }
.from(b)
.leftJoin(m).on(b.ID.eq(m.BOOK_ID))
.leftJoin(d).on(b.ID.eq(d.BOOK_ID))
.leftJoin(r).on(b.ID.eq(r.BOOK_ID)).and(readProgressCondition(userId))
.leftJoin(bt).on(b.ID.eq(bt.BOOK_ID))
.leftJoin(rlb).on(b.ID.eq(rlb.BOOK_ID))
.apply { if (joinConditions.tag) leftJoin(bt).on(b.ID.eq(bt.BOOK_ID)) }
.apply { if (joinConditions.selectReadListNumber) leftJoin(rlb).on(b.ID.eq(rlb.BOOK_ID)) }

private fun ResultQuery<Record>.fetchAndMap() =
fetch()
Expand All @@ -206,11 +206,11 @@ class BookDtoDao(
private fun BookSearchWithReadProgress.toCondition(): Condition {
var c: Condition = DSL.trueCondition()

libraryIds?.let { c = c.and(b.LIBRARY_ID.`in`(it)) }
seriesIds?.let { c = c.and(b.SERIES_ID.`in`(it)) }
if (!libraryIds.isNullOrEmpty()) c = c.and(b.LIBRARY_ID.`in`(libraryIds))
if (!seriesIds.isNullOrEmpty()) c = c.and(b.SERIES_ID.`in`(seriesIds))
searchTerm?.let { c = c.and(d.TITLE.containsIgnoreCase(it)) }
mediaStatus?.let { c = c.and(m.STATUS.`in`(it)) }
tags?.let { tags -> c = c.and(lower(bt.TAG).`in`(tags.map { it.toLowerCase() })) }
if (!mediaStatus.isNullOrEmpty()) c = c.and(m.STATUS.`in`(mediaStatus))
if (!tags.isNullOrEmpty()) c = c.and(lower(bt.TAG).`in`(tags.map { it.toLowerCase() }))

if (readStatus != null) {
val cr = readStatus.map {
Expand All @@ -227,6 +227,16 @@ class BookDtoDao(
return c
}

private fun BookSearchWithReadProgress.toJoinConditions() =
JoinConditions(
tag = !tags.isNullOrEmpty()
)

private data class JoinConditions(
val selectReadListNumber: Boolean = false,
val tag: Boolean = false
)

private fun BookRecord.toDto(media: MediaDto, metadata: BookMetadataDto, readProgress: ReadProgressDto?) =
BookDto(
id = id,
Expand Down
66 changes: 44 additions & 22 deletions komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesDtoDao.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,17 +67,16 @@ class SeriesDtoDao(

override fun findAll(search: SeriesSearchWithReadProgress, userId: String, pageable: Pageable): Page<SeriesDto> {
val conditions = search.toCondition()

val having = search.readStatus?.toCondition() ?: DSL.trueCondition()

return findAll(conditions, having, userId, pageable)
return findAll(conditions, having, userId, pageable, search.toJoinConditions())
}

override fun findByCollectionId(collectionId: String, userId: String, pageable: Pageable): Page<SeriesDto> {
val conditions = cs.COLLECTION_ID.eq(collectionId)
val having = DSL.trueCondition()

return findAll(conditions, having, userId, pageable, true)
return findAll(conditions, having, userId, pageable, JoinConditions(selectCollectionNumber = true, collection = true))
}

override fun findRecentlyUpdated(search: SeriesSearchWithReadProgress, userId: String, pageable: Pageable): Page<SeriesDto> {
Expand All @@ -86,7 +85,7 @@ class SeriesDtoDao(

val having = search.readStatus?.toCondition() ?: DSL.trueCondition()

return findAll(conditions, having, userId, pageable)
return findAll(conditions, having, userId, pageable, search.toJoinConditions())
}

override fun findByIdOrNull(seriesId: String, userId: String): SeriesDto? =
Expand All @@ -97,27 +96,36 @@ class SeriesDtoDao(
.firstOrNull()


private fun selectBase(userId: String, selectCollectionNumber: Boolean = false): SelectOnConditionStep<Record> =
private fun selectBase(
userId: String,
joinConditions: JoinConditions = JoinConditions()
): SelectOnConditionStep<Record> =
dsl.selectDistinct(*groupFields)
.select(DSL.countDistinct(b.ID).`as`(BOOKS_COUNT))
.apply { if (selectCollectionNumber) select(cs.NUMBER) }
.apply { if (joinConditions.selectCollectionNumber) select(cs.NUMBER) }
.from(s)
.leftJoin(b).on(s.ID.eq(b.SERIES_ID))
.leftJoin(d).on(s.ID.eq(d.SERIES_ID))
.leftJoin(r).on(b.ID.eq(r.BOOK_ID)).and(readProgressCondition(userId))
.leftJoin(g).on(s.ID.eq(g.SERIES_ID))
.leftJoin(st).on(s.ID.eq(st.SERIES_ID))
.leftJoin(cs).on(s.ID.eq(cs.SERIES_ID))

private fun findAll(conditions: Condition, having: Condition, userId: String, pageable: Pageable, selectCollectionNumber: Boolean = false): Page<SeriesDto> {
.apply { if (joinConditions.genre) leftJoin(g).on(s.ID.eq(g.SERIES_ID)) }
.apply { if (joinConditions.tag) leftJoin(st).on(s.ID.eq(st.SERIES_ID)) }
.apply { if (joinConditions.collection) leftJoin(cs).on(s.ID.eq(cs.SERIES_ID)) }

private fun findAll(
conditions: Condition,
having: Condition,
userId: String,
pageable: Pageable,
joinConditions: JoinConditions = JoinConditions()
): Page<SeriesDto> {
val count = dsl.selectDistinct(s.ID)
.from(s)
.leftJoin(b).on(s.ID.eq(b.SERIES_ID))
.leftJoin(d).on(s.ID.eq(d.SERIES_ID))
.leftJoin(r).on(b.ID.eq(r.BOOK_ID)).and(readProgressCondition(userId))
.leftJoin(g).on(s.ID.eq(g.SERIES_ID))
.leftJoin(st).on(s.ID.eq(st.SERIES_ID))
.leftJoin(cs).on(s.ID.eq(cs.SERIES_ID))
.apply { if (joinConditions.genre) leftJoin(g).on(s.ID.eq(g.SERIES_ID)) }
.apply { if (joinConditions.tag) leftJoin(st).on(s.ID.eq(st.SERIES_ID)) }
.apply { if (joinConditions.collection) leftJoin(cs).on(s.ID.eq(cs.SERIES_ID)) }
.where(conditions)
.groupBy(s.ID)
.having(having)
Expand All @@ -126,7 +134,7 @@ class SeriesDtoDao(

val orderBy = pageable.sort.toOrderBy(sorts)

val dtos = selectBase(userId, selectCollectionNumber)
val dtos = selectBase(userId, joinConditions)
.where(conditions)
.groupBy(*groupFields)
.having(having)
Expand Down Expand Up @@ -182,18 +190,32 @@ class SeriesDtoDao(
private fun SeriesSearchWithReadProgress.toCondition(): Condition {
var c: Condition = DSL.trueCondition()

libraryIds?.let { c = c.and(s.LIBRARY_ID.`in`(it)) }
collectionIds?.let { c = c.and(cs.COLLECTION_ID.`in`(it)) }
if (!libraryIds.isNullOrEmpty()) c = c.and(s.LIBRARY_ID.`in`(libraryIds))
if (!collectionIds.isNullOrEmpty()) c = c.and(cs.COLLECTION_ID.`in`(collectionIds))
searchTerm?.let { c = c.and(d.TITLE.containsIgnoreCase(it)) }
metadataStatus?.let { c = c.and(d.STATUS.`in`(it)) }
publishers?.let { publishers -> c = c.and(lower(d.PUBLISHER).`in`(publishers.map { it.toLowerCase() })) }
languages?.let { languages -> c = c.and(lower(d.LANGUAGE).`in`(languages.map { it.toLowerCase() })) }
genres?.let { genres -> c = c.and(lower(g.GENRE).`in`(genres.map { it.toLowerCase() })) }
tags?.let { tags -> c = c.and(lower(st.TAG).`in`(tags.map { it.toLowerCase() })) }
if (!metadataStatus.isNullOrEmpty()) c = c.and(d.STATUS.`in`(metadataStatus))
if (!publishers.isNullOrEmpty()) c = c.and(lower(d.PUBLISHER).`in`(publishers.map { it.toLowerCase() }))
if (!languages.isNullOrEmpty()) c = c.and(lower(d.LANGUAGE).`in`(languages.map { it.toLowerCase() }))
if (!genres.isNullOrEmpty()) c = c.and(lower(g.GENRE).`in`(genres.map { it.toLowerCase() }))
if (!tags.isNullOrEmpty()) c = c.and(lower(st.TAG).`in`(tags.map { it.toLowerCase() }))

return c
}

private fun SeriesSearchWithReadProgress.toJoinConditions() =
JoinConditions(
genre = !genres.isNullOrEmpty(),
tag = !tags.isNullOrEmpty(),
collection = !collectionIds.isNullOrEmpty()
)

private data class JoinConditions(
val selectCollectionNumber: Boolean = false,
val genre: Boolean = false,
val tag: Boolean = false,
val collection: Boolean = false
)

private fun Collection<ReadStatus>.toCondition(): Condition =
map {
when (it) {
Expand Down

0 comments on commit 6eb7669

Please sign in to comment.