Skip to content

Commit 72c5e83

Browse files
msasikanthmsasikanth
authored
Move post content and full article HTML to separate table (#1281)
* Add `FullArticleFetcher` to fetch HTML content * Move post raw content to a separate table The `rawContent` field has been removed from the `Post` table and moved to a new `postContent` table. This change helps to keep the main `Post` table lean, as `rawContent` can be large and is not always needed. The `postContent` table will store the `rawContent` and `htmlContent` (for full article view) for each post. This change also involves: - Updating database queries and triggers to reflect the new table structure. - Modifying `RssRepository` to handle the new `postContent` table. - Creating a new `PostContentRepository` to manage data in the `postContent` table. - Updating data models (`Post`, `PostWithMetadata`) to remove `rawContent`. - Creating a new `PostContent` data model. * Refactor Reader JS to not fetch full article content The functionality to fetch the full article content was removed from the `parseReaderContent` JavaScript function. Instead, the HTML content is now passed directly as a parameter. This change simplifies the JavaScript logic and ensures that the reader component relies on the caller to provide the complete HTML content. Corresponding updates were made in the Android and iOS `ReaderWebView` implementations to pass the content directly. * Run code formatting * Remove image from formatted Reddit post The image was previously displayed above the post content. This change removes the image from the formatted Reddit post since we already display the hero image in the detail view this always gets removed in later step, so removing it when formatting the post itself. * Remove `fetchFullArticle` parameter from `ReaderWebView` The `fetchFullArticle` parameter was unused and has been removed from the `ReaderWebView` composable function and its platform-specific implementations. * Add vertical padding to progress indicator in reader screen * Pass post ID to `ReaderPageViewModel` The `ReaderPageViewModel` now takes the post ID as a constructor parameter. This allows the view model to load the post content and full article HTML directly, instead of relying on the `ReaderScreen` to pass it down. * Load full article content in reader screen when the button is clicked * Refactor `ReaderPageViewModel`
1 parent ad05548 commit 72c5e83

File tree

21 files changed

+431
-93
lines changed

21 files changed

+431
-93
lines changed

‎core/data/src/commonMain/kotlin/dev/sasikanth/rss/reader/data/di/DataComponent.kt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import dev.sasikanth.rss.reader.data.database.Bookmark
2121
import dev.sasikanth.rss.reader.data.database.Feed
2222
import dev.sasikanth.rss.reader.data.database.FeedGroup
2323
import dev.sasikanth.rss.reader.data.database.Post
24+
import dev.sasikanth.rss.reader.data.database.PostContent
2425
import dev.sasikanth.rss.reader.data.database.ReaderDatabase
2526
import dev.sasikanth.rss.reader.data.database.adapter.DateAdapter
2627
import dev.sasikanth.rss.reader.data.database.adapter.DurationAdapter
@@ -55,6 +56,10 @@ interface DataComponent :
5556
createdAtAdapter = DateAdapter,
5657
updatedAtAdapter = DateAdapter,
5758
pinnedAtAdapter = DateAdapter
59+
),
60+
postContentAdapter =
61+
PostContent.Adapter(
62+
createdAtAdapter = DateAdapter,
5863
)
5964
)
6065
}
@@ -150,4 +155,6 @@ interface DataComponent :
150155
fun providesFeedGroupFeedQueries(database: ReaderDatabase) = database.feedGroupFeedQueries
151156

152157
@Provides fun providesUserQueries(database: ReaderDatabase) = database.userQueries
158+
159+
@Provides fun providesPostContentQueries(database: ReaderDatabase) = database.postContentQueries
153160
}

‎core/data/src/commonMain/kotlin/dev/sasikanth/rss/reader/data/repository/PostContentRepository.kt‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Copyright 2025 Sasikanth Miriyampalli
3+
*
4+
* Licensed under the GPL, Version 3.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.gnu.org/licenses/gpl-3.0.en.html
9+
*
10+
*/
11+
12+
package dev.sasikanth.rss.reader.data.repository
13+
14+
import app.cash.sqldelight.coroutines.asFlow
15+
import app.cash.sqldelight.coroutines.mapToOne
16+
import dev.sasikanth.rss.reader.core.model.local.PostContent
17+
import dev.sasikanth.rss.reader.data.database.PostContentQueries
18+
import dev.sasikanth.rss.reader.util.DispatchersProvider
19+
import kotlinx.coroutines.flow.Flow
20+
import kotlinx.coroutines.withContext
21+
import me.tatarka.inject.annotations.Inject
22+
23+
@Inject
24+
class PostContentRepository(
25+
private val postContentQueries: PostContentQueries,
26+
private val dispatcherProvider: DispatchersProvider,
27+
) {
28+
29+
fun postContent(postId: String): Flow<PostContent> {
30+
return postContentQueries
31+
.getByPostId(postId) { id, rawContent, htmlContent, createdAt ->
32+
PostContent(id, rawContent, htmlContent)
33+
}
34+
.asFlow()
35+
.mapToOne(dispatcherProvider.databaseRead)
36+
}
37+
38+
suspend fun updateFullArticleContent(postId: String, htmlContent: String?) {
39+
withContext(dispatcherProvider.databaseWrite) {
40+
postContentQueries.updateHtmlContent(htmlContent, postId)
41+
}
42+
}
43+
}

‎core/data/src/commonMain/kotlin/dev/sasikanth/rss/reader/data/repository/RssRepository.kt‎

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import dev.sasikanth.rss.reader.data.database.FeedGroupFeedQueries
3636
import dev.sasikanth.rss.reader.data.database.FeedGroupQueries
3737
import dev.sasikanth.rss.reader.data.database.FeedQueries
3838
import dev.sasikanth.rss.reader.data.database.FeedSearchFTSQueries
39+
import dev.sasikanth.rss.reader.data.database.PostContentQueries
3940
import dev.sasikanth.rss.reader.data.database.PostQueries
4041
import dev.sasikanth.rss.reader.data.database.PostSearchFTSQueries
4142
import dev.sasikanth.rss.reader.data.database.SourceQueries
@@ -59,6 +60,7 @@ class RssRepository(
5960
private val transactionRunner: TransactionRunner,
6061
private val feedQueries: FeedQueries,
6162
private val postQueries: PostQueries,
63+
private val postContentQueries: PostContentQueries,
6264
private val postSearchFTSQueries: PostSearchFTSQueries,
6365
private val bookmarkQueries: BookmarkQueries,
6466
private val feedSearchFTSQueries: FeedSearchFTSQueries,
@@ -96,22 +98,31 @@ class RssRepository(
9698
feedLastCleanUpAt?.toEpochMilliseconds()
9799
?: Instant.DISTANT_PAST.toEpochMilliseconds()
98100

99-
feedPayload.posts.forEach { post ->
100-
if (post.date < feedLastCleanUpAtEpochMilli) return@forEach
101+
feedPayload.posts.forEach { postPayload ->
102+
if (postPayload.date < feedLastCleanUpAtEpochMilli) return@forEach
101103

104+
val postId = nameBasedUuidOf(postPayload.link).toString()
102105
postQueries.upsert(
103-
id = nameBasedUuidOf(post.link).toString(),
106+
id = postId,
104107
sourceId = feedId,
105-
title = post.title,
106-
description = post.description,
107-
imageUrl = post.imageUrl,
108-
date = Instant.fromEpochMilliseconds(post.date),
108+
title = postPayload.title,
109+
description = postPayload.description,
110+
imageUrl = postPayload.imageUrl,
111+
date = Instant.fromEpochMilliseconds(postPayload.date),
109112
syncedAt = Clock.System.now(),
110-
link = post.link,
111-
commnetsLink = post.commentsLink,
112-
rawContent = post.rawContent,
113-
isDateParsedCorrectly = post.isDateParsedCorrectly,
113+
link = postPayload.link,
114+
commnetsLink = postPayload.commentsLink,
115+
isDateParsedCorrectly = postPayload.isDateParsedCorrectly,
114116
)
117+
118+
if (postPayload.rawContent != null) {
119+
postContentQueries.upsert(
120+
id = postId,
121+
rawContent = postPayload.rawContent,
122+
htmlContent = null,
123+
createdAt = Clock.System.now(),
124+
)
125+
}
115126
}
116127
}
117128
}
@@ -232,7 +243,6 @@ class RssRepository(
232243
sourceId,
233244
title,
234245
description,
235-
rawContent,
236246
imageUrl,
237247
date,
238248
link,
@@ -249,7 +259,6 @@ class RssRepository(
249259
sourceId = sourceId,
250260
title = title,
251261
description = description,
252-
rawContent = rawContent,
253262
imageUrl = imageUrl,
254263
date = date,
255264
link = link,
@@ -536,7 +545,6 @@ class RssRepository(
536545
sourceId = sourceId,
537546
title = title,
538547
description = description,
539-
rawContent = description,
540548
imageUrl = imageUrl,
541549
date = date,
542550
link = link,

‎core/data/src/commonMain/kotlin/dev/sasikanth/rss/reader/data/repository/WidgetDataRepository.kt‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@ class WidgetDataRepository(
7272
sourceId,
7373
title,
7474
description,
75-
rawContent,
7675
imageUrl,
7776
date,
7877
link,
@@ -110,7 +109,6 @@ class WidgetDataRepository(
110109
sourceId,
111110
title,
112111
description,
113-
rawContent,
114112
imageUrl,
115113
date,
116114
link,
@@ -150,7 +148,6 @@ class WidgetDataRepository(
150148
sourceId,
151149
title,
152150
description,
153-
rawContent,
154151
imageUrl,
155152
date,
156153
link,
@@ -166,7 +163,6 @@ class WidgetDataRepository(
166163
sourceId = sourceId,
167164
title = title,
168165
description = description,
169-
rawContent = rawContent,
170166
imageUrl = imageUrl,
171167
date = date,
172168
link = link,

‎core/data/src/commonMain/sqldelight/dev/sasikanth/rss/reader/data/database/BlockedWords.sq‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,7 @@ BEGIN
1212
SET isHidden = 1
1313
WHERE
1414
(title LIKE '%' || new.content || '%' OR
15-
description LIKE '%' || new.content || '%' OR
16-
rawContent LIKE '%' || new.content || '%') AND
15+
description LIKE '%' || new.content || '%') AND
1716
isHidden = 0;
1817
END;
1918

@@ -26,8 +25,7 @@ BEGIN
2625
SELECT 1
2726
FROM blockedWord
2827
WHERE (post.title LIKE '%' || blockedWord.content || '%' OR
29-
post.description LIKE '%' || blockedWord.content || '%' OR
30-
post.rawContent LIKE '%' || blockedWord.content || '%')
28+
post.description LIKE '%' || blockedWord.content || '%')
3129
)
3230
AND isHidden = 1;
3331
END;

‎core/data/src/commonMain/sqldelight/dev/sasikanth/rss/reader/data/database/Post.sq‎

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ CREATE TABLE post(
66
sourceId TEXT NOT NULL,
77
title TEXT NOT NULL,
88
description TEXT NOT NULL,
9-
rawContent TEXT,
109
imageUrl TEXT,
1110
date INTEGER AS Instant NOT NULL,
1211
syncedAt INTEGER AS Instant NOT NULL,
@@ -31,52 +30,48 @@ WHEN (
3130
FROM blockedWord
3231
WHERE
3332
new.title LIKE '%' || blockedWord.content || '%' OR
34-
new.description LIKE '%' || blockedWord.content || '%' OR
35-
new.rawContent LIKE '%' || blockedWord.content || '%'
33+
new.description LIKE '%' || blockedWord.content || '%'
3634
) IS NOT NULL
3735
BEGIN
3836
UPDATE post SET isHidden = 1 WHERE id = new.id;
3937
END;
4038

4139
CREATE TRIGGER hide_post_if_blocked_word_is_present_BEFORE_UPDATE
42-
BEFORE UPDATE OF title, description, rawContent ON post
40+
BEFORE UPDATE OF title, description ON post -- Removed rawContent
4341
FOR EACH ROW
4442
WHEN (
4543
SELECT 1
4644
FROM blockedWord
4745
WHERE
4846
(new.title LIKE '%' || blockedWord.content || '%' OR
49-
new.description LIKE '%' || blockedWord.content || '%' OR
50-
new.rawContent LIKE '%' || blockedWord.content || '%') AND
47+
new.description LIKE '%' || blockedWord.content || '%') AND
5148
new.isHidden == 0
5249
) IS NOT NULL
5350
BEGIN
5451
UPDATE post SET isHidden = 1 WHERE id = new.id;
5552
END;
5653

5754
CREATE TRIGGER unhide_post_if_no_blocked_words_present_and_post_is_hidden_BEFORE_UPDATE
58-
UPDATE OF title, description, rawContent ON post
55+
UPDATE OF title, description ON post
5956
FOR EACH ROW
6057
WHEN (
6158
SELECT 1
6259
FROM blockedWord
6360
WHERE
6461
(new.title LIKE '%' || blockedWord.content || '%' OR
65-
new.description LIKE '%' || blockedWord.content || '%' OR
66-
new.rawContent LIKE '%' || blockedWord.content || '%')
62+
new.description LIKE '%' || blockedWord.content || '%')
6763
) IS NULL AND old.isHidden == 1
6864
BEGIN
6965
UPDATE post SET isHidden = 0 WHERE id = new.id;
7066
END;
7167

7268
upsert:
73-
INSERT INTO post(id, sourceId, title, description, rawContent, imageUrl, date, syncedAt, link, commentsLink)
74-
VALUES (:id, :sourceId, :title, :description, :rawContent, :imageUrl, :date, :syncedAt, :link, :commnetsLink)
69+
INSERT INTO post(id, sourceId, title, description, imageUrl, date, syncedAt, link, commentsLink)
70+
VALUES (:id, :sourceId, :title, :description, :imageUrl, :date, :syncedAt, :link, :commnetsLink)
7571
ON CONFLICT(id, sourceId) DO
7672
UPDATE SET
7773
title = excluded.title,
7874
description = excluded.description,
79-
rawContent = excluded.rawContent,
8075
imageUrl = excluded.imageUrl,
8176
date = CASE WHEN :isDateParsedCorrectly AND date < excluded.date THEN excluded.date ELSE date END,
8277
read = CASE WHEN :isDateParsedCorrectly AND date < excluded.date THEN excluded.read ELSE read END;
@@ -97,7 +92,6 @@ WITH featuredPosts AS (
9792
sourceId,
9893
post.title,
9994
post.description,
100-
post.rawContent,
10195
post.imageUrl,
10296
post.date,
10397
post.link,
@@ -131,7 +125,6 @@ SELECT
131125
sourceId,
132126
post.title,
133127
post.description,
134-
post.rawContent,
135128
post.imageUrl,
136129
post.date,
137130
post.link,
@@ -243,7 +236,6 @@ SELECT
243236
sourceId,
244237
post.title,
245238
post.description,
246-
post.rawContent,
247239
post.imageUrl,
248240
post.date,
249241
post.link,

‎core/data/src/commonMain/sqldelight/dev/sasikanth/rss/reader/data/database/PostContent.sq‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import kotlin.time.Instant;
2+
3+
CREATE TABLE postContent (
4+
id TEXT NOT NULL PRIMARY KEY,
5+
rawContent TEXT,
6+
htmlContent TEXT,
7+
createdAt INTEGER AS Instant NOT NULL
8+
);
9+
10+
upsert:
11+
INSERT OR REPLACE INTO postContent (id, rawContent, htmlContent, createdAt)
12+
VALUES (:id, :rawContent, :htmlContent, :createdAt);
13+
14+
getByPostId:
15+
SELECT * FROM postContent
16+
WHERE id = :id;
17+
18+
updateHtmlContent:
19+
UPDATE postContent SET htmlContent = :htmlContent
20+
WHERE id = :id;

‎core/data/src/commonMain/sqldelight/dev/sasikanth/rss/reader/data/database/PostSearchFTS.sq‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ SELECT
3333
sourceId,
3434
post_search.title,
3535
post_search.description,
36-
post.rawContent,
3736
post.imageUrl,
3837
post.date,
3938
post.link,

0 commit comments

Comments
 (0)