From 22310eecfde6c813d57694cc8b12af242745e757 Mon Sep 17 00:00:00 2001 From: Akhilesh Thite Date: Thu, 9 May 2024 21:16:28 +0530 Subject: [PATCH 1/9] feat: implement random sort and dynamic timeline sorting via dropdown --- db.js | 25 +++++++++++++ index.html | 12 +++++- theme-selector.js | 2 +- timeline.css | 34 ++++++++++++++++- timeline.js | 95 +++++++++++++++++++++++++++++++++++++++-------- 5 files changed, 149 insertions(+), 19 deletions(-) diff --git a/db.js b/db.js index 9dcd007..53be54a 100644 --- a/db.js +++ b/db.js @@ -250,6 +250,31 @@ export class ActivityPubDB extends EventTarget { await tx.done } + async * searchNotesRandom ({ limit = DEFAULT_LIMIT } = {}) { + const tx = this.db.transaction(NOTES_STORE, 'readonly') + const store = tx.objectStore(NOTES_STORE) + const totalNotes = await store.count() + + for (let i = 0; i < limit; i++) { + const randomSkip = Math.floor(Math.random() * totalNotes) + let cursor = await store.openCursor() + if (cursor) { + if (randomSkip > 0) { // Only advance if randomSkip is greater than 0 + await cursor.advance(randomSkip) + if (cursor) { + yield cursor.value + cursor = await cursor.continue() + } + } else { + // If randomSkip is 0, yield the first item directly + yield cursor.value + cursor = await cursor.continue() // Continue to the next for proper iteration + } + } + } + await tx.done + } + async ingestActor (url, isInitial = false) { console.log(`Starting ingestion for actor from URL: ${url}`) const actor = await this.getActor(url) diff --git a/index.html b/index.html index 54d695c..d99a83c 100644 --- a/index.html +++ b/index.html @@ -12,7 +12,17 @@
- +
+
+ + +
+ +
diff --git a/theme-selector.js b/theme-selector.js index 6eadb1e..87ceaeb 100644 --- a/theme-selector.js +++ b/theme-selector.js @@ -39,7 +39,7 @@ class ThemeSelector extends HTMLElement { const style = document.createElement('style') style.textContent = ` select { - padding: 4px; + padding: 2px; margin: 6px 0; border: 1px solid var(--rdp-border-color); border-radius: 4px; diff --git a/timeline.css b/timeline.css index e70aabf..6f8d6df 100644 --- a/timeline.css +++ b/timeline.css @@ -9,6 +9,35 @@ body { background: var(--bg-color); } +.reader-container { + margin-top: 20px; + display: flex; + flex-direction: column; + align-items: flex-start; +} + +.sort-container { + width: 100%; + display: flex; + justify-content: flex-start; +} + +.sort-container label { + color: var(--rdp-text-color); + font-size: 0.875rem; + margin-right: 4px; + display: flex; + align-items: center; + margin-left: 48px; +} + +.sort-dropdown { + padding: 2px; + border: 1px solid var(--rdp-border-color); + border-radius: 4px; + width: 75px; +} + reader-timeline { flex: 1; max-width: 600px; @@ -17,9 +46,12 @@ reader-timeline { } @media screen and (max-width: 768px) { + .reader-container { + margin-top: 160px; + } + reader-timeline { width: 100%; max-width: 100%; - margin-top: 150px; } } diff --git a/timeline.js b/timeline.js index b42477b..3ea8be1 100644 --- a/timeline.js +++ b/timeline.js @@ -6,12 +6,15 @@ class ReaderTimeline extends HTMLElement { skip = 0 limit = 32 hasMoreItems = true + sort = 'latest' loadMoreBtn = null + randomNotes = [] // Cache for random notes + randomIndex = 0 // Current index in the random notes cache constructor () { super() this.loadMoreBtn = document.createElement('button') - this.loadMoreBtn.textContent = 'Load More..' + this.loadMoreBtn.textContent = 'Load More...' this.loadMoreBtn.className = 'load-more-btn' this.loadMoreBtnWrapper = document.createElement('div') @@ -22,9 +25,42 @@ class ReaderTimeline extends HTMLElement { } connectedCallback () { + this.initializeSortOrder() this.initializeDefaultFollowedActors().then(() => this.initTimeline()) } + initializeSortOrder () { + const params = new URLSearchParams(window.location.search) + this.sort = params.get('sort') || 'latest' + + const sortOrderSelect = document.getElementById('sortOrder') + if (sortOrderSelect) { + sortOrderSelect.value = this.sort + sortOrderSelect.addEventListener('change', (event) => { + this.sort = event.target.value + this.updateURL() + this.resetTimeline() + }) + } + } + + updateURL () { + const url = new URL(window.location) + url.searchParams.set('sort', this.sort) + window.history.pushState({}, '', url) + } + + async resetTimeline () { + this.skip = 0 + this.randomIndex = 0 + this.randomNotes = [] + this.hasMoreItems = true + while (this.firstChild) { + this.removeChild(this.firstChild) + } + this.loadMore() + } + async initializeDefaultFollowedActors () { const defaultActors = [ 'https://social.distributed.press/v1/@announcements@social.distributed.press/', @@ -35,14 +71,9 @@ class ReaderTimeline extends HTMLElement { // "https://staticpub.mauve.moe/about.jsonld", ] - // Check if followed actors have already been initialized const hasFollowedActors = await db.hasFollowedActors() if (!hasFollowedActors) { - await Promise.all( - defaultActors.map(async (actorUrl) => { - await db.followActor(actorUrl) - }) - ) + await Promise.all(defaultActors.map(actorUrl => db.followActor(actorUrl))) } } @@ -56,20 +87,52 @@ class ReaderTimeline extends HTMLElement { } async loadMore () { - // Remove the button before loading more items this.loadMoreBtnWrapper.remove() - let count = 0 - for await (const note of db.searchNotes({}, { skip: this.skip, limit: this.limit })) { - count++ - this.appendNoteElement(note) + + if (this.sort === 'random' && this.randomNotes.length === 0) { + const allNotes = [] + for await (const note of db.searchNotesRandom(this.limit)) { + allNotes.push(note) + } + this.randomNotes = allNotes.sort(() => Math.random() - 0.5) + } + + const notesToShow = this.sort === 'random' + ? this.randomNotes.slice(this.randomIndex, this.randomIndex + this.limit) + : await this.fetchSortedNotes() + + for (const note of notesToShow) { + if (note) { + this.appendNoteElement(note) + count++ + } } - // Update skip value and determine if there are more items - this.skip += this.limit - this.hasMoreItems = count === this.limit + this.updateIndexes(count) + this.appendLoadMoreIfNeeded() + } + + async fetchSortedNotes () { + const notesGenerator = db.searchNotes({}, { skip: this.skip, limit: this.limit, sort: this.sort === 'oldest' ? 1 : -1 }) + const notes = [] + for await (const note of notesGenerator) { + notes.push(note) + } + return notes + } + + updateIndexes (count) { + if (this.sort === 'random') { + this.randomIndex += this.limit + this.hasMoreItems = this.randomIndex < this.randomNotes.length + } else { + this.skip += this.limit + this.hasMoreItems = count === this.limit + } + } - // Append the button at the end if there are more items + appendLoadMoreIfNeeded () { if (this.hasMoreItems) { this.appendChild(this.loadMoreBtnWrapper) } From 6de5757a4462adfacb131c83a3b97f311b4e7f75 Mon Sep 17 00:00:00 2001 From: Akhilesh Thite Date: Tue, 14 May 2024 18:55:17 +0530 Subject: [PATCH 2/9] refactor: implement robust random access and efficient cursor management for searchNotesRandom --- db.js | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/db.js b/db.js index 53be54a..af8fd53 100644 --- a/db.js +++ b/db.js @@ -255,20 +255,23 @@ export class ActivityPubDB extends EventTarget { const store = tx.objectStore(NOTES_STORE) const totalNotes = await store.count() - for (let i = 0; i < limit; i++) { + if (totalNotes === 0) return // Early exit if no notes are present + + let cursor = await store.openCursor() + const uniqueIndexes = new Set() + + while (uniqueIndexes.size < limit && uniqueIndexes.size < totalNotes) { const randomSkip = Math.floor(Math.random() * totalNotes) - let cursor = await store.openCursor() - if (cursor) { - if (randomSkip > 0) { // Only advance if randomSkip is greater than 0 + if (!uniqueIndexes.has(randomSkip)) { + uniqueIndexes.add(randomSkip) + if (randomSkip > 0) { + // Move the cursor to the randomSkip position if not already there await cursor.advance(randomSkip) - if (cursor) { - yield cursor.value - cursor = await cursor.continue() - } - } else { - // If randomSkip is 0, yield the first item directly + } + if (cursor) { yield cursor.value - cursor = await cursor.continue() // Continue to the next for proper iteration + // After yielding, reset the cursor to the start for the next random pick + cursor = await store.openCursor() } } } From 4a2e8f9387f2ce36418d10fd48fcb71d932e6dd7 Mon Sep 17 00:00:00 2001 From: Akhilesh Thite Date: Tue, 14 May 2024 21:16:58 +0530 Subject: [PATCH 3/9] perf: optimize random note fetching and streamline timeline updates --- db.js | 7 +++++++ timeline.js | 48 ++++++++++++++++++++++++------------------------ 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/db.js b/db.js index af8fd53..32b02e0 100644 --- a/db.js +++ b/db.js @@ -191,6 +191,13 @@ export class ActivityPubDB extends EventTarget { } } + async getTotalNotesCount () { + const tx = this.db.transaction(NOTES_STORE, 'readonly') + const store = tx.objectStore(NOTES_STORE) + const totalNotes = await store.count() + return totalNotes + } + async getActivity (url) { try { return this.db.get(ACTIVITIES_STORE, url) diff --git a/timeline.js b/timeline.js index 3ea8be1..56540bd 100644 --- a/timeline.js +++ b/timeline.js @@ -7,9 +7,9 @@ class ReaderTimeline extends HTMLElement { limit = 32 hasMoreItems = true sort = 'latest' + totalNotesCount = 0 + loadedNotesCount = 0 loadMoreBtn = null - randomNotes = [] // Cache for random notes - randomIndex = 0 // Current index in the random notes cache constructor () { super() @@ -24,7 +24,7 @@ class ReaderTimeline extends HTMLElement { this.loadMoreBtn.addEventListener('click', () => this.loadMore()) } - connectedCallback () { + async connectedCallback () { this.initializeSortOrder() this.initializeDefaultFollowedActors().then(() => this.initTimeline()) } @@ -52,8 +52,8 @@ class ReaderTimeline extends HTMLElement { async resetTimeline () { this.skip = 0 - this.randomIndex = 0 - this.randomNotes = [] + this.totalNotesCount = await db.getTotalNotesCount() + this.loadedNotesCount = 0 this.hasMoreItems = true while (this.firstChild) { this.removeChild(this.firstChild) @@ -78,35 +78,35 @@ class ReaderTimeline extends HTMLElement { } async initTimeline () { + this.loadMore() // Start loading notes immediately + if (!hasLoaded) { hasLoaded = true const followedActors = await db.getFollowedActors() - await Promise.all(followedActors.map(({ url }) => db.ingestActor(url))) + // Ingest actors in the background without waiting for them + Promise.all(followedActors.map(({ url }) => db.ingestActor(url))) + .then(() => console.log('All followed actors have been ingested')) + .catch(error => console.error('Error ingesting followed actors:', error)) } - this.loadMore() } async loadMore () { this.loadMoreBtnWrapper.remove() let count = 0 - if (this.sort === 'random' && this.randomNotes.length === 0) { - const allNotes = [] - for await (const note of db.searchNotesRandom(this.limit)) { - allNotes.push(note) - } - this.randomNotes = allNotes.sort(() => Math.random() - 0.5) - } - - const notesToShow = this.sort === 'random' - ? this.randomNotes.slice(this.randomIndex, this.randomIndex + this.limit) - : await this.fetchSortedNotes() - - for (const note of notesToShow) { - if (note) { + if (this.sort === 'random') { + for await (const note of db.searchNotesRandom({ limit: this.limit })) { this.appendNoteElement(note) count++ } + } else { + const notesToShow = await this.fetchSortedNotes() + for (const note of notesToShow) { + if (note) { + this.appendNoteElement(note) + count++ + } + } } this.updateIndexes(count) @@ -124,10 +124,10 @@ class ReaderTimeline extends HTMLElement { updateIndexes (count) { if (this.sort === 'random') { - this.randomIndex += this.limit - this.hasMoreItems = this.randomIndex < this.randomNotes.length + this.loadedNotesCount += count + this.hasMoreItems = this.loadedNotesCount < this.totalNotesCount } else { - this.skip += this.limit + this.skip += count this.hasMoreItems = count === this.limit } } From 4299062d863bee91d81f72217900351f4b04d4de Mon Sep 17 00:00:00 2001 From: Akhilesh Thite Date: Thu, 23 May 2024 17:37:09 +0530 Subject: [PATCH 4/9] chore: rename updateIndexes function to updateHasMore --- timeline.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/timeline.js b/timeline.js index 56540bd..2281502 100644 --- a/timeline.js +++ b/timeline.js @@ -109,7 +109,7 @@ class ReaderTimeline extends HTMLElement { } } - this.updateIndexes(count) + this.updateHasMore(count) this.appendLoadMoreIfNeeded() } @@ -122,7 +122,7 @@ class ReaderTimeline extends HTMLElement { return notes } - updateIndexes (count) { + updateHasMore (count) { if (this.sort === 'random') { this.loadedNotesCount += count this.hasMoreItems = this.loadedNotesCount < this.totalNotesCount From b0acbb0d1366fe8323a7d7d0bacc97bfd0e7e45d Mon Sep 17 00:00:00 2001 From: Akhilesh Thite Date: Thu, 23 May 2024 23:11:29 +0530 Subject: [PATCH 5/9] refactor: allow duplicates and always open new cursor for random notes fetch --- db.js | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/db.js b/db.js index 32b02e0..8b7a32b 100644 --- a/db.js +++ b/db.js @@ -227,31 +227,24 @@ export class ActivityPubDB extends EventTarget { await tx.done() } - async * searchNotes ({ attributedTo } = {}, { skip = 0, limit = DEFAULT_LIMIT, sort = -1 } = {}) { + async * searchNotesRandom ({ limit = DEFAULT_LIMIT } = {}) { const tx = this.db.transaction(NOTES_STORE, 'readonly') - let count = 0 - const direction = sort > 0 ? 'next' : 'prev' // 'prev' for descending order - let cursor = null - - const indexName = attributedTo ? ATTRIBUTED_TO_FIELD + ', published' : PUBLISHED_FIELD + const store = tx.objectStore(NOTES_STORE) + const totalNotes = await store.count() - const index = tx.store.index(indexName) + if (totalNotes === 0) return // Early exit if no notes are present - if (attributedTo) { - cursor = await index.openCursor([attributedTo], direction) - } else { - cursor = await index.openCursor(null, direction) - } + for (let i = 0; i < limit; i++) { + const randomSkip = Math.floor(Math.random() * totalNotes) + const cursor = await store.openCursor() - // Skip the required entries - if (skip) await cursor.advance(skip) + if (randomSkip > 0) { + await cursor.advance(randomSkip) + } - // Collect the required limit of entries - while (cursor) { - if (count >= limit) break - count++ - yield cursor.value - cursor = await cursor.continue() + if (cursor) { + yield cursor.value + } } await tx.done From 529e158dd6c0dbcd40a7ad178c4c923a117ad34e Mon Sep 17 00:00:00 2001 From: Akhilesh Thite Date: Thu, 30 May 2024 18:24:09 +0530 Subject: [PATCH 6/9] refactor: merge searchNotesRandom functionality into searchNotes with sort=0 for randomization --- db.js | 65 ++++++++++++++++++++++++----------------------------- timeline.js | 2 +- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/db.js b/db.js index 8b7a32b..42b5c25 100644 --- a/db.js +++ b/db.js @@ -227,54 +227,47 @@ export class ActivityPubDB extends EventTarget { await tx.done() } - async * searchNotesRandom ({ limit = DEFAULT_LIMIT } = {}) { + async * searchNotes ({ attributedTo } = {}, { skip = 0, limit = DEFAULT_LIMIT, sort = -1 } = {}) { const tx = this.db.transaction(NOTES_STORE, 'readonly') - const store = tx.objectStore(NOTES_STORE) - const totalNotes = await store.count() - - if (totalNotes === 0) return // Early exit if no notes are present - - for (let i = 0; i < limit; i++) { - const randomSkip = Math.floor(Math.random() * totalNotes) - const cursor = await store.openCursor() - - if (randomSkip > 0) { - await cursor.advance(randomSkip) - } - - if (cursor) { - yield cursor.value - } - } - - await tx.done - } - - async * searchNotesRandom ({ limit = DEFAULT_LIMIT } = {}) { - const tx = this.db.transaction(NOTES_STORE, 'readonly') - const store = tx.objectStore(NOTES_STORE) - const totalNotes = await store.count() + let count = 0 + const direction = sort > 0 ? 'next' : (sort === 0 ? 'next' : 'prev') // 'prev' for descending order + let cursor = null - if (totalNotes === 0) return // Early exit if no notes are present + const indexName = attributedTo ? ATTRIBUTED_TO_FIELD + ', published' : PUBLISHED_FIELD - let cursor = await store.openCursor() - const uniqueIndexes = new Set() + const index = tx.store.index(indexName) - while (uniqueIndexes.size < limit && uniqueIndexes.size < totalNotes) { - const randomSkip = Math.floor(Math.random() * totalNotes) - if (!uniqueIndexes.has(randomSkip)) { - uniqueIndexes.add(randomSkip) + if (sort === 0) { // Random sort + const totalNotes = await index.count() + for (let i = 0; i < limit; i++) { + const randomSkip = Math.floor(Math.random() * totalNotes) + cursor = await index.openCursor() if (randomSkip > 0) { - // Move the cursor to the randomSkip position if not already there await cursor.advance(randomSkip) } if (cursor) { yield cursor.value - // After yielding, reset the cursor to the start for the next random pick - cursor = await store.openCursor() } } + } else { + if (attributedTo) { + cursor = await index.openCursor([attributedTo], direction) + } else { + cursor = await index.openCursor(null, direction) + } + + // Skip the required entries + if (skip) await cursor.advance(skip) + + // Collect the required limit of entries + while (cursor) { + if (count >= limit) break + count++ + yield cursor.value + cursor = await cursor.continue() + } } + await tx.done } diff --git a/timeline.js b/timeline.js index 2281502..d746d00 100644 --- a/timeline.js +++ b/timeline.js @@ -95,7 +95,7 @@ class ReaderTimeline extends HTMLElement { let count = 0 if (this.sort === 'random') { - for await (const note of db.searchNotesRandom({ limit: this.limit })) { + for await (const note of db.searchNotes({ limit: this.limit })) { this.appendNoteElement(note) count++ } From cbbf2f7610a6e5a57e89bcd993f6c02237f93761 Mon Sep 17 00:00:00 2001 From: Akhilesh Thite Date: Thu, 30 May 2024 18:27:04 +0530 Subject: [PATCH 7/9] chore: add a TODO to consider removing duplicates in the random sort for better UX in the future --- db.js | 1 + 1 file changed, 1 insertion(+) diff --git a/db.js b/db.js index 42b5c25..4fc1816 100644 --- a/db.js +++ b/db.js @@ -238,6 +238,7 @@ export class ActivityPubDB extends EventTarget { const index = tx.store.index(indexName) if (sort === 0) { // Random sort + // TODO: Consider removing duplicates in the future to improve UX const totalNotes = await index.count() for (let i = 0; i < limit; i++) { const randomSkip = Math.floor(Math.random() * totalNotes) From e029220348bbeb2a31ad398ccaba8b2896674ae8 Mon Sep 17 00:00:00 2001 From: Akhilesh Thite Date: Tue, 4 Jun 2024 01:18:31 +0530 Subject: [PATCH 8/9] fix: pass dynamic sort parameter for random note fetching in timeline --- .vscode/settings.json | 3 +++ timeline.js | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..6f3a291 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "liveServer.settings.port": 5501 +} \ No newline at end of file diff --git a/timeline.js b/timeline.js index d746d00..fb7a685 100644 --- a/timeline.js +++ b/timeline.js @@ -95,7 +95,7 @@ class ReaderTimeline extends HTMLElement { let count = 0 if (this.sort === 'random') { - for await (const note of db.searchNotes({ limit: this.limit })) { + for await (const note of db.searchNotes({}, { limit: this.limit, sort: this.sort === 'random' ? 0 : (this.sort === 'oldest' ? 1 : -1) })) { this.appendNoteElement(note) count++ } From 63a49dd519b6656c994acf88f45072a646394d53 Mon Sep 17 00:00:00 2001 From: Mauve Signweaver Date: Wed, 5 Jun 2024 13:09:02 -0400 Subject: [PATCH 9/9] remove vscode folder --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 6f3a291..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "liveServer.settings.port": 5501 -} \ No newline at end of file