Merge pull request #1048 from WorldBrain/feature/improved-remote-inde…

…xing

Allow remote apps to add page to lists while indexing a page

src/background-script/setup.ts

@@ -191,6 +191,15 @@ export function createBackgroundModules(options: {
           }).processor
         : undefined
 
+    const customLists = new CustomListBackground({
+        storageManager,
+        queryTabs: bindMethod(options.browserAPIs.tabs, 'query'),
+        windows: options.browserAPIs.windows,
+        searchIndex: search.searchIndex,
+        pageStorage: pages.storage,
+        localBrowserStorage: options.browserAPIs.storage.local,
+    })
+
     return {
         auth,
         social,
@@ -208,14 +217,7 @@ export function createBackgroundModules(options: {
         }),
         search,
         eventLog: new EventLogBackground({ storageManager }),
-        customLists: new CustomListBackground({
-            storageManager,
-            queryTabs: bindMethod(options.browserAPIs.tabs, 'query'),
-            windows: options.browserAPIs.windows,
-            searchIndex: search.searchIndex,
-            pageStorage: pages.storage,
-            localBrowserStorage: options.browserAPIs.storage.local,
-        }),
+        customLists,
         tags,
         bookmarks,
         backupModule: new backup.BackupBackgroundModule({
@@ -243,18 +245,51 @@ export function createBackgroundModules(options: {
             localBrowserStorage: options.browserAPIs.storage.local,
             fetchPageData: options.fetchPageDataProcessor,
             storePageContent,
-            addVisit: (visit) => pages.addVisit(visit.url, visit.time),
+            addVisit: (visit) =>
+                pages.addVisit(visit.normalizedUrl, visit.time),
             addBookmark: async (bookmark) => {
-                if (!(await bookmarks.storage.pageHasBookmark(bookmark.url))) {
-                    await bookmarks.addBookmark(bookmark)
+                if (
+                    !(await bookmarks.storage.pageHasBookmark(
+                        bookmark.normalizedUrl,
+                    ))
+                ) {
+                    await bookmarks.addBookmark({
+                        url: bookmark.normalizedUrl,
+                        time: bookmark.time,
+                    })
                 }
             },
-            addTags: (params) =>
-                Promise.all(
-                    params.tags.map((tag) =>
-                        tags.addTagToPage({ url: params.url, tag }),
-                    ),
-                ),
+            addTags: async (params) => {
+                const existingTags = await tags.storage.fetchPageTags({
+                    url: params.normalizedUrl,
+                })
+                await Promise.all(
+                    params.tags.map(async (tag) => {
+                        if (!existingTags.includes(tag)) {
+                            await tags.addTagToPage({
+                                url: params.normalizedUrl,
+                                tag,
+                            })
+                        }
+                    }),
+                )
+            },
+            addToLists: async (params) => {
+                const existingEntries = await customLists.storage.fetchListIdsByUrl(
+                    params.normalizedUrl,
+                )
+                await Promise.all(
+                    params.lists.map(async (listId) => {
+                        if (!existingEntries.includes(listId)) {
+                            await customLists.storage.insertPageToList({
+                                listId,
+                                pageUrl: params.normalizedUrl,
+                                fullUrl: params.fullUrl,
+                            })
+                        }
+                    }),
+                )
+            },
         }),
         features: new FeatureOptIns(),
         pages,

src/custom-lists/background/storage.ts

@@ -211,6 +211,11 @@ export default class CustomListStorage extends StorageModule {
         return this.operation('findListEntriesByListId', { listId })
     }
 
+    async fetchListIdsByUrl(url: string): Promise<number[]> {
+        const entries = await this.operation('findListEntriesByUrl', { url })
+        return entries.map((entry) => entry.listId)
+    }
+
     async fetchListPagesByUrl({ url }: { url: string }) {
         const pages = await this.operation('findListEntriesByUrl', { url })
 

src/storex-hub/background/index.ts

@@ -7,6 +7,7 @@ import {
     StorexHubCallbacks_v0,
     HandleRemoteCallResult_v0,
 } from '@worldbrain/storex-hub/lib/public-api'
+import { normalizeUrl } from '@worldbrain/memex-url-utils'
 import {
     FetchPageProcessor,
     PageContent,
@@ -32,9 +33,26 @@ export class StorexHubBackground {
             fetchPageData: FetchPageProcessor
             storePageContent: (content: PageContent) => Promise<void>
             localBrowserStorage: Storage.LocalStorageArea
-            addVisit: (visit: { url: string; time: number }) => void
-            addBookmark: (bookmark: { url: string; time: number }) => void
-            addTags: (params: { url: string; tags: string[] }) => void
+            addVisit: (visit: {
+                normalizedUrl: string
+                fullUrl: string
+                time: number
+            }) => Promise<void>
+            addBookmark: (bookmark: {
+                normalizedUrl: string
+                fullUrl: string
+                time: number
+            }) => Promise<void>
+            addTags: (params: {
+                normalizedUrl: string
+                fullUrl: string
+                tags: string[]
+            }) => Promise<void>
+            addToLists: (params: {
+                normalizedUrl: string
+                fullUrl: string
+                lists: number[]
+            }) => Promise<void>
         },
     ) {
         this.logger = console['log'].bind(console)
@@ -101,10 +119,13 @@ export class StorexHubBackground {
             return { status: 'invalid-args' }
         }
 
+        const fullUrl = args.url
+        const normalizedUrl = normalizeUrl(fullUrl)
+
         let processedData: PipelineRes
         try {
             processedData = await this.dependencies.fetchPageData.process(
-                args.url,
+                fullUrl,
             )
         } catch (e) {
             return {
@@ -126,7 +147,8 @@ export class StorexHubBackground {
 
         try {
             await this.dependencies.addVisit({
-                url: args.url,
+                fullUrl,
+                normalizedUrl,
                 time: args.visitTime ?? Date.now(),
             })
         } catch (e) {
@@ -144,7 +166,8 @@ export class StorexHubBackground {
                         ? null
                         : args.bookmark.creationTime
                 await this.dependencies.addBookmark({
-                    url: args.url,
+                    fullUrl,
+                    normalizedUrl,
                     time: time || Date.now(),
                 })
             }
@@ -159,7 +182,8 @@ export class StorexHubBackground {
         try {
             if (args.tags) {
                 await this.dependencies.addTags({
-                    url: args.url,
+                    fullUrl,
+                    normalizedUrl,
                     tags: args.tags,
                 })
             }
@@ -171,6 +195,22 @@ export class StorexHubBackground {
             }
         }
 
+        try {
+            if (args.lists) {
+                await this.dependencies.addToLists({
+                    fullUrl,
+                    normalizedUrl,
+                    lists: args.lists,
+                })
+            }
+        } catch (e) {
+            return {
+                status: 'internal-error',
+                errorStatus: 'could-not-list',
+                errorText: `Error while adding page to lists: ${e}`,
+            }
+        }
+
         return { status: 'success', result: {} }
     }
 

src/storex-hub/background/types.ts

@@ -3,4 +3,5 @@ export interface IndexPageArgs {
     visitTime?: number
     bookmark?: true | { creationTime: number }
     tags?: string[]
+    lists?: number[]
 }