fix file based import

WorldBrain · Mar 27, 2024 · 9bd7bdf · 9bd7bdf
1 parent 9f11a73
commit 9bd7bdf
Show file tree

Hide file tree

Showing 6 changed files with 25 additions and 11 deletions.
diff --git a/external/@worldbrain/memex-common b/external/@worldbrain/memex-common
diff --git a/src/imports/background/data-sources.js b/src/imports/background/data-sources.js
@@ -9,6 +9,7 @@ import {
     startOfDay,
     WEEK_IN_MS,
 } from '@worldbrain/memex-common/lib/utils/date-time'
+import { parseHTML } from 'linkedom/worker'
 
 export default class ImportDataSources {
     static LOOKBACK_WEEKS = 12 // Browser history is limited to the last 3 months
@@ -116,11 +117,17 @@ export default class ImportDataSources {
             allowTypes[TYPE.OTHERS] === SERVICES.POCKET ||
             allowTypes[TYPE.OTHERS] === SERVICES.NETSCAPE
         ) {
-            contents = await loadBlob({
-                url,
-                timeout: 10000,
-                responseType: 'document',
-            })
+            if (url.startsWith('data:text/html;base64,')) {
+                const base64Content = url.split('data:text/html;base64,')[1]
+                const htmlContent = atob(base64Content)
+                contents = parseHTML(htmlContent).document
+            } else {
+                contents = await loadBlob({
+                    url,
+                    timeout: 10000,
+                    responseType: 'document',
+                })
+            }
         }
 
         if (allowTypes[TYPE.OTHERS] === SERVICES.POCKET) {

diff --git a/src/imports/background/service-parsers/netscape-parser.ts b/src/imports/background/service-parsers/netscape-parser.ts
@@ -80,7 +80,6 @@ const parseShallowList: ServiceParser = (doc) => {
         ),
     )
     for (let header of headers) {
-        console.log(header)
         const collectionName =
             header.textContent !== ''
                 ? header.textContent

diff --git a/src/options/imports/container.jsx b/src/options/imports/container.jsx
@@ -103,13 +103,19 @@ class ImportContainer extends Component {
 
     handleInputFile = (event) => {
         const input = event.target
+
         if (!input.files[0]) {
             return
         }
         const file = input.files[0]
-        this.props.boundActions.prepareImport()
-        this.props.boundActions.setBlobUrl(URL.createObjectURL(file))
-        setTimeout(() => this.props.recalcEsts(), 500)
+
+        const reader = new FileReader()
+        reader.onloadend = () => {
+            this.props.boundActions.setBlobUrl(reader.result)
+        }
+        reader.readAsDataURL(file)
+
+        setTimeout(() => this.props.recalcEsts(), 3000)
     }
 
     renderCancelButton() {

diff --git a/src/options/imports/reducer.js b/src/options/imports/reducer.js
@@ -101,7 +101,7 @@ const finishImportsReducer = ({ loading = false, finish = true }) => (
 const prepareImportReducer = (state) => {
     return {
         ...state,
-        importStatus: STATUS.LOADING,
+        // importStatus: STATUS.LOADING,
         loadingMsg: 'Calculating size of history & bookmarks',
     }
 }

diff --git a/src/page-indexing/background/index.ts b/src/page-indexing/background/index.ts
@@ -930,13 +930,15 @@ export class PageIndexingBackground {
             content = pageData.content
             htmlBody = pageData.htmlBody
             favIconURI = pageData.favIconURI
+            // title = pageData.title ?? props.metaData?.pageTitle
 
             await this.storeDocContent(normalizeUrl(props.fullUrl), {
                 htmlBody,
             })
 
             pageDoc.favIconURI = favIconURI
             pageDoc.content.title = content?.title || props.metaData?.pageTitle
+
             pageDoc.content.fullText = content?.fullText
         } else {
             const pdfData = await this.options.fetchPdfData(props.fullUrl)