refactor(core): optimize the usage of indexedDB (#33)

apps/monkey/src/Ctrl.vue

@@ -21,8 +21,8 @@ const progressText = computed(() => () => `${postStore.posts.length}/${postStore
 const pauseFn = ref<() => void>()
 const resumeFn = ref<() => void>()
 
-function exportDatas() {
-  const res = exportData(postStore.posts)
+async function exportDatas() {
+  const res = await exportData(postStore.posts)
   if (!res)
     return
   const scripts = 'https://github.com/Chilfish/Weibo-archiver/raw/monkey/scripts.zip'

apps/web/nuxt.config.ts

@@ -18,6 +18,10 @@ export default defineNuxtConfig({
     '@bg-dev/nuxt-naiveui',
   ],
 
+  alias: {
+    '@core': join(root, 'packages/core/src'),
+  },
+
   imports: {
     dirs: [
       'stores',

apps/web/src/layouts/default.vue

@@ -4,13 +4,11 @@
 
 <template>
   <app-main>
-    <n-modal-provider>
-      <nuxt-loading-indicator />
-      <main-header />
-      <img-viewer />
-      <main class="mt-16 h-full">
-        <slot />
-      </main>
-    </n-modal-provider>
+    <nuxt-loading-indicator />
+    <main-header />
+    <img-viewer />
+    <main class="mt-16 h-full">
+      <slot />
+    </main>
   </app-main>
 </template>

packages/core/package.json

@@ -8,9 +8,9 @@
   "dependencies": {
     "@vueuse/core": "^10.9.0",
     "conf": "^12.0.0",
-    "idb-keyval": "^6.2.1",
+    "fuse.js": "^7.0.0",
+    "idb": "^8.0.0",
     "ofetch": "^1.3.3",
-    "pinia": "^2.1.7",
-    "unstorage": "^1.10.1"
+    "pinia": "^2.1.7"
   }
 }

packages/core/src/stores/post.ts

@@ -1,30 +1,40 @@
 import { defineStore } from 'pinia'
 import { useRoute } from 'vue-router'
 import type { Post } from '@types'
-import { getMany, set as setDB, setMany } from 'idb-keyval'
+import type { FuseResult } from 'fuse.js'
+import {
+  addDBPosts,
+  buildSearch,
+  clearDB,
+  getAllDBPosts,
+  getDBPost,
+  getDBPosts,
+  getPostCount,
+  getSize,
+} from '../utils/storage'
 
 export const usePostStore = defineStore('post', () => {
-  console.log('post pinia store created')
-
-  const ids = ref([] as string[])
-  const posts = shallowRef([] as Post[])
-
   const route = useRoute()
 
-  const viewImg = ref(imgViewSrc)
-
   const curPage = ref(Number(route.query.page) || 1)
-  const pageSize = ref(Number(route.query.pageSize) || 10) // 每页显示的帖子数量 ppp
+  const pageSize = ref(Number(route.query.pageSize) || 10)
 
-  // 总帖子数
-  const total = ref(ids.value.length)
+  const seachFn = ref<(text: string) => FuseResult<{
+    time: number
+    text: string
+  }>[]>()
+
+  // 该结果的总帖子数
+  const total = ref(0)
+
+  // DB 中的帖子总数
+  const totalDB = ref(0)
 
   const pages = computed(() => {
     return Math.ceil(total.value / pageSize.value)
   })
 
   function reset() {
-    viewImg.value = imgViewSrc
     curPage.value = 1
     pageSize.value = 20
   }
@@ -36,78 +46,68 @@ export const usePostStore = defineStore('post', () => {
    */
   async function set(
     data: Post[],
-    replace = false,
+    isReplace = false,
   ) {
     if (!data[0]?.user)
       throw new Error('数据格式错误，可能要重新导入')
 
-    let after = data
-    const _posts = toValue(posts)
+    const { count, search } = await addDBPosts(data, isReplace)
+    totalDB.value = count
+    seachFn.value = search
+  }
 
-    if (!replace) {
-      after = [..._posts, ...data].filter((post, index, self) => {
-        return index === self.findIndex(t => t.id === post.id)
-      })
+  async function searchPost(
+    query: string,
+    page: number,
+    pageSize: number,
+  ) {
+    if (!seachFn.value) {
+      const posts = await getAllDBPosts()
+      const { search } = buildSearch(posts)
+      seachFn.value = search
     }
 
-    const _ids = after.map(post => `post-${post.id}`)
+    const result = seachFn.value(query)
+      .map(r => r.item)
+      .sort((a, b) => b.time - a.time)
+
+    const count = result.length
+    const start = (page - 1) * pageSize
+    const end = start + pageSize
+    const posts = result.slice(start, end)
 
-    await setDB('ids', _ids)
-    await setMany(after.map(post => [`post-${post.id}`, post]))
-    ids.value = _ids
-    total.value = after.length
-    posts.value = after
+    return { posts, count }
   }
 
   async function get(page?: number) {
-    if (ids.value.length === 0)
-      return []
-
-    let p = page
-    if (!p)
-      p = curPage.value
-    const sliceDis = [(p - 1) * pageSize.value, p * pageSize.value]
+    const p = page || curPage.value
 
     const path = route.path
     const query = route.query.q as string
 
-    if (posts.value.length === 0) {
-      posts.value = (
-        await getMany<Post>(ids.value)
-          .then(res =>
-            res.filter(Boolean).sort((a, b) => b.id - a.id),
-          )
-      )
-    }
+    let result: Post[] = []
 
     if (path === '/post') {
-      total.value = ids.value.length
-      return posts.value.slice(...sliceDis)
+      result = await getDBPosts(p, pageSize.value)
     }
     else {
-      const data = await searchText(query)
-      total.value = data.length
-      return data.slice(...sliceDis)
+      const { posts, count } = await searchPost(query, p, pageSize.value)
+
+      result = await getDBPost(posts.map(p => p.time))
+      total.value = count
     }
+
+    return result
   }
 
-  // TODO: 优化
-  async function searchText(p: string): Promise<Post[]> {
-    const res = posts.value.filter((post) => {
-      const word = p.toLowerCase().trim().replace(/ /g, '')
-      const regex = new RegExp(word, 'igm')
-      return regex.test(post.text)
-        || (post.card && regex.test(post.card?.title))
-        || (post.retweeted_status && regex.test(post.retweeted_status?.text))
-    })
-
-    return res
+  async function updateTotal() {
+    total.value = await getPostCount()
+    totalDB.value = total.value
   }
 
   return {
-    ids,
-    viewImg,
     total,
+    totalDB,
     pages,
     pageSize,
     curPage,
@@ -116,6 +116,9 @@ export const usePostStore = defineStore('post', () => {
     set,
     reset,
 
-    searchText,
+    clearDB,
+    getAll: getAllDBPosts,
+    updateTotal,
+    getSize,
   }
 })

packages/core/src/utils/export.ts

@@ -3,7 +3,7 @@ import fileSaver from 'file-saver'
 import type { Post } from '@types'
 import { imgsParser } from './parse'
 
-export function exportData(posts: Post[]) {
+export async function exportData(posts: Post[]) {
   if (!posts[0]) {
     window.$message.warning('没有数据可以导出')
     return false
@@ -23,16 +23,16 @@ export function exportData(posts: Post[]) {
     .join(',\n') // csv 格式
   zip.file('imgs.csv', imgsData)
 
-  zip
-    .generateAsync({ type: 'blob' })
-    .then((zipFile) => {
-      window.$message.success('导出成功，正在下载数据...')
-      fileSaver.saveAs(zipFile, `weibo-archiver-${name}.zip`)
-    })
-    .catch((err) => {
-      window.$message.error('导出失败')
-      console.error('导出失败', err)
-    })
+  try {
+    const zipFile = await zip.generateAsync({ type: 'blob' })
+    window.$message.success('导出成功，正在下载数据...')
+    fileSaver.saveAs(zipFile, `weibo-archiver-${name}.zip`)
+  }
+  catch (err) {
+    window.$message.error('导出失败')
+    console.error('导出失败', err)
+    return false
+  }
 
   return true
 }

packages/core/src/utils/index.ts

@@ -7,7 +7,6 @@ export * from './protocol'
 export * from './fetch'
 export * from './image'
 export * from './export'
-export * from './storage'
 
 export const isElectron = import.meta.env.VITE_IS_ELECTRON === 'true'