fix(monkey): 将数据暂存到 indexedDB 而不是内存里 (#38)

README.md

@@ -30,7 +30,7 @@
 
 需要浏览器装有 [Tampermonkey](https://tampermonkey.net/) 或 [Violentmonkey](https://violentmonkey.github.io/) 插件。
 
-安装至油猴脚本：[weibo-archiver.user.js][releases]（如果下载缓慢可用境内加速的 [镜像地址]）。在用户个人主页（必须是通过点击头像来进入）刷新后将自动启动脚本，点击开始后将开始获取数据。期间请不要刷新或切换页面，否则就要重新获取
+安装至油猴脚本：[weibo-archiver.user.js][releases]（如果下载缓慢可用境内加速的 [镜像地址]）。在用户个人主页（必须是通过点击头像来进入）刷新后将自动启动脚本，点击开始后将开始获取数据。支持断点续传，可恢复到上次的进度。
 
 更多操作细节可见 [使用教程]
 

apps/monkey/src/Config.vue

@@ -2,7 +2,7 @@
 import { storeToRefs } from 'pinia'
 import { useConfigStore } from './stores'
 
-const { state: config } = storeToRefs(useConfigStore())
+const { config } = storeToRefs(useConfigStore())
 
 const dateRange = computed({
   get() {

apps/monkey/src/Ctrl.vue

@@ -16,10 +16,10 @@ const isStart = ref(false)
 const isStop = ref(false)
 const isFinish = ref(false)
 
-const { state: config } = storeToRefs(configStore)
+const { config } = storeToRefs(configStore)
 
-const percentage = computed(() => postStore.posts.length / postStore.total * 100)
-const progressText = computed(() => () => `${postStore.posts.length}/${postStore.total} 条`)
+const percentage = computed(() => postStore.fetchedCount / postStore.total * 100)
+const progressText = computed(() => () => `${postStore.fetchedCount}/${postStore.total} 条`)
 
 const pauseFn = ref<() => void>()
 const resumeFn = ref<() => void>()
@@ -45,7 +45,8 @@ async function saveUserInfo() {
  * 导出数据
  */
 async function exportDatas() {
-  const res = await exportData(postStore.posts)
+  const posts = await postStore.getAll()
+  const res = await exportData(posts)
   if (!res)
     return
   const scripts = 'https://github.com/Chilfish/Weibo-archiver/raw/monkey/scripts.zip'
@@ -59,17 +60,21 @@ async function start() {
     duration: 5000,
   })
 
-  postStore.reset()
   isStart.value = true
   isFinish.value = false
   isStop.value = false
 
+  // 如果是恢复暂停
+  if (resumeFn.value) {
+    resumeFn.value()
+    return
+  }
+
+  postStore.setDB()
   const { pause, resume } = await fetchPosts({
-    fetchOptions: config.value,
-    startPage: () => postStore.fetchedPage + 1,
-    isFetchAll: config.value.isFetchAll,
-    setTotal: total => postStore.total = total,
+    fetchOptions: () => config.value,
     addPosts: postStore.add,
+    setTotal: total => postStore.total = total,
     stopCondition: () => {
       const finished = postStore.fetchedPage >= postStore.pages
 
@@ -86,17 +91,22 @@ async function start() {
   resumeFn.value = resume
 }
 
-watch(isStop, (val, oldVal) => {
-  if (val === oldVal)
-    return
+window.$message = useMessage()
 
-  if (val)
-    pauseFn.value?.()
-  else
-    resumeFn.value?.()
-})
+async function reset() {
+  await postStore.reset()
+  configStore.reset()
+  isStart.value = false
+  isStop.value = false
+  isFinish.value = false
 
-window.$message = useMessage()
+  message.success('数据已重置')
+}
+
+function stop() {
+  isStop.value = true
+  pauseFn.value?.()
+}
 
 /**
  * 初始化用户信息
@@ -110,6 +120,7 @@ async function init() {
     uid,
     name,
   })
+  configStore.init(uid)
 }
 
 onMounted(async () => {
@@ -119,7 +130,7 @@ onMounted(async () => {
 
 <template>
   <div
-    v-show="!configStore.isMinimize"
+    v-show="!config.isMinimize"
     class="card w-32rem shadow-xl"
   >
     <div class="flex items-center justify-between">
@@ -135,15 +146,6 @@ onMounted(async () => {
       </button>
     </div>
 
-    <n-alert
-      type="warning"
-      closable
-    >
-      <p>
-        请勿刷新或关闭页面，否则导致已有的数据丢失而重头来过
-      </p>
-    </n-alert>
-
     <n-alert type="info">
       <p>
         导出完毕后，可在
@@ -166,13 +168,12 @@ onMounted(async () => {
       {{ progressText() }}
     </n-progress>
 
-    <div class="btns flex gap-4">
+    <div class="btns flex gap-2">
       <button
         v-show="!isStart || (!isFinish && isStop)"
         @click="start"
       >
-        {{ isStart ? '重新开始' : '开始' }}
-        (获取 <strong>{{ config.isFetchAll ? '全部' : '部分' }}</strong> 微博)
+        {{ config.fetchedCount ? '继续' : '开始' }}获取 <strong>{{ config.isFetchAll ? '全部' : '部分' }}</strong> 微博
       </button>
 
       <div
@@ -183,10 +184,10 @@ onMounted(async () => {
       </div>
 
       <button
-        v-show="isStart"
-        @click="isStop = !isStop"
+        v-show="isStart && !isStop"
+        @click="stop"
       >
-        {{ isStop ? '继续' : '暂停' }}
+        暂停
       </button>
 
       <button
@@ -201,6 +202,12 @@ onMounted(async () => {
       >
         同步信息
       </button>
+
+      <button
+        @click="reset"
+      >
+        重置数据
+      </button>
     </div>
   </div>
 

apps/monkey/src/stores/configStore.ts

@@ -1,15 +1,22 @@
 import { defineStore } from 'pinia'
 import type { FetchOptions } from '@types'
+import { useStorage } from '@vueuse/core'
+
+type Config = FetchOptions & {
+  isMinimize: boolean
+}
 
 export const useConfigStore = defineStore('config', () => {
   const now = Date.now()
-  const KEY_MINIMIZE = 'archiver-isMinimize'
-
-  const isMinimize = ref(localStorage.getItem(KEY_MINIMIZE) === 'true')
+  const KEY = 'weibo-archiver'
 
-  const state = reactive<FetchOptions>({
+  const initConfig: Config = {
+    isMinimize: true,
     uid: '',
     name: '',
+    since_id: '',
+    curPage: 0,
+    fetchedCount: 0,
     isFetchAll: true,
     largePic: true,
     repostPic: true,
@@ -18,21 +25,34 @@ export const useConfigStore = defineStore('config', () => {
     hasFavorite: true,
     commentCount: 6,
     dateRange: [now, now],
-  })
+  }
 
-  function setConfig(config: Partial<FetchOptions>) {
-    Object.assign(state, config)
+  const config = useStorage<Config>(KEY, initConfig, localStorage, { mergeDefaults: true })
+
+  function setConfig(_config: Partial<FetchOptions>) {
+    Object.assign(config.value, _config)
   }
 
   function toggleMinimize() {
-    isMinimize.value = !isMinimize.value
-    localStorage.setItem(KEY_MINIMIZE, isMinimize.value.toString())
+    config.value.isMinimize = !config.value.isMinimize
+  }
+
+  function init(curUid: string) {
+    if (config.value.uid !== curUid) {
+      config.value.since_id = ''
+      config.value.uid = curUid
+      config.value.curPage = 0
+    }
   }
 
   return {
-    state,
-    isMinimize,
+    config,
     setConfig,
     toggleMinimize,
+    init,
+    reset: () => {
+      const { uid, name, isMinimize } = config.value
+      config.value = { ...initConfig, uid, name, isMinimize }
+    },
   }
 })

apps/monkey/src/stores/postStore.ts

@@ -1,12 +1,19 @@
-import { defineStore } from 'pinia'
-import type { Post } from '@types'
+import { defineStore, storeToRefs } from 'pinia'
+import type { Post, UID } from '@types'
+import { EmptyIDB, IDB } from '@core/utils/storage'
+import { useConfigStore } from './configStore'
 
 export const usePostStore = defineStore('post', () => {
-  /* 获取到的所有帖子 */
-  const posts = shallowRef([] as Post[])
+  /* 获取到的所有帖子，但会卡内存 */
+  // const posts = shallowRef([] as Post[])
+
+  const configStore = useConfigStore()
+  const { config } = storeToRefs(configStore)
 
   /* 已获取的页数 */
-  const fetchedPage = ref(0)
+  const fetchedPage = toRef(config.value.curPage)
+  /* 已获取到的数量 */
+  const fetchedCount = ref(config.value.fetchedCount)
   /* 每页的帖子数量 */
   const pageSize = ref(20)
 
@@ -15,35 +22,71 @@ export const usePostStore = defineStore('post', () => {
   /* 总页数 */
   const pages = computed(() => Math.ceil(total.value / pageSize.value))
 
+  const idb = ref<IDB>(new EmptyIDB())
+
+  function setDB() {
+    const wrappedUid = `uid-${config.value.uid}` as UID
+    idb.value = new IDB(wrappedUid)
+  }
+
+  /**
+   * 等待 IDB 初始化完成
+   */
+  async function waitIDB() {
+    const dbName = `uid-${config.value.uid}`
+
+    while (idb.value.name !== dbName)
+      await new Promise(r => setTimeout(r, 300))
+  }
+
   /**
    * 重置
    */
-  function reset() {
-    posts.value = []
+  async function reset() {
     total.value = 0
     pageSize.value = 20
     fetchedPage.value = 0
+    fetchedCount.value = 0
+
+    setDB()
+    await waitIDB()
+    await idb.value.clearDB()
   }
 
   /**
    * 添加帖子
    */
-  function add(newPosts: Post[]) {
-    // pageSize.value = newPosts.length
-    posts.value = [...posts.value, ...newPosts]
-    triggerRef(posts)
+  async function add(newPosts: Post[], since_id: string) {
+    await waitIDB()
+
+    const { count } = await idb.value.addDBPosts(newPosts, false, false)
+    fetchedCount.value = count
     fetchedPage.value++
     pageSize.value = (newPosts.length || 20)
+
+    configStore.setConfig({
+      since_id,
+      curPage: fetchedPage.value,
+      fetchedCount: fetchedCount.value,
+    })
+  }
+
+  async function getAll() {
+    await waitIDB()
+    const posts = await idb.value.getAllDBPosts()
+    return posts
   }
 
   return {
-    posts,
     total,
     pages,
     pageSize,
     fetchedPage,
+    fetchedCount,
 
+    setDB,
     add,
     reset,
+    getAll,
   }
 })