refactor(core): Move options for parse and fetch logic out

apps/desktop/package.json

@@ -39,6 +39,7 @@
     "@weibo-archiver/core": "workspace:*",
     "@weibo-archiver/database": "workspace:*",
     "@weibo-archiver/ui": "workspace:*",
+    "conf": "^12.0.0",
     "custom-electron-titlebar": "^4.2.8",
     "electron-context-menu": "^3.6.1",
     "electron-log": "^5.1.1",

packages/core/src/utils/config.ts → apps/desktop/utils/config.ts

@@ -21,15 +21,14 @@ const config = new Config<AppConfig>({
     theme: 'light',
     useCdn: false,
     fetchOptions: {
-      now,
       uid: '',
       name: '',
       cookie: '',
       isFetchAll: true,
-      picLarge: true,
+      largePic: true,
       repostPic: true,
-      repost: true,
-      comment: true,
+      hasComment: true,
+      hasRepost: true,
       commentCount: 10,
       dateRange: [now, now],
     },
@@ -40,7 +39,6 @@ const path = config.path.replace(/config\.json$/, '')
 
 config.set('configPath', path)
 config.set('version', version)
-config.set('fetchOptions.now', now)
 config.set('fetchOptions.dateRange', [now, now])
 
 export default config

apps/monkey/src/Config.vue

@@ -1,19 +1,18 @@
 <script setup lang="ts">
+import { storeToRefs } from 'pinia'
 import { useConfigStore } from './stores'
 
-const configStore = useConfigStore()
+const { state: config } = storeToRefs(useConfigStore())
 
 const dateRange = computed({
   get() {
-    return configStore.state.dateRange
+    return config.value.dateRange
   },
   set(val: [number, number]) {
-    configStore.state.dateRange = val ?? [Date.now(), Date.now()]
-    configStore.state.isFetchAll = false
+    config.value.dateRange = val ?? [Date.now(), Date.now()]
+    config.value.isFetchAll = false
   },
 })
-
-configStore.setConfig({ now: Date.now() })
 </script>
 
 <template>
@@ -30,42 +29,42 @@ configStore.setConfig({ now: Date.now() })
 
     <div class="center flex-wrap justify-start gap-2">
       <n-checkbox
-        v-model:checked="configStore.state.picLarge"
+        v-model:checked="config.largePic"
         label="导出原图"
       />
       <n-checkbox
-        v-model:checked="configStore.state.comment"
+        v-model:checked="config.hasComment"
         label="包含评论"
       />
       <n-checkbox
-        v-model:checked="configStore.state.repost"
+        v-model:checked="config.hasRepost"
         label="包含转发的微博"
       />
       <n-checkbox
-        v-show="configStore.state.repost"
-        v-model:checked="configStore.state.repostPic"
+        v-show="config.hasRepost"
+        v-model:checked="config.repostPic"
         label="导出转发的图片"
       />
       <button
         class="bg-#18a058 py-1 btn hover:bg-green-7"
         @click="() => {
-          configStore.state.isFetchAll = true
+          config.isFetchAll = true
         }"
       >
         重置为所有微博
       </button>
     </div>
 
     <div
-      v-show="configStore.state.comment"
+      v-show="config.hasComment"
       class="flex items-center gap-4"
     >
-      <span>要获取的评论数（最多15条）</span>
+      <span>要获取的评论数（最多20条）</span>
       <n-input-number
-        v-model="configStore.state.commentCount"
-        :default-value="10"
+        v-model="config.commentCount"
+        :default-value="6"
         :min="0"
-        :max="15"
+        :max="20"
       />
     </div>
   </div>

apps/monkey/src/Ctrl.vue

@@ -61,6 +61,7 @@ async function start() {
   isStop.value = false
 
   const { pause, resume } = await fetchPosts({
+    fetchOptions: config.value,
     startPage: () => postStore.fetchedPage + 1,
     isFetchAll: config.value.isFetchAll,
     setTotal: total => postStore.total = total,

apps/monkey/src/stores/configStore.ts

@@ -13,17 +13,16 @@ export const useConfigStore = defineStore('config', () => {
     uid: '',
     name: '',
     isFetchAll: true,
-    picLarge: true,
+    largePic: true,
     repostPic: true,
-    repost: true,
-    comment: true,
-    commentCount: 10,
+    hasRepost: true,
+    hasComment: true,
+    commentCount: 6,
     dateRange: [now, now],
-    isMinimize: false,
-    now,
+    isMinimize: true,
   })
 
-  // 判断 key 是否都相等
+  // 判断 key 是否都相等，如果相等则初始化配置
   function initConfig() {
     const keys = Object.keys(state)
     const localKeys = Object.keys(localData)

apps/monkey/vite.config.ts

@@ -61,7 +61,6 @@ export default defineConfig({
           'https://weibo.com/u/*',
           'https://weibo.com/n/*',
           'https://weibo.chilfish.top/*',
-          'http://localhost:3334/*',
         ],
         grant: [
           'GM_setValue',

packages/core/package.json

@@ -7,7 +7,6 @@
   },
   "dependencies": {
     "@vueuse/core": "^10.9.0",
-    "conf": "^12.0.0",
     "dayjs": "^1.11.10",
     "fuse.js": "^7.0.0",
     "idb": "^8.0.0",

packages/core/src/services/postService.ts

@@ -1,12 +1,11 @@
 import type {
   Comment,
+  FetchOptions,
   FetchReturn,
-  LoopFetchParams,
   Post,
   PostMeta,
 } from '@types'
 import {
-  getOptions,
   postsParser,
   weiFetch,
 } from '../utils'
@@ -18,17 +17,18 @@ import {
 let since_id = ''
 
 /**
- * 有没有可能，获取全部微博可以用 rangePosts 来实现？
- * 不设 start 参数就行……实际上并不行，会有缺失的微博
+ * 有没有可能，获取全部微博可以用 rangePosts 来实现？不设 start 参数……
+ * 实际上并不行，会有缺失的微博
  */
-export async function fetchAllPosts(page = 1): FetchReturn {
+export async function fetchAllPosts(
+  uid: string,
+  page = 1,
+): FetchReturn {
   if (page === 0)
     return null
   else if (page === 1)
     since_id = ''
 
-  const { uid } = await getOptions()
-
   const { data } = await weiFetch<{ data: PostMeta }>(`/statuses/mymblog`, {
     params: {
       uid,
@@ -43,35 +43,32 @@ export async function fetchAllPosts(page = 1): FetchReturn {
   else
     return null
 
-  return {
-    ...data,
-    list: await postsParser(data.list),
-  }
+  return data
 }
 
-export async function fetchRangePosts(page = 1): FetchReturn {
-  const { uid, dateRange, repost } = await getOptions()
-  const [start, end] = dateRange
-
+export async function fetchRangePosts(
+  uid: string,
+  start: number,
+  end: number,
+  page = 1,
+  hasRepost = true,
+): FetchReturn {
   const { data } = await weiFetch<{ data: PostMeta }>(`/statuses/searchProfile`, {
     params: {
       uid,
       page,
       starttime: start / 1000,
       endtime: end / 1000,
       hasori: 1, // 是否包含原创
-      hasret: repost ? 1 : 0, // 是否包含转发
+      hasret: hasRepost ? 1 : 0, // 是否包含转发
       hastext: 1, // 是否包含文字
       haspic: 1, // 是否包含图片
       hasvideo: 1, // 是否包含视频
       hasmusic: 1, // 是否包含音乐
     },
   })
 
-  return {
-    ...data,
-    list: await postsParser(data.list || []),
-  }
+  return data
 }
 
 export async function fetchLongText(
@@ -81,7 +78,9 @@ export async function fetchLongText(
 
   if (post.isLongText) {
     await delay()
-    const { data } = await weiFetch<{ data: { longTextContent: string } }>(`/statuses/longtext`, {
+    const { data } = await weiFetch<{
+      data: { longTextContent: string }
+    }>(`/statuses/longtext`, {
       params: {
         id: post.id,
       },
@@ -94,40 +93,41 @@ export async function fetchLongText(
 }
 
 /**
- * 获取前 n 条评论
+ * 获取评论
+ * @param postId 微博 id
+ * @param uid 用户 id
+ * @param isShowBulletin 必填字段，区分旧微博和新微博
+ * @param count 获取数量
  */
-export async function fetchComments(post: Post): Promise<Comment[]> {
-  const { commentCount, comment, picLarge } = await getOptions()
-  const imgSize = picLarge ? 'woriginal' : 'large'
-
-  if (!post.user || post.comments_count === 0 || !comment)
-    return []
-
+export async function fetchComments(
+  postId: string,
+  uid: string,
+  isShowBulletin: number,
+  count: number,
+): Promise<Comment[]> {
   await delay(3000)
   const { data } = await weiFetch<{ data: Comment[] }>(`/statuses/buildComments`, {
     params: {
-      id: post.id,
-      is_show_bulletin: post.is_show_bulletin, // 必填字段，区分旧微博和新微博
+      id: postId,
+      is_show_bulletin: isShowBulletin,
       flow: 0, // 热评
       is_reload: 1, // 获取详情页的评论
       is_mix: 0,
       count: 10,
       fetch_level: 0,
       locale: 'zh_CN',
-      uid: post.user.id,
+      uid,
     },
   })
 
   if (!data)
     return []
 
-  return filterComments(
-    data.slice(0, commentCount),
-    imgSize,
-  )
+  return filterComments(data.slice(0, count))
 }
 
 interface FetchPosts {
+  fetchOptions: FetchOptions
   startPage: () => number
   isFetchAll: boolean
   setTotal: (total: number) => void
@@ -139,14 +139,19 @@ interface FetchPosts {
  * 爬取微博
  */
 export async function fetchPosts(
-  { startPage, isFetchAll, setTotal, addPosts, stopCondition }: FetchPosts,
+  { fetchOptions, startPage, isFetchAll, setTotal, addPosts, stopCondition }: FetchPosts,
 ) {
+  const { uid, dateRange, hasRepost } = fetchOptions
+  const [start, end] = dateRange
+
   async function fetching() {
     const page = startPage()
     const res = isFetchAll
-      ? await fetchAllPosts(page)
-      : await fetchRangePosts(page)
-    addPosts(res?.list || [])
+      ? await fetchAllPosts(uid, page)
+      : await fetchRangePosts(uid, start, end, page, hasRepost)
+    const list = await postsParser(res?.list || [], fetchOptions)
+
+    addPosts(list)
     console.log(`已获取第 ${page} 页`)
     return res
   }
@@ -173,29 +178,3 @@ export async function fetchPosts(
     resume,
   }
 }
-
-/**
- * @deprecated use {@link usePausableLoop} instead
- */
-export async function loopFetcher(
-  { start, stopFn, fetchFn, onResult, onEnd, isAbort }: LoopFetchParams,
-) {
-  let page = start + 1
-  while (!stopFn()) {
-    await delay()
-    if (isAbort)
-      return
-
-    const data = await fetchFn?.(page)
-    onResult(data!.list)
-
-    // 无数据时，直接退出
-    if (data!.list.length === 0) {
-      await onEnd?.()
-      return
-    }
-    page++
-  }
-
-  await onEnd?.()
-}

packages/core/src/utils/fetch.ts

@@ -1,20 +1,14 @@
 import type { FetchOptions } from 'ofetch'
 import { ofetch } from 'ofetch'
-import { getOptions } from './index'
 
 export const aborter = new AbortController()
 
 export async function weiFetch<T extends { data: any }>(
   path: string,
   options?: FetchOptions<'json'>,
 ) {
-  const cookie = typeof document !== 'undefined'
-    ? document.cookie
-    : (await getOptions()).cookie
-
   return ofetch<T>(`https://weibo.com/ajax${path}`, {
     headers: {
-      'Cookie': cookie,
       'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0',
     },
     retry: 3,