refactor: Make mod loading more progressive

xmcl-electron-app/preload/service.ts

@@ -117,13 +117,20 @@ async function receive(_result: any, states: Record<string, WeakRef<MutableState
 function createServiceChannels(): ServiceChannels {
   const gc = new FinalizationRegistry<string>((id) => {
     delete states[id]
-    ipcRenderer.invoke('deref', id)
+    ipcRenderer.invoke('unref', id)
     console.log(`deref ${id}`)
   })
   const servicesEmitters = new Map<ServiceKey<any>, WeakRef<EventEmitter>>()
   const states: Record<string, WeakRef<MutableState<object>>> = {}
   const pendingCommits: Record<string, { type: string; payload: any }[]> = {}
 
+  ipcRenderer.on('state-validating', (_, { id, validating }) => {
+    const state = states[id]?.deref()
+    if (state) {
+      (state as any)[kEmitter].emit('state-validating', validating)
+    }
+  })
+
   ipcRenderer.on('service-event', (_, { service, event, args }) => {
     const emitter = servicesEmitters.get(service)?.deref()
     if (emitter) {

xmcl-keystone-ui/src/components/HomeCard.vue

@@ -37,8 +37,6 @@
     </v-card-text>
     <v-card-actions>
       <v-btn
-        :disabled="refreshing"
-        :loading="refreshing"
         color="teal accent-4"
         text
         @click="emit('navigate')"

xmcl-keystone-ui/src/composables/instanceMods.ts

@@ -16,6 +16,7 @@ export function useInstanceMods(instancePath: Ref<string>, instanceRuntime: Ref<
     return mods as any
   }, class extends InstanceModsState {
     override instanceModUpdates(ops: [Resource, number][]) {
+      console.log('instanceModUpdates', ops.length)
       for (const o of ops) {
         markRaw(o[0])
       }
@@ -29,17 +30,27 @@ export function useInstanceMods(instancePath: Ref<string>, instanceRuntime: Ref<
 
   const enabledModCounts = computed(() => mods.value.filter(v => v.enabled).length)
 
+  function reset() {
+    mods.value = []
+    modsIconsMap.value = {}
+    provideRuntime.value = {}
+  }
+  watch(instancePath, (v, prev) => {
+    if (v !== prev) {
+      reset()
+    }
+  })
   watch([computed(() => state.value?.mods), java], () => {
     if (!state.value?.mods) {
-      mods.value = []
+      reset()
       return
     }
     console.log('update instance mods by state')
     updateItems(state.value?.mods, instanceRuntime.value)
   })
   watch(instanceRuntime, () => {
     if (!state.value?.mods) {
-      mods.value = []
+      reset()
       return
     }
     console.log('update instance mods by runtime')

xmcl-keystone-ui/src/composables/modSearch.ts

@@ -141,7 +141,13 @@ export function useLocalModsSearch(keyword: Ref<string>, modLoaderFilters: Ref<M
   const instances = computed(() => result.value[1])
 
   async function processCachedMod() {
-    modFiles.value = keyword.value ? (await getResourcesByKeyword(keyword.value, ResourceDomain.Mods)).filter(isValidResource).map(r => getModFileFromResource(r, runtime.value)) : []
+    if (keyword.value) {
+      const searched = await getResourcesByKeyword(keyword.value, ResourceDomain.Mods)
+      const resources = searched.filter(isValidResource).map(r => getModFileFromResource(r, runtime.value))
+      modFiles.value = resources
+    } else {
+      modFiles.value = []
+    }
   }
 
   const loadingCached = ref(false)
@@ -179,7 +185,7 @@ const getOptifineAsMod = () => {
   return result
 }
 
-export function useModsSearch(runtime: Ref<InstanceData['runtime']>, instanceMods: Ref<ModFile[]>) {
+export function useModsSearch(runtime: Ref<InstanceData['runtime']>, instanceMods: Ref<ModFile[]>, isValidating: Ref<boolean>) {
   const modLoaderFilters = ref([] as ModLoaderFilter[])
   const curseforgeCategory = ref(undefined as number | undefined)
   const modrinthCategories = ref([] as string[])
@@ -210,7 +216,7 @@ export function useModsSearch(runtime: Ref<InstanceData['runtime']>, instanceMod
   const { loadMoreModrinth, loadingModrinth, modrinth, modrinthError } = useModrinthSearch('mod', keyword, modLoaderFilters, modrinthCategories, modrinthSort, runtime)
   const { loadMoreCurseforge, loadingCurseforge, curseforge, curseforgeError } = useCurseforgeSearch<ProjectEntry<ModFile>>(CurseforgeBuiltinClassId.mod, keyword, modLoaderFilters, curseforgeCategory, curseforgeSort, runtime)
   const { cached: cachedMods, instances, loadingCached } = useLocalModsSearch(keyword, modLoaderFilters, runtime, instanceMods)
-  const loading = computed(() => loadingModrinth.value || loadingCurseforge.value || loadingCached.value)
+  const loading = computed(() => loadingModrinth.value || loadingCurseforge.value || loadingCached.value || isValidating.value)
 
   const all = useAggregateProjects<ProjectEntry<ModFile>>(
     modrinth,
@@ -222,7 +228,7 @@ export function useModsSearch(runtime: Ref<InstanceData['runtime']>, instanceMod
   const items = useProjectsFilterSearch(
     keyword,
     all,
-    computed(() => keyword.value.length > 0 || modrinthCategories.value.length > 0 || curseforgeCategory.value !== undefined),
+    computed(() => (keyword.value.length > 0 && (modrinth.value.length > 0 || curseforge.value.length > 0)) || modrinthCategories.value.length > 0 || curseforgeCategory.value !== undefined),
     isCurseforgeActive,
     isModrinthActive,
   )

xmcl-keystone-ui/src/composables/syncableState.ts

@@ -26,6 +26,10 @@ export function useState<T extends object>(fetcher: (abortSignal: AbortSignal) =
       data.subscribeAll((mutation, payload) => {
         ((Type.prototype as any)?.[mutation] as Function)?.call(state.value, payload)
       })
+      // @ts-ignore
+      data.subscribe('state-validating', (v) => {
+        isValidating.value = v as any
+      })
       state.value = data
     } catch (e) {
       if (signal.aborted) { return }

xmcl-keystone-ui/src/windows/main/Context.ts

@@ -78,7 +78,7 @@ export default defineComponent({
     const task = useLaunchTask(instance.path, instance.runtime, instanceVersion.versionHeader)
     const instanceLaunch = useInstanceLaunch(instance.instance, instanceVersion.resolvedVersion, instanceJava.java, user.userProfile, settings)
 
-    const modsSearch = useModsSearch(instance.runtime, instanceMods.mods)
+    const modsSearch = useModsSearch(instance.runtime, instanceMods.mods, instanceMods.isValidating)
     const modUpgrade = useModUpgrade(instance.path, instance.runtime, modsSearch.all)
 
     const resourcePackSearch = useResourcePackSearch(instance.runtime, resourcePacks.enabled, resourcePacks.disabled)

xmcl-runtime-api/src/util/MutableState.ts

@@ -13,8 +13,10 @@ export type MutableState<T> = T & {
   readonly id: string
 
   subscribe<K extends keyof Mutations<T>>(key: K, listener: (payload: Mutations<T>[K]) => void): MutableState<T>
+  subscribe(key: 'state-validating', listener: (v: boolean) => void): MutableState<T>
 
   unsubscribe<K extends keyof Mutations<T>>(key: K, listener: (payload: Mutations<T>[K]) => void): MutableState<T>
+  unsubscribe(key: 'state-validating', listener: (v: boolean) => void): MutableState<T>
 
   subscribeAll<K extends keyof Mutations<T>>(listener: (mutation: K, payload: Mutations<T>[keyof Mutations<T>]) => void): MutableState<T>
 

xmcl-runtime/instance/InstanceOptionsService.ts

@@ -34,10 +34,10 @@ export class InstanceOptionsService extends AbstractService implements IInstance
   async watch(path: string) {
     requireString(path)
     const stateManager = await this.app.registry.get(ServiceStateManager)
-    return stateManager.registerOrGet(getInstanceGameOptionKey(path), async () => {
+    return stateManager.registerOrGet(getInstanceGameOptionKey(path), async ({ defineAsyncOperation }) => {
       const state = new GameOptionsState()
 
-      const loadShaderOptions = async (path: string) => {
+      const loadShaderOptions = defineAsyncOperation(async (path: string) => {
         try {
           const result = await this.getShaderOptions(path)
           state.shaderPackSet(result.shaderPack)
@@ -47,9 +47,9 @@ export class InstanceOptionsService extends AbstractService implements IInstance
             this.warn(e)
           }
         }
-      }
+      })
 
-      const loadOptions = async (path: string) => {
+      const loadOptions = defineAsyncOperation(async (path: string) => {
         try {
           const result = await this.getGameOptions(path)
           state.gameOptionsSet(result)
@@ -59,7 +59,7 @@ export class InstanceOptionsService extends AbstractService implements IInstance
             this.warn(e)
           }
         }
-      }
+      })
 
       this.log(`Start to watch instance options.txt in ${path}`)
 

xmcl-runtime/instance/InstanceServerInfoService.ts

@@ -16,12 +16,12 @@ export class InstanceServerInfoService extends AbstractService implements IInsta
 
   async watch(path: string) {
     const stateManager = await this.app.registry.get(ServiceStateManager)
-    return stateManager.registerOrGet(getServerInfoKey(path), async () => {
+    return stateManager.registerOrGet(getServerInfoKey(path), async ({ defineAsyncOperation }) => {
       const state = new ServerInfoState()
 
       const serversPath = join(path, 'servers.dat')
 
-      const update = async () => {
+      const update = defineAsyncOperation(async () => {
         if (await exists(serversPath)) {
           const serverDat = await readFile(serversPath)
           const infos = /* await readInfo(serverDat) */ undefined as any
@@ -30,13 +30,12 @@ export class InstanceServerInfoService extends AbstractService implements IInsta
         } else {
           this.log('No server data found in instance.')
         }
-      }
+      })
       const watcher = watch(path, (event, filePath) => {
         if (event === 'update') {
           update()
         }
       })
-
       await update()
 
       return [state, () => {

xmcl-runtime/instanceIO/InstanceFileOperationHandler.ts

@@ -169,7 +169,6 @@ export class InstanceFileOperationHandler {
         url: urls,
         destination,
         pendingFile: pending,
-        skipRevalidate: true,
         validator: sha1
           ? {
             hash: sha1,

xmcl-runtime/mod/InstanceModsService.ts

@@ -37,61 +37,40 @@ export class InstanceModsService extends AbstractService implements IInstanceMod
     // TODO: make this excpetion as this is a bad request
     if (!instancePath) throw new AnyError('WatchModError', 'Cannot watch instance mods on empty path')
     const stateManager = await this.app.registry.get(ServiceStateManager)
-    return stateManager.registerOrGet(getInstanceModStateKey(instancePath), async (onDestroy) => {
+    return stateManager.registerOrGet(getInstanceModStateKey(instancePath), async ({ defineAsyncOperation }) => {
       const updateMod = new AggregateExecutor<InstanceModUpdatePayload, InstanceModUpdatePayload[]>(v => v,
         (all) => {
           state.instanceModUpdates(all)
         },
         500)
 
-      const scan = async (dir: string) => {
-        const files = await readdirIfPresent(dir)
-
-        const fileArgs = files.filter((file) => !shouldIgnoreFile(file)).map((file) => join(dir, file))
-
-        const resources = await this.resourceService.importResources(fileArgs.map(f => ({ path: f, domain: ResourceDomain.Mods })), true)
-        return resources.map((r, i) => ({ ...r, path: fileArgs[i] }))
-      }
-
       const state = new InstanceModsState()
-      const listener = this.resourceService as IResourceService
-      const onResourceUpdate = async (res: PartialResourceHash[]) => {
-        if (res) {
-          updateMod.push([res, InstanceModUpdatePayloadAction.Update])
-        } else {
-          this.error(new AnyError('InstanceModUpdateError', 'Cannot update instance mods as the resource is empty'))
-        }
-      }
-
-      listener
-        .on('resourceUpdate', onResourceUpdate)
+      const pending: Set<string> = new Set()
 
-      const basePath = join(instancePath, 'mods')
-      await ensureDir(basePath)
-      await this.resourceService.whenReady(ResourceDomain.Mods)
-      const initializing = scan(basePath)
-      state.mods = await initializing
-
-      const processUpdate = async (filePath: string, retryLimit = 7) => {
+      const processUpdate = defineAsyncOperation(async (filePath: string, retryLimit = 7) => {
         try {
+          if (pending.has(filePath)) return
+          pending.add(filePath)
+
           const [resource] = await this.resourceService.importResources([{ path: filePath, domain: ResourceDomain.Mods }], true)
           if (resource && isModResource(resource)) {
             this.log(`Instance mod add ${filePath}`)
-            updateMod.push([resource, InstanceModUpdatePayloadAction.Upsert])
           } else {
             this.warn(`Non mod resource added in /mods directory! ${filePath}`)
           }
+          updateMod.push([resource, InstanceModUpdatePayloadAction.Upsert])
+          pending.delete(filePath)
         } catch (e) {
           if (isSystemError(e) && (e.code === 'EMFILE' || e.code === 'EBUSY') && retryLimit > 0) {
             // Retry
             setTimeout(() => processUpdate(filePath, retryLimit - 1), Math.random() * 2000 + 1000)
           } else {
             this.error(new AnyError('InstanceModAddError', `Fail to add instance mod ${filePath}`, { cause: e }))
+            pending.delete(filePath)
           }
         }
-      }
-
-      const processRemove = async (filePath: string) => {
+      })
+      const processRemove = (filePath: string) => {
         const target = state.mods.find(r => r.path === filePath)
         if (target) {
           this.log(`Instance mod remove ${filePath}`)
@@ -101,6 +80,36 @@ export class InstanceModsService extends AbstractService implements IInstanceMod
         }
       }
 
+      const listener = this.resourceService as IResourceService
+      const onResourceUpdate = (res: PartialResourceHash[]) => {
+        if (res) {
+          updateMod.push([res, InstanceModUpdatePayloadAction.Update])
+        } else {
+          this.error(new AnyError('InstanceModUpdateError', 'Cannot update instance mods as the resource is empty'))
+        }
+      }
+      listener
+        .on('resourceUpdate', onResourceUpdate)
+
+      const basePath = join(instancePath, 'mods')
+      await ensureDir(basePath)
+      await this.resourceService.whenReady(ResourceDomain.Mods)
+      const scan = async (dir: string) => {
+        const files = (await readdirIfPresent(dir))
+          .filter((file) => !shouldIgnoreFile(file))
+          .map((file) => join(dir, file))
+
+        const peekCount = 32
+        const peekChunks = files.slice(0, peekCount)
+        for (const file of files.slice(peekCount)) {
+          processUpdate(file)
+        }
+
+        const resources = await this.resourceService.importResources(peekChunks.map(f => ({ path: f, domain: ResourceDomain.Mods })), true)
+        return resources.map((r, i) => ({ ...r, path: peekChunks[i] }))
+      }
+      state.mods = await scan(basePath)
+
       const watcher = watch(basePath, async (event, filePath) => {
         if (shouldIgnoreFile(filePath) || filePath === basePath) return
         if (event === 'update') {
@@ -112,8 +121,8 @@ export class InstanceModsService extends AbstractService implements IInstanceMod
 
       watcher.on('close', () => {
         this.log(`Unwatch on instance mods: ${basePath}`)
-        onDestroy()
       })
+
       this.log(`Mounted on instance mods: ${basePath}`)
 
       return [state, () => {
@@ -122,7 +131,6 @@ export class InstanceModsService extends AbstractService implements IInstanceMod
           .removeListener('resourceUpdate', onResourceUpdate)
       }, async () => {
         // relvaidate
-        await initializing.catch(() => undefined)
         const files = await readdirIfPresent(basePath)
         const expectFiles = files.filter((file) => !shouldIgnoreFile(file)).map((file) => join(basePath, file))
         const current = state.mods.length