diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index dccc0cb..fefc161 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -52,8 +52,8 @@ export class Client { refresh() { Object.keys(this.ctx.root[Context.internal]).forEach(async (name) => { - if (!name.startsWith('console.')) return - const key = name.slice(8) + if (!name.startsWith('console.services.')) return + const key = name.slice(17) const service = this.ctx.get(name) as DataService if (!service) return if (await this.ctx.serial('console/intercept', this, service.options)) { diff --git a/packages/core/src/service.ts b/packages/core/src/service.ts index ca74a4a..a693596 100644 --- a/packages/core/src/service.ts +++ b/packages/core/src/service.ts @@ -9,7 +9,6 @@ export namespace DataService { } export abstract class DataService extends Service { - static filter = false static inject = ['console'] public async get(forced?: boolean, client?: Client): Promise { diff --git a/packages/registry/package.json b/packages/registry/package.json new file mode 100644 index 0000000..eb77826 --- /dev/null +++ b/packages/registry/package.json @@ -0,0 +1,36 @@ +{ + "name": "@cordisjs/registry", + "description": "Scan Package Manager for Koishi Plugins", + "version": "7.0.3", + "main": "lib/index.js", + "typings": "lib/index.d.ts", + "files": [ + "lib", + "src" + ], + "author": "Shigma ", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/cordisjs/webui.git", + "directory": "packages/registry" + }, + "bugs": { + "url": "https://github.com/cordisjs/webui/issues" + }, + "keywords": [ + "market", + "registry", + "npm", + "package", + "search" + ], + "devDependencies": { + "@types/semver": "^7.5.8" + }, + "dependencies": { + "cosmokit": "^1.6.2", + "p-map": "^4.0.0", + "semver": "^7.6.0" + } +} diff --git a/packages/registry/src/index.ts b/packages/registry/src/index.ts new file mode 100644 index 0000000..4361693 --- /dev/null +++ b/packages/registry/src/index.ts @@ -0,0 +1,160 @@ +import { compare, intersects } from 'semver' +import { Awaitable, defineProperty, Dict, Time } from 'cosmokit' +import { Registry, RemotePackage, SearchObject, SearchResult } from './types' +import { conclude } from './utils' +import pMap from 'p-map' + +export * from './local' +export * from './types' +export * from './utils' + +export interface CollectConfig { + step?: number + margin?: number + timeout?: number + ignored?: string[] + endpoint?: string +} + +export interface AnalyzeConfig { + version: string + concurrency?: number + before?(object: SearchObject): void + onRegistry?(registry: Registry, versions: RemotePackage[]): Awaitable + onSuccess?(object: SearchObject, versions: RemotePackage[]): Awaitable + onFailure?(name: string, reason: any): Awaitable + onSkipped?(name: string): Awaitable + after?(object: SearchObject): void +} + +export interface ScanConfig extends CollectConfig, AnalyzeConfig { + request(url: string): Promise +} + +const stopWords = [ + 'cordis', + 'plugin', +] + +export interface RequestConfig { + timeout?: number +} + +export default interface Scanner extends SearchResult { + progress: number +} + +export default class Scanner { + private cache: Dict + + constructor(public request: (url: string, config?: RequestConfig) => Promise) { + defineProperty(this, 'progress', 0) + defineProperty(this, 'cache', {}) + } + + private async search(offset: number, config: CollectConfig) { + const { step = 250, timeout = Time.second * 30 } = config + const result = await this.request(`/-/v1/search?text=cordis+plugin&size=${step}&from=${offset}`, { timeout }) + this.version = result.version + for (const object of result.objects) { + this.cache[object.package.name] = object + } + return result.total + } + + public async collect(config: CollectConfig = {}) { + const { step = 250, margin = 25, ignored = [] } = config + this.cache = {} + this.time = new Date().toUTCString() + const total = await this.search(0, config) + for (let offset = Object.values(this.cache).length; offset < total; offset += step - margin) { + await this.search(offset - margin, config) + } + this.objects = Object.values(this.cache).filter((object) => { + const { name, date } = object.package + // `date` can be `undefined` due to a bug in https://registry.npmjs.org + return date && !object.ignored && !ignored.includes(name) && Scanner.isPlugin(name) + }) + this.total = this.objects.length + } + + static isPlugin(name: string) { + const official = /^@cordisjs\/plugin-[0-9a-z-]+$/.test(name) + const community = /(^|\/)cordis-plugin-[0-9a-z-]+$/.test(name) + return official || community + } + + static isCompatible(range: string, remote: Pick) { + const { peerDependencies = {} } = remote + const declaredVersion = peerDependencies['cordis'] + try { + return declaredVersion && intersects(range, declaredVersion) + } catch {} + } + + public async process(object: SearchObject, range: string, onRegistry: AnalyzeConfig['onRegistry']) { + const { name } = object.package + const official = name.startsWith('@cordisjs/plugin-') + const registry = await this.request(`/${name}`) + const compatible = Object.values(registry.versions).filter((remote) => { + return Scanner.isCompatible(range, remote) + }).sort((a, b) => compare(b.version, a.version)) + + await onRegistry?.(registry, compatible) + const versions = compatible.filter(item => !item.deprecated) + if (!versions.length) return + + const latest = registry.versions[versions[0].version] + const manifest = conclude(latest) + const times = compatible.map(item => registry.time[item.version]).sort() + + object.shortname = name.replace(/(cordis-|^@cordisjs\/)plugin-/, '') + object.verified = official + object.manifest = manifest + object.insecure = manifest.insecure + object.category = manifest.category + object.createdAt = times[0] + object.updatedAt = times[times.length - 1] + object.package.contributors ??= latest.author ? [latest.author] : [] + object.package.keywords = (latest.keywords ?? []) + .map(keyword => keyword.toLowerCase()) + .filter((keyword) => { + return !keyword.includes(':') + && !object.shortname.includes(keyword) + && !stopWords.some(word => keyword.includes(word)) + }) + return versions + } + + public async analyze(config: AnalyzeConfig) { + const { concurrency = 5, version, before, onSuccess, onFailure, onSkipped, onRegistry, after } = config + + const result = await pMap(this.objects, async (object) => { + if (object.ignored) return + before?.(object) + const { name } = object.package + try { + const versions = await this.process(object, version, onRegistry) + if (versions) { + await onSuccess?.(object, versions) + return versions + } else { + object.ignored = true + await onSkipped?.(name) + } + } catch (error) { + object.ignored = true + await onFailure?.(name, error) + } finally { + this.progress += 1 + after?.(object) + } + }, { concurrency }) + + return result.filter(isNonNullable) + } +} + +function isNonNullable(value: T): value is Exclude { + return value !== null && value !== undefined +} diff --git a/packages/registry/src/local.ts b/packages/registry/src/local.ts new file mode 100644 index 0000000..bd0bd99 --- /dev/null +++ b/packages/registry/src/local.ts @@ -0,0 +1,88 @@ +/// + +import { defineProperty, Dict, pick } from 'cosmokit' +import { dirname } from 'path' +import { readdir, readFile } from 'fs/promises' +import { PackageJson, SearchObject, SearchResult } from './types' +import { conclude } from './utils' + +export interface LocalScanner extends SearchResult {} + +export class LocalScanner { + private cache: Dict> + private task: Promise + + constructor(public baseDir: string) { + defineProperty(this, 'cache', {}) + } + + onError(reason: any, name: string) {} + + async _collect() { + this.cache = {} + let root = this.baseDir + const tasks: Promise[] = [] + while (1) { + tasks.push(this.loadDirectory(root)) + const parent = dirname(root) + if (root === parent) break + root = parent + } + await Promise.all(tasks) + return Promise.all(Object.values(this.cache)) + } + + async collect(forced = false) { + if (forced) delete this.task + this.objects = await (this.task ||= this._collect()) + } + + private async loadDirectory(baseDir: string) { + const base = baseDir + '/node_modules' + const files = await readdir(base).catch(() => []) + for (const name of files) { + if (name.startsWith('cordis-plugin-')) { + this.cache[name] ||= this.loadPackage(name) + } else if (name.startsWith('@')) { + const base2 = base + '/' + name + const files = await readdir(base2).catch(() => []) + for (const name2 of files) { + if (name === '@cordisjs' && name2.startsWith('plugin-') || name2.startsWith('cordis-plugin-')) { + this.cache[name + '/' + name2] ||= this.loadPackage(name + '/' + name2) + } + } + } + } + } + + private async loadPackage(name: string) { + try { + return await this.parsePackage(name) + } catch (error) { + this.onError(error, name) + } + } + + private async loadManifest(name: string) { + const filename = require.resolve(name + '/package.json') + const meta: PackageJson = JSON.parse(await readFile(filename, 'utf8')) + meta.peerDependencies ||= {} + meta.peerDependenciesMeta ||= {} + return [meta, !filename.includes('node_modules')] as const + } + + protected async parsePackage(name: string) { + const [data, workspace] = await this.loadManifest(name) + return { + workspace, + manifest: conclude(data), + shortname: data.name.replace(/(cordis-|^@cordisjs\/)plugin-/, ''), + package: pick(data, [ + 'name', + 'version', + 'peerDependencies', + 'peerDependenciesMeta', + ]), + } as SearchObject + } +} diff --git a/packages/registry/src/types.ts b/packages/registry/src/types.ts new file mode 100644 index 0000000..8e80347 --- /dev/null +++ b/packages/registry/src/types.ts @@ -0,0 +1,158 @@ +import { Dict } from 'cosmokit' + +export interface User { + name?: string + email: string + url?: string + username?: string +} + +export interface BasePackage { + name: string + version: string + description: string +} + +export type DependencyKey = 'dependencies' | 'devDependencies' | 'peerDependencies' | 'optionalDependencies' +export type DependencyMetaKey = 'deprecated' | 'peerDependencies' | 'peerDependenciesMeta' + +export interface PackageJson extends BasePackage, Partial>> { + main?: string + module?: string + browser?: string + bin?: string | Dict + scripts?: Dict + exports?: PackageJson.Exports + cordis?: Partial + keywords: string[] + engines?: Dict + os?: string[] + cpu?: string[] + overrides?: Dict + peerDependenciesMeta?: Dict +} + +export namespace PackageJson { + export type Exports = string | { [key: string]: Exports } + export type Overrides = string | { [key: string]: Overrides } + + export interface PeerMeta { + optional?: boolean + } +} + +export interface IconSvg { + type: 'svg' + viewBox: string + pathData: string +} + +export interface Manifest { + icon?: IconSvg + hidden?: boolean + preview?: boolean + insecure?: boolean + browser?: boolean + category?: string + public?: string[] + exports?: Dict + description: string | Dict + service: Manifest.Service + locales: string[] +} + +export namespace Manifest { + export interface Service { + required: string[] + optional: string[] + implements: string[] + } +} + +export interface RemotePackage extends PackageJson { + deprecated?: string + author?: User + contributors?: User[] + maintainers: User[] + license: string + dist: RemotePackage.Dist +} + +export namespace RemotePackage { + export interface Dist { + shasum: string + integrity: string + tarball: string + fileCount: number + unpackedSize: number + } +} + +export interface Registry extends BasePackage { + versions: Dict + time: Dict + license: string + readme: string + readmeFilename: string +} + +export interface DatedPackage extends BasePackage { + date: string +} + +export interface SearchPackage extends DatedPackage, Pick { + // `links` is absent in npmmirror + links?: Dict + author?: User + contributors?: User[] + keywords: string[] + publisher: User + maintainers: User[] + // npmmirror only + versions?: string[] + 'dist-tags'?: Dict +} + +export interface SearchObject { + shortname: string + package: SearchPackage + searchScore: number + score: Score + rating: number + verified: boolean + workspace?: boolean + category?: string + portable?: boolean + insecure?: boolean + ignored?: boolean + license: string + manifest: Manifest + createdAt: string + updatedAt: string + publishSize?: number + installSize?: number + downloads?: { + lastMonth: number + } +} + +export interface Score { + final: number + detail: Score.Detail +} + +export namespace Score { + export interface Detail { + quality: number + popularity: number + maintenance: number + } +} + +export interface SearchResult { + total: number + time: string + objects: SearchObject[] + version?: number + forceTime?: number +} diff --git a/packages/registry/src/utils.ts b/packages/registry/src/utils.ts new file mode 100644 index 0000000..d647b0d --- /dev/null +++ b/packages/registry/src/utils.ts @@ -0,0 +1,75 @@ +import { Dict } from 'cosmokit' +import { Manifest, PackageJson } from './types' + +interface Ensure { + (value: any): T | undefined + (value: any, fallback: T): T +} + +export namespace Ensure { + export const array: Ensure = (value: any, fallback?: any) => { + if (!Array.isArray(value)) return fallback + return value.filter(x => typeof x === 'string') + } + + export const dict: Ensure> = (value: any, fallback?: any) => { + if (typeof value !== 'object' || value === null) return fallback + return Object.entries(value).reduce>((dict, [key, value]) => { + if (typeof value === 'string') dict[key] = value + return dict + }, {}) + } + + // https://github.com/microsoft/TypeScript/issues/15713#issuecomment-499474386 + const primitive = (type: string): Ensure => (value: any, fallback?: T) => { + if (typeof value !== type) return fallback + return value + } + + export const boolean = primitive('boolean') + export const number = primitive('number') + export const string = primitive('string') +} + +export function conclude(meta: PackageJson) { + const manifest: Manifest = { + hidden: Ensure.boolean(meta.cordis?.hidden), + preview: Ensure.boolean(meta.cordis?.preview), + insecure: Ensure.boolean(meta.cordis?.insecure), + browser: Ensure.boolean(meta.cordis?.browser), + category: Ensure.string(meta.cordis?.category), + public: Ensure.array(meta.cordis?.public), + description: Ensure.dict(meta.cordis?.description) || Ensure.string(meta.description, ''), + locales: Ensure.array(meta.cordis?.locales, []), + service: { + required: Ensure.array(meta.cordis?.service?.required, []), + optional: Ensure.array(meta.cordis?.service?.optional, []), + implements: Ensure.array(meta.cordis?.service?.implements, []), + }, + } + + if (typeof manifest.description === 'string') { + manifest.description = manifest.description.slice(0, 1024) + } else if (manifest.description) { + for (const key in manifest.description) { + manifest.description[key] = manifest.description[key].slice(0, 1024) + } + } + + meta.keywords = Ensure.array(meta.keywords, []).filter((keyword) => { + if (!keyword.includes(':')) return true + if (keyword === 'market:hidden') { + manifest.hidden = true + } else if (keyword.startsWith('required:')) { + manifest.service.required.push(keyword.slice(9)) + } else if (keyword.startsWith('optional:')) { + manifest.service.optional.push(keyword.slice(9)) + } else if (keyword.startsWith('impl:')) { + manifest.service.implements.push(keyword.slice(5)) + } else if (keyword.startsWith('locale:')) { + manifest.locales.push(keyword.slice(7)) + } + }) + + return manifest +} diff --git a/packages/registry/tsconfig.json b/packages/registry/tsconfig.json new file mode 100644 index 0000000..42cbe59 --- /dev/null +++ b/packages/registry/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.base", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib", + }, + "include": [ + "src", + ], +} diff --git a/plugins/insight/package.json b/plugins/insight/package.json index b27a12b..6c7b296 100644 --- a/plugins/insight/package.json +++ b/plugins/insight/package.json @@ -1,6 +1,6 @@ { "name": "@cordisjs/plugin-insight", - "description": "Show plugin dependency graph for Koishi", + "description": "Show plugin dependency graph for Cordis", "version": "3.5.1", "main": "lib/index.cjs", "types": "lib/index.d.ts",