From 1d2f8052971ff6a5a7d524d4e112a3814507dab6 Mon Sep 17 00:00:00 2001 From: Qiming Zhao Date: Wed, 19 Oct 2022 01:35:21 +0800 Subject: [PATCH 01/13] feat(extension): add dependency module --- src/__tests__/helper.ts | 21 + .../modules/extensionDependency.test.ts | 383 ++++++++++++++++ src/__tests__/modules/installer.test.ts | 35 +- src/extension/dependency.ts | 417 ++++++++++++++++++ src/extension/installer.ts | 86 +--- src/model/download.ts | 22 +- 6 files changed, 849 insertions(+), 115 deletions(-) create mode 100644 src/__tests__/modules/extensionDependency.test.ts create mode 100644 src/extension/dependency.ts diff --git a/src/__tests__/helper.ts b/src/__tests__/helper.ts index a754551f66c..7434dd8cb46 100644 --- a/src/__tests__/helper.ts +++ b/src/__tests__/helper.ts @@ -1,3 +1,4 @@ +import net from 'net' /* eslint-disable @typescript-eslint/no-unsafe-return */ /* eslint-disable @typescript-eslint/no-unsafe-call */ import { Buffer, Neovim, Window } from '@chemzqm/neovim' @@ -364,4 +365,24 @@ export function makeLine(length) { return result } +export function getPort(): Promise { + let port = 7080 + let fn = cb => { + let server = net.createServer() + server.listen(port, () => { + server.once('close', () => { + cb(port) + }) + server.close() + }) + server.on('error', () => { + port++ + fn(cb) + }) + } + return new Promise(resolve => { + fn(resolve) + }) +} + export default new Helper() diff --git a/src/__tests__/modules/extensionDependency.test.ts b/src/__tests__/modules/extensionDependency.test.ts new file mode 100644 index 00000000000..1354d93b912 --- /dev/null +++ b/src/__tests__/modules/extensionDependency.test.ts @@ -0,0 +1,383 @@ +import fs from 'fs' +import tar from 'tar' +import http, { Server } from 'http' +import os from 'os' +import path from 'path' +import { URL } from 'url' +import { v4 as uuid } from 'uuid' +import { checkFileSha1, DependenciesInstaller, DependencyItem, findItem, getModuleInfo, getRegistries, getVersion, readDependencies, shouldRetry, untar, VersionInfo } from '../../extension/dependency' +import { Dependencies } from '../../extension/installer' +import { writeJson, remove, loadJson } from '../../util/fs' +import helper, { getPort } from '../helper' + +process.env.NO_PROXY = '*' + +describe('utils', () => { + it('should getRegistries', () => { + let u = new URL('https://registry.npmjs.org') + expect(getRegistries(u).length).toBe(2) + u = new URL('https://registry.yarnpkg.com') + expect(getRegistries(u).length).toBe(2) + u = new URL('https://example.com') + expect(getRegistries(u).length).toBe(3) + }) + + it('should checkFileSha1', async () => { + let not_exists = path.join(os.tmpdir(), 'not_exists') + let checked = await checkFileSha1(not_exists, 'shasum') + expect(checked).toBe(false) + let tarfile = path.resolve(__dirname, '../test.tar.gz') + checked = await checkFileSha1(tarfile, 'bf0d88712fc3dbf6e3ab9a6968c0b4232779dbc4') + expect(checked).toBe(true) + }) + + it('should untar files', async () => { + let tarfile = path.resolve(__dirname, '../test.tar.gz') + let folder = path.join(os.tmpdir(), `test-${uuid()}`) + await untar(folder, tarfile, 0) + let file = path.join(folder, 'test.js') + expect(fs.existsSync(file)).toBe(true) + await remove(folder) + }) + + it('should throw on untar error', async () => { + let fn = async () => { + let file = path.join(os.tmpdir(), `note_exists_${uuid()}`) + let folder = path.join(os.tmpdir(), `test-${uuid()}`) + await untar(folder, file, 0) + } + await expect(fn()).rejects.toThrow(Error) + }) + + it('should throw when item not found', async () => { + expect(() => { + findItem('name', '^1.0.1', []) + }).toThrow() + }) + + it('should getModuleInfo', () => { + expect(() => { + getModuleInfo('{') + }).toThrow() + expect(() => { + getModuleInfo('{}') + }).toThrow() + expect(() => { + getModuleInfo('{"name": "name"}') + }).toThrow() + let obj: any = { name: 'name', version: '1.0.0', versions: {} } + expect(getModuleInfo(JSON.stringify(obj))).toBeDefined() + obj = { name: 'name', 'dist-tags': { latest: '1.0.0' }, versions: {} } + expect(getModuleInfo(JSON.stringify(obj))).toBeDefined() + }) + + it('should check retry', () => { + expect(shouldRetry({})).toBe(false) + expect(shouldRetry({ message: 'message' })).toBe(false) + expect(shouldRetry({ message: 'timeout' })).toBe(true) + expect(shouldRetry({ message: 'ECONNRESET' })).toBe(true) + }) + + it('should readDependencies', () => { + let dir = path.join(os.tmpdir(), uuid()) + fs.mkdirSync(dir, { recursive: true }) + let filepath = path.join(dir, 'package.json') + writeJson(filepath, { dependencies: { "coc.nvim": ">= 0.0.80", "is-number": "^1.0.0" } }) + let res = readDependencies(dir) + expect(res).toEqual({ 'is-number': '^1.0.0' }) + }) + + it('should getVersion', () => { + expect(getVersion('>= 1.0.0', ['1.0.0', '2.0.0', '2.0.1'], '2.0.1')).toBe('2.0.1') + expect(getVersion('^1.0.0', ['1.0.0', '1.1.0', '2.0.1'])).toBe('1.1.0') + expect(getVersion('^3.0.0', ['1.0.0'])).toBeUndefined() + }) +}) + +describe('DependenciesInstaller', () => { + let httpPort: number + let server: Server + let jsonResponses: Map = new Map() + let url: URL + let dirs: string[] = [] + let createFiles = false + let timer + + beforeAll(async () => { + httpPort = await getPort() + url = new URL(`http://127.0.0.1:${httpPort}`) + server = await createServer(httpPort) + }) + + afterEach(async () => { + jsonResponses.clear() + for (let dir of dirs) { + await remove(dir) + } + dirs = [] + }) + + afterAll(() => { + clearTimeout(timer) + if (server) server.close() + }) + + async function createTarFile(name: string, version: string): Promise { + let folder = path.join(os.tmpdir(), uuid()) + fs.mkdirSync(folder, { recursive: true }) + fs.writeFileSync(path.join(folder, 'index.js'), '', 'utf8') + writeJson(path.join(folder, 'package.json'), { name, version, dependencies: {} }) + let file = path.join(os.tmpdir(), uuid(), `${name}.${version}.tgz`) + fs.mkdirSync(path.dirname(file), { recursive: true }) + await tar.create({ file, gzip: true, cwd: path.dirname(folder) }, [path.basename(folder)]) + return file + } + + async function createServer(port: number): Promise { + return await new Promise(resolve => { + const server = http.createServer(async (req, res) => { + for (let [url, text] of jsonResponses.entries()) { + if (req.url == url) { + res.writeHead(200, { 'Content-Type': 'application/json;charset=utf8' }) + res.end(text) + return + } + } + if (req.url.endsWith('/slow')) { + timer = setTimeout(() => { + res.writeHead(100) + res.end('abc') + }, 300) + return + } + if (req.url.endsWith('.tgz')) { + res.setHeader('Content-Disposition', 'attachment; filename="file.tgz"') + res.setHeader('Content-Type', 'application/octet-stream') + let tarfile: string + if (createFiles) { + let parts = req.url.slice(1).replace(/\.tgz/, '').split('-') + tarfile = await createTarFile(parts[0], parts[1]) + } else { + tarfile = path.resolve(__dirname, '../test.tar.gz') + } + let stat = fs.statSync(tarfile) + res.setHeader('Content-Length', stat.size) + res.writeHead(200) + let stream = fs.createReadStream(tarfile) + stream.pipe(res) + } + }) + server.listen(port, () => { + resolve(server) + }) + }) + } + + function create(root?: string, onMessage?: (msg: string) => void): DependenciesInstaller { + if (!root) { + root = path.join(os.tmpdir(), uuid()) + fs.mkdirSync(root) + dirs.push(root) + } + let registry = new URL(`http://127.0.0.1:${httpPort}`) + onMessage = onMessage ?? function() {} + return new DependenciesInstaller(registry, root, onMessage) + } + + function createVersion(name: string, version: string, dependencies?: Dependencies): VersionInfo { + return { + name, + version, + dependencies, + dist: { + shasum: '', + integrity: '', + tarball: `http://127.0.0.1:${httpPort}/${name}-${version}.tgz`, + } + } + } + + function addJsonData(): void { + // a => b, c, d + // c => b, d + // b => d + jsonResponses.set('/a', JSON.stringify({ + name: 'a', + versions: { + '0.0.1': createVersion('a', '0.0.1', { b: '^1.0.0', c: '^2.0.0', d: '>= 0.0.1' }) + } + })) + jsonResponses.set('/b', JSON.stringify({ + name: 'b', + versions: { + '1.0.0': createVersion('b', '1.0.0', {}), + '2.0.0': createVersion('b', '2.0.0', { d: '^1.0.0' }), + '3.0.0': createVersion('b', '3.0.0', { d: '^1.0.0' }), + } + })) + jsonResponses.set('/c', JSON.stringify({ + name: 'c', + versions: { + '1.0.0': createVersion('c', '1.0.0', {}), + '2.0.0': createVersion('c', '2.0.0', { b: '^2.0.0', d: '^1.0.0' }), + '3.0.0': createVersion('c', '3.0.0', { b: '^3.0.0', d: '^1.0.0' }), + } + })) + jsonResponses.set('/d', JSON.stringify({ + name: 'd', + versions: { + '1.0.0': createVersion('d', '1.0.0') + } + })) + } + + it('should retry fetch', async () => { + let install = create() + let fn = async () => { + await install.fetch(new URL('/', url), { timeout: 10 }, 3) + } + await expect(fn()).rejects.toThrow(Error) + jsonResponses.set('/json', '{"result": "ok"}') + let res = await install.fetch(new URL('/json', url), {}, 1) + expect(res).toEqual({ result: 'ok' }) + }) + + it('should cancel request', async () => { + let install = create() + let p = install.fetch(new URL('/slow', url), {}, 1) + await helper.wait(10) + let fn = async () => { + install.cancel() + await p + } + await expect(fn()).rejects.toThrow(Error) + }) + + it('should throw when unable to load info', async () => { + let install = create() + let fn = async () => { + await install.loadInfo(url, 'foo', 10) + } + await expect(fn()).rejects.toThrow(Error) + }) + + it('should fetchInfos', async () => { + addJsonData() + let install = create() + await install.fetchInfos({ a: '^0.0.1' }) + expect(install.resolvedInfos.size).toBe(4) + expect(install.resolvedVersions).toEqual([ + { name: 'a', requirement: '^0.0.1', version: '0.0.1' }, + { name: 'b', requirement: '^1.0.0', version: '1.0.0' }, + { name: 'c', requirement: '^2.0.0', version: '2.0.0' }, + { name: 'b', requirement: '^2.0.0', version: '2.0.0' }, + { name: 'd', requirement: '^1.0.0', version: '1.0.0' }, + { name: 'd', requirement: '>= 0.0.1', version: '1.0.0' } + ]) + }) + + it('should linkDependencies', async () => { + addJsonData() + let install = create() + await install.fetchInfos({ a: '^0.0.1' }) + let items: DependencyItem[] = [] + install.linkDependencies(undefined, items) + expect(items).toEqual([]) + install.linkDependencies({ a: '^0.0.1' }, items) + expect(items.length).toBe(5) + }) + + it('should retry download', async () => { + let install = create() + let fn = async () => { + await install.download(new URL('res', url), 'res', '', 3, 10) + } + await expect(fn()).rejects.toThrow(Error) + fn = async () => { + await install.download(new URL('test.tgz', url), 'test.tgz', 'badsum') + } + await expect(fn()).rejects.toThrow(Error) + let res = await install.download(new URL('test.tgz', url), 'test.tgz', '') + expect(fs.existsSync(res)).toBe(true) + fs.unlinkSync(res) + res = await install.download(new URL('test.tgz', url), 'test.tgz', 'bf0d88712fc3dbf6e3ab9a6968c0b4232779dbc4') + expect(fs.existsSync(res)).toBe(true) + fs.unlinkSync(res) + }) + + it('should check exists and download items', async () => { + let items: DependencyItem[] = [] + items.push({ + integrity: '', + name: 'foo', + resolved: `http://127.0.0.1:${httpPort}/foo.tgz`, + satisfiedVersions: [], + shasum: 'bf0d88712fc3dbf6e3ab9a6968c0b4232779dbc4', + version: '0.0.1' + }) + items.push({ + integrity: '', + name: 'bar', + resolved: `http://127.0.0.1:${httpPort}/bar.tgz`, + satisfiedVersions: ['^0.0.1'], + shasum: 'bf0d88712fc3dbf6e3ab9a6968c0b4232779dbc4', + version: '0.0.2' + }) + let install = create() + let dest = path.join(install.modulesRoot, '.cache') + fs.mkdirSync(dest, { recursive: true }) + let tarfile = path.resolve(__dirname, '../test.tar.gz') + fs.copyFileSync(tarfile, path.join(dest, `foo.0.0.1.tgz`)) + let res = await install.downloadItems(items) + expect(res.size).toBe(2) + }) + + it('should throw on error', async () => { + let items: DependencyItem[] = [] + items.push({ + integrity: '', + name: 'bar', + resolved: `http://127.0.0.1:${httpPort}/bar.tgz`, + satisfiedVersions: [], + shasum: 'badsum', + version: '0.0.2' + }) + let install = create() + let fn = async () => { + await install.downloadItems(items) + } + await expect(fn()).rejects.toThrow(Error) + }) + + it('should no nothing if no dependencies', async () => { + let msg: string + let install = create(undefined, s => { + msg = s + }) + let directory = path.join(os.tmpdir(), uuid()) + let file = path.join(directory, 'package.json') + writeJson(file, { dependencies: {} }) + await install.installDependencies(directory) + expect(msg).toMatch('No dependencies') + fs.rmSync(directory, { recursive: true }) + }) + + it('should install dependencies ', async () => { + createFiles = true + addJsonData() + let install = create() + let directory = path.join(os.tmpdir(), uuid()) + fs.mkdirSync(directory, { recursive: true }) + let file = path.join(directory, 'package.json') + writeJson(file, { dependencies: { a: '^0.0.1' } }) + await install.installDependencies(directory) + let folder = path.join(directory, 'node_modules') + let res = fs.readdirSync(folder) + expect(res).toEqual(['a', 'b', 'c', 'd']) + let obj = loadJson(path.join(folder, 'b/package.json')) as any + expect(obj.version).toBe('1.0.0') + obj = loadJson(path.join(folder, 'c/node_modules/b/package.json')) as any + expect(obj.version).toBe('2.0.0') + fs.rmSync(directory, { recursive: true }) + }) +}) diff --git a/src/__tests__/modules/installer.test.ts b/src/__tests__/modules/installer.test.ts index 41ca3aa4cee..0dc6363019e 100644 --- a/src/__tests__/modules/installer.test.ts +++ b/src/__tests__/modules/installer.test.ts @@ -2,7 +2,7 @@ import fs from 'fs' import os from 'os' import path from 'path' import { v4 as uuid } from 'uuid' -import { getDependencies, Info, Installer, isNpmCommand, isYarn, registryUrl } from '../../extension/installer' +import { Info, Installer, registryUrl } from '../../extension/installer' import { remove } from '../../util/fs' const rcfile = path.join(os.tmpdir(), '.npmrc') @@ -14,15 +14,6 @@ afterEach(() => { }) describe('utils', () => { - it('should getDependencies', async () => { - expect(getDependencies({})).toEqual([]) - expect(getDependencies({ dependencies: { 'coc.nvim': '0.0.1' } })).toEqual([]) - }) - - it('should check command is npm or yarn', async () => { - expect(isNpmCommand('npm')).toBe(true) - expect(isYarn('yarnpkg')).toBe(true) - }) it('should get registry url', async () => { const getUrl = () => { @@ -69,13 +60,6 @@ describe('Installer', () => { }) describe('getInfo()', () => { - it('should get install arguments', async () => { - let installer = new Installer(__dirname, 'npm', 'https://github.com/') - expect(installer.getInstallArguments('pnpm', 'https://github.com/')).toEqual(['install', '--production', '--config.strict-peer-dependencies=false']) - expect(installer.getInstallArguments('npm', '')).toEqual(['install', '--ignore-scripts', '--no-lockfile', '--omit=dev', '--legacy-peer-deps', '--no-global']) - expect(installer.getInstallArguments('yarn', '')).toEqual(['install', '--ignore-scripts', '--no-lockfile', '--production', '--ignore-engines']) - }) - it('should getInfo from url', async () => { let installer = new Installer(__dirname, 'npm', 'https://github.com/') let spy = jest.spyOn(installer, 'getInfoFromUri').mockImplementation(() => { @@ -411,23 +395,8 @@ describe('Installer', () => { installer.on('message', () => { called = true }) - await installer.installDependencies(tmpfolder, ['a', 'b']) + await installer.installDependencies(tmpfolder) expect(called).toBe(true) }) - - it('should reject on install error', async () => { - let npm = path.resolve(__dirname, '../npm') - tmpfolder = path.join(os.tmpdir(), uuid()) - fs.mkdirSync(tmpfolder) - let installer = new Installer(tmpfolder, npm, 'coc-omni') - let spy = jest.spyOn(installer, 'getInstallArguments').mockImplementation(() => { - return ['--error'] - }) - let fn = async () => { - await installer.installDependencies(tmpfolder, ['a', 'b']) - } - await expect(fn()).rejects.toThrow(Error) - spy.mockRestore() - }) }) }) diff --git a/src/extension/dependency.ts b/src/extension/dependency.ts new file mode 100644 index 00000000000..c1aab5d93b7 --- /dev/null +++ b/src/extension/dependency.ts @@ -0,0 +1,417 @@ +import { createHash } from 'crypto' +import fs, { createReadStream } from 'fs' +import path from 'path' +import semver from 'semver' +import tar from 'tar' +import { URL } from 'url' +import { CancellationTokenSource } from 'vscode-languageserver-protocol' +import download from '../model/download' +import fetch, { FetchOptions } from '../model/fetch' +import { concurrent } from '../util' +import { loadJson, writeJson } from '../util/fs' +const logger = require('../util/logger')('extension-dependency') + +export interface Dependencies { [key: string]: string } + +// The information we cares about +export interface VersionInfo { + name: string + version: string + dependencies?: Dependencies + dist: { + integrity: string // starts with sha512-, base64 string + shasum: string // sha1 hash, hex string + tarball: string // download url + } +} + +interface ModuleInfo { + name: string + latest?: string + versions: { + [version: string]: VersionInfo + } +} + +interface ResolvedVersion { + name: string + requirement: string + version: string +} + +export interface DependencyItem { + name: string + version: string + resolved: string // download url + shasum: string + integrity: string + satisfiedVersions: string[] + dependencies?: { + [key: string]: string + } +} + +const NPM_REGISTRY = new URL('https://registry.npmjs.org') +const YARN_REGISTRY = new URL('https://registry.yarnpkg.com') +const DEV_DEPENDENCIES = ['coc.nvim', 'webpack', 'esbuild'] +const INFO_TIMEOUT = global.__TEST__ ? 100 : 10000 + +function toFilename(item: DependencyItem): string { + return `${item.name}.${item.version}.tgz` +} + +export function findItem(name: string, requirement: string, items: ReadonlyArray): DependencyItem { + let item = items.find(o => o.name === name && o.satisfiedVersions.includes(requirement)) + if (!item) throw new Error(`item not found for: ${name} ${requirement}`) + return item +} + +export function getRegistries(registry: URL): URL[] { + let urls: URL[] = [registry] + if (registry.host !== NPM_REGISTRY.host) urls.push(NPM_REGISTRY) + if (registry.host !== YARN_REGISTRY.host) urls.push(YARN_REGISTRY) + return urls +} + +export function getModuleInfo(text: string): ModuleInfo { + let obj + try { + obj = JSON.parse(text) as any + } catch (e) { + throw new Error(`Invalid JSON data, ${e}`) + } + if (typeof obj.name !== 'string' || obj.versions == null) throw new Error(`Invalid JSON data, name or versions not found`) + return { + name: obj.name, + latest: obj['dist-tags']?.latest, + versions: obj.versions + } as ModuleInfo +} + +export function shouldRetry(error: any): boolean { + let message = error.message + if (typeof message !== 'string') return false + if (message.includes('timeout') || + message.includes('Invalid JSON') || + message.includes('Bad shasum') || + message.includes('ECONNRESET')) return true + return false +} + +export function readDependencies(directory: string): { [key: string]: string } { + let jsonfile = path.join(directory, 'package.json') + let obj = loadJson(jsonfile) as any + let dependencies = obj.dependencies as { [key: string]: string } + for (let key of Object.keys(dependencies ?? {})) { + if (DEV_DEPENDENCIES.includes(key) || key.startsWith('@types/')) delete dependencies[key] + } + return dependencies +} + +export function getVersion(requirement: string, versions: string[], latest?: string): string | undefined { + if (latest && semver.satisfies(latest, requirement)) return latest + let sorted = semver.rsort(versions.filter(v => semver.valid(v, { includePrerelease: false }))) + for (let v of sorted) { + if (semver.satisfies(v, requirement)) return v + } +} + +export async function untar(dest: string, tarfile: string, strip = 1): Promise { + if (!fs.existsSync(tarfile)) throw new Error(`${tarfile} not exists`) + fs.rmSync(dest, { recursive: true, force: true }) + fs.mkdirSync(dest, { recursive: true }) + await new Promise((resolve, reject) => { + const input = createReadStream(tarfile) + input.on('error', reject) + let stream = tar.x({ strip, C: dest }) + input.pipe(stream) + stream.on('error', reject) + stream.on('finish', () => { + resolve() + }) + }) +} + +export async function checkFileSha1(filepath: string, shasum: string): Promise { + const hash = createHash('sha1') + if (!fs.existsSync(filepath)) return Promise.resolve(false) + return new Promise(resolve => { + const input = createReadStream(filepath) + input.on('error', () => { + resolve(false) + }) + input.on('readable', () => { + // Only one element is going to be produced by the hash stream. + const data = input.read() + if (data) + hash.update(data) + else { + resolve(hash.digest('hex') == shasum) + } + }) + }) +} + +export class DependenciesInstaller { + public resolvedVersions: ResolvedVersion[] = [] + public resolvedInfos: Map = new Map() + private tokenSource: CancellationTokenSource = new CancellationTokenSource() + constructor( + private registry: URL, + public readonly modulesRoot: string, + private onMessage: (msg: string) => void + ) { + } + + private get dest(): string { + return path.join(this.modulesRoot, '.cache') + } + + public async installDependencies(directory: string): Promise { + // TODO reuse resolved.json + let dependencies = readDependencies(directory) + // no need to install + if (!dependencies || Object.keys(dependencies).length == 0) { + this.onMessage(`No dependencies`) + return + } + this.onMessage('Resolving dependencies.') + await this.fetchInfos(dependencies) + this.onMessage('Linking dependencies.') + // create DependencyItems + let items: DependencyItem[] = [] + this.linkDependencies(dependencies, items) + if (items.length > 0) { + let filepath = path.join(directory, 'resolved.json') + writeJson(filepath, items) + } + this.onMessage('Downloading dependencies.') + await this.downloadItems(items) + this.onMessage('Extract modules.') + await this.extractDependencies(items, dependencies, directory) + this.onMessage('Done') + } + + public async extractDependencies(items: DependencyItem[], dependencies: Dependencies, directory: string): Promise { + items.sort((a, b) => b.satisfiedVersions.length - a.satisfiedVersions.length) + let rootPackages: Set = new Set() + let rootItems: DependencyItem[] = [] + const addToRoot = (item: DependencyItem) => { + if (!rootPackages.has(item.name)) { + rootPackages.add(item.name) + rootItems.push(item) + } + } + // Top level dependencies + for (let [key, requirement] of Object.entries(dependencies)) { + let item = findItem(key, requirement, items) + addToRoot(item) + } + items.forEach(item => { + addToRoot(item) + }) + rootPackages.clear() + + let err: unknown + await concurrent(rootItems, async item => { + try { + let filename = toFilename(item) + let tarfile = path.join(this.dest, filename) + let dest = path.join(directory, 'node_modules', item.name) + await untar(dest, tarfile) + } catch (e) { + err = e + } + }, 5) + if (err) throw err + for (let item of rootItems) { + let folder = path.join(directory, 'node_modules', item.name) + await this.extractFor(item, items, rootItems, folder) + } + } + + /** + * Recursive extract dependencies for item in folder + */ + public async extractFor(item: DependencyItem, items: ReadonlyArray, rootItems: ReadonlyArray, folder: string): Promise { + // Deps to install, name to item + let deps: Map = new Map() + for (let [name, requirement] of Object.entries(item.dependencies ?? {})) { + let idx = rootItems.findIndex(o => o.name == name && o.satisfiedVersions.includes(requirement)) + if (idx == -1) deps.set(name, findItem(name, requirement, items)) + } + if (deps.size === 0) return + let newRoot: DependencyItem[] = [] + await Promise.all(Array.from(deps.values()).map(item => { + let tarfile = path.join(this.dest, toFilename(item)) + let dest = path.join(folder, 'node_modules', item.name) + newRoot.push(item) + return untar(dest, tarfile) + })) + newRoot.push(...rootItems) + for (let item of deps.values()) { + if (item.dependencies) { + let dest = path.join(folder, 'node_modules', item.name) + await this.extractFor(item, items, newRoot, dest) + } + } + } + + public linkDependencies(dependencies: Dependencies | undefined, items: DependencyItem[]): void { + if (!dependencies) return + for (let [name, requirement] of Object.entries(dependencies)) { + let version = this.resolveVersion(name, requirement) + let item = items.find(o => o.name === name && o.version === version) + if (item) { + if (!item.satisfiedVersions.includes(requirement)) item.satisfiedVersions.push(requirement) + } else { + let info = this.resolvedInfos.get(name) + let versionInfo = info ? info.versions[version] : undefined + if (!versionInfo) throw new Error(`Version data not found for "${name}@${version}"`) + let { dist } = versionInfo + items.push({ + name, + version, + resolved: dist.tarball, + shasum: dist.shasum, + integrity: dist.integrity, + satisfiedVersions: [requirement], + dependencies: versionInfo.dependencies + }) + this.linkDependencies(versionInfo.dependencies, items) + } + } + } + + private resolveVersion(name: string, requirement: string): string { + let item = this.resolvedVersions.find(o => o.name == name && o.requirement == requirement) + if (item) return item.version + let info = this.resolvedInfos.get(name) + if (info) { + let version = getVersion(requirement, Object.keys(info.versions), info.latest) + if (version) { + this.resolvedVersions.push({ name, requirement, version }) + return version + } + } + throw new Error(`No valid version found for "${name}" ${requirement}`) + } + + /** + * Recursive fetch + */ + public async fetchInfos(dependencies: Dependencies): Promise { + let keys = Object.keys(dependencies) + if (keys.length === 0) return + let fetched: Map = new Map() + await Promise.all(keys.map(key => { + if (this.resolvedInfos.has(key)) { + fetched.set(key, this.resolvedInfos.get(key)) + return Promise.resolve() + } + return this.loadInfo(this.registry, key, INFO_TIMEOUT).then(info => { + fetched.set(key, info) + this.resolvedInfos.set(key, info) + }) + })) + for (let [key, info] of fetched.entries()) { + let requirement = dependencies[key] + let version = this.resolveVersion(key, requirement) + let deps = info.versions[version].dependencies + if (deps) await this.fetchInfos(deps) + } + } + + /** + * Concurrent download necessary dependencies + */ + public async downloadItems(items: DependencyItem[], retry = 3): Promise> { + let res: Map = new Map() + let total = items.length + let finished = 0 + let err: unknown + await concurrent(items, async item => { + try { + let filename = toFilename(item) + let filepath = path.join(this.dest, filename) + let checked = await checkFileSha1(filepath, item.shasum) + let onFinish = () => { + res.set(filename, filepath) + finished++ + this.onMessage(`Downloaded ${finished}/${total}`) + } + if (checked) { + onFinish() + } else { + // 5min timeout + await this.download(new URL(item.resolved), filename, item.shasum, retry, global.__TEST__ ? 1000 : 5 * 60 * 1000) + onFinish() + } + } catch (e) { + err = e + } + }, 3) + if (finished !== total) throw new Error(err ? err.toString() : 'unknown error') + return res + } + + public async fetch(url: string | URL, options: FetchOptions = {}, retry = 1): Promise { + for (let i = 0; i < retry; i++) { + try { + return await fetch(url, options, this.tokenSource.token) + } catch (e) { + if (i == retry - 1 || !shouldRetry(e)) { + throw e + } else { + this.onMessage(`Network issue, retry fetch for ${url}`) + } + } + } + } + + // Try different registries + public async loadInfo(registry: URL, name: string, timeout = 100): Promise { + let info: ModuleInfo + for (let url of getRegistries(registry)) { + try { + let buffer = await this.fetch(new URL(name, url), { timeout, buffer: true }) as Buffer + info = getModuleInfo(buffer.toString()) + return info + } catch (e) { + this.onMessage(`Error on fetch ${url.hostname}/${name}: ${e}`) + } + } + if (!info) throw new Error(`Unable to fetch info for "${name}"`) + return info + } + + public async download(url: string | URL, filename: string, shasum: string, retry = 1, timeout?: number): Promise { + for (let i = 0; i < retry; i++) { + try { + let fullpath = path.join(this.dest, filename) + await download(url, { + dest: this.dest, + filename, + extract: false, + timeout + }, this.tokenSource.token) + if (shasum) { + let checked = await checkFileSha1(fullpath, shasum) + if (!checked) throw new Error(`Bad shasum for ${filename}`) + } + return fullpath + } catch (e) { + if (i == retry - 1 || !shouldRetry(e)) { + throw e + } else { + this.onMessage(`Network issue, retry download for ${url}`) + } + } + } + } + + public cancel(): void { + this.tokenSource.cancel() + this.tokenSource = new CancellationTokenSource() + } +} diff --git a/src/extension/installer.ts b/src/extension/installer.ts index 782671b1805..57949be17c1 100644 --- a/src/extension/installer.ts +++ b/src/extension/installer.ts @@ -1,19 +1,17 @@ 'use strict' -import { spawn } from 'child_process' import { EventEmitter } from 'events' import fs from 'fs' import os from 'os' import path from 'path' -import readline from 'readline' import semver from 'semver' -import { v4 as uuid } from 'uuid' import { URL } from 'url' +import { v4 as uuid } from 'uuid' import download, { DownloadOptions } from '../model/download' import fetch, { FetchOptions } from '../model/fetch' -import workspace from '../workspace' import { loadJson } from '../util/fs' +import workspace from '../workspace' +import { DependenciesInstaller } from './dependency' const logger = require('../util/logger')('extension-installer') -const local_dependencies = ['coc.nvim', 'esbuild', 'webpack', '@types/node'] export interface Info { 'dist.tarball'?: string @@ -54,21 +52,6 @@ export function registryUrl(home = os.homedir()): URL { return res ?? new URL('https://registry.npmjs.org') } -export function isNpmCommand(exePath: string): boolean { - let name = path.basename(exePath) - return name === 'npm' || name === 'npm.CMD' -} - -export function isYarn(exePath: string) { - let name = path.basename(exePath) - return ['yarn', 'yarn.CMD', 'yarnpkg', 'yarnpkg.CMD'].includes(name) -} - -function isPnpm(exePath: string) { - let name = path.basename(exePath) - return name === 'pnpm' || name === 'pnpm.CMD' -} - function isSymbolicLink(folder: string): boolean { if (fs.existsSync(folder)) { let stat = fs.lstatSync(folder) @@ -204,58 +187,12 @@ export class Installer extends EventEmitter implements IInstaller { return path.dirname(jsonFile) } - public getInstallArguments(exePath: string, url: string | undefined): string[] { - let args = ['install', '--ignore-scripts', '--no-lockfile'] - if (url && url.startsWith('https://github.com')) { - args = ['install'] - } - if (isNpmCommand(exePath)) { - args.push('--omit=dev') - args.push('--legacy-peer-deps') - args.push('--no-global') - } - if (isYarn(exePath)) { - args.push('--production') - args.push('--ignore-engines') - } - if (isPnpm(exePath)) { - args.push('--production') - args.push('--config.strict-peer-dependencies=false') - } - return args - } - - private readLines(key: string, stream: NodeJS.ReadableStream): void { - const rl = readline.createInterface({ - input: stream - }) - rl.on('line', line => { - this.log(`${key} ${line}`, true) - }) - } - - public installDependencies(folder: string, dependencies: string[]): Promise { - if (dependencies.length == 0) return Promise.resolve() - return new Promise((resolve, reject) => { - let args = this.getInstallArguments(this.npm, this.url) - this.log(`Installing dependencies by: ${this.npm} ${args.join(' ')}.`) - const child = spawn(this.npm, args, { - cwd: folder, - env: Object.assign(process.env, { NODE_ENV: 'production' }) - }) - this.readLines('[npm stdout]', child.stdout) - this.readLines('[npm stderr]', child.stderr) - child.stderr.setEncoding('utf8') - child.stdout.setEncoding('utf8') - child.on('error', reject) - child.on('exit', code => { - if (code) { - reject(new Error(`${this.npm} install exited with ${code}`)) - return - } - resolve() - }) + public async installDependencies(folder: string): Promise { + let registry = registryUrl() + let installer = new DependenciesInstaller(registry, this.root, msg => { + this.log(msg) }) + await installer.installDependencies(folder) } public async doInstall(info: Info): Promise { @@ -274,8 +211,7 @@ export class Installer extends EventEmitter implements IInstaller { onProgress: p => this.log(`Download progress ${p}%`, true), }) this.log(`Extension download at ${downloadFolder}`) - let obj = loadJson(path.join(downloadFolder, 'package.json')) as any - await this.installDependencies(downloadFolder, getDependencies(obj)) + await this.installDependencies(downloadFolder) } catch (e) { fs.rmSync(downloadFolder, { recursive: true, force: true }) throw e @@ -296,7 +232,3 @@ export class Installer extends EventEmitter implements IInstaller { return await fetch(url, options) } } - -export function getDependencies(obj: { dependencies?: { [key: string]: string } }): string[] { - return Object.keys(obj.dependencies ?? {}).filter(id => !local_dependencies.includes(id)) -} diff --git a/src/model/download.ts b/src/model/download.ts index 79f677e3316..3d296b3ab3b 100644 --- a/src/model/download.ts +++ b/src/model/download.ts @@ -7,7 +7,7 @@ import path from 'path' import tar from 'tar' import unzip from 'unzip-stream' import { URL } from 'url' -import { v1 as uuidv1 } from 'uuid' +import { v4 as uuidv4 } from 'uuid' import { CancellationToken } from 'vscode-languageserver-protocol' import { FetchOptions, getRequestModule, resolveRequestOptions, toURL } from './fetch' const logger = require('../util/logger')('model-download') @@ -17,6 +17,10 @@ export interface DownloadOptions extends Omit { * Folder that contains downloaded file or extracted files by untar or unzip */ dest: string + /** + * filename in dest, Could contains path separator, will be removed when exists. + */ + filename?: string /** * algorithm for check etag. */ @@ -50,7 +54,7 @@ export function getEtag(headers: IncomingHttpHeaders): string | undefined { export default function download(urlInput: string | URL, options: DownloadOptions, token?: CancellationToken): Promise { let url = toURL(urlInput) let { etagAlgorithm } = options - let { dest, onProgress, extract } = options + let { dest, onProgress, extract, filename } = options if (!dest || !path.isAbsolute(dest)) { throw new Error(`Expect absolute file path for dest option.`) } @@ -62,6 +66,14 @@ export default function download(urlInput: string | URL, options: DownloadOption throw new Error(`${dest} exists, but not directory!`) } } + if (filename) { + let fullpath = path.join(dest, filename) + if (fs.existsSync(fullpath)) { + fs.rmSync(fullpath, { force: true, recursive: true }) + } else { + fs.mkdirSync(path.dirname(fullpath), { recursive: true }) + } + } let mod = getRequestModule(url) let opts = resolveRequestOptions(url, options) if (!opts.agent && options.agent) opts.agent = options.agent @@ -109,7 +121,7 @@ export default function download(urlInput: string | URL, options: DownloadOption if (hash) hash.update(chunk) if (hasTotal) { let percent = (cur / total * 100).toFixed(1) - typeof onProgress === 'function' ? onProgress(percent) : logger.info(`Download ${url} progress ${percent}%`) + if (typeof onProgress === 'function') onProgress(percent) } }) res.on('end', () => { @@ -124,7 +136,7 @@ export default function download(urlInput: string | URL, options: DownloadOption } else if (extract === 'unzip') { stream = res.pipe(unzip.Extract({ path: dest })) } else { - dest = path.join(dest, `${uuidv1()}${extname}`) + dest = path.join(dest, filename ?? `${uuidv4()}${extname}`) stream = res.pipe(fs.createWriteStream(dest)) } stream.on('finish', () => { @@ -138,7 +150,7 @@ export default function download(urlInput: string | URL, options: DownloadOption logger.info(`Downloaded ${url} => ${dest}`) setTimeout(() => { resolve(dest) - }, 100) + }, global.__TEST__ ? 20 : 100) }) stream.on('error', reject) } else { From ea7fc3f5da009e5e6ef07da67d8e9478e0731472 Mon Sep 17 00:00:00 2001 From: Qiming Zhao Date: Thu, 20 Oct 2022 01:26:31 +0800 Subject: [PATCH 02/13] chore(extension): more tests --- .../modules/extensionDependency.test.ts | 58 ++++++-- src/__tests__/modules/extensions.test.ts | 3 + src/extension/dependency.ts | 127 ++++++++---------- src/extension/index.ts | 9 +- src/index.ts | 1 + 5 files changed, 110 insertions(+), 88 deletions(-) diff --git a/src/__tests__/modules/extensionDependency.test.ts b/src/__tests__/modules/extensionDependency.test.ts index 1354d93b912..c6520294d53 100644 --- a/src/__tests__/modules/extensionDependency.test.ts +++ b/src/__tests__/modules/extensionDependency.test.ts @@ -1,13 +1,13 @@ import fs from 'fs' -import tar from 'tar' import http, { Server } from 'http' import os from 'os' import path from 'path' +import tar from 'tar' import { URL } from 'url' import { v4 as uuid } from 'uuid' -import { checkFileSha1, DependenciesInstaller, DependencyItem, findItem, getModuleInfo, getRegistries, getVersion, readDependencies, shouldRetry, untar, VersionInfo } from '../../extension/dependency' +import { checkFileSha1, DependenciesInstaller, DependencyItem, findItem, getModuleInfo, getRegistries, getVersion, readDependencies, shouldRetry, untar, validVersionInfo, VersionInfo } from '../../extension/dependency' import { Dependencies } from '../../extension/installer' -import { writeJson, remove, loadJson } from '../../util/fs' +import { loadJson, remove, writeJson } from '../../util/fs' import helper, { getPort } from '../helper' process.env.NO_PROXY = '*' @@ -22,6 +22,19 @@ describe('utils', () => { expect(getRegistries(u).length).toBe(3) }) + it('should check valid versionInfo', async () => { + expect(validVersionInfo(null)).toBe(false) + expect(validVersionInfo({ name: 3 })).toBe(false) + expect(validVersionInfo({ name: 'name', version: '', dist: {} })).toBe(false) + expect(validVersionInfo({ + name: 'name', version: '1.0.0', dist: { + tarball: '', + integrity: '', + shasum: '' + } + })).toBe(true) + }) + it('should checkFileSha1', async () => { let not_exists = path.join(os.tmpdir(), 'not_exists') let checked = await checkFileSha1(not_exists, 'shasum') @@ -29,6 +42,14 @@ describe('utils', () => { let tarfile = path.resolve(__dirname, '../test.tar.gz') checked = await checkFileSha1(tarfile, 'bf0d88712fc3dbf6e3ab9a6968c0b4232779dbc4') expect(checked).toBe(true) + // throw on error + let bigfile = path.join(os.tmpdir(), 'bigfile') + let buf = Buffer.allocUnsafe(1024 * 1024) + fs.writeFileSync(bigfile, buf) + let p = checkFileSha1(bigfile, '') + fs.unlinkSync(bigfile) + let res = await p + expect(res).toBe(false) }) it('should untar files', async () => { @@ -259,6 +280,10 @@ describe('DependenciesInstaller', () => { await install.loadInfo(url, 'foo', 10) } await expect(fn()).rejects.toThrow(Error) + fn = async () => { + await install.loadInfo(url, 'bar') + } + await expect(fn()).rejects.toThrow(Error) }) it('should fetchInfos', async () => { @@ -266,14 +291,6 @@ describe('DependenciesInstaller', () => { let install = create() await install.fetchInfos({ a: '^0.0.1' }) expect(install.resolvedInfos.size).toBe(4) - expect(install.resolvedVersions).toEqual([ - { name: 'a', requirement: '^0.0.1', version: '0.0.1' }, - { name: 'b', requirement: '^1.0.0', version: '1.0.0' }, - { name: 'c', requirement: '^2.0.0', version: '2.0.0' }, - { name: 'b', requirement: '^2.0.0', version: '2.0.0' }, - { name: 'd', requirement: '^1.0.0', version: '1.0.0' }, - { name: 'd', requirement: '>= 0.0.1', version: '1.0.0' } - ]) }) it('should linkDependencies', async () => { @@ -305,6 +322,25 @@ describe('DependenciesInstaller', () => { fs.unlinkSync(res) }) + it('should throw when unable to resolve version', async () => { + let install = create() + expect(() => { + install.resolveVersion('foo', '^1.0.0') + }).toThrow() + install.resolvedInfos.set('foo', { + name: 'foo', + versions: { + '2.0.0': {} as any + } + }) + expect(() => { + install.resolveVersion('foo', '^1.0.0') + }).toThrow() + expect(() => { + install.resolveVersion('foo', '^2.0.0') + }).toThrow() + }) + it('should check exists and download items', async () => { let items: DependencyItem[] = [] items.push({ diff --git a/src/__tests__/modules/extensions.test.ts b/src/__tests__/modules/extensions.test.ts index 6a6ad75b4df..3263fa7694b 100644 --- a/src/__tests__/modules/extensions.test.ts +++ b/src/__tests__/modules/extensions.test.ts @@ -207,9 +207,12 @@ describe('extensions', () => { let link = path.join(extensions.modulesFolder, 'test-link') fs.mkdirSync(folder, { recursive: true }) fs.symlinkSync(folder, link) + let cacheFolder = path.join(extensions.modulesFolder, '.cache') + fs.mkdirSync(cacheFolder, { recursive: true }) extensions.cleanModulesFolder() expect(fs.existsSync(folder)).toBe(false) expect(fs.existsSync(link)).toBe(false) + expect(fs.existsSync(cacheFolder)).toBe(true) }) it('should install global extension', async () => { diff --git a/src/extension/dependency.ts b/src/extension/dependency.ts index c1aab5d93b7..1eb605cf113 100644 --- a/src/extension/dependency.ts +++ b/src/extension/dependency.ts @@ -9,6 +9,8 @@ import download from '../model/download' import fetch, { FetchOptions } from '../model/fetch' import { concurrent } from '../util' import { loadJson, writeJson } from '../util/fs' +import { objectLiteral } from '../util/is' +import { Mutex } from '../util/mutex' const logger = require('../util/logger')('extension-dependency') export interface Dependencies { [key: string]: string } @@ -33,12 +35,6 @@ interface ModuleInfo { } } -interface ResolvedVersion { - name: string - requirement: string - version: string -} - export interface DependencyItem { name: string version: string @@ -53,8 +49,10 @@ export interface DependencyItem { const NPM_REGISTRY = new URL('https://registry.npmjs.org') const YARN_REGISTRY = new URL('https://registry.yarnpkg.com') +const TAOBAO_REGISTRY = new URL('https://registry.npmmirror.com') const DEV_DEPENDENCIES = ['coc.nvim', 'webpack', 'esbuild'] const INFO_TIMEOUT = global.__TEST__ ? 100 : 10000 +const DOWNLOAD_TIMEOUT = global.__TEST__ ? 500 : 3 * 60 * 1000 function toFilename(item: DependencyItem): string { return `${item.name}.${item.version}.tgz` @@ -73,6 +71,14 @@ export function getRegistries(registry: URL): URL[] { return urls } +export function validVersionInfo(info: any): info is VersionInfo { + if (!info) return false + if (typeof info.name !== 'string' || typeof info.version !== 'string' || !info.dist) return false + let { tarball, integrity, shasum } = info.dist + if (typeof tarball !== 'string' || typeof integrity !== 'string' || typeof shasum !== 'string') return false + return true +} + export function getModuleInfo(text: string): ModuleInfo { let obj try { @@ -80,7 +86,7 @@ export function getModuleInfo(text: string): ModuleInfo { } catch (e) { throw new Error(`Invalid JSON data, ${e}`) } - if (typeof obj.name !== 'string' || obj.versions == null) throw new Error(`Invalid JSON data, name or versions not found`) + if (typeof obj.name !== 'string' || !objectLiteral(obj.versions)) throw new Error(`Invalid JSON data, name or versions not found`) return { name: obj.name, latest: obj['dist-tags']?.latest, @@ -109,7 +115,7 @@ export function readDependencies(directory: string): { [key: string]: string } { } export function getVersion(requirement: string, versions: string[], latest?: string): string | undefined { - if (latest && semver.satisfies(latest, requirement)) return latest + if (latest && validVersionInfo(versions[latest]) && semver.satisfies(latest, requirement)) return latest let sorted = semver.rsort(versions.filter(v => semver.valid(v, { includePrerelease: false }))) for (let v of sorted) { if (semver.satisfies(v, requirement)) return v @@ -137,7 +143,7 @@ export async function checkFileSha1(filepath: string, shasum: string): Promise { const input = createReadStream(filepath) - input.on('error', () => { + input.on('error', e => { resolve(false) }) input.on('readable', () => { @@ -152,8 +158,9 @@ export async function checkFileSha1(filepath: string, shasum: string): Promise = new Map() private tokenSource: CancellationTokenSource = new CancellationTokenSource() constructor( @@ -168,28 +175,29 @@ export class DependenciesInstaller { } public async installDependencies(directory: string): Promise { - // TODO reuse resolved.json let dependencies = readDependencies(directory) // no need to install if (!dependencies || Object.keys(dependencies).length == 0) { this.onMessage(`No dependencies`) return } - this.onMessage('Resolving dependencies.') - await this.fetchInfos(dependencies) - this.onMessage('Linking dependencies.') - // create DependencyItems - let items: DependencyItem[] = [] - this.linkDependencies(dependencies, items) - if (items.length > 0) { + this.onMessage('Waiting for install dependencies.') + // TODO reuse resolved.json + await mutex.use(async () => { + this.onMessage('Resolving dependencies.') + await this.fetchInfos(dependencies) + this.onMessage('Linking dependencies.') + // create DependencyItems + let items: DependencyItem[] = [] + this.linkDependencies(dependencies, items) let filepath = path.join(directory, 'resolved.json') writeJson(filepath, items) - } - this.onMessage('Downloading dependencies.') - await this.downloadItems(items) - this.onMessage('Extract modules.') - await this.extractDependencies(items, dependencies, directory) - this.onMessage('Done') + this.onMessage('Downloading dependencies.') + await this.downloadItems(items) + this.onMessage('Extract modules.') + await this.extractDependencies(items, dependencies, directory) + this.onMessage('Done') + }) } public async extractDependencies(items: DependencyItem[], dependencies: Dependencies, directory: string): Promise { @@ -211,19 +219,12 @@ export class DependenciesInstaller { addToRoot(item) }) rootPackages.clear() - - let err: unknown - await concurrent(rootItems, async item => { - try { - let filename = toFilename(item) - let tarfile = path.join(this.dest, filename) - let dest = path.join(directory, 'node_modules', item.name) - await untar(dest, tarfile) - } catch (e) { - err = e - } - }, 5) - if (err) throw err + for (let item of rootItems) { + let filename = toFilename(item) + let tarfile = path.join(this.dest, filename) + let dest = path.join(directory, 'node_modules', item.name) + await untar(dest, tarfile) + } for (let item of rootItems) { let folder = path.join(directory, 'node_modules', item.name) await this.extractFor(item, items, rootItems, folder) @@ -250,25 +251,20 @@ export class DependenciesInstaller { })) newRoot.push(...rootItems) for (let item of deps.values()) { - if (item.dependencies) { - let dest = path.join(folder, 'node_modules', item.name) - await this.extractFor(item, items, newRoot, dest) - } + let dest = path.join(folder, 'node_modules', item.name) + await this.extractFor(item, items, newRoot, dest) } } public linkDependencies(dependencies: Dependencies | undefined, items: DependencyItem[]): void { if (!dependencies) return for (let [name, requirement] of Object.entries(dependencies)) { - let version = this.resolveVersion(name, requirement) - let item = items.find(o => o.name === name && o.version === version) + let versionInfo = this.resolveVersion(name, requirement) + let item = items.find(o => o.name === name && o.version === versionInfo.version) if (item) { if (!item.satisfiedVersions.includes(requirement)) item.satisfiedVersions.push(requirement) } else { - let info = this.resolvedInfos.get(name) - let versionInfo = info ? info.versions[version] : undefined - if (!versionInfo) throw new Error(`Version data not found for "${name}@${version}"`) - let { dist } = versionInfo + let { dist, version } = versionInfo items.push({ name, version, @@ -283,15 +279,14 @@ export class DependenciesInstaller { } } - private resolveVersion(name: string, requirement: string): string { - let item = this.resolvedVersions.find(o => o.name == name && o.requirement == requirement) - if (item) return item.version + public resolveVersion(name: string, requirement: string): VersionInfo { let info = this.resolvedInfos.get(name) if (info) { let version = getVersion(requirement, Object.keys(info.versions), info.latest) if (version) { - this.resolvedVersions.push({ name, requirement, version }) - return version + let versionInfo = info.versions[version] + versionInfo.version = version + if (validVersionInfo(versionInfo)) return versionInfo } } throw new Error(`No valid version found for "${name}" ${requirement}`) @@ -300,25 +295,18 @@ export class DependenciesInstaller { /** * Recursive fetch */ - public async fetchInfos(dependencies: Dependencies): Promise { - let keys = Object.keys(dependencies) + public async fetchInfos(dependencies: Dependencies | undefined): Promise { + let keys = Object.keys(dependencies ?? {}) if (keys.length === 0) return - let fetched: Map = new Map() await Promise.all(keys.map(key => { - if (this.resolvedInfos.has(key)) { - fetched.set(key, this.resolvedInfos.get(key)) - return Promise.resolve() - } + if (this.resolvedInfos.has(key)) return Promise.resolve() return this.loadInfo(this.registry, key, INFO_TIMEOUT).then(info => { - fetched.set(key, info) this.resolvedInfos.set(key, info) }) })) - for (let [key, info] of fetched.entries()) { - let requirement = dependencies[key] - let version = this.resolveVersion(key, requirement) - let deps = info.versions[version].dependencies - if (deps) await this.fetchInfos(deps) + for (let key of keys) { + let versionInfo = this.resolveVersion(key, dependencies[key]) + await this.fetchInfos(versionInfo.dependencies) } } @@ -338,13 +326,13 @@ export class DependenciesInstaller { let onFinish = () => { res.set(filename, filepath) finished++ - this.onMessage(`Downloaded ${finished}/${total}`) + this.onMessage(`Downloaded ${filename} ${finished}/${total}`) } if (checked) { onFinish() } else { // 5min timeout - await this.download(new URL(item.resolved), filename, item.shasum, retry, global.__TEST__ ? 1000 : 5 * 60 * 1000) + await this.download(new URL(item.resolved), filename, item.shasum, retry, DOWNLOAD_TIMEOUT) onFinish() } } catch (e) { @@ -355,7 +343,7 @@ export class DependenciesInstaller { return res } - public async fetch(url: string | URL, options: FetchOptions = {}, retry = 1): Promise { + public async fetch(url: string | URL, options: FetchOptions, retry = 1): Promise { for (let i = 0; i < retry; i++) { try { return await fetch(url, options, this.tokenSource.token) @@ -381,8 +369,7 @@ export class DependenciesInstaller { this.onMessage(`Error on fetch ${url.hostname}/${name}: ${e}`) } } - if (!info) throw new Error(`Unable to fetch info for "${name}"`) - return info + throw new Error(`Unable to fetch info for "${name}"`) } public async download(url: string | URL, filename: string, shasum: string, retry = 1, timeout?: number): Promise { diff --git a/src/extension/index.ts b/src/extension/index.ts index 9514f8576ed..1af4d90795e 100644 --- a/src/extension/index.ts +++ b/src/extension/index.ts @@ -305,14 +305,9 @@ export class Extensions { if (!fs.existsSync(this.modulesFolder)) return let files = fs.readdirSync(this.modulesFolder) for (let file of files) { - if (folders.includes(file)) continue + if (folders.includes(file) || file === '.cache') continue let p = path.join(this.modulesFolder, file) - let stat = fs.lstatSync(p) - if (stat.isSymbolicLink()) { - fs.unlinkSync(p) - } else if (stat.isDirectory()) { - fs.rmSync(p, { recursive: true, force: true }) - } + fs.rmSync(p, { recursive: true, force: true }) } } diff --git a/src/index.ts b/src/index.ts index 0b7d5de8c5c..c5a7d86cd08 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,6 +23,7 @@ import BasicList from './list/basic' import { Mutex } from './util/mutex' import { URI } from 'vscode-uri' import { + CodeAction, CodeActionKind, CodeActionTriggerKind, Disposable, From 323016b6e425bd9713a0e9ea9e6571cdf50eb492 Mon Sep 17 00:00:00 2001 From: Qiming Zhao Date: Thu, 20 Oct 2022 14:11:06 +0800 Subject: [PATCH 03/13] fix invalid versionInfo --- src/extension/dependency.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/extension/dependency.ts b/src/extension/dependency.ts index 1eb605cf113..62132ed72bd 100644 --- a/src/extension/dependency.ts +++ b/src/extension/dependency.ts @@ -115,7 +115,7 @@ export function readDependencies(directory: string): { [key: string]: string } { } export function getVersion(requirement: string, versions: string[], latest?: string): string | undefined { - if (latest && validVersionInfo(versions[latest]) && semver.satisfies(latest, requirement)) return latest + if (latest && semver.satisfies(latest, requirement)) return latest let sorted = semver.rsort(versions.filter(v => semver.valid(v, { includePrerelease: false }))) for (let v of sorted) { if (semver.satisfies(v, requirement)) return v @@ -285,7 +285,6 @@ export class DependenciesInstaller { let version = getVersion(requirement, Object.keys(info.versions), info.latest) if (version) { let versionInfo = info.versions[version] - versionInfo.version = version if (validVersionInfo(versionInfo)) return versionInfo } } From 7d208e067777ba98a6416f4ab2d425ebb263dcba Mon Sep 17 00:00:00 2001 From: Qiming Zhao Date: Thu, 20 Oct 2022 20:29:46 +0800 Subject: [PATCH 04/13] find best registry --- data/schema.json | 6 - doc/coc-config.txt | 11 -- doc/coc.txt | 8 +- src/__tests__/client/configuration.test.ts | 10 +- .../modules/extensionDependency.test.ts | 93 +++++++++---- src/__tests__/modules/extensions.test.ts | 19 +-- src/__tests__/modules/installer.test.ts | 70 ++++++---- src/__tests__/modules/util.test.ts | 35 +++++ src/__tests__/tree/treeView.test.ts | 2 +- src/extension/dependency.ts | 125 ++++++++++-------- src/extension/index.ts | 28 ++-- src/extension/installer.ts | 46 ++++--- src/list/source/extensions.ts | 19 --- src/util/ping.ts | 101 ++++++++++++++ 14 files changed, 364 insertions(+), 209 deletions(-) create mode 100644 src/util/ping.ts diff --git a/data/schema.json b/data/schema.json index 01861d1be64..7120c6b0d0a 100644 --- a/data/schema.json +++ b/data/schema.json @@ -1421,12 +1421,6 @@ "scope": "application", "description": "Winblend option of notification window, neovim only." }, - "npm.binPath": { - "type": "string", - "scope": "application", - "default": "npm", - "description": "Command or absolute path to npm or yarn." - }, "outline.autoPreview": { "type": "boolean", "scope": "application", diff --git a/doc/coc-config.txt b/doc/coc-config.txt index 9e7ec1342da..b401820386c 100644 --- a/doc/coc-config.txt +++ b/doc/coc-config.txt @@ -17,7 +17,6 @@ Inlay hint |coc-config-inlayHint| Links |coc-config-links| List |coc-config-list| Notification |coc-config-notification| -Npm |coc-config-npm| Outline |coc-config-outline| Pull diagnostics |coc-config-pullDiagnostic| Refactor |coc-config-refactor| @@ -790,16 +789,6 @@ Notification~ Scope: `application`, default: `30` ------------------------------------------------------------------------------- -Npm~ - *coc-config-npm* -"npm.binPath" *coc-config-npm-binPath* - - Command or absolute path to npm or yarn for global extension - install/uninstall. - - Scope: `application`, default: `"npm"` - ------------------------------------------------------------------------------ Outline~ *coc-config-outline* diff --git a/doc/coc.txt b/doc/coc.txt index d7b08bc1393..13f73ea7762 100644 --- a/doc/coc.txt +++ b/doc/coc.txt @@ -177,11 +177,8 @@ Use |:CocInstall| to install coc extensions from vim's command line. To make coc.nvim install extensions on startup, use |g:coc_global_extensions|. -To use package manager other than npm (like `yarn` or `pnpm`), use -|coc-config-npm-binPath|. - -To customize npm registry for coc.nvim add `coc.nvim:registry` in your -`~/.npmrc`, like: +To customize node modules registry for coc.nvim add `coc.nvim:registry` in +your `~/.npmrc`, like: > coc.nvim:registry=https://registry.mycompany.org/ < @@ -3642,7 +3639,6 @@ extensions *coc-list-extensions* - 'enable' enable extension. - 'lock' lock/unlock extension to current version. - 'doc' view extension's README doc. - - 'fix' fix dependencies in terminal buffer. - 'reload' reload extension. - 'uninstall' uninstall extension. diff --git a/src/__tests__/client/configuration.test.ts b/src/__tests__/client/configuration.test.ts index 9582995f3d1..4a80c5c312a 100644 --- a/src/__tests__/client/configuration.test.ts +++ b/src/__tests__/client/configuration.test.ts @@ -119,7 +119,7 @@ describe('publish configuration feature', () => { it('should send configuration for specific sections', async () => { let client: LanguageClient let called = false - client = createClient(['coc.preferences', 'npm', 'unknown'], { + client = createClient(['coc.preferences', 'http', 'unknown'], { workspace: { didChangeConfiguration: (sections, next) => { called = true @@ -139,12 +139,12 @@ describe('publish configuration feature', () => { return changed != null }, true) expect(changed.settings.coc).toBeDefined() - expect(changed.settings.npm).toBeDefined() + expect(changed.settings.http).toBeDefined() let { configurations } = workspace - configurations.updateMemoryConfig({ 'npm.binPath': 'cnpm' }) + configurations.updateMemoryConfig({ 'http.proxyStrictSSL': false }) await helper.waitValue(() => { - return changed.settings.npm?.binPath - }, 'cnpm') + return changed.settings.http?.proxyStrictSSL + }, false) await client.stop() }) diff --git a/src/__tests__/modules/extensionDependency.test.ts b/src/__tests__/modules/extensionDependency.test.ts index c6520294d53..bca1c0347e8 100644 --- a/src/__tests__/modules/extensionDependency.test.ts +++ b/src/__tests__/modules/extensionDependency.test.ts @@ -5,23 +5,15 @@ import path from 'path' import tar from 'tar' import { URL } from 'url' import { v4 as uuid } from 'uuid' -import { checkFileSha1, DependenciesInstaller, DependencyItem, findItem, getModuleInfo, getRegistries, getVersion, readDependencies, shouldRetry, untar, validVersionInfo, VersionInfo } from '../../extension/dependency' +import { checkFileSha1, DependencySession, DependenciesInstaller, DependencyItem, findItem, getModuleInfo, getVersion, readDependencies, shouldRetry, untar, validVersionInfo, VersionInfo } from '../../extension/dependency' import { Dependencies } from '../../extension/installer' +import { CancellationError } from '../../util/errors' import { loadJson, remove, writeJson } from '../../util/fs' import helper, { getPort } from '../helper' process.env.NO_PROXY = '*' describe('utils', () => { - it('should getRegistries', () => { - let u = new URL('https://registry.npmjs.org') - expect(getRegistries(u).length).toBe(2) - u = new URL('https://registry.yarnpkg.com') - expect(getRegistries(u).length).toBe(2) - u = new URL('https://example.com') - expect(getRegistries(u).length).toBe(3) - }) - it('should check valid versionInfo', async () => { expect(validVersionInfo(null)).toBe(false) expect(validVersionInfo({ name: 3 })).toBe(false) @@ -194,7 +186,7 @@ describe('DependenciesInstaller', () => { }) } - function create(root?: string, onMessage?: (msg: string) => void): DependenciesInstaller { + function create(root: string | undefined, directory: string, onMessage?: (msg: string) => void): DependenciesInstaller { if (!root) { root = path.join(os.tmpdir(), uuid()) fs.mkdirSync(root) @@ -202,7 +194,8 @@ describe('DependenciesInstaller', () => { } let registry = new URL(`http://127.0.0.1:${httpPort}`) onMessage = onMessage ?? function() {} - return new DependenciesInstaller(registry, root, onMessage) + let session = new DependencySession(registry, root) + return session.createInstaller(directory, onMessage) } function createVersion(name: string, version: string, dependencies?: Dependencies): VersionInfo { @@ -252,8 +245,50 @@ describe('DependenciesInstaller', () => { })) } + it('should throw on cancel', async () => { + let root = path.join(os.tmpdir(), uuid()) + fs.mkdirSync(root) + dirs.push(root) + let registry = new URL(`http://127.0.0.1:${httpPort}`) + let session = new DependencySession(registry, root) + let directory = path.join(os.tmpdir(), uuid()) + dirs.push(directory) + writeJson(path.join(directory, 'package.json'), { dependencies: { foo: '>= 0.0.1' } }) + let other = path.join(os.tmpdir(), uuid()) + dirs.push(other) + writeJson(path.join(other, 'package.json'), { dependencies: { bar: '>= 0.0.1' } }) + let one = session.createInstaller(directory, () => {}) + let two = session.createInstaller(other, () => {}) + let spy = jest.spyOn(one, 'fetchInfos').mockImplementation(() => { + return new Promise((resolve, reject) => { + one.token.onCancellationRequested(() => { + clearTimeout(timer) + reject(new CancellationError()) + }) + let timer = setTimeout(() => { + resolve() + }, 500) + }) + }) + let p = one.installDependencies() + let err + two.installDependencies().catch(error => { + err = error + }) + await helper.wait(30) + session.cancel() + let fn = async () => { + await p + } + await expect(fn()).rejects.toThrow(Error) + spy.mockRestore() + await helper.waitValue(() => { + return err != null + }, true) + }) + it('should retry fetch', async () => { - let install = create() + let install = create(undefined, '') let fn = async () => { await install.fetch(new URL('/', url), { timeout: 10 }, 3) } @@ -264,7 +299,7 @@ describe('DependenciesInstaller', () => { }) it('should cancel request', async () => { - let install = create() + let install = create(undefined, '') let p = install.fetch(new URL('/slow', url), {}, 1) await helper.wait(10) let fn = async () => { @@ -275,7 +310,7 @@ describe('DependenciesInstaller', () => { }) it('should throw when unable to load info', async () => { - let install = create() + let install = create(undefined, '') let fn = async () => { await install.loadInfo(url, 'foo', 10) } @@ -288,14 +323,14 @@ describe('DependenciesInstaller', () => { it('should fetchInfos', async () => { addJsonData() - let install = create() + let install = create(undefined, '') await install.fetchInfos({ a: '^0.0.1' }) expect(install.resolvedInfos.size).toBe(4) }) it('should linkDependencies', async () => { addJsonData() - let install = create() + let install = create(undefined, '') await install.fetchInfos({ a: '^0.0.1' }) let items: DependencyItem[] = [] install.linkDependencies(undefined, items) @@ -305,7 +340,7 @@ describe('DependenciesInstaller', () => { }) it('should retry download', async () => { - let install = create() + let install = create(undefined, '') let fn = async () => { await install.download(new URL('res', url), 'res', '', 3, 10) } @@ -323,7 +358,7 @@ describe('DependenciesInstaller', () => { }) it('should throw when unable to resolve version', async () => { - let install = create() + let install = create(undefined, '') expect(() => { install.resolveVersion('foo', '^1.0.0') }).toThrow() @@ -359,12 +394,12 @@ describe('DependenciesInstaller', () => { shasum: 'bf0d88712fc3dbf6e3ab9a6968c0b4232779dbc4', version: '0.0.2' }) - let install = create() + let install = create(undefined, '') let dest = path.join(install.modulesRoot, '.cache') fs.mkdirSync(dest, { recursive: true }) let tarfile = path.resolve(__dirname, '../test.tar.gz') fs.copyFileSync(tarfile, path.join(dest, `foo.0.0.1.tgz`)) - let res = await install.downloadItems(items) + let res = await install.downloadItems(items, 1) expect(res.size).toBe(2) }) @@ -378,22 +413,22 @@ describe('DependenciesInstaller', () => { shasum: 'badsum', version: '0.0.2' }) - let install = create() + let install = create(undefined, '') let fn = async () => { - await install.downloadItems(items) + await install.downloadItems(items, 2) } await expect(fn()).rejects.toThrow(Error) }) it('should no nothing if no dependencies', async () => { let msg: string - let install = create(undefined, s => { - msg = s - }) let directory = path.join(os.tmpdir(), uuid()) let file = path.join(directory, 'package.json') writeJson(file, { dependencies: {} }) - await install.installDependencies(directory) + let install = create(undefined, directory, s => { + msg = s + }) + await install.installDependencies() expect(msg).toMatch('No dependencies') fs.rmSync(directory, { recursive: true }) }) @@ -401,12 +436,12 @@ describe('DependenciesInstaller', () => { it('should install dependencies ', async () => { createFiles = true addJsonData() - let install = create() let directory = path.join(os.tmpdir(), uuid()) fs.mkdirSync(directory, { recursive: true }) let file = path.join(directory, 'package.json') + let install = create(undefined, directory) writeJson(file, { dependencies: { a: '^0.0.1' } }) - await install.installDependencies(directory) + await install.installDependencies() let folder = path.join(directory, 'node_modules') let res = fs.readdirSync(folder) expect(res).toEqual(['a', 'b', 'c', 'd']) diff --git a/src/__tests__/modules/extensions.test.ts b/src/__tests__/modules/extensions.test.ts index 3263fa7694b..442d63b6c07 100644 --- a/src/__tests__/modules/extensions.test.ts +++ b/src/__tests__/modules/extensions.test.ts @@ -2,7 +2,6 @@ import fs from 'fs' import os from 'os' import path from 'path' import { v4 as uuid } from 'uuid' -import which from 'which' import commandManager from '../../commands' import extensions, { Extensions, toUrl } from '../../extension' import { writeFile, writeJson } from '../../util/fs' @@ -37,7 +36,8 @@ describe('extensions', () => { expect(extensions.onDidActiveExtension).toBeDefined() expect(extensions.onDidUnloadExtension).toBeDefined() expect(extensions.schemes).toBeDefined() - expect(extensions.creteInstaller('npm', 'id')).toBeDefined() + let res = await extensions.creteInstaller('id') + expect(res).toBeDefined() }) it('should get extensions stat', async () => { @@ -94,21 +94,6 @@ describe('extensions', () => { s.mockRestore() }) - it('should use absolute path for npm', async () => { - let res = extensions.npm - expect(path.isAbsolute(res)).toBe(true) - }) - - it('should not throw when npm not found', async () => { - let spy = jest.spyOn(which, 'sync').mockImplementation(() => { - throw new Error('not executable') - }) - let res = extensions.npm - expect(res).toBeNull() - await extensions.updateExtensions() - spy.mockRestore() - }) - it('should get all extensions', () => { let list = extensions.all expect(Array.isArray(list)).toBe(true) diff --git a/src/__tests__/modules/installer.test.ts b/src/__tests__/modules/installer.test.ts index 0dc6363019e..32e0ed4b60a 100644 --- a/src/__tests__/modules/installer.test.ts +++ b/src/__tests__/modules/installer.test.ts @@ -3,7 +3,9 @@ import os from 'os' import path from 'path' import { v4 as uuid } from 'uuid' import { Info, Installer, registryUrl } from '../../extension/installer' +import { DependencySession } from '../../extension/dependency' import { remove } from '../../util/fs' +import { URL } from 'url' const rcfile = path.join(os.tmpdir(), '.npmrc') let tmpfolder: string @@ -13,28 +15,41 @@ afterEach(() => { } }) +function createSession(root: string, registry?: string): DependencySession { + return new DependencySession(new URL(registry ?? 'https://registry.npmjs.org/'), root) +} + describe('utils', () => { it('should get registry url', async () => { + let url = await registryUrl(os.tmpdir(), [], 50) + expect(url).toBeDefined() const getUrl = () => { return registryUrl(os.tmpdir()) } fs.rmSync(rcfile, { force: true, recursive: true }) - expect(getUrl().toString()).toBe('https://registry.npmjs.org/') + async function check(res: string): Promise { + let url = await registryUrl(os.tmpdir(), [new URL('https://registry.npmjs.org')], 10) + expect(url.toString()).toBe(res) + } + await check('https://registry.npmjs.org/') fs.writeFileSync(rcfile, '', 'utf8') - expect(getUrl().toString()).toBe('https://registry.npmjs.org/') + await check('https://registry.npmjs.org/') fs.writeFileSync(rcfile, 'coc.nvim:registry=https://example.org', 'utf8') - expect(getUrl().toString()).toBe('https://example.org/') + await check('https://example.org/') fs.writeFileSync(rcfile, '#coc.nvim:registry=https://example.org', 'utf8') - expect(getUrl().toString()).toBe('https://registry.npmjs.org/') + await check('https://registry.npmjs.org/') fs.writeFileSync(rcfile, 'coc.nvim:registry=example.org', 'utf8') - expect(getUrl().toString()).toBe('https://registry.npmjs.org/') + url = await getUrl() + expect(url).toBeDefined() + url = await registryUrl(os.tmpdir(), [new URL('https://127.0.0.1')], 50) + expect(url).toBeDefined() fs.rmSync(rcfile, { force: true, recursive: true }) }) it('should parse name & version', async () => { const getInfo = (def: string): { name?: string, version?: string } => { - let installer = new Installer(__dirname, 'npm', def) + let installer = new Installer(createSession(__dirname), def) return installer.info } expect(getInfo('https://github.com')).toEqual({ name: undefined, version: undefined }) @@ -47,7 +62,7 @@ describe('utils', () => { describe('Installer', () => { describe('fetch() & download()', () => { it('should throw with invalid url', async () => { - let installer = new Installer(__dirname, 'npm', 'foo') + let installer = new Installer(createSession(__dirname), 'foo') let fn = async () => { await installer.fetch('url') } @@ -61,7 +76,7 @@ describe('Installer', () => { describe('getInfo()', () => { it('should getInfo from url', async () => { - let installer = new Installer(__dirname, 'npm', 'https://github.com/') + let installer = new Installer(createSession(__dirname), 'https://github.com/') let spy = jest.spyOn(installer, 'getInfoFromUri').mockImplementation(() => { return Promise.resolve({ name: 'vue-vscode-snippets', version: '1.0.0' }) }) @@ -71,7 +86,7 @@ describe('Installer', () => { }) it('should use latest version', async () => { - let installer = new Installer(__dirname, 'npm', 'coc-omni') + let installer = new Installer(createSession(__dirname), 'coc-omni') let spy = jest.spyOn(installer, 'fetch').mockImplementation(url => { expect(url.toString()).toMatch('coc-omni') return Promise.resolve(JSON.stringify({ @@ -92,7 +107,7 @@ describe('Installer', () => { }) it('should throw when version not found', async () => { - let installer = new Installer(__dirname, 'npm', 'coc-omni@1.0.2') + let installer = new Installer(createSession(__dirname), 'coc-omni@1.0.2') let spy = jest.spyOn(installer, 'fetch').mockImplementation(() => { return Promise.resolve(JSON.stringify({ name: 'coc-omni', @@ -114,7 +129,7 @@ describe('Installer', () => { }) it('should throw when not coc.nvim extension', async () => { - let installer = new Installer(__dirname, 'npm', 'coc-omni') + let installer = new Installer(createSession(__dirname), 'coc-omni') let spy = jest.spyOn(installer, 'fetch').mockImplementation(() => { return Promise.resolve(JSON.stringify({ name: 'coc-omni', @@ -137,7 +152,7 @@ describe('Installer', () => { describe('getInfoFromUri()', () => { it('should throw for url that not supported', async () => { - let installer = new Installer(__dirname, 'npm', 'https://example.com') + let installer = new Installer(createSession(__dirname), 'https://example.com') let fn = async () => { await installer.getInfoFromUri() } @@ -145,7 +160,7 @@ describe('Installer', () => { }) it('should get info from url #1', async () => { - let installer = new Installer(__dirname, 'npm', 'https://github.com/sdras/vue-vscode-snippets') + let installer = new Installer(createSession(__dirname), 'https://github.com/sdras/vue-vscode-snippets') let spy = jest.spyOn(installer, 'fetch').mockImplementation(() => { return Promise.resolve(JSON.stringify({ name: 'vue-vscode-snippets', version: '1.0.0' })) }) @@ -155,7 +170,7 @@ describe('Installer', () => { }) it('should get info from url #2', async () => { - let installer = new Installer(__dirname, 'npm', 'https://github.com/sdras/vue-vscode-snippets@main') + let installer = new Installer(createSession(__dirname), 'https://github.com/sdras/vue-vscode-snippets@main') let spy = jest.spyOn(installer, 'fetch').mockImplementation(() => { return Promise.resolve({ name: 'vue-vscode-snippets', version: '1.0.0', engines: { coc: '>=0.0.1' } }) }) @@ -173,7 +188,7 @@ describe('Installer', () => { fs.unlinkSync(tmpfolder) } fs.symlinkSync(__dirname, tmpfolder, 'dir') - let installer = new Installer(os.tmpdir(), 'npm', 'foo') + let installer = new Installer(createSession(os.tmpdir()), 'foo') let res = await installer.doInstall({ name: 'foo' }) expect(res).toBe(false) let val = await installer.update() @@ -182,7 +197,7 @@ describe('Installer', () => { it('should update from url', async () => { let url = 'https://github.com/sdras/vue-vscode-snippets@main' - let installer = new Installer(__dirname, 'npm', url) + let installer = new Installer(createSession(__dirname), url) let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ version: '1.0.0', name: 'vue-vscode-snippets' }) }) @@ -197,7 +212,7 @@ describe('Installer', () => { it('should skip update when current version is latest', async () => { tmpfolder = path.join(os.tmpdir(), 'coc-pairs') - let installer = new Installer(os.tmpdir(), 'npm', 'coc-pairs') + let installer = new Installer(createSession(os.tmpdir()), 'coc-pairs') let version = '1.0.0' let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ version }) @@ -212,7 +227,7 @@ describe('Installer', () => { it('should skip update when version not satisfies', async () => { tmpfolder = path.join(os.tmpdir(), 'coc-pairs') - let installer = new Installer(os.tmpdir(), 'npm', 'coc-pairs') + let installer = new Installer(createSession(os.tmpdir()), 'coc-pairs') let version = '2.0.0' let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ version, 'engines.coc': '>=99.0.0' }) @@ -228,7 +243,7 @@ describe('Installer', () => { it('should return undefined when update not performed', async () => { tmpfolder = path.join(os.tmpdir(), 'coc-pairs') - let installer = new Installer(os.tmpdir(), 'npm', 'coc-pairs') + let installer = new Installer(createSession(os.tmpdir()), 'coc-pairs') let version = '2.0.0' let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ version }) @@ -246,7 +261,7 @@ describe('Installer', () => { it('should update extension', async () => { tmpfolder = path.join(os.tmpdir(), 'coc-pairs') - let installer = new Installer(os.tmpdir(), 'npm', 'coc-pairs') + let installer = new Installer(createSession(os.tmpdir()), 'coc-pairs') let version = '2.0.0' let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ version, name: 'coc-pairs' }) @@ -266,7 +281,7 @@ describe('Installer', () => { describe('install()', () => { it('should throw when version not match required', async () => { - let installer = new Installer(__dirname, 'npm', 'coc-omni') + let installer = new Installer(createSession(__dirname), 'coc-pairs') let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ name: 'coc-omni', @@ -283,7 +298,7 @@ describe('Installer', () => { }) it('should return install info', async () => { - let installer = new Installer(__dirname, 'npm', 'coc-omni') + let installer = new Installer(createSession(__dirname), 'coc-omni') let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ name: 'coc-omni', @@ -303,7 +318,7 @@ describe('Installer', () => { it('should throw and remove folder when download failed', async () => { tmpfolder = path.join(os.tmpdir(), uuid()) - let installer = new Installer(tmpfolder, 'npm', 'coc-omni') + let installer = new Installer(createSession(tmpfolder), 'coc-omni') let folder: string let option: any let spy = jest.spyOn(installer, 'download').mockImplementation((_url, opt) => { @@ -325,7 +340,7 @@ describe('Installer', () => { it('should revert folder when download failed', async () => { tmpfolder = path.join(os.tmpdir(), uuid()) - let installer = new Installer(tmpfolder, 'npm', 'coc-omni') + let installer = new Installer(createSession(tmpfolder), 'coc-omni') let f = path.join(tmpfolder, 'coc-omni') fs.mkdirSync(f, { recursive: true }) fs.writeFileSync(path.join(f, 'package.json'), '{}', 'utf8') @@ -344,7 +359,7 @@ describe('Installer', () => { it('should install new extension', async () => { tmpfolder = path.join(os.tmpdir(), uuid()) - let installer = new Installer(tmpfolder, 'npm', 'coc-omni') + let installer = new Installer(createSession(tmpfolder), 'coc-omni') let f = path.join(tmpfolder, 'coc-omni') let spy = jest.spyOn(installer, 'download').mockImplementation((_url, option) => { if (option.onProgress) { @@ -365,7 +380,7 @@ describe('Installer', () => { it('should install new version', async () => { tmpfolder = path.join(os.tmpdir(), uuid()) - let installer = new Installer(tmpfolder, 'npm', 'coc-omni') + let installer = new Installer(createSession(tmpfolder), 'coc-omni') let f = path.join(tmpfolder, 'coc-omni') fs.mkdirSync(f, { recursive: true }) fs.writeFileSync(path.join(f, 'package.json'), '{}', 'utf8') @@ -387,10 +402,9 @@ describe('Installer', () => { }) it('should install dependencies', async () => { - let npm = path.resolve(__dirname, '../npm') tmpfolder = path.join(os.tmpdir(), uuid()) fs.mkdirSync(tmpfolder) - let installer = new Installer(tmpfolder, npm, 'coc-omni') + let installer = new Installer(createSession(tmpfolder), 'coc-omni') let called = false installer.on('message', () => { called = true diff --git a/src/__tests__/modules/util.test.ts b/src/__tests__/modules/util.test.ts index 56d90d8dc82..3860ec9d080 100644 --- a/src/__tests__/modules/util.test.ts +++ b/src/__tests__/modules/util.test.ts @@ -1,10 +1,12 @@ import style from 'ansi-styles' import * as assert from 'assert' import { spawn } from 'child_process' +import fs from 'fs' import os from 'os' import path from 'path' import vm from 'vm' import { CancellationTokenSource, Color, Position, Range, SymbolKind, TextDocumentEdit, TextEdit, WorkspaceEdit } from 'vscode-languageserver-protocol' +import which from 'which' import { LinesTextDocument } from '../../model/textdocument' import { ConfigurationScope } from '../../types' import { concurrent, delay, disposeAll, wait } from '../../util' @@ -24,6 +26,7 @@ import { Extensions, IJSONContributionRegistry } from '../../util/jsonRegistry' import * as lodash from '../../util/lodash' import { Mutex } from '../../util/mutex' import * as objects from '../../util/object' +import * as ping from '../../util/ping' import * as platform from '../../util/platform' import * as positions from '../../util/position' import { executable, isRunning, runCommand, terminate } from '../../util/processes' @@ -1347,4 +1350,36 @@ describe('diff', () => { }, token) }) }) + + describe('ping', () => { + it('should get ping config', async () => { + let check = (platform: NodeJS.Platform) => { + let res = ping.getPing(platform) + if (res) { + expect(fs.existsSync(res.bin)).toBe(true) + } + } + check('darwin') + check('linux') + check('win32') + check('android') + check('freebsd') + let spy = jest.spyOn(which, 'sync').mockImplementation(() => { + throw Error('not found') + }) + check('freebsd') + spy.mockRestore() + }) + + it('should find best host', async () => { + let res = await ping.findBestHost([], 500) + expect(res).toBeUndefined() + res = await ping.findBestHost(['www.baidu.com', 'www.google.com'], 1) + expect(res).toBeUndefined() + res = await ping.findBestHost(['www.baidu.com', 'www.google.com'], 500, 'not_exists_bin') + expect(res).toBeUndefined() + res = await ping.findBestHost(['127.0.0.1', 'registry.npmjs.org', 'registry.yarnpkg.com', 'registry.npmmirror.com'], 500) + expect(res).toBeDefined() + }) + }) }) diff --git a/src/__tests__/tree/treeView.test.ts b/src/__tests__/tree/treeView.test.ts index b1965ca41cb..842eb5e3941 100644 --- a/src/__tests__/tree/treeView.test.ts +++ b/src/__tests__/tree/treeView.test.ts @@ -295,7 +295,7 @@ describe('TreeView', () => { createTreeView(defaultDef, { canSelectMany: true }) await treeView.show() await events.race(['TextChanged']) - let selection: TreeNode[] + let selection: TreeNode[] = [] treeView.onDidChangeSelection(e => { selection = e.selection }) diff --git a/src/extension/dependency.ts b/src/extension/dependency.ts index 62132ed72bd..c0dab10498f 100644 --- a/src/extension/dependency.ts +++ b/src/extension/dependency.ts @@ -4,10 +4,10 @@ import path from 'path' import semver from 'semver' import tar from 'tar' import { URL } from 'url' -import { CancellationTokenSource } from 'vscode-languageserver-protocol' +import { CancellationToken, CancellationTokenSource } from 'vscode-languageserver-protocol' import download from '../model/download' import fetch, { FetchOptions } from '../model/fetch' -import { concurrent } from '../util' +import { CancellationError } from '../util/errors' import { loadJson, writeJson } from '../util/fs' import { objectLiteral } from '../util/is' import { Mutex } from '../util/mutex' @@ -47,9 +47,6 @@ export interface DependencyItem { } } -const NPM_REGISTRY = new URL('https://registry.npmjs.org') -const YARN_REGISTRY = new URL('https://registry.yarnpkg.com') -const TAOBAO_REGISTRY = new URL('https://registry.npmmirror.com') const DEV_DEPENDENCIES = ['coc.nvim', 'webpack', 'esbuild'] const INFO_TIMEOUT = global.__TEST__ ? 100 : 10000 const DOWNLOAD_TIMEOUT = global.__TEST__ ? 500 : 3 * 60 * 1000 @@ -64,13 +61,6 @@ export function findItem(name: string, requirement: string, items: ReadonlyArray return item } -export function getRegistries(registry: URL): URL[] { - let urls: URL[] = [registry] - if (registry.host !== NPM_REGISTRY.host) urls.push(NPM_REGISTRY) - if (registry.host !== YARN_REGISTRY.host) urls.push(YARN_REGISTRY) - return urls -} - export function validVersionInfo(info: any): info is VersionInfo { if (!info) return false if (typeof info.name !== 'string' || typeof info.version !== 'string' || !info.dist) return false @@ -122,6 +112,9 @@ export function getVersion(requirement: string, versions: string[], latest?: str } } +/** + * Extract tarfile to dest folder, with strip option. + */ export async function untar(dest: string, tarfile: string, strip = 1): Promise { if (!fs.existsSync(tarfile)) throw new Error(`${tarfile} not exists`) fs.rmSync(dest, { recursive: true, force: true }) @@ -161,20 +154,26 @@ export async function checkFileSha1(filepath: string, shasum: string): Promise = new Map() private tokenSource: CancellationTokenSource = new CancellationTokenSource() constructor( private registry: URL, + public readonly resolvedInfos: Map, public readonly modulesRoot: string, + private directory: string, private onMessage: (msg: string) => void ) { } + public get token(): CancellationToken { + return this.tokenSource.token + } + private get dest(): string { return path.join(this.modulesRoot, '.cache') } - public async installDependencies(directory: string): Promise { + public async installDependencies(): Promise { + let { directory, tokenSource } = this let dependencies = readDependencies(directory) // no need to install if (!dependencies || Object.keys(dependencies).length == 0) { @@ -182,8 +181,10 @@ export class DependenciesInstaller { return } this.onMessage('Waiting for install dependencies.') + let token = tokenSource.token // TODO reuse resolved.json await mutex.use(async () => { + if (token.isCancellationRequested) throw new CancellationError() this.onMessage('Resolving dependencies.') await this.fetchInfos(dependencies) this.onMessage('Linking dependencies.') @@ -192,8 +193,8 @@ export class DependenciesInstaller { this.linkDependencies(dependencies, items) let filepath = path.join(directory, 'resolved.json') writeJson(filepath, items) - this.onMessage('Downloading dependencies.') - await this.downloadItems(items) + this.onMessage(`Downloading dependencies to ${this.dest}.`) + await this.downloadItems(items, 3) this.onMessage('Extract modules.') await this.extractDependencies(items, dependencies, directory) this.onMessage('Done') @@ -292,7 +293,7 @@ export class DependenciesInstaller { } /** - * Recursive fetch + * Recursive fetch module info */ public async fetchInfos(dependencies: Dependencies | undefined): Promise { let keys = Object.keys(dependencies ?? {}) @@ -312,36 +313,42 @@ export class DependenciesInstaller { /** * Concurrent download necessary dependencies */ - public async downloadItems(items: DependencyItem[], retry = 3): Promise> { + public async downloadItems(items: DependencyItem[], retry: number): Promise> { let res: Map = new Map() let total = items.length let finished = 0 - let err: unknown - await concurrent(items, async item => { - try { - let filename = toFilename(item) - let filepath = path.join(this.dest, filename) - let checked = await checkFileSha1(filepath, item.shasum) - let onFinish = () => { - res.set(filename, filepath) - finished++ - this.onMessage(`Downloaded ${filename} ${finished}/${total}`) - } + for (let item of items) { + let ts = Date.now() + let filename = toFilename(item) + let filepath = path.join(this.dest, filename) + let checked = await checkFileSha1(filepath, item.shasum) + let onFinish = () => { + res.set(filename, filepath) + finished++ if (checked) { - onFinish() + this.onMessage(`Downloaded ${filename} cost ${Date.now() - ts}ms [${finished}/${total}]`) } else { - // 5min timeout - await this.download(new URL(item.resolved), filename, item.shasum, retry, DOWNLOAD_TIMEOUT) - onFinish() + this.onMessage(`File ${filename} exists [${finished}/${total}]`) } - } catch (e) { - err = e } - }, 3) - if (finished !== total) throw new Error(err ? err.toString() : 'unknown error') + if (checked) { + onFinish() + } else { + await this.download(new URL(item.resolved), filename, item.shasum, retry, DOWNLOAD_TIMEOUT) + onFinish() + } + } return res } + /** + * Fetch module info + */ + public async loadInfo(registry: URL, name: string, timeout = 100): Promise { + let buffer = await this.fetch(new URL(name, registry), { timeout, buffer: true }) as Buffer + return getModuleInfo(buffer.toString()) + } + public async fetch(url: string | URL, options: FetchOptions, retry = 1): Promise { for (let i = 0; i < retry; i++) { try { @@ -356,21 +363,9 @@ export class DependenciesInstaller { } } - // Try different registries - public async loadInfo(registry: URL, name: string, timeout = 100): Promise { - let info: ModuleInfo - for (let url of getRegistries(registry)) { - try { - let buffer = await this.fetch(new URL(name, url), { timeout, buffer: true }) as Buffer - info = getModuleInfo(buffer.toString()) - return info - } catch (e) { - this.onMessage(`Error on fetch ${url.hostname}/${name}: ${e}`) - } - } - throw new Error(`Unable to fetch info for "${name}"`) - } - + /** + * Download tgz file with sha1 check. + */ public async download(url: string | URL, filename: string, shasum: string, retry = 1, timeout?: number): Promise { for (let i = 0; i < retry; i++) { try { @@ -401,3 +396,29 @@ export class DependenciesInstaller { this.tokenSource = new CancellationTokenSource() } } + +export class DependencySession { + private resolvedInfos: Map = new Map() + private installers: Set = new Set() + constructor( + public readonly registry: URL, + public readonly modulesRoot: string + ) { + } + + public createInstaller(directory: string, onMessage: (msg: string) => void): DependenciesInstaller { + let installer = new DependenciesInstaller(this.registry, this.resolvedInfos, this.modulesRoot, directory, onMessage) + this.installers.add(installer) + return installer + } + + /** + * Cancel all installer + */ + public cancel(): void { + for (let item of this.installers) { + item.cancel() + } + this.installers.clear() + } +} diff --git a/src/extension/index.ts b/src/extension/index.ts index 1af4d90795e..0332751501b 100644 --- a/src/extension/index.ts +++ b/src/extension/index.ts @@ -2,7 +2,6 @@ import fs from 'fs' import path from 'path' import { Event } from 'vscode-languageserver-protocol' -import which from 'which' import commandManager from '../commands' import type { OutputChannel } from '../types' import { concurrent } from '../util' @@ -12,7 +11,8 @@ import { isUrl } from '../util/is' import { executable } from '../util/processes' import window from '../window' import workspace from '../workspace' -import { IInstaller, Installer } from './installer' +import { DependencySession } from './dependency' +import { IInstaller, Installer, registryUrl } from './installer' import { API, Extension, ExtensionInfo, ExtensionItem, ExtensionManager, ExtensionState } from './manager' import { checkExtensionRoot, ExtensionStat, loadExtensionJson } from './stat' import { InstallBuffer, InstallChannel, InstallUI } from './ui' @@ -139,37 +139,27 @@ export class Extensions { return await this.manager.call(id, method, args) } - public get npm(): string { - let npm = workspace.getConfiguration('npm', null).get('binPath', 'npm') - npm = workspace.expand(npm) - for (let exe of [npm, 'yarnpkg', 'yarn', 'npm']) { - if (executable(exe)) return which.sync(exe) - } - void window.showErrorMessage(`Can't find npm or yarn in your $PATH`) - return null - } - private createInstallerUI(isUpdate: boolean, silent: boolean): InstallUI { return silent ? new InstallChannel(isUpdate, this.outputChannel) : new InstallBuffer(isUpdate) } - public creteInstaller(npm: string, def: string): IInstaller { - return new Installer(this.modulesFolder, npm, def) + public async creteInstaller(def: string): Promise { + let url = await registryUrl() + return new Installer(new DependencySession(url, this.modulesFolder), def) } /** * Install extensions, can be called without initialize. */ public async installExtensions(list: string[]): Promise { - let { npm } = this - if (!npm || list.length == 0) return + if (list.length == 0) return list = distinct(list) let installBuffer = this.createInstallerUI(false, false) await Promise.resolve(installBuffer.start(list)) let fn = async (key: string): Promise => { try { installBuffer.startProgress(key) - let installer = this.creteInstaller(npm, key) + let installer = await this.creteInstaller(key) installer.on('message', (msg, isProgress) => { installBuffer.addMessage(key, msg, isProgress) }) @@ -193,8 +183,6 @@ export class Extensions { * Update global extensions */ public async updateExtensions(silent = false): Promise { - let { npm } = this - if (!npm) return let stats = this.globalExtensionStats() stats = stats.filter(s => { if (s.isLocked || s.state === 'disabled') { @@ -212,7 +200,7 @@ export class Extensions { try { installBuffer.startProgress(id) let url = stat.exotic ? stat.uri : null - let installer = this.creteInstaller(npm, id) + let installer = await this.creteInstaller(id) installer.on('message', (msg, isProgress) => { installBuffer.addMessage(id, msg, isProgress) }) diff --git a/src/extension/installer.ts b/src/extension/installer.ts index 57949be17c1..3ad2c10673a 100644 --- a/src/extension/installer.ts +++ b/src/extension/installer.ts @@ -8,9 +8,11 @@ import { URL } from 'url' import { v4 as uuid } from 'uuid' import download, { DownloadOptions } from '../model/download' import fetch, { FetchOptions } from '../model/fetch' +import { isFalsyOrEmpty } from '../util/array' import { loadJson } from '../util/fs' +import { findBestHost } from '../util/ping' import workspace from '../workspace' -import { DependenciesInstaller } from './dependency' +import { DependencySession } from './dependency' const logger = require('../util/logger')('extension-installer') export interface Info { @@ -30,11 +32,19 @@ export interface InstallResult { url?: string } -export function registryUrl(home = os.homedir()): URL { +const TAOBAO_REGISTRY = new URL('https://registry.npmmirror.com') +const NPM_REGISTRY = new URL('https://registry.npmjs.org') +const YARN_REGISTRY = new URL('https://registry.yarnpkg.com') +const PINGTIMEOUT = global.__TEST__ ? 50 : 500 + +/** + * Find the user configured registry or the best one + */ +export async function registryUrl(home = os.homedir(), registries?: URL[], timeout = PINGTIMEOUT): Promise { let res: URL let filepath = path.join(home, '.npmrc') - if (fs.existsSync(filepath)) { - try { + try { + if (fs.existsSync(filepath)) { let content = fs.readFileSync(filepath, 'utf8') let uri: string for (let line of content.split(/\r?\n/)) { @@ -45,11 +55,16 @@ export function registryUrl(home = os.homedir()): URL { } } if (uri) res = new URL(uri) - } catch (e) { - logger.debug('Error on parse .npmrc:', e) } + if (res) return res + registries = isFalsyOrEmpty(registries) ? [TAOBAO_REGISTRY, NPM_REGISTRY, YARN_REGISTRY] : registries + const hosts = registries.map(o => o.hostname) + let host = await findBestHost(hosts, timeout) + return host == null ? NPM_REGISTRY : registries[hosts.indexOf(host)] + } catch (e) { + logger.debug('Error on get registry', e) + return NPM_REGISTRY } - return res ?? new URL('https://registry.npmjs.org') } function isSymbolicLink(folder: string): boolean { @@ -73,8 +88,7 @@ export class Installer extends EventEmitter implements IInstaller { private url: string private version: string constructor( - private root: string, - private npm: string, + private dependencySession: DependencySession, // could be url or name@version or name private def: string ) { @@ -92,13 +106,17 @@ export class Installer extends EventEmitter implements IInstaller { } } + private get root(): string { + return this.dependencySession.modulesRoot + } + public get info() { return { name: this.name, version: this.version } } public async getInfo(): Promise { if (this.url) return await this.getInfoFromUri() - let registry = registryUrl() + let registry = this.dependencySession.registry this.log(`Get info from ${registry}`) let buffer = await this.fetch(new URL(this.name, registry), { timeout: 10000, buffer: true }) let res = JSON.parse(buffer.toString()) @@ -146,7 +164,6 @@ export class Installer extends EventEmitter implements IInstaller { } public async install(): Promise { - this.log(`Using npm from: ${this.npm}`) let info = await this.getInfo() logger.info(`Fetched info of ${this.def}`, info) let { name, version } = info @@ -170,7 +187,6 @@ export class Installer extends EventEmitter implements IInstaller { let obj = loadJson(path.join(folder, 'package.json')) as any version = obj.version } - this.log(`Using npm from: ${this.npm}`) let info = await this.getInfo() if (version && info.version && semver.gte(version, info.version)) { this.log(`Current version ${version} is up to date.`) @@ -188,11 +204,11 @@ export class Installer extends EventEmitter implements IInstaller { } public async installDependencies(folder: string): Promise { - let registry = registryUrl() - let installer = new DependenciesInstaller(registry, this.root, msg => { + let { dependencySession } = this + let installer = dependencySession.createInstaller(folder, msg => { this.log(msg) }) - await installer.installDependencies(folder) + await installer.installDependencies() } public async doInstall(info: Info): Promise { diff --git a/src/list/source/extensions.ts b/src/list/source/extensions.ts index 73429665a08..e993912479d 100644 --- a/src/list/source/extensions.ts +++ b/src/list/source/extensions.ts @@ -77,25 +77,6 @@ export default class ExtensionList extends BasicList { await extensions.manager.reloadExtension(id) }, { persist: true, reload: true }) - this.addAction('fix', async item => { - let { root, isLocal } = item.data - let { npm } = extensions - if (isLocal) { - void window.showWarningMessage(`Can't fix for local extension.`) - return - } - if (!npm) return - let folder = path.join(root, 'node_modules') - fs.rmSync(folder, { recursive: true, force: true }) - let terminal = await window.createTerminal({ - cwd: root - }) - let shown = await terminal.show(false) - if (!shown) return - workspace.nvim.command(`startinsert`, true) - terminal.sendText(`${npm} install --production --ignore-scripts --no-lockfile`, true) - }) - this.addMultipleAction('uninstall', async items => { let ids = [] for (let item of items) { diff --git a/src/util/ping.ts b/src/util/ping.ts new file mode 100644 index 00000000000..fc1e9002ea6 --- /dev/null +++ b/src/util/ping.ts @@ -0,0 +1,101 @@ +import { spawn } from 'child_process' +import fs from 'fs' +import which from 'which' + +interface PingConfig { + bin: string + args: string[] + regmatch: RegExp +} + +let pingConfig: PingConfig | undefined + +export function getPing(platform = process.platform): PingConfig | undefined { + let config: PingConfig + if (/^win/.test(platform)) { + config = { + bin: 'c:/windows/system32/ping.exe', + args: ['-n', '1', '-w', '5000'], + regmatch: /[><=]([0-9.]+?)\s?ms/ + } + } else if (/^linux/.test(platform)) { + config = { + bin: '/bin/ping', + args: ['-n', '-w', '2', '-c', '1'], + regmatch: /=([0-9.]+?) ms/ + } + } else if (/^darwin/.test(platform)) { + config = { + bin: '/sbin/ping', + args: ['-n', '-t', '2', '-c', '1'], + regmatch: /=([0-9.]+?) ms/ + } + } else { + try { + let bin = which.sync('ping') + config = { + bin, + args: ['-n', '-w', '2', '-c', '1'], + regmatch: /=([0-9.]+?) ms/ + } + } catch (_e) { + // not found + } + } + if (!config || !fs.existsSync(config.bin)) return undefined + return config +} + +export async function findBestHost(hosts: string[], timeout: number, bin?: string): Promise { + let config = pingConfig ?? getPing() + pingConfig = config + if (hosts.length === 0 || !config) return undefined + if (hosts.length === 1) return hosts[0] + const check = (host: string) => { + return new Promise(resolve => { + let args = pingConfig.args.concat([host]) + let stdout = '' + let cp = spawn(bin ?? pingConfig.bin, args) + let finished = false + let timer = setTimeout(() => { + finished = true + cp.kill() + resolve(undefined) + }, timeout) + cp.on('error', _e => { + finished = true + clearTimeout(timer) + resolve(undefined) + }) + let onEnd = () => { + clearTimeout(timer) + if (finished) return + finished = true + let ms = stdout.match(config.regmatch) + let milliseconds = (ms && ms[1]) ? Number(ms[1]) : undefined + resolve(milliseconds) + } + cp.stdout.on('data', data => { + stdout = stdout + data.toString() + }) + cp.stdout.on('end', () => { + onEnd() + }) + cp.on('exit', () => { + onEnd() + }) + }) + } + let minIndex = -1 + let minValue: undefined | number + await Promise.all(hosts.map((host, idx) => { + return check(host).then(val => { + if (val === undefined) return + if (!minValue || val < minValue) { + minValue = val + minIndex = idx + } + }) + })) + return minIndex == -1 ? undefined : hosts[minIndex] +} From 4225a1706c81566afaaab23b4a800ddc7f2173cb Mon Sep 17 00:00:00 2001 From: Qiming Zhao Date: Thu, 20 Oct 2022 22:58:04 +0800 Subject: [PATCH 05/13] cancel support for install process --- src/__tests__/modules/extensions.test.ts | 100 +++++++++++++++++++++-- src/extension/dependency.ts | 6 +- src/extension/index.ts | 77 +++++++++++++---- src/extension/installer.ts | 23 ++++-- src/extension/ui.ts | 3 +- src/model/fetch.ts | 4 +- src/util/index.ts | 7 +- typings/index.d.ts | 2 +- 8 files changed, 181 insertions(+), 41 deletions(-) diff --git a/src/__tests__/modules/extensions.test.ts b/src/__tests__/modules/extensions.test.ts index 442d63b6c07..487e414e8dd 100644 --- a/src/__tests__/modules/extensions.test.ts +++ b/src/__tests__/modules/extensions.test.ts @@ -1,9 +1,12 @@ import fs from 'fs' import os from 'os' import path from 'path' +import { URL } from 'url' import { v4 as uuid } from 'uuid' +import { CancellationTokenSource } from 'vscode-languageserver-protocol' import commandManager from '../../commands' import extensions, { Extensions, toUrl } from '../../extension' +import { CancellationError } from '../../util/errors' import { writeFile, writeJson } from '../../util/fs' import helper from '../helper' @@ -36,7 +39,7 @@ describe('extensions', () => { expect(extensions.onDidActiveExtension).toBeDefined() expect(extensions.onDidUnloadExtension).toBeDefined() expect(extensions.schemes).toBeDefined() - let res = await extensions.creteInstaller('id') + let res = extensions.createInstaller(new URL('https://github.com'), 'id') expect(res).toBeDefined() }) @@ -107,13 +110,18 @@ describe('extensions', () => { }) it('should catch error when installExtensions', async () => { - let spy = jest.spyOn(extensions, 'creteInstaller').mockImplementation(() => { + let spy = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { return { on: (_key, cb) => { cb('msg', false) }, install: () => { return Promise.resolve({ name: 'name', url: 'http://e', version: '1.0.0' }) + }, + update: () => { + return '' + }, + dispose: () => { } } as any }) @@ -125,15 +133,51 @@ describe('extensions', () => { s.mockRestore() }) + it('should cancel extension install', async () => { + let called = false + let spy = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { + let tokenSource = new CancellationTokenSource() + let token = tokenSource.token + return { + on: (_key, cb) => { + cb('msg', false) + }, + update: () => { + return Promise.resolve('') + }, + install: () => { + called = true + return new Promise((_resolve, reject) => { + token.onCancellationRequested(() => { + reject(new CancellationError()) + }) + }) + }, + dispose: () => { + tokenSource.cancel() + } + } + }) + let p = extensions.installExtensions(['abc@1.0.0', 'def@2.0.0']) + await helper.waitValue(() => { + return called + }, true) + extensions.cancelInstallers() + await p + spy.mockRestore() + }) + it('should catch error on updateExtensions', async () => { let spy = jest.spyOn(extensions, 'globalExtensionStats').mockImplementation(() => { return [{ id: 'test' }] as any }) - let s = jest.spyOn(extensions, 'creteInstaller').mockImplementation(() => { + let s = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { return { on: () => {}, update: () => { return Promise.resolve(path.join(os.tmpdir(), uuid())) + }, + dispose: () => { } } as any }) @@ -142,11 +186,49 @@ describe('extensions', () => { s.mockRestore() }) + it('should cancel extension update', async () => { + let s = jest.spyOn(extensions, 'globalExtensionStats').mockImplementation(() => { + return [{ id: 'test' }, { id: 'foo' }] as any + }) + let called = false + let spy = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { + let tokenSource = new CancellationTokenSource() + let token = tokenSource.token + return { + on: (_key, cb) => { + cb('msg', false) + }, + update: () => { + called = true + return new Promise((_resolve, reject) => { + token.onCancellationRequested(() => { + reject(new CancellationError()) + }) + }) + }, + install: () => { + return Promise.reject(new CancellationError()) + }, + dispose: () => { + tokenSource.cancel() + } + } + }) + let p = extensions.updateExtensions(true) + await helper.waitValue(() => { + return called + }, true) + extensions.cancelInstallers() + await p + s.mockRestore() + spy.mockRestore() + }) + it('should update enabled extensions', async () => { let spy = jest.spyOn(extensions, 'globalExtensionStats').mockImplementation(() => { return [{ id: 'test' }, { id: 'global', isLocked: true }, { id: 'disabled', state: 'disabled' }] as any }) - let s = jest.spyOn(extensions, 'creteInstaller').mockImplementation(() => { + let s = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { return { on: (_key, cb) => { cb('msg', false) @@ -154,6 +236,8 @@ describe('extensions', () => { update: async () => { await helper.wait(1) return '' + }, + dispose: () => { } } as any }) @@ -167,7 +251,7 @@ describe('extensions', () => { return [{ id: 'test', exotic: true, uri: 'http://example.com' }] as any }) let called = false - let s = jest.spyOn(extensions, 'creteInstaller').mockImplementation(() => { + let s = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { return { on: (_key, cb) => { cb('msg', false) @@ -177,6 +261,8 @@ describe('extensions', () => { called = true expect(url).toBe('http://example.com') return '' + }, + dispose: () => { } } as any }) @@ -202,7 +288,7 @@ describe('extensions', () => { it('should install global extension', async () => { let folder = path.join(extensions.modulesFolder, 'coc-omni') - let spy = jest.spyOn(extensions, 'creteInstaller').mockImplementation(() => { + let spy = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { return { on: () => {}, install: async () => { @@ -211,6 +297,8 @@ describe('extensions', () => { await writeFile(file, JSON.stringify({ name: 'coc-omni', engines: { coc: '>=0.0.1' }, version: '0.0.1' }, null, 2)) await writeFile(path.join(folder, 'index.js'), 'exports.activate = () => {}') return { name: 'coc-omni', version: '1.0.0', folder } + }, + dispose: () => { } } as any }) diff --git a/src/extension/dependency.ts b/src/extension/dependency.ts index c0dab10498f..88dfd5fd4d0 100644 --- a/src/extension/dependency.ts +++ b/src/extension/dependency.ts @@ -1,5 +1,6 @@ import { createHash } from 'crypto' import fs, { createReadStream } from 'fs' +import bytes from 'bytes' import path from 'path' import semver from 'semver' import tar from 'tar' @@ -326,9 +327,10 @@ export class DependenciesInstaller { res.set(filename, filepath) finished++ if (checked) { - this.onMessage(`Downloaded ${filename} cost ${Date.now() - ts}ms [${finished}/${total}]`) + this.onMessage(`File ${filename} already exists [${finished}/${total}]`) } else { - this.onMessage(`File ${filename} exists [${finished}/${total}]`) + let stat = fs.statSync(filepath) + this.onMessage(`Downloaded ${filename} (${bytes(stat.size)}) cost ${Date.now() - ts}ms [${finished}/${total}]`) } } if (checked) { diff --git a/src/extension/index.ts b/src/extension/index.ts index 0332751501b..a6306d4365f 100644 --- a/src/extension/index.ts +++ b/src/extension/index.ts @@ -1,14 +1,16 @@ 'use strict' import fs from 'fs' import path from 'path' -import { Event } from 'vscode-languageserver-protocol' +import { URL } from 'url' +import { CancellationTokenSource, Disposable, Event } from 'vscode-languageserver-protocol' import commandManager from '../commands' +import events from '../events' import type { OutputChannel } from '../types' -import { concurrent } from '../util' +import { concurrent, disposeAll } from '../util' import { distinct } from '../util/array' +import { isCancellationError } from '../util/errors' import '../util/extensions' import { isUrl } from '../util/is' -import { executable } from '../util/processes' import window from '../window' import workspace from '../workspace' import { DependencySession } from './dependency' @@ -37,10 +39,18 @@ export class Extensions { public readonly manager: ExtensionManager public readonly states: ExtensionStat public readonly modulesFolder = path.join(EXTENSIONS_FOLDER, 'node_modules') + private disposables: Disposable[] = [] constructor() { checkExtensionRoot(EXTENSIONS_FOLDER) this.states = new ExtensionStat(EXTENSIONS_FOLDER) this.manager = new ExtensionManager(this.states, EXTENSIONS_FOLDER) + events.on('VimLeavePre', () => { + this.cancelInstallers() + }) + } + + public cancelInstallers() { + disposeAll(this.disposables) } public async init(): Promise { @@ -139,13 +149,17 @@ export class Extensions { return await this.manager.call(id, method, args) } - private createInstallerUI(isUpdate: boolean, silent: boolean): InstallUI { - return silent ? new InstallChannel(isUpdate, this.outputChannel) : new InstallBuffer(isUpdate) + private createInstallerUI(isUpdate: boolean, silent: boolean, disposables: Disposable[]): InstallUI { + return silent ? new InstallChannel(isUpdate, this.outputChannel) : new InstallBuffer(isUpdate, async () => { + if (disposables.length > 0) { + disposeAll(disposables) + void window.showWarningMessage(`Extension install process canceled`) + } + }) } - public async creteInstaller(def: string): Promise { - let url = await registryUrl() - return new Installer(new DependencySession(url, this.modulesFolder), def) + public createInstaller(registry: URL, def: string): IInstaller { + return new Installer(new DependencySession(registry, this.modulesFolder), def) } /** @@ -153,17 +167,27 @@ export class Extensions { */ public async installExtensions(list: string[]): Promise { if (list.length == 0) return + this.cancelInstallers() list = distinct(list) - let installBuffer = this.createInstallerUI(false, false) + let disposables: Disposable[] = this.disposables = [] + let installBuffer = this.createInstallerUI(false, false, disposables) + let tokenSource = new CancellationTokenSource() + disposables.push(Disposable.create(() => { + tokenSource.cancel() + })) await Promise.resolve(installBuffer.start(list)) + let registry = await registryUrl() let fn = async (key: string): Promise => { + let installer: IInstaller try { installBuffer.startProgress(key) - let installer = await this.creteInstaller(key) + installer = this.createInstaller(registry, key) + disposables.push(installer) installer.on('message', (msg, isProgress) => { installBuffer.addMessage(key, msg, isProgress) }) let result = await installer.install() + disposables = disposables.filter(o => o !== installer) installBuffer.finishProgress(key, true) this.states.addExtension(result.name, result.url ? result.url : `>=${result.version}`) let ms = key.match(/@[\d.]+$/) @@ -172,17 +196,21 @@ export class Extensions { } catch (err: any) { installBuffer.addMessage(key, err.message) installBuffer.finishProgress(key, false) - void window.showErrorMessage(`Error on install ${key}: ${err}`) - logger.error(`Error on install ${key}`, err) + if (!isCancellationError(err)) { + void window.showErrorMessage(`Error on install ${key}: ${err}`) + logger.error(`Error on install ${key}`, err) + } } } - await concurrent(list, fn) + await concurrent(list, fn, 3, tokenSource.token) + disposables.splice(0, disposables.length) } /** * Update global extensions */ public async updateExtensions(silent = false): Promise { + this.cancelInstallers() let stats = this.globalExtensionStats() stats = stats.filter(s => { if (s.isLocked || s.state === 'disabled') { @@ -193,28 +221,40 @@ export class Extensions { }) this.states.setLastUpdate() this.cleanModulesFolder() - let installBuffer = this.createInstallerUI(true, silent) + let registry = await registryUrl() + let disposables: Disposable[] = this.disposables = [] + let installBuffer = this.createInstallerUI(true, silent, disposables) + let tokenSource = new CancellationTokenSource() + disposables.push(Disposable.create(() => { + tokenSource.cancel() + })) await Promise.resolve(installBuffer.start(stats.map(o => o.id))) let fn = async (stat: ExtensionInfo): Promise => { let { id } = stat + let installer: IInstaller try { installBuffer.startProgress(id) let url = stat.exotic ? stat.uri : null - let installer = await this.creteInstaller(id) + installer = this.createInstaller(registry, id) + disposables.push(installer) installer.on('message', (msg, isProgress) => { installBuffer.addMessage(id, msg, isProgress) }) let directory = await installer.update(url) + disposables = disposables.filter(o => o !== installer) installBuffer.finishProgress(id, true) if (directory) await this.manager.loadExtension(directory) } catch (err: any) { installBuffer.addMessage(id, err.message) installBuffer.finishProgress(id, false) - void window.showErrorMessage(`Error on update ${id}: ${err}`) - logger.error(`Error on update ${id}`, err) + if (!isCancellationError(err)) { + void window.showErrorMessage(`Error on update ${id}: ${err}`) + logger.error(`Error on update ${id}`, err) + } } } - await concurrent(stats, fn, silent ? 1 : 3) + await concurrent(stats, fn, silent ? 1 : 3, tokenSource.token) + disposables.splice(0, disposables.length) } /** @@ -300,6 +340,7 @@ export class Extensions { } public dispose(): void { + this.cancelInstallers() this.manager.dispose() } } diff --git a/src/extension/installer.ts b/src/extension/installer.ts index 3ad2c10673a..672304aa5ed 100644 --- a/src/extension/installer.ts +++ b/src/extension/installer.ts @@ -6,6 +6,7 @@ import path from 'path' import semver from 'semver' import { URL } from 'url' import { v4 as uuid } from 'uuid' +import { CancellationTokenSource } from 'vscode-languageserver-protocol' import download, { DownloadOptions } from '../model/download' import fetch, { FetchOptions } from '../model/fetch' import { isFalsyOrEmpty } from '../util/array' @@ -41,9 +42,8 @@ const PINGTIMEOUT = global.__TEST__ ? 50 : 500 * Find the user configured registry or the best one */ export async function registryUrl(home = os.homedir(), registries?: URL[], timeout = PINGTIMEOUT): Promise { - let res: URL - let filepath = path.join(home, '.npmrc') try { + let filepath = path.join(home, '.npmrc') if (fs.existsSync(filepath)) { let content = fs.readFileSync(filepath, 'utf8') let uri: string @@ -54,9 +54,8 @@ export async function registryUrl(home = os.homedir(), registries?: URL[], timeo uri = ms[2] } } - if (uri) res = new URL(uri) + if (uri) return new URL(uri) } - if (res) return res registries = isFalsyOrEmpty(registries) ? [TAOBAO_REGISTRY, NPM_REGISTRY, YARN_REGISTRY] : registries const hosts = registries.map(o => o.hostname) let host = await findBestHost(hosts, timeout) @@ -81,12 +80,14 @@ export interface IInstaller { on(event: 'message', cb: (msg: string, isProgress: boolean) => void): void install(): Promise update(url?: string): Promise + dispose(): void } export class Installer extends EventEmitter implements IInstaller { private name: string private url: string private version: string + private tokenSource = new CancellationTokenSource() constructor( private dependencySession: DependencySession, // could be url or name@version or name @@ -204,10 +205,13 @@ export class Installer extends EventEmitter implements IInstaller { } public async installDependencies(folder: string): Promise { - let { dependencySession } = this + let { dependencySession, tokenSource } = this let installer = dependencySession.createInstaller(folder, msg => { this.log(msg) }) + tokenSource.token.onCancellationRequested(() => { + installer.cancel() + }) await installer.installDependencies() } @@ -241,10 +245,15 @@ export class Installer extends EventEmitter implements IInstaller { } public async download(url: string, options: DownloadOptions): Promise { - return await download(url, options) + return await download(url, options, this.tokenSource.token) } public async fetch(url: string | URL, options: FetchOptions = {}): Promise { - return await fetch(url, options) + return await fetch(url, options, this.tokenSource.token) + } + + public dispose(): void { + this.tokenSource.cancel() + this.tokenSource = new CancellationTokenSource() } } diff --git a/src/extension/ui.ts b/src/extension/ui.ts index 23be439e411..b7455255c96 100644 --- a/src/extension/ui.ts +++ b/src/extension/ui.ts @@ -60,7 +60,7 @@ export class InstallBuffer implements InstallUI { private interval: NodeJS.Timer public bufnr: number - constructor(private isUpdate: boolean) { + constructor(private isUpdate: boolean, onClose = () => {}) { let floatFactory = window.createFloatFactory({ modes: ['n'] }) this.disposables.push(floatFactory) let fn = debounce(async (bufnr, cursor) => { @@ -77,6 +77,7 @@ export class InstallBuffer implements InstallUI { events.on('BufUnload', bufnr => { if (bufnr === this.bufnr) { this.dispose() + onClose() } }, null, this.disposables) } diff --git a/src/model/fetch.ts b/src/model/fetch.ts index 45949f71ba9..e1e3c064220 100644 --- a/src/model/fetch.ts +++ b/src/model/fetch.ts @@ -8,7 +8,7 @@ import { ParsedUrlQueryInput, stringify } from 'querystring' import { Readable } from 'stream' import { URL } from 'url' import { CancellationToken } from 'vscode-languageserver-protocol' -import { CancellationError } from '../util/errors' +import { CancellationError, isCancellationError } from '../util/errors' import { objectLiteral } from '../util/is' import workspace from '../workspace' const logger = require('../util/logger')('model-fetch') @@ -267,7 +267,7 @@ export default function fetch(urlInput: string | URL, options: FetchOptions = {} let url = toURL(urlInput) let opts = resolveRequestOptions(url, options) return request(url, options.data, opts, token).catch(err => { - logger.error(`Fetch error for ${url}:`, opts, err) + if (!isCancellationError(err)) logger.error(`Fetch error for ${url}:`, opts, err) if (opts.agent && opts.agent.proxy) { let { proxy } = opts.agent throw new Error(`Request failed using proxy ${proxy.host}: ${err.message}`) diff --git a/src/util/index.ts b/src/util/index.ts index bdd68b6eb1f..1cf16a492e8 100644 --- a/src/util/index.ts +++ b/src/util/index.ts @@ -1,7 +1,5 @@ 'use strict' -interface Disposable { - dispose(): void -} +import { CancellationToken, Disposable } from 'vscode-languageserver-protocol' export const CONFIG_FILE_NAME = 'coc-settings.json' @@ -48,13 +46,14 @@ export function delay(func: () => void, defaultDelay: number): ((ms?: number) => return fn as any } -export function concurrent(arr: T[], fn: (val: T) => Promise, limit = 3): Promise { +export function concurrent(arr: T[], fn: (val: T) => Promise, limit = 3, token?: CancellationToken): Promise { if (arr.length == 0) return Promise.resolve() let finished = 0 let total = arr.length let remain = arr.slice() return new Promise(resolve => { let run = (val): void => { + if (token && token.isCancellationRequested) return resolve() let cb = () => { finished = finished + 1 if (finished == total) { diff --git a/typings/index.d.ts b/typings/index.d.ts index 6b240302dfa..61bad8a15eb 100644 --- a/typings/index.d.ts +++ b/typings/index.d.ts @@ -5261,7 +5261,7 @@ declare module 'coc.nvim' { /** * Concurrent run async functions with limit support. */ - export function concurrent(arr: T[], fn: (val: T) => Promise, limit?: number): Promise + export function concurrent(arr: T[], fn: (val: T) => Promise, limit?: number, token?: CancellationToken): Promise /** * Create promise resolved after ms milliseconds. From 8adfc127db755e9f57a2bbe4d28db1189198d21f Mon Sep 17 00:00:00 2001 From: Qiming Zhao Date: Fri, 21 Oct 2022 00:34:05 +0800 Subject: [PATCH 06/13] cancel support for extension install --- .../modules/extensionDependency.test.ts | 26 +++++----- src/extension/dependency.ts | 18 ++----- src/extension/index.ts | 49 ++++++++++++------- src/extension/installer.ts | 2 +- src/extension/ui.ts | 23 ++++++++- src/util/index.ts | 16 +++--- 6 files changed, 80 insertions(+), 54 deletions(-) diff --git a/src/__tests__/modules/extensionDependency.test.ts b/src/__tests__/modules/extensionDependency.test.ts index bca1c0347e8..e9d4b0cae78 100644 --- a/src/__tests__/modules/extensionDependency.test.ts +++ b/src/__tests__/modules/extensionDependency.test.ts @@ -254,11 +254,7 @@ describe('DependenciesInstaller', () => { let directory = path.join(os.tmpdir(), uuid()) dirs.push(directory) writeJson(path.join(directory, 'package.json'), { dependencies: { foo: '>= 0.0.1' } }) - let other = path.join(os.tmpdir(), uuid()) - dirs.push(other) - writeJson(path.join(other, 'package.json'), { dependencies: { bar: '>= 0.0.1' } }) let one = session.createInstaller(directory, () => {}) - let two = session.createInstaller(other, () => {}) let spy = jest.spyOn(one, 'fetchInfos').mockImplementation(() => { return new Promise((resolve, reject) => { one.token.onCancellationRequested(() => { @@ -271,20 +267,26 @@ describe('DependenciesInstaller', () => { }) }) let p = one.installDependencies() - let err - two.installDependencies().catch(error => { - err = error - }) await helper.wait(30) - session.cancel() + one.cancel() let fn = async () => { await p } await expect(fn()).rejects.toThrow(Error) spy.mockRestore() - await helper.waitValue(() => { - return err != null - }, true) + }) + + it('should throw when Cancellation requested', async () => { + let install = create(undefined, '') + install.cancel() + let fn = async () => { + await install.fetch(new URL('/', url), { timeout: 10 }, 3) + } + await expect(fn()).rejects.toThrow(CancellationError) + fn = async () => { + await install.download(new URL('/', url), 'filename', '') + } + await expect(fn()).rejects.toThrow(CancellationError) }) it('should retry fetch', async () => { diff --git a/src/extension/dependency.ts b/src/extension/dependency.ts index 88dfd5fd4d0..6c01aff16a8 100644 --- a/src/extension/dependency.ts +++ b/src/extension/dependency.ts @@ -352,6 +352,7 @@ export class DependenciesInstaller { } public async fetch(url: string | URL, options: FetchOptions, retry = 1): Promise { + if (this.tokenSource.token.isCancellationRequested) throw new CancellationError() for (let i = 0; i < retry; i++) { try { return await fetch(url, options, this.tokenSource.token) @@ -369,6 +370,7 @@ export class DependenciesInstaller { * Download tgz file with sha1 check. */ public async download(url: string | URL, filename: string, shasum: string, retry = 1, timeout?: number): Promise { + if (this.tokenSource.token.isCancellationRequested) throw new CancellationError() for (let i = 0; i < retry; i++) { try { let fullpath = path.join(this.dest, filename) @@ -395,13 +397,11 @@ export class DependenciesInstaller { public cancel(): void { this.tokenSource.cancel() - this.tokenSource = new CancellationTokenSource() } } export class DependencySession { private resolvedInfos: Map = new Map() - private installers: Set = new Set() constructor( public readonly registry: URL, public readonly modulesRoot: string @@ -409,18 +409,6 @@ export class DependencySession { } public createInstaller(directory: string, onMessage: (msg: string) => void): DependenciesInstaller { - let installer = new DependenciesInstaller(this.registry, this.resolvedInfos, this.modulesRoot, directory, onMessage) - this.installers.add(installer) - return installer - } - - /** - * Cancel all installer - */ - public cancel(): void { - for (let item of this.installers) { - item.cancel() - } - this.installers.clear() + return new DependenciesInstaller(this.registry, this.resolvedInfos, this.modulesRoot, directory, onMessage) } } diff --git a/src/extension/index.ts b/src/extension/index.ts index a6306d4365f..becc121492f 100644 --- a/src/extension/index.ts +++ b/src/extension/index.ts @@ -172,8 +172,16 @@ export class Extensions { let disposables: Disposable[] = this.disposables = [] let installBuffer = this.createInstallerUI(false, false, disposables) let tokenSource = new CancellationTokenSource() + let installers: Map = new Map() + installBuffer.onDidCancel(key => { + let item = installers.get(key) + if (item) item.dispose() + }) disposables.push(Disposable.create(() => { tokenSource.cancel() + for (let item of installers.values()) { + item.dispose() + } })) await Promise.resolve(installBuffer.start(list)) let registry = await registryUrl() @@ -182,28 +190,24 @@ export class Extensions { try { installBuffer.startProgress(key) installer = this.createInstaller(registry, key) - disposables.push(installer) + installers.set(key, installer) installer.on('message', (msg, isProgress) => { installBuffer.addMessage(key, msg, isProgress) }) + logger.debug('install:', key) let result = await installer.install() - disposables = disposables.filter(o => o !== installer) installBuffer.finishProgress(key, true) this.states.addExtension(result.name, result.url ? result.url : `>=${result.version}`) let ms = key.match(/@[\d.]+$/) if (ms != null) this.states.setLocked(result.name, true) await this.manager.loadExtension(result.folder) } catch (err: any) { - installBuffer.addMessage(key, err.message) - installBuffer.finishProgress(key, false) - if (!isCancellationError(err)) { - void window.showErrorMessage(`Error on install ${key}: ${err}`) - logger.error(`Error on install ${key}`, err) - } + this.onInstallError(key, installBuffer, err) } } await concurrent(list, fn, 3, tokenSource.token) - disposables.splice(0, disposables.length) + let len = disposables.length + disposables.splice(0, len) } /** @@ -223,11 +227,19 @@ export class Extensions { this.cleanModulesFolder() let registry = await registryUrl() let disposables: Disposable[] = this.disposables = [] + let installers: Map = new Map() let installBuffer = this.createInstallerUI(true, silent, disposables) let tokenSource = new CancellationTokenSource() disposables.push(Disposable.create(() => { tokenSource.cancel() + for (let item of installers.values()) { + item.dispose() + } })) + installBuffer.onDidCancel(key => { + let item = installers.get(key) + if (item) item.dispose() + }) await Promise.resolve(installBuffer.start(stats.map(o => o.id))) let fn = async (stat: ExtensionInfo): Promise => { let { id } = stat @@ -236,27 +248,30 @@ export class Extensions { installBuffer.startProgress(id) let url = stat.exotic ? stat.uri : null installer = this.createInstaller(registry, id) - disposables.push(installer) + installers.set(id, installer) installer.on('message', (msg, isProgress) => { installBuffer.addMessage(id, msg, isProgress) }) let directory = await installer.update(url) - disposables = disposables.filter(o => o !== installer) installBuffer.finishProgress(id, true) if (directory) await this.manager.loadExtension(directory) } catch (err: any) { - installBuffer.addMessage(id, err.message) - installBuffer.finishProgress(id, false) - if (!isCancellationError(err)) { - void window.showErrorMessage(`Error on update ${id}: ${err}`) - logger.error(`Error on update ${id}`, err) - } + this.onInstallError(id, installBuffer, err) } } await concurrent(stats, fn, silent ? 1 : 3, tokenSource.token) disposables.splice(0, disposables.length) } + private onInstallError(id: string, installBuffer: InstallUI, err: Error): void { + installBuffer.addMessage(id, err.message) + installBuffer.finishProgress(id, false) + if (!isCancellationError(err)) { + void window.showErrorMessage(`Error on install ${id}: ${err}`) + logger.error(`Error on update ${id}`, err) + } + } + /** * Get all extension states */ diff --git a/src/extension/installer.ts b/src/extension/installer.ts index 672304aa5ed..e2a6c21569b 100644 --- a/src/extension/installer.ts +++ b/src/extension/installer.ts @@ -165,6 +165,7 @@ export class Installer extends EventEmitter implements IInstaller { } public async install(): Promise { + this.emit('message', `fetch info of ${this.def}`, false) let info = await this.getInfo() logger.info(`Fetched info of ${this.def}`, info) let { name, version } = info @@ -254,6 +255,5 @@ export class Installer extends EventEmitter implements IInstaller { public dispose(): void { this.tokenSource.cancel() - this.tokenSource = new CancellationTokenSource() } } diff --git a/src/extension/ui.ts b/src/extension/ui.ts index b7455255c96..5ba86951827 100644 --- a/src/extension/ui.ts +++ b/src/extension/ui.ts @@ -1,6 +1,6 @@ 'use strict' import { debounce } from 'debounce' -import { Disposable } from 'vscode-languageserver-protocol' +import { Disposable, Emitter, Event } from 'vscode-languageserver-protocol' import events from '../events' import { frames } from '../model/status' import { HighlightItem, OutputChannel } from '../types' @@ -18,6 +18,7 @@ export enum State { } export interface InstallUI { + onDidCancel: Event start(names: string[]): void | Promise addMessage(name: string, msg: string, isProgress?: boolean): void startProgress(name: string): void @@ -25,6 +26,8 @@ export interface InstallUI { } export class InstallChannel implements InstallUI { + private readonly _onDidCancel = new Emitter() + public readonly onDidCancel: Event = this._onDidCancel.event constructor(private isUpdate: boolean, private channel: OutputChannel) { } @@ -59,6 +62,8 @@ export class InstallBuffer implements InstallUI { private names: string[] = [] private interval: NodeJS.Timer public bufnr: number + private readonly _onDidCancel = new Emitter() + public readonly onDidCancel: Event = this._onDidCancel.event constructor(private isUpdate: boolean, onClose = () => {}) { let floatFactory = window.createFloatFactory({ modes: ['n'] }) @@ -163,6 +168,21 @@ export class InstallBuffer implements InstallUI { return this.interval == null } + public async onTab(): Promise { + let line = await workspace.nvim.eval(`line(".")-1`) as number + let name = this.names[line - 2] + if (!name) return + let state = this.statMap.get(name) + let couldCancel = state === State.Progressing + let actions: string[] = [] + if (couldCancel) actions.push('Cancel') + if (actions.length === 0) return + let idx = await window.showMenuPicker(['Cancel']) + if (idx === 0) { + this._onDidCancel.fire(name) + } + } + // draw frame public draw(): void { let { remains, bufnr } = this @@ -198,6 +218,7 @@ export class InstallBuffer implements InstallUI { if (!isSync) nvim.command('nnoremap q :q', true) this.highlight() let res = await nvim.resumeNotification() + workspace.registerLocalKeymap('n', '', this.onTab.bind(this)) this.bufnr = res[0][1] as number this.interval = setInterval(() => { this.draw() diff --git a/src/util/index.ts b/src/util/index.ts index 1cf16a492e8..9a1a6f4f8b7 100644 --- a/src/util/index.ts +++ b/src/util/index.ts @@ -46,11 +46,11 @@ export function delay(func: () => void, defaultDelay: number): ((ms?: number) => return fn as any } -export function concurrent(arr: T[], fn: (val: T) => Promise, limit = 3, token?: CancellationToken): Promise { +export function concurrent(arr: ReadonlyArray, fn: (val: T) => Promise, limit = 3, token?: CancellationToken): Promise { if (arr.length == 0) return Promise.resolve() let finished = 0 let total = arr.length - let remain = arr.slice() + let curr = 0 return new Promise(resolve => { let run = (val): void => { if (token && token.isCancellationRequested) return resolve() @@ -58,16 +58,16 @@ export function concurrent(arr: T[], fn: (val: T) => Promise, limit = 3 finished = finished + 1 if (finished == total) { resolve() - } else if (remain.length) { - let next = remain.shift() - run(next) + } else if (curr < total - 1) { + curr++ + run(arr[curr]) } } fn(val).then(cb, cb) } - for (let i = 0; i < Math.min(limit, remain.length); i++) { - let val = remain.shift() - run(val) + curr = Math.min(limit, total) - 1 + for (let i = 0; i <= curr; i++) { + run(arr[i]) } }) } From 7d04edabdcaaec846a1daf4a6cb604145fa3aae5 Mon Sep 17 00:00:00 2001 From: Qiming Zhao Date: Fri, 21 Oct 2022 00:59:03 +0800 Subject: [PATCH 07/13] use promisify --- src/extension/dependency.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/extension/dependency.ts b/src/extension/dependency.ts index 6c01aff16a8..9edd17ca37a 100644 --- a/src/extension/dependency.ts +++ b/src/extension/dependency.ts @@ -1,10 +1,11 @@ +import bytes from 'bytes' import { createHash } from 'crypto' import fs, { createReadStream } from 'fs' -import bytes from 'bytes' import path from 'path' import semver from 'semver' import tar from 'tar' import { URL } from 'url' +import { promisify } from 'util' import { CancellationToken, CancellationTokenSource } from 'vscode-languageserver-protocol' import download from '../model/download' import fetch, { FetchOptions } from '../model/fetch' @@ -95,7 +96,10 @@ export function shouldRetry(error: any): boolean { return false } -export function readDependencies(directory: string): { [key: string]: string } { +/** + * Production dependencies in directory + */ +export function readDependencies(directory: string): Dependencies { let jsonfile = path.join(directory, 'package.json') let obj = loadJson(jsonfile) as any let dependencies = obj.dependencies as { [key: string]: string } @@ -118,8 +122,8 @@ export function getVersion(requirement: string, versions: string[], latest?: str */ export async function untar(dest: string, tarfile: string, strip = 1): Promise { if (!fs.existsSync(tarfile)) throw new Error(`${tarfile} not exists`) - fs.rmSync(dest, { recursive: true, force: true }) - fs.mkdirSync(dest, { recursive: true }) + await promisify(fs.rm)(dest, { recursive: true, force: true }) + await promisify(fs.mkdir)(dest, { recursive: true }) await new Promise((resolve, reject) => { const input = createReadStream(tarfile) input.on('error', reject) From 09b6ac4d9a7793373d8e75eff5e866b9c25bb243 Mon Sep 17 00:00:00 2001 From: Qiming Zhao Date: Thu, 27 Oct 2022 22:53:27 +0800 Subject: [PATCH 08/13] chore(package): remove unused types --- package.json | 1 - yarn.lock | 7 ------- 2 files changed, 8 deletions(-) diff --git a/package.json b/package.json index 8ae36ca6678..1cabd7a64c2 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,6 @@ "@types/jest": "^27.0.3", "@types/marked": "^4.0.1", "@types/minimatch": "^5.1.2", - "@types/mkdirp": "^1.0.1", "@types/node": "14.14", "@types/semver": "^7.3.4", "@types/tar": "^4.0.5", diff --git a/yarn.lock b/yarn.lock index a7e01155cda..5ffdd9c9985 100644 --- a/yarn.lock +++ b/yarn.lock @@ -844,13 +844,6 @@ dependencies: "@types/node" "*" -"@types/mkdirp@^1.0.1": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-1.0.2.tgz#8d0bad7aa793abe551860be1f7ae7f3198c16666" - integrity sha512-o0K1tSO0Dx5X6xlU5F1D6625FawhC3dU3iqr25lluNv/+/QIVH8RLNEiVokgIZo+mz+87w/3Mkg/VvQS+J51fQ== - dependencies: - "@types/node" "*" - "@types/node-int64@*": version "0.4.29" resolved "https://registry.yarnpkg.com/@types/node-int64/-/node-int64-0.4.29.tgz#8c7c16a7c1195ae4f8beaa903b0018ac66291d16" From 6b80790f760824dac41dca3b6a8ad5b4ea211efd Mon Sep 17 00:00:00 2001 From: Qiming Zhao Date: Fri, 28 Oct 2022 16:54:29 +0800 Subject: [PATCH 09/13] add checkModule method --- src/__tests__/modules/extensionAll.test.ts | 40 ++++++++++++++++++ src/extension/dependency.ts | 47 +++++++++++++++++++--- src/model/fetch.ts | 2 +- 3 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 src/__tests__/modules/extensionAll.test.ts diff --git a/src/__tests__/modules/extensionAll.test.ts b/src/__tests__/modules/extensionAll.test.ts new file mode 100644 index 00000000000..d20ca751efc --- /dev/null +++ b/src/__tests__/modules/extensionAll.test.ts @@ -0,0 +1,40 @@ +import os from 'os' +import { URL } from 'url' +import { DependencySession } from '../../extension/dependency' +import fetch from '../../model/fetch' + +process.env.NO_PROXY = '*' + +/** + * Test dependencies for all coc.nvim extensions + */ +describe('Test dependencies ', () => { + it('should test extensions', async () => { + let names: string[] = [] + let obj = await fetch(`https://registry.npmjs.com/-/v1/search?text=keywords:coc.nvim&size=200&from=0`) + for (let item of obj['objects']) { + names.push(item['package'].name) + } + obj = await fetch(`https://registry.npmjs.com/-/v1/search?text=keywords:coc.nvim&size=200&from=200`) + for (let item of obj['objects']) { + let name = item['package'].name + if (!names.includes(name)) { + names.push(name) + } + } + // optionalDependencies + console.log(`total: ${names.length}`) + let registry = new URL('https://registry.npmjs.org/') + let session = new DependencySession(registry, os.tmpdir()) + + for (let name of names) { + console.log(`Checking module ${name}`) + try { + let dep = session.createInstaller(os.tmpdir(), () => {}) + await dep.checkModule(names[3]) + } catch (e) { + console.error(`Error with ${name}`, e) + } + } + }, 120 * 1000 * 60) +}) diff --git a/src/extension/dependency.ts b/src/extension/dependency.ts index 9edd17ca37a..5a53e906af0 100644 --- a/src/extension/dependency.ts +++ b/src/extension/dependency.ts @@ -13,6 +13,7 @@ import { CancellationError } from '../util/errors' import { loadJson, writeJson } from '../util/fs' import { objectLiteral } from '../util/is' import { Mutex } from '../util/mutex' +import { toObject } from '../util/object' const logger = require('../util/logger')('extension-dependency') export interface Dependencies { [key: string]: string } @@ -22,6 +23,7 @@ export interface VersionInfo { name: string version: string dependencies?: Dependencies + optionalDependencies?: Dependencies dist: { integrity: string // starts with sha512-, base64 string shasum: string // sha1 hash, hex string @@ -50,8 +52,8 @@ export interface DependencyItem { } const DEV_DEPENDENCIES = ['coc.nvim', 'webpack', 'esbuild'] -const INFO_TIMEOUT = global.__TEST__ ? 100 : 10000 -const DOWNLOAD_TIMEOUT = global.__TEST__ ? 500 : 3 * 60 * 1000 +const INFO_TIMEOUT = 10000 +const DOWNLOAD_TIMEOUT = 3 * 60 * 1000 function toFilename(item: DependencyItem): string { return `${item.name}.${item.version}.tgz` @@ -89,13 +91,29 @@ export function getModuleInfo(text: string): ModuleInfo { export function shouldRetry(error: any): boolean { let message = error.message if (typeof message !== 'string') return false - if (message.includes('timeout') || - message.includes('Invalid JSON') || + if (message.includes('Invalid JSON') || message.includes('Bad shasum') || message.includes('ECONNRESET')) return true return false } +function checkDeps(obj: any): void { + if (obj.optionalDependencies) { + console.log(`${obj.name} have optionalDependencies: ${JSON.stringify(obj.optionalDependencies)}`) + } + let dependencies = toObject(obj.dependencies) as { [key: string]: string } + let keys: string[] = [] + for (let [key, value] of Object.entries(dependencies)) { + if (!semver.validRange(value)) { + console.log(`${obj.name} have invalid dependency version: ${key}:${value}`) + keys.push(key) + } + } + keys.forEach(key => { + delete dependencies[key] + }) +} + /** * Production dependencies in directory */ @@ -103,9 +121,10 @@ export function readDependencies(directory: string): Dependencies { let jsonfile = path.join(directory, 'package.json') let obj = loadJson(jsonfile) as any let dependencies = obj.dependencies as { [key: string]: string } - for (let key of Object.keys(dependencies ?? {})) { + for (let key of Object.keys(toObject(dependencies))) { if (DEV_DEPENDENCIES.includes(key) || key.startsWith('@types/')) delete dependencies[key] } + checkDeps(obj) return dependencies } @@ -177,6 +196,23 @@ export class DependenciesInstaller { return path.join(this.modulesRoot, '.cache') } + public async checkModule(name: string): Promise { + let info = await this.loadInfo(this.registry, name, INFO_TIMEOUT) + let extensionInfo = info.versions[info.latest] + if (!extensionInfo) { + console.log(`Can not get latest version for ${name}`) + return + } + checkDeps(extensionInfo) + let dependencies = toObject(extensionInfo.dependencies) as Dependencies + for (let key of Object.keys(dependencies)) { + if (DEV_DEPENDENCIES.includes(key) || key.startsWith('@types/')) delete dependencies[key] + } + await this.fetchInfos(dependencies) + let items: DependencyItem[] = [] + this.linkDependencies(dependencies, items) + } + public async installDependencies(): Promise { let { directory, tokenSource } = this let dependencies = readDependencies(directory) @@ -266,6 +302,7 @@ export class DependenciesInstaller { if (!dependencies) return for (let [name, requirement] of Object.entries(dependencies)) { let versionInfo = this.resolveVersion(name, requirement) + checkDeps(versionInfo) let item = items.find(o => o.name === name && o.version === versionInfo.version) if (item) { if (!item.satisfiedVersions.includes(requirement)) item.satisfiedVersions.push(requirement) diff --git a/src/model/fetch.ts b/src/model/fetch.ts index e1e3c064220..9095d933cf7 100644 --- a/src/model/fetch.ts +++ b/src/model/fetch.ts @@ -244,7 +244,7 @@ export function request(url: URL, data: any, opts: any, token?: CancellationToke } }) req.on('timeout', () => { - req.destroy(new Error(`Request timeout after ${opts.timeout}ms`)) + req.destroy(new Error(`Request ${url} timeout after ${opts.timeout}ms`)) }) if (data) req.write(getText(data)) if (opts.timeout) req.setTimeout(opts.timeout) From 9024602181744ba2115e4c04f811f6c19bf4ea2a Mon Sep 17 00:00:00 2001 From: Qiming Zhao Date: Fri, 28 Oct 2022 17:06:33 +0800 Subject: [PATCH 10/13] fix bad name --- src/__tests__/modules/extensionAll.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__tests__/modules/extensionAll.test.ts b/src/__tests__/modules/extensionAll.test.ts index d20ca751efc..f76cc29c65d 100644 --- a/src/__tests__/modules/extensionAll.test.ts +++ b/src/__tests__/modules/extensionAll.test.ts @@ -31,7 +31,7 @@ describe('Test dependencies ', () => { console.log(`Checking module ${name}`) try { let dep = session.createInstaller(os.tmpdir(), () => {}) - await dep.checkModule(names[3]) + await dep.checkModule(name) } catch (e) { console.error(`Error with ${name}`, e) } From 10817c0518dc5a011081fa02b5ba50b23b8922fa Mon Sep 17 00:00:00 2001 From: Qiming Zhao Date: Fri, 28 Oct 2022 17:12:22 +0800 Subject: [PATCH 11/13] avoid out of memory --- src/__tests__/modules/extensionAll.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/__tests__/modules/extensionAll.test.ts b/src/__tests__/modules/extensionAll.test.ts index f76cc29c65d..b433f52c853 100644 --- a/src/__tests__/modules/extensionAll.test.ts +++ b/src/__tests__/modules/extensionAll.test.ts @@ -2,6 +2,7 @@ import os from 'os' import { URL } from 'url' import { DependencySession } from '../../extension/dependency' import fetch from '../../model/fetch' +import { waitImmediate } from '../../util' process.env.NO_PROXY = '*' @@ -25,9 +26,10 @@ describe('Test dependencies ', () => { // optionalDependencies console.log(`total: ${names.length}`) let registry = new URL('https://registry.npmjs.org/') - let session = new DependencySession(registry, os.tmpdir()) for (let name of names) { + await waitImmediate() + let session = new DependencySession(registry, os.tmpdir()) console.log(`Checking module ${name}`) try { let dep = session.createInstaller(os.tmpdir(), () => {}) From 8267570b62233683e2d6d928164b3f3aa8b3a8c6 Mon Sep 17 00:00:00 2001 From: Qiming Zhao Date: Fri, 28 Oct 2022 19:48:23 +0800 Subject: [PATCH 12/13] fix recursive fetch infos --- src/__tests__/modules/extensionAll.test.ts | 5 ++-- .../modules/extensionDependency.test.ts | 8 +++--- src/extension/dependency.ts | 27 ++++++++++++------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/__tests__/modules/extensionAll.test.ts b/src/__tests__/modules/extensionAll.test.ts index b433f52c853..2f5636879f3 100644 --- a/src/__tests__/modules/extensionAll.test.ts +++ b/src/__tests__/modules/extensionAll.test.ts @@ -23,13 +23,12 @@ describe('Test dependencies ', () => { names.push(name) } } - // optionalDependencies console.log(`total: ${names.length}`) - let registry = new URL('https://registry.npmjs.org/') + let registry = new URL('https://registry.npmmirror.com/') + let session = new DependencySession(registry, os.tmpdir()) for (let name of names) { await waitImmediate() - let session = new DependencySession(registry, os.tmpdir()) console.log(`Checking module ${name}`) try { let dep = session.createInstaller(os.tmpdir(), () => {}) diff --git a/src/__tests__/modules/extensionDependency.test.ts b/src/__tests__/modules/extensionDependency.test.ts index e9d4b0cae78..f25c2d152ff 100644 --- a/src/__tests__/modules/extensionDependency.test.ts +++ b/src/__tests__/modules/extensionDependency.test.ts @@ -96,8 +96,8 @@ describe('utils', () => { fs.mkdirSync(dir, { recursive: true }) let filepath = path.join(dir, 'package.json') writeJson(filepath, { dependencies: { "coc.nvim": ">= 0.0.80", "is-number": "^1.0.0" } }) - let res = readDependencies(dir) - expect(res).toEqual({ 'is-number': '^1.0.0' }) + let { dependencies } = readDependencies(dir) + expect(dependencies).toEqual({ 'is-number': '^1.0.0' }) }) it('should getVersion', () => { @@ -326,14 +326,14 @@ describe('DependenciesInstaller', () => { it('should fetchInfos', async () => { addJsonData() let install = create(undefined, '') - await install.fetchInfos({ a: '^0.0.1' }) + await install.fetchInfos('x', '0.0.1', { a: '^0.0.1' }) expect(install.resolvedInfos.size).toBe(4) }) it('should linkDependencies', async () => { addJsonData() let install = create(undefined, '') - await install.fetchInfos({ a: '^0.0.1' }) + await install.fetchInfos('x', '0.0.1', { a: '^0.0.1' }) let items: DependencyItem[] = [] install.linkDependencies(undefined, items) expect(items).toEqual([]) diff --git a/src/extension/dependency.ts b/src/extension/dependency.ts index 5a53e906af0..d98abf0037d 100644 --- a/src/extension/dependency.ts +++ b/src/extension/dependency.ts @@ -52,7 +52,7 @@ export interface DependencyItem { } const DEV_DEPENDENCIES = ['coc.nvim', 'webpack', 'esbuild'] -const INFO_TIMEOUT = 10000 +const INFO_TIMEOUT = 30000 const DOWNLOAD_TIMEOUT = 3 * 60 * 1000 function toFilename(item: DependencyItem): string { @@ -117,7 +117,7 @@ function checkDeps(obj: any): void { /** * Production dependencies in directory */ -export function readDependencies(directory: string): Dependencies { +export function readDependencies(directory: string): { name: string, version: string, dependencies: Dependencies } { let jsonfile = path.join(directory, 'package.json') let obj = loadJson(jsonfile) as any let dependencies = obj.dependencies as { [key: string]: string } @@ -125,7 +125,7 @@ export function readDependencies(directory: string): Dependencies { if (DEV_DEPENDENCIES.includes(key) || key.startsWith('@types/')) delete dependencies[key] } checkDeps(obj) - return dependencies + return { name: obj.name, version: obj.version, dependencies } } export function getVersion(requirement: string, versions: string[], latest?: string): string | undefined { @@ -179,6 +179,7 @@ const mutex = new Mutex() export class DependenciesInstaller { private tokenSource: CancellationTokenSource = new CancellationTokenSource() + private fetched: string[] = [] constructor( private registry: URL, public readonly resolvedInfos: Map, @@ -208,14 +209,16 @@ export class DependenciesInstaller { for (let key of Object.keys(dependencies)) { if (DEV_DEPENDENCIES.includes(key) || key.startsWith('@types/')) delete dependencies[key] } - await this.fetchInfos(dependencies) + console.log('fetch infos') + await this.fetchInfos(name, info.latest, dependencies) + console.log('linking') let items: DependencyItem[] = [] this.linkDependencies(dependencies, items) } public async installDependencies(): Promise { let { directory, tokenSource } = this - let dependencies = readDependencies(directory) + let { dependencies, name, version } = readDependencies(directory) // no need to install if (!dependencies || Object.keys(dependencies).length == 0) { this.onMessage(`No dependencies`) @@ -227,7 +230,7 @@ export class DependenciesInstaller { await mutex.use(async () => { if (token.isCancellationRequested) throw new CancellationError() this.onMessage('Resolving dependencies.') - await this.fetchInfos(dependencies) + await this.fetchInfos(name, version, dependencies) this.onMessage('Linking dependencies.') // create DependencyItems let items: DependencyItem[] = [] @@ -323,6 +326,7 @@ export class DependenciesInstaller { } public resolveVersion(name: string, requirement: string): VersionInfo { + if (!semver.validRange(requirement)) throw new Error(`Unsupported version range "${requirement}" of "${name}"`) let info = this.resolvedInfos.get(name) if (info) { let version = getVersion(requirement, Object.keys(info.versions), info.latest) @@ -337,9 +341,12 @@ export class DependenciesInstaller { /** * Recursive fetch module info */ - public async fetchInfos(dependencies: Dependencies | undefined): Promise { + public async fetchInfos(name: string, version: string, dependencies: Dependencies | undefined): Promise { let keys = Object.keys(dependencies ?? {}) if (keys.length === 0) return + let id = `${name}-${version}` + if (this.fetched.includes(id)) return + this.fetched.push(id) await Promise.all(keys.map(key => { if (this.resolvedInfos.has(key)) return Promise.resolve() return this.loadInfo(this.registry, key, INFO_TIMEOUT).then(info => { @@ -348,7 +355,7 @@ export class DependenciesInstaller { })) for (let key of keys) { let versionInfo = this.resolveVersion(key, dependencies[key]) - await this.fetchInfos(versionInfo.dependencies) + await this.fetchInfos(versionInfo.name, versionInfo.version, versionInfo.dependencies) } } @@ -388,13 +395,13 @@ export class DependenciesInstaller { * Fetch module info */ public async loadInfo(registry: URL, name: string, timeout = 100): Promise { - let buffer = await this.fetch(new URL(name, registry), { timeout, buffer: true }) as Buffer + let buffer = await this.fetch(new URL(name, registry), { timeout, buffer: true }, 3) as Buffer return getModuleInfo(buffer.toString()) } public async fetch(url: string | URL, options: FetchOptions, retry = 1): Promise { - if (this.tokenSource.token.isCancellationRequested) throw new CancellationError() for (let i = 0; i < retry; i++) { + if (this.tokenSource.token.isCancellationRequested) throw new CancellationError() try { return await fetch(url, options, this.tokenSource.token) } catch (e) { From 39b10c25befcb29014ca125ec923a964db630186 Mon Sep 17 00:00:00 2001 From: Qiming Zhao Date: Fri, 28 Oct 2022 21:32:00 +0800 Subject: [PATCH 13/13] remove code for checkModule --- src/__tests__/client/integration.test.ts | 10 ++--- src/__tests__/modules/extensionAll.test.ts | 41 ------------------- .../modules/extensionDependency.test.ts | 3 ++ src/extension/dependency.ts | 36 +++++----------- 4 files changed, 18 insertions(+), 72 deletions(-) delete mode 100644 src/__tests__/modules/extensionAll.test.ts diff --git a/src/__tests__/client/integration.test.ts b/src/__tests__/client/integration.test.ts index c8e442b4999..b92b6ea22ee 100644 --- a/src/__tests__/client/integration.test.ts +++ b/src/__tests__/client/integration.test.ts @@ -214,8 +214,9 @@ describe('Client events', () => { await client.sendNotification('showDocument', { uri: 'lsptest:///1', takeFocus: false }) await client.sendNotification('showDocument', { uri: uri.toString() }) await client.sendNotification('showDocument', { uri: uri.toString(), selection: Range.create(0, 0, 1, 0) }) - await helper.wait(300) - expect(client.hasPendingResponse).toBe(false) + await helper.waitValue(() => { + return client.hasPendingResponse + }, false) await client.stop() }) @@ -628,7 +629,7 @@ describe('Client integration', () => { } await expect(fn()).rejects.toThrow(Error) spy.mockRestore() - await helper.wait(10) + await helper.wait(30) }) }) @@ -649,9 +650,6 @@ describe('SettingMonitor', () => { return client.state }, lsclient.State.Stopped) helper.updateConfiguration('html.enabled', true) - await helper.waitValue(() => { - return client.state != lsclient.State.Stopped - }, true) await client.onReady() disposable.dispose() }) diff --git a/src/__tests__/modules/extensionAll.test.ts b/src/__tests__/modules/extensionAll.test.ts deleted file mode 100644 index 2f5636879f3..00000000000 --- a/src/__tests__/modules/extensionAll.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import os from 'os' -import { URL } from 'url' -import { DependencySession } from '../../extension/dependency' -import fetch from '../../model/fetch' -import { waitImmediate } from '../../util' - -process.env.NO_PROXY = '*' - -/** - * Test dependencies for all coc.nvim extensions - */ -describe('Test dependencies ', () => { - it('should test extensions', async () => { - let names: string[] = [] - let obj = await fetch(`https://registry.npmjs.com/-/v1/search?text=keywords:coc.nvim&size=200&from=0`) - for (let item of obj['objects']) { - names.push(item['package'].name) - } - obj = await fetch(`https://registry.npmjs.com/-/v1/search?text=keywords:coc.nvim&size=200&from=200`) - for (let item of obj['objects']) { - let name = item['package'].name - if (!names.includes(name)) { - names.push(name) - } - } - console.log(`total: ${names.length}`) - let registry = new URL('https://registry.npmmirror.com/') - let session = new DependencySession(registry, os.tmpdir()) - - for (let name of names) { - await waitImmediate() - console.log(`Checking module ${name}`) - try { - let dep = session.createInstaller(os.tmpdir(), () => {}) - await dep.checkModule(name) - } catch (e) { - console.error(`Error with ${name}`, e) - } - } - }, 120 * 1000 * 60) -}) diff --git a/src/__tests__/modules/extensionDependency.test.ts b/src/__tests__/modules/extensionDependency.test.ts index f25c2d152ff..d915c3347d4 100644 --- a/src/__tests__/modules/extensionDependency.test.ts +++ b/src/__tests__/modules/extensionDependency.test.ts @@ -82,6 +82,9 @@ describe('utils', () => { expect(getModuleInfo(JSON.stringify(obj))).toBeDefined() obj = { name: 'name', 'dist-tags': { latest: '1.0.0' }, versions: {} } expect(getModuleInfo(JSON.stringify(obj))).toBeDefined() + obj = { name: 'name', 'dist-tags': { latest: '1.0.0' }, versions: { '0.0.9': {}, '0.0.8': {} } } + let res = getModuleInfo(JSON.stringify(obj)) + expect(res.latest).toBe('0.0.9') }) it('should check retry', () => { diff --git a/src/extension/dependency.ts b/src/extension/dependency.ts index d98abf0037d..97bdf249aac 100644 --- a/src/extension/dependency.ts +++ b/src/extension/dependency.ts @@ -52,8 +52,8 @@ export interface DependencyItem { } const DEV_DEPENDENCIES = ['coc.nvim', 'webpack', 'esbuild'] -const INFO_TIMEOUT = 30000 -const DOWNLOAD_TIMEOUT = 3 * 60 * 1000 +const INFO_TIMEOUT = global.__TEST__ ? 100 : 20000 +const DOWNLOAD_TIMEOUT = global.__TEST__ ? 500 : 3 * 60 * 1000 function toFilename(item: DependencyItem): string { return `${item.name}.${item.version}.tgz` @@ -73,6 +73,9 @@ export function validVersionInfo(info: any): info is VersionInfo { return true } +/** + * Get required info form json text. + */ export function getModuleInfo(text: string): ModuleInfo { let obj try { @@ -81,9 +84,12 @@ export function getModuleInfo(text: string): ModuleInfo { throw new Error(`Invalid JSON data, ${e}`) } if (typeof obj.name !== 'string' || !objectLiteral(obj.versions)) throw new Error(`Invalid JSON data, name or versions not found`) + let versions = Object.keys(obj.versions) + let latest = obj['dist-tags']?.latest + if (latest && versions[latest] == null) latest = semver.rsort(versions)[0] return { name: obj.name, - latest: obj['dist-tags']?.latest, + latest, versions: obj.versions } as ModuleInfo } @@ -91,7 +97,8 @@ export function getModuleInfo(text: string): ModuleInfo { export function shouldRetry(error: any): boolean { let message = error.message if (typeof message !== 'string') return false - if (message.includes('Invalid JSON') || + if (message.includes('timeout') || + message.includes('Invalid JSON') || message.includes('Bad shasum') || message.includes('ECONNRESET')) return true return false @@ -124,7 +131,6 @@ export function readDependencies(directory: string): { name: string, version: st for (let key of Object.keys(toObject(dependencies))) { if (DEV_DEPENDENCIES.includes(key) || key.startsWith('@types/')) delete dependencies[key] } - checkDeps(obj) return { name: obj.name, version: obj.version, dependencies } } @@ -197,25 +203,6 @@ export class DependenciesInstaller { return path.join(this.modulesRoot, '.cache') } - public async checkModule(name: string): Promise { - let info = await this.loadInfo(this.registry, name, INFO_TIMEOUT) - let extensionInfo = info.versions[info.latest] - if (!extensionInfo) { - console.log(`Can not get latest version for ${name}`) - return - } - checkDeps(extensionInfo) - let dependencies = toObject(extensionInfo.dependencies) as Dependencies - for (let key of Object.keys(dependencies)) { - if (DEV_DEPENDENCIES.includes(key) || key.startsWith('@types/')) delete dependencies[key] - } - console.log('fetch infos') - await this.fetchInfos(name, info.latest, dependencies) - console.log('linking') - let items: DependencyItem[] = [] - this.linkDependencies(dependencies, items) - } - public async installDependencies(): Promise { let { directory, tokenSource } = this let { dependencies, name, version } = readDependencies(directory) @@ -305,7 +292,6 @@ export class DependenciesInstaller { if (!dependencies) return for (let [name, requirement] of Object.entries(dependencies)) { let versionInfo = this.resolveVersion(name, requirement) - checkDeps(versionInfo) let item = items.find(o => o.name === name && o.version === versionInfo.version) if (item) { if (!item.satisfiedVersions.includes(requirement)) item.satisfiedVersions.push(requirement)