diff --git a/data/schema.json b/data/schema.json index 01861d1be64..7120c6b0d0a 100644 --- a/data/schema.json +++ b/data/schema.json @@ -1421,12 +1421,6 @@ "scope": "application", "description": "Winblend option of notification window, neovim only." }, - "npm.binPath": { - "type": "string", - "scope": "application", - "default": "npm", - "description": "Command or absolute path to npm or yarn." - }, "outline.autoPreview": { "type": "boolean", "scope": "application", diff --git a/doc/coc-config.txt b/doc/coc-config.txt index 9e7ec1342da..b401820386c 100644 --- a/doc/coc-config.txt +++ b/doc/coc-config.txt @@ -17,7 +17,6 @@ Inlay hint |coc-config-inlayHint| Links |coc-config-links| List |coc-config-list| Notification |coc-config-notification| -Npm |coc-config-npm| Outline |coc-config-outline| Pull diagnostics |coc-config-pullDiagnostic| Refactor |coc-config-refactor| @@ -790,16 +789,6 @@ Notification~ Scope: `application`, default: `30` ------------------------------------------------------------------------------- -Npm~ - *coc-config-npm* -"npm.binPath" *coc-config-npm-binPath* - - Command or absolute path to npm or yarn for global extension - install/uninstall. - - Scope: `application`, default: `"npm"` - ------------------------------------------------------------------------------ Outline~ *coc-config-outline* diff --git a/doc/coc.txt b/doc/coc.txt index d7b08bc1393..13f73ea7762 100644 --- a/doc/coc.txt +++ b/doc/coc.txt @@ -177,11 +177,8 @@ Use |:CocInstall| to install coc extensions from vim's command line. To make coc.nvim install extensions on startup, use |g:coc_global_extensions|. -To use package manager other than npm (like `yarn` or `pnpm`), use -|coc-config-npm-binPath|. - -To customize npm registry for coc.nvim add `coc.nvim:registry` in your -`~/.npmrc`, like: +To customize node modules registry for coc.nvim add `coc.nvim:registry` in +your `~/.npmrc`, like: > coc.nvim:registry=https://registry.mycompany.org/ < @@ -3642,7 +3639,6 @@ extensions *coc-list-extensions* - 'enable' enable extension. - 'lock' lock/unlock extension to current version. - 'doc' view extension's README doc. - - 'fix' fix dependencies in terminal buffer. - 'reload' reload extension. - 'uninstall' uninstall extension. diff --git a/package.json b/package.json index 8ae36ca6678..1cabd7a64c2 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,6 @@ "@types/jest": "^27.0.3", "@types/marked": "^4.0.1", "@types/minimatch": "^5.1.2", - "@types/mkdirp": "^1.0.1", "@types/node": "14.14", "@types/semver": "^7.3.4", "@types/tar": "^4.0.5", diff --git a/src/__tests__/client/configuration.test.ts b/src/__tests__/client/configuration.test.ts index 9582995f3d1..4a80c5c312a 100644 --- a/src/__tests__/client/configuration.test.ts +++ b/src/__tests__/client/configuration.test.ts @@ -119,7 +119,7 @@ describe('publish configuration feature', () => { it('should send configuration for specific sections', async () => { let client: LanguageClient let called = false - client = createClient(['coc.preferences', 'npm', 'unknown'], { + client = createClient(['coc.preferences', 'http', 'unknown'], { workspace: { didChangeConfiguration: (sections, next) => { called = true @@ -139,12 +139,12 @@ describe('publish configuration feature', () => { return changed != null }, true) expect(changed.settings.coc).toBeDefined() - expect(changed.settings.npm).toBeDefined() + expect(changed.settings.http).toBeDefined() let { configurations } = workspace - configurations.updateMemoryConfig({ 'npm.binPath': 'cnpm' }) + configurations.updateMemoryConfig({ 'http.proxyStrictSSL': false }) await helper.waitValue(() => { - return changed.settings.npm?.binPath - }, 'cnpm') + return changed.settings.http?.proxyStrictSSL + }, false) await client.stop() }) diff --git a/src/__tests__/client/integration.test.ts b/src/__tests__/client/integration.test.ts index c8e442b4999..b92b6ea22ee 100644 --- a/src/__tests__/client/integration.test.ts +++ b/src/__tests__/client/integration.test.ts @@ -214,8 +214,9 @@ describe('Client events', () => { await client.sendNotification('showDocument', { uri: 'lsptest:///1', takeFocus: false }) await client.sendNotification('showDocument', { uri: uri.toString() }) await client.sendNotification('showDocument', { uri: uri.toString(), selection: Range.create(0, 0, 1, 0) }) - await helper.wait(300) - expect(client.hasPendingResponse).toBe(false) + await helper.waitValue(() => { + return client.hasPendingResponse + }, false) await client.stop() }) @@ -628,7 +629,7 @@ describe('Client integration', () => { } await expect(fn()).rejects.toThrow(Error) spy.mockRestore() - await helper.wait(10) + await helper.wait(30) }) }) @@ -649,9 +650,6 @@ describe('SettingMonitor', () => { return client.state }, lsclient.State.Stopped) helper.updateConfiguration('html.enabled', true) - await helper.waitValue(() => { - return client.state != lsclient.State.Stopped - }, true) await client.onReady() disposable.dispose() }) diff --git a/src/__tests__/helper.ts b/src/__tests__/helper.ts index a754551f66c..7434dd8cb46 100644 --- a/src/__tests__/helper.ts +++ b/src/__tests__/helper.ts @@ -1,3 +1,4 @@ +import net from 'net' /* eslint-disable @typescript-eslint/no-unsafe-return */ /* eslint-disable @typescript-eslint/no-unsafe-call */ import { Buffer, Neovim, Window } from '@chemzqm/neovim' @@ -364,4 +365,24 @@ export function makeLine(length) { return result } +export function getPort(): Promise { + let port = 7080 + let fn = cb => { + let server = net.createServer() + server.listen(port, () => { + server.once('close', () => { + cb(port) + }) + server.close() + }) + server.on('error', () => { + port++ + fn(cb) + }) + } + return new Promise(resolve => { + fn(resolve) + }) +} + export default new Helper() diff --git a/src/__tests__/modules/extensionDependency.test.ts b/src/__tests__/modules/extensionDependency.test.ts new file mode 100644 index 00000000000..d915c3347d4 --- /dev/null +++ b/src/__tests__/modules/extensionDependency.test.ts @@ -0,0 +1,459 @@ +import fs from 'fs' +import http, { Server } from 'http' +import os from 'os' +import path from 'path' +import tar from 'tar' +import { URL } from 'url' +import { v4 as uuid } from 'uuid' +import { checkFileSha1, DependencySession, DependenciesInstaller, DependencyItem, findItem, getModuleInfo, getVersion, readDependencies, shouldRetry, untar, validVersionInfo, VersionInfo } from '../../extension/dependency' +import { Dependencies } from '../../extension/installer' +import { CancellationError } from '../../util/errors' +import { loadJson, remove, writeJson } from '../../util/fs' +import helper, { getPort } from '../helper' + +process.env.NO_PROXY = '*' + +describe('utils', () => { + it('should check valid versionInfo', async () => { + expect(validVersionInfo(null)).toBe(false) + expect(validVersionInfo({ name: 3 })).toBe(false) + expect(validVersionInfo({ name: 'name', version: '', dist: {} })).toBe(false) + expect(validVersionInfo({ + name: 'name', version: '1.0.0', dist: { + tarball: '', + integrity: '', + shasum: '' + } + })).toBe(true) + }) + + it('should checkFileSha1', async () => { + let not_exists = path.join(os.tmpdir(), 'not_exists') + let checked = await checkFileSha1(not_exists, 'shasum') + expect(checked).toBe(false) + let tarfile = path.resolve(__dirname, '../test.tar.gz') + checked = await checkFileSha1(tarfile, 'bf0d88712fc3dbf6e3ab9a6968c0b4232779dbc4') + expect(checked).toBe(true) + // throw on error + let bigfile = path.join(os.tmpdir(), 'bigfile') + let buf = Buffer.allocUnsafe(1024 * 1024) + fs.writeFileSync(bigfile, buf) + let p = checkFileSha1(bigfile, '') + fs.unlinkSync(bigfile) + let res = await p + expect(res).toBe(false) + }) + + it('should untar files', async () => { + let tarfile = path.resolve(__dirname, '../test.tar.gz') + let folder = path.join(os.tmpdir(), `test-${uuid()}`) + await untar(folder, tarfile, 0) + let file = path.join(folder, 'test.js') + expect(fs.existsSync(file)).toBe(true) + await remove(folder) + }) + + it('should throw on untar error', async () => { + let fn = async () => { + let file = path.join(os.tmpdir(), `note_exists_${uuid()}`) + let folder = path.join(os.tmpdir(), `test-${uuid()}`) + await untar(folder, file, 0) + } + await expect(fn()).rejects.toThrow(Error) + }) + + it('should throw when item not found', async () => { + expect(() => { + findItem('name', '^1.0.1', []) + }).toThrow() + }) + + it('should getModuleInfo', () => { + expect(() => { + getModuleInfo('{') + }).toThrow() + expect(() => { + getModuleInfo('{}') + }).toThrow() + expect(() => { + getModuleInfo('{"name": "name"}') + }).toThrow() + let obj: any = { name: 'name', version: '1.0.0', versions: {} } + expect(getModuleInfo(JSON.stringify(obj))).toBeDefined() + obj = { name: 'name', 'dist-tags': { latest: '1.0.0' }, versions: {} } + expect(getModuleInfo(JSON.stringify(obj))).toBeDefined() + obj = { name: 'name', 'dist-tags': { latest: '1.0.0' }, versions: { '0.0.9': {}, '0.0.8': {} } } + let res = getModuleInfo(JSON.stringify(obj)) + expect(res.latest).toBe('0.0.9') + }) + + it('should check retry', () => { + expect(shouldRetry({})).toBe(false) + expect(shouldRetry({ message: 'message' })).toBe(false) + expect(shouldRetry({ message: 'timeout' })).toBe(true) + expect(shouldRetry({ message: 'ECONNRESET' })).toBe(true) + }) + + it('should readDependencies', () => { + let dir = path.join(os.tmpdir(), uuid()) + fs.mkdirSync(dir, { recursive: true }) + let filepath = path.join(dir, 'package.json') + writeJson(filepath, { dependencies: { "coc.nvim": ">= 0.0.80", "is-number": "^1.0.0" } }) + let { dependencies } = readDependencies(dir) + expect(dependencies).toEqual({ 'is-number': '^1.0.0' }) + }) + + it('should getVersion', () => { + expect(getVersion('>= 1.0.0', ['1.0.0', '2.0.0', '2.0.1'], '2.0.1')).toBe('2.0.1') + expect(getVersion('^1.0.0', ['1.0.0', '1.1.0', '2.0.1'])).toBe('1.1.0') + expect(getVersion('^3.0.0', ['1.0.0'])).toBeUndefined() + }) +}) + +describe('DependenciesInstaller', () => { + let httpPort: number + let server: Server + let jsonResponses: Map = new Map() + let url: URL + let dirs: string[] = [] + let createFiles = false + let timer + + beforeAll(async () => { + httpPort = await getPort() + url = new URL(`http://127.0.0.1:${httpPort}`) + server = await createServer(httpPort) + }) + + afterEach(async () => { + jsonResponses.clear() + for (let dir of dirs) { + await remove(dir) + } + dirs = [] + }) + + afterAll(() => { + clearTimeout(timer) + if (server) server.close() + }) + + async function createTarFile(name: string, version: string): Promise { + let folder = path.join(os.tmpdir(), uuid()) + fs.mkdirSync(folder, { recursive: true }) + fs.writeFileSync(path.join(folder, 'index.js'), '', 'utf8') + writeJson(path.join(folder, 'package.json'), { name, version, dependencies: {} }) + let file = path.join(os.tmpdir(), uuid(), `${name}.${version}.tgz`) + fs.mkdirSync(path.dirname(file), { recursive: true }) + await tar.create({ file, gzip: true, cwd: path.dirname(folder) }, [path.basename(folder)]) + return file + } + + async function createServer(port: number): Promise { + return await new Promise(resolve => { + const server = http.createServer(async (req, res) => { + for (let [url, text] of jsonResponses.entries()) { + if (req.url == url) { + res.writeHead(200, { 'Content-Type': 'application/json;charset=utf8' }) + res.end(text) + return + } + } + if (req.url.endsWith('/slow')) { + timer = setTimeout(() => { + res.writeHead(100) + res.end('abc') + }, 300) + return + } + if (req.url.endsWith('.tgz')) { + res.setHeader('Content-Disposition', 'attachment; filename="file.tgz"') + res.setHeader('Content-Type', 'application/octet-stream') + let tarfile: string + if (createFiles) { + let parts = req.url.slice(1).replace(/\.tgz/, '').split('-') + tarfile = await createTarFile(parts[0], parts[1]) + } else { + tarfile = path.resolve(__dirname, '../test.tar.gz') + } + let stat = fs.statSync(tarfile) + res.setHeader('Content-Length', stat.size) + res.writeHead(200) + let stream = fs.createReadStream(tarfile) + stream.pipe(res) + } + }) + server.listen(port, () => { + resolve(server) + }) + }) + } + + function create(root: string | undefined, directory: string, onMessage?: (msg: string) => void): DependenciesInstaller { + if (!root) { + root = path.join(os.tmpdir(), uuid()) + fs.mkdirSync(root) + dirs.push(root) + } + let registry = new URL(`http://127.0.0.1:${httpPort}`) + onMessage = onMessage ?? function() {} + let session = new DependencySession(registry, root) + return session.createInstaller(directory, onMessage) + } + + function createVersion(name: string, version: string, dependencies?: Dependencies): VersionInfo { + return { + name, + version, + dependencies, + dist: { + shasum: '', + integrity: '', + tarball: `http://127.0.0.1:${httpPort}/${name}-${version}.tgz`, + } + } + } + + function addJsonData(): void { + // a => b, c, d + // c => b, d + // b => d + jsonResponses.set('/a', JSON.stringify({ + name: 'a', + versions: { + '0.0.1': createVersion('a', '0.0.1', { b: '^1.0.0', c: '^2.0.0', d: '>= 0.0.1' }) + } + })) + jsonResponses.set('/b', JSON.stringify({ + name: 'b', + versions: { + '1.0.0': createVersion('b', '1.0.0', {}), + '2.0.0': createVersion('b', '2.0.0', { d: '^1.0.0' }), + '3.0.0': createVersion('b', '3.0.0', { d: '^1.0.0' }), + } + })) + jsonResponses.set('/c', JSON.stringify({ + name: 'c', + versions: { + '1.0.0': createVersion('c', '1.0.0', {}), + '2.0.0': createVersion('c', '2.0.0', { b: '^2.0.0', d: '^1.0.0' }), + '3.0.0': createVersion('c', '3.0.0', { b: '^3.0.0', d: '^1.0.0' }), + } + })) + jsonResponses.set('/d', JSON.stringify({ + name: 'd', + versions: { + '1.0.0': createVersion('d', '1.0.0') + } + })) + } + + it('should throw on cancel', async () => { + let root = path.join(os.tmpdir(), uuid()) + fs.mkdirSync(root) + dirs.push(root) + let registry = new URL(`http://127.0.0.1:${httpPort}`) + let session = new DependencySession(registry, root) + let directory = path.join(os.tmpdir(), uuid()) + dirs.push(directory) + writeJson(path.join(directory, 'package.json'), { dependencies: { foo: '>= 0.0.1' } }) + let one = session.createInstaller(directory, () => {}) + let spy = jest.spyOn(one, 'fetchInfos').mockImplementation(() => { + return new Promise((resolve, reject) => { + one.token.onCancellationRequested(() => { + clearTimeout(timer) + reject(new CancellationError()) + }) + let timer = setTimeout(() => { + resolve() + }, 500) + }) + }) + let p = one.installDependencies() + await helper.wait(30) + one.cancel() + let fn = async () => { + await p + } + await expect(fn()).rejects.toThrow(Error) + spy.mockRestore() + }) + + it('should throw when Cancellation requested', async () => { + let install = create(undefined, '') + install.cancel() + let fn = async () => { + await install.fetch(new URL('/', url), { timeout: 10 }, 3) + } + await expect(fn()).rejects.toThrow(CancellationError) + fn = async () => { + await install.download(new URL('/', url), 'filename', '') + } + await expect(fn()).rejects.toThrow(CancellationError) + }) + + it('should retry fetch', async () => { + let install = create(undefined, '') + let fn = async () => { + await install.fetch(new URL('/', url), { timeout: 10 }, 3) + } + await expect(fn()).rejects.toThrow(Error) + jsonResponses.set('/json', '{"result": "ok"}') + let res = await install.fetch(new URL('/json', url), {}, 1) + expect(res).toEqual({ result: 'ok' }) + }) + + it('should cancel request', async () => { + let install = create(undefined, '') + let p = install.fetch(new URL('/slow', url), {}, 1) + await helper.wait(10) + let fn = async () => { + install.cancel() + await p + } + await expect(fn()).rejects.toThrow(Error) + }) + + it('should throw when unable to load info', async () => { + let install = create(undefined, '') + let fn = async () => { + await install.loadInfo(url, 'foo', 10) + } + await expect(fn()).rejects.toThrow(Error) + fn = async () => { + await install.loadInfo(url, 'bar') + } + await expect(fn()).rejects.toThrow(Error) + }) + + it('should fetchInfos', async () => { + addJsonData() + let install = create(undefined, '') + await install.fetchInfos('x', '0.0.1', { a: '^0.0.1' }) + expect(install.resolvedInfos.size).toBe(4) + }) + + it('should linkDependencies', async () => { + addJsonData() + let install = create(undefined, '') + await install.fetchInfos('x', '0.0.1', { a: '^0.0.1' }) + let items: DependencyItem[] = [] + install.linkDependencies(undefined, items) + expect(items).toEqual([]) + install.linkDependencies({ a: '^0.0.1' }, items) + expect(items.length).toBe(5) + }) + + it('should retry download', async () => { + let install = create(undefined, '') + let fn = async () => { + await install.download(new URL('res', url), 'res', '', 3, 10) + } + await expect(fn()).rejects.toThrow(Error) + fn = async () => { + await install.download(new URL('test.tgz', url), 'test.tgz', 'badsum') + } + await expect(fn()).rejects.toThrow(Error) + let res = await install.download(new URL('test.tgz', url), 'test.tgz', '') + expect(fs.existsSync(res)).toBe(true) + fs.unlinkSync(res) + res = await install.download(new URL('test.tgz', url), 'test.tgz', 'bf0d88712fc3dbf6e3ab9a6968c0b4232779dbc4') + expect(fs.existsSync(res)).toBe(true) + fs.unlinkSync(res) + }) + + it('should throw when unable to resolve version', async () => { + let install = create(undefined, '') + expect(() => { + install.resolveVersion('foo', '^1.0.0') + }).toThrow() + install.resolvedInfos.set('foo', { + name: 'foo', + versions: { + '2.0.0': {} as any + } + }) + expect(() => { + install.resolveVersion('foo', '^1.0.0') + }).toThrow() + expect(() => { + install.resolveVersion('foo', '^2.0.0') + }).toThrow() + }) + + it('should check exists and download items', async () => { + let items: DependencyItem[] = [] + items.push({ + integrity: '', + name: 'foo', + resolved: `http://127.0.0.1:${httpPort}/foo.tgz`, + satisfiedVersions: [], + shasum: 'bf0d88712fc3dbf6e3ab9a6968c0b4232779dbc4', + version: '0.0.1' + }) + items.push({ + integrity: '', + name: 'bar', + resolved: `http://127.0.0.1:${httpPort}/bar.tgz`, + satisfiedVersions: ['^0.0.1'], + shasum: 'bf0d88712fc3dbf6e3ab9a6968c0b4232779dbc4', + version: '0.0.2' + }) + let install = create(undefined, '') + let dest = path.join(install.modulesRoot, '.cache') + fs.mkdirSync(dest, { recursive: true }) + let tarfile = path.resolve(__dirname, '../test.tar.gz') + fs.copyFileSync(tarfile, path.join(dest, `foo.0.0.1.tgz`)) + let res = await install.downloadItems(items, 1) + expect(res.size).toBe(2) + }) + + it('should throw on error', async () => { + let items: DependencyItem[] = [] + items.push({ + integrity: '', + name: 'bar', + resolved: `http://127.0.0.1:${httpPort}/bar.tgz`, + satisfiedVersions: [], + shasum: 'badsum', + version: '0.0.2' + }) + let install = create(undefined, '') + let fn = async () => { + await install.downloadItems(items, 2) + } + await expect(fn()).rejects.toThrow(Error) + }) + + it('should no nothing if no dependencies', async () => { + let msg: string + let directory = path.join(os.tmpdir(), uuid()) + let file = path.join(directory, 'package.json') + writeJson(file, { dependencies: {} }) + let install = create(undefined, directory, s => { + msg = s + }) + await install.installDependencies() + expect(msg).toMatch('No dependencies') + fs.rmSync(directory, { recursive: true }) + }) + + it('should install dependencies ', async () => { + createFiles = true + addJsonData() + let directory = path.join(os.tmpdir(), uuid()) + fs.mkdirSync(directory, { recursive: true }) + let file = path.join(directory, 'package.json') + let install = create(undefined, directory) + writeJson(file, { dependencies: { a: '^0.0.1' } }) + await install.installDependencies() + let folder = path.join(directory, 'node_modules') + let res = fs.readdirSync(folder) + expect(res).toEqual(['a', 'b', 'c', 'd']) + let obj = loadJson(path.join(folder, 'b/package.json')) as any + expect(obj.version).toBe('1.0.0') + obj = loadJson(path.join(folder, 'c/node_modules/b/package.json')) as any + expect(obj.version).toBe('2.0.0') + fs.rmSync(directory, { recursive: true }) + }) +}) diff --git a/src/__tests__/modules/extensions.test.ts b/src/__tests__/modules/extensions.test.ts index 6a6ad75b4df..487e414e8dd 100644 --- a/src/__tests__/modules/extensions.test.ts +++ b/src/__tests__/modules/extensions.test.ts @@ -1,10 +1,12 @@ import fs from 'fs' import os from 'os' import path from 'path' +import { URL } from 'url' import { v4 as uuid } from 'uuid' -import which from 'which' +import { CancellationTokenSource } from 'vscode-languageserver-protocol' import commandManager from '../../commands' import extensions, { Extensions, toUrl } from '../../extension' +import { CancellationError } from '../../util/errors' import { writeFile, writeJson } from '../../util/fs' import helper from '../helper' @@ -37,7 +39,8 @@ describe('extensions', () => { expect(extensions.onDidActiveExtension).toBeDefined() expect(extensions.onDidUnloadExtension).toBeDefined() expect(extensions.schemes).toBeDefined() - expect(extensions.creteInstaller('npm', 'id')).toBeDefined() + let res = extensions.createInstaller(new URL('https://github.com'), 'id') + expect(res).toBeDefined() }) it('should get extensions stat', async () => { @@ -94,21 +97,6 @@ describe('extensions', () => { s.mockRestore() }) - it('should use absolute path for npm', async () => { - let res = extensions.npm - expect(path.isAbsolute(res)).toBe(true) - }) - - it('should not throw when npm not found', async () => { - let spy = jest.spyOn(which, 'sync').mockImplementation(() => { - throw new Error('not executable') - }) - let res = extensions.npm - expect(res).toBeNull() - await extensions.updateExtensions() - spy.mockRestore() - }) - it('should get all extensions', () => { let list = extensions.all expect(Array.isArray(list)).toBe(true) @@ -122,13 +110,18 @@ describe('extensions', () => { }) it('should catch error when installExtensions', async () => { - let spy = jest.spyOn(extensions, 'creteInstaller').mockImplementation(() => { + let spy = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { return { on: (_key, cb) => { cb('msg', false) }, install: () => { return Promise.resolve({ name: 'name', url: 'http://e', version: '1.0.0' }) + }, + update: () => { + return '' + }, + dispose: () => { } } as any }) @@ -140,15 +133,51 @@ describe('extensions', () => { s.mockRestore() }) + it('should cancel extension install', async () => { + let called = false + let spy = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { + let tokenSource = new CancellationTokenSource() + let token = tokenSource.token + return { + on: (_key, cb) => { + cb('msg', false) + }, + update: () => { + return Promise.resolve('') + }, + install: () => { + called = true + return new Promise((_resolve, reject) => { + token.onCancellationRequested(() => { + reject(new CancellationError()) + }) + }) + }, + dispose: () => { + tokenSource.cancel() + } + } + }) + let p = extensions.installExtensions(['abc@1.0.0', 'def@2.0.0']) + await helper.waitValue(() => { + return called + }, true) + extensions.cancelInstallers() + await p + spy.mockRestore() + }) + it('should catch error on updateExtensions', async () => { let spy = jest.spyOn(extensions, 'globalExtensionStats').mockImplementation(() => { return [{ id: 'test' }] as any }) - let s = jest.spyOn(extensions, 'creteInstaller').mockImplementation(() => { + let s = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { return { on: () => {}, update: () => { return Promise.resolve(path.join(os.tmpdir(), uuid())) + }, + dispose: () => { } } as any }) @@ -157,11 +186,49 @@ describe('extensions', () => { s.mockRestore() }) + it('should cancel extension update', async () => { + let s = jest.spyOn(extensions, 'globalExtensionStats').mockImplementation(() => { + return [{ id: 'test' }, { id: 'foo' }] as any + }) + let called = false + let spy = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { + let tokenSource = new CancellationTokenSource() + let token = tokenSource.token + return { + on: (_key, cb) => { + cb('msg', false) + }, + update: () => { + called = true + return new Promise((_resolve, reject) => { + token.onCancellationRequested(() => { + reject(new CancellationError()) + }) + }) + }, + install: () => { + return Promise.reject(new CancellationError()) + }, + dispose: () => { + tokenSource.cancel() + } + } + }) + let p = extensions.updateExtensions(true) + await helper.waitValue(() => { + return called + }, true) + extensions.cancelInstallers() + await p + s.mockRestore() + spy.mockRestore() + }) + it('should update enabled extensions', async () => { let spy = jest.spyOn(extensions, 'globalExtensionStats').mockImplementation(() => { return [{ id: 'test' }, { id: 'global', isLocked: true }, { id: 'disabled', state: 'disabled' }] as any }) - let s = jest.spyOn(extensions, 'creteInstaller').mockImplementation(() => { + let s = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { return { on: (_key, cb) => { cb('msg', false) @@ -169,6 +236,8 @@ describe('extensions', () => { update: async () => { await helper.wait(1) return '' + }, + dispose: () => { } } as any }) @@ -182,7 +251,7 @@ describe('extensions', () => { return [{ id: 'test', exotic: true, uri: 'http://example.com' }] as any }) let called = false - let s = jest.spyOn(extensions, 'creteInstaller').mockImplementation(() => { + let s = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { return { on: (_key, cb) => { cb('msg', false) @@ -192,6 +261,8 @@ describe('extensions', () => { called = true expect(url).toBe('http://example.com') return '' + }, + dispose: () => { } } as any }) @@ -207,14 +278,17 @@ describe('extensions', () => { let link = path.join(extensions.modulesFolder, 'test-link') fs.mkdirSync(folder, { recursive: true }) fs.symlinkSync(folder, link) + let cacheFolder = path.join(extensions.modulesFolder, '.cache') + fs.mkdirSync(cacheFolder, { recursive: true }) extensions.cleanModulesFolder() expect(fs.existsSync(folder)).toBe(false) expect(fs.existsSync(link)).toBe(false) + expect(fs.existsSync(cacheFolder)).toBe(true) }) it('should install global extension', async () => { let folder = path.join(extensions.modulesFolder, 'coc-omni') - let spy = jest.spyOn(extensions, 'creteInstaller').mockImplementation(() => { + let spy = jest.spyOn(extensions, 'createInstaller').mockImplementation(() => { return { on: () => {}, install: async () => { @@ -223,6 +297,8 @@ describe('extensions', () => { await writeFile(file, JSON.stringify({ name: 'coc-omni', engines: { coc: '>=0.0.1' }, version: '0.0.1' }, null, 2)) await writeFile(path.join(folder, 'index.js'), 'exports.activate = () => {}') return { name: 'coc-omni', version: '1.0.0', folder } + }, + dispose: () => { } } as any }) diff --git a/src/__tests__/modules/installer.test.ts b/src/__tests__/modules/installer.test.ts index 41ca3aa4cee..32e0ed4b60a 100644 --- a/src/__tests__/modules/installer.test.ts +++ b/src/__tests__/modules/installer.test.ts @@ -2,8 +2,10 @@ import fs from 'fs' import os from 'os' import path from 'path' import { v4 as uuid } from 'uuid' -import { getDependencies, Info, Installer, isNpmCommand, isYarn, registryUrl } from '../../extension/installer' +import { Info, Installer, registryUrl } from '../../extension/installer' +import { DependencySession } from '../../extension/dependency' import { remove } from '../../util/fs' +import { URL } from 'url' const rcfile = path.join(os.tmpdir(), '.npmrc') let tmpfolder: string @@ -13,37 +15,41 @@ afterEach(() => { } }) -describe('utils', () => { - it('should getDependencies', async () => { - expect(getDependencies({})).toEqual([]) - expect(getDependencies({ dependencies: { 'coc.nvim': '0.0.1' } })).toEqual([]) - }) +function createSession(root: string, registry?: string): DependencySession { + return new DependencySession(new URL(registry ?? 'https://registry.npmjs.org/'), root) +} - it('should check command is npm or yarn', async () => { - expect(isNpmCommand('npm')).toBe(true) - expect(isYarn('yarnpkg')).toBe(true) - }) +describe('utils', () => { it('should get registry url', async () => { + let url = await registryUrl(os.tmpdir(), [], 50) + expect(url).toBeDefined() const getUrl = () => { return registryUrl(os.tmpdir()) } fs.rmSync(rcfile, { force: true, recursive: true }) - expect(getUrl().toString()).toBe('https://registry.npmjs.org/') + async function check(res: string): Promise { + let url = await registryUrl(os.tmpdir(), [new URL('https://registry.npmjs.org')], 10) + expect(url.toString()).toBe(res) + } + await check('https://registry.npmjs.org/') fs.writeFileSync(rcfile, '', 'utf8') - expect(getUrl().toString()).toBe('https://registry.npmjs.org/') + await check('https://registry.npmjs.org/') fs.writeFileSync(rcfile, 'coc.nvim:registry=https://example.org', 'utf8') - expect(getUrl().toString()).toBe('https://example.org/') + await check('https://example.org/') fs.writeFileSync(rcfile, '#coc.nvim:registry=https://example.org', 'utf8') - expect(getUrl().toString()).toBe('https://registry.npmjs.org/') + await check('https://registry.npmjs.org/') fs.writeFileSync(rcfile, 'coc.nvim:registry=example.org', 'utf8') - expect(getUrl().toString()).toBe('https://registry.npmjs.org/') + url = await getUrl() + expect(url).toBeDefined() + url = await registryUrl(os.tmpdir(), [new URL('https://127.0.0.1')], 50) + expect(url).toBeDefined() fs.rmSync(rcfile, { force: true, recursive: true }) }) it('should parse name & version', async () => { const getInfo = (def: string): { name?: string, version?: string } => { - let installer = new Installer(__dirname, 'npm', def) + let installer = new Installer(createSession(__dirname), def) return installer.info } expect(getInfo('https://github.com')).toEqual({ name: undefined, version: undefined }) @@ -56,7 +62,7 @@ describe('utils', () => { describe('Installer', () => { describe('fetch() & download()', () => { it('should throw with invalid url', async () => { - let installer = new Installer(__dirname, 'npm', 'foo') + let installer = new Installer(createSession(__dirname), 'foo') let fn = async () => { await installer.fetch('url') } @@ -69,15 +75,8 @@ describe('Installer', () => { }) describe('getInfo()', () => { - it('should get install arguments', async () => { - let installer = new Installer(__dirname, 'npm', 'https://github.com/') - expect(installer.getInstallArguments('pnpm', 'https://github.com/')).toEqual(['install', '--production', '--config.strict-peer-dependencies=false']) - expect(installer.getInstallArguments('npm', '')).toEqual(['install', '--ignore-scripts', '--no-lockfile', '--omit=dev', '--legacy-peer-deps', '--no-global']) - expect(installer.getInstallArguments('yarn', '')).toEqual(['install', '--ignore-scripts', '--no-lockfile', '--production', '--ignore-engines']) - }) - it('should getInfo from url', async () => { - let installer = new Installer(__dirname, 'npm', 'https://github.com/') + let installer = new Installer(createSession(__dirname), 'https://github.com/') let spy = jest.spyOn(installer, 'getInfoFromUri').mockImplementation(() => { return Promise.resolve({ name: 'vue-vscode-snippets', version: '1.0.0' }) }) @@ -87,7 +86,7 @@ describe('Installer', () => { }) it('should use latest version', async () => { - let installer = new Installer(__dirname, 'npm', 'coc-omni') + let installer = new Installer(createSession(__dirname), 'coc-omni') let spy = jest.spyOn(installer, 'fetch').mockImplementation(url => { expect(url.toString()).toMatch('coc-omni') return Promise.resolve(JSON.stringify({ @@ -108,7 +107,7 @@ describe('Installer', () => { }) it('should throw when version not found', async () => { - let installer = new Installer(__dirname, 'npm', 'coc-omni@1.0.2') + let installer = new Installer(createSession(__dirname), 'coc-omni@1.0.2') let spy = jest.spyOn(installer, 'fetch').mockImplementation(() => { return Promise.resolve(JSON.stringify({ name: 'coc-omni', @@ -130,7 +129,7 @@ describe('Installer', () => { }) it('should throw when not coc.nvim extension', async () => { - let installer = new Installer(__dirname, 'npm', 'coc-omni') + let installer = new Installer(createSession(__dirname), 'coc-omni') let spy = jest.spyOn(installer, 'fetch').mockImplementation(() => { return Promise.resolve(JSON.stringify({ name: 'coc-omni', @@ -153,7 +152,7 @@ describe('Installer', () => { describe('getInfoFromUri()', () => { it('should throw for url that not supported', async () => { - let installer = new Installer(__dirname, 'npm', 'https://example.com') + let installer = new Installer(createSession(__dirname), 'https://example.com') let fn = async () => { await installer.getInfoFromUri() } @@ -161,7 +160,7 @@ describe('Installer', () => { }) it('should get info from url #1', async () => { - let installer = new Installer(__dirname, 'npm', 'https://github.com/sdras/vue-vscode-snippets') + let installer = new Installer(createSession(__dirname), 'https://github.com/sdras/vue-vscode-snippets') let spy = jest.spyOn(installer, 'fetch').mockImplementation(() => { return Promise.resolve(JSON.stringify({ name: 'vue-vscode-snippets', version: '1.0.0' })) }) @@ -171,7 +170,7 @@ describe('Installer', () => { }) it('should get info from url #2', async () => { - let installer = new Installer(__dirname, 'npm', 'https://github.com/sdras/vue-vscode-snippets@main') + let installer = new Installer(createSession(__dirname), 'https://github.com/sdras/vue-vscode-snippets@main') let spy = jest.spyOn(installer, 'fetch').mockImplementation(() => { return Promise.resolve({ name: 'vue-vscode-snippets', version: '1.0.0', engines: { coc: '>=0.0.1' } }) }) @@ -189,7 +188,7 @@ describe('Installer', () => { fs.unlinkSync(tmpfolder) } fs.symlinkSync(__dirname, tmpfolder, 'dir') - let installer = new Installer(os.tmpdir(), 'npm', 'foo') + let installer = new Installer(createSession(os.tmpdir()), 'foo') let res = await installer.doInstall({ name: 'foo' }) expect(res).toBe(false) let val = await installer.update() @@ -198,7 +197,7 @@ describe('Installer', () => { it('should update from url', async () => { let url = 'https://github.com/sdras/vue-vscode-snippets@main' - let installer = new Installer(__dirname, 'npm', url) + let installer = new Installer(createSession(__dirname), url) let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ version: '1.0.0', name: 'vue-vscode-snippets' }) }) @@ -213,7 +212,7 @@ describe('Installer', () => { it('should skip update when current version is latest', async () => { tmpfolder = path.join(os.tmpdir(), 'coc-pairs') - let installer = new Installer(os.tmpdir(), 'npm', 'coc-pairs') + let installer = new Installer(createSession(os.tmpdir()), 'coc-pairs') let version = '1.0.0' let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ version }) @@ -228,7 +227,7 @@ describe('Installer', () => { it('should skip update when version not satisfies', async () => { tmpfolder = path.join(os.tmpdir(), 'coc-pairs') - let installer = new Installer(os.tmpdir(), 'npm', 'coc-pairs') + let installer = new Installer(createSession(os.tmpdir()), 'coc-pairs') let version = '2.0.0' let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ version, 'engines.coc': '>=99.0.0' }) @@ -244,7 +243,7 @@ describe('Installer', () => { it('should return undefined when update not performed', async () => { tmpfolder = path.join(os.tmpdir(), 'coc-pairs') - let installer = new Installer(os.tmpdir(), 'npm', 'coc-pairs') + let installer = new Installer(createSession(os.tmpdir()), 'coc-pairs') let version = '2.0.0' let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ version }) @@ -262,7 +261,7 @@ describe('Installer', () => { it('should update extension', async () => { tmpfolder = path.join(os.tmpdir(), 'coc-pairs') - let installer = new Installer(os.tmpdir(), 'npm', 'coc-pairs') + let installer = new Installer(createSession(os.tmpdir()), 'coc-pairs') let version = '2.0.0' let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ version, name: 'coc-pairs' }) @@ -282,7 +281,7 @@ describe('Installer', () => { describe('install()', () => { it('should throw when version not match required', async () => { - let installer = new Installer(__dirname, 'npm', 'coc-omni') + let installer = new Installer(createSession(__dirname), 'coc-pairs') let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ name: 'coc-omni', @@ -299,7 +298,7 @@ describe('Installer', () => { }) it('should return install info', async () => { - let installer = new Installer(__dirname, 'npm', 'coc-omni') + let installer = new Installer(createSession(__dirname), 'coc-omni') let spy = jest.spyOn(installer, 'getInfo').mockImplementation(() => { return Promise.resolve({ name: 'coc-omni', @@ -319,7 +318,7 @@ describe('Installer', () => { it('should throw and remove folder when download failed', async () => { tmpfolder = path.join(os.tmpdir(), uuid()) - let installer = new Installer(tmpfolder, 'npm', 'coc-omni') + let installer = new Installer(createSession(tmpfolder), 'coc-omni') let folder: string let option: any let spy = jest.spyOn(installer, 'download').mockImplementation((_url, opt) => { @@ -341,7 +340,7 @@ describe('Installer', () => { it('should revert folder when download failed', async () => { tmpfolder = path.join(os.tmpdir(), uuid()) - let installer = new Installer(tmpfolder, 'npm', 'coc-omni') + let installer = new Installer(createSession(tmpfolder), 'coc-omni') let f = path.join(tmpfolder, 'coc-omni') fs.mkdirSync(f, { recursive: true }) fs.writeFileSync(path.join(f, 'package.json'), '{}', 'utf8') @@ -360,7 +359,7 @@ describe('Installer', () => { it('should install new extension', async () => { tmpfolder = path.join(os.tmpdir(), uuid()) - let installer = new Installer(tmpfolder, 'npm', 'coc-omni') + let installer = new Installer(createSession(tmpfolder), 'coc-omni') let f = path.join(tmpfolder, 'coc-omni') let spy = jest.spyOn(installer, 'download').mockImplementation((_url, option) => { if (option.onProgress) { @@ -381,7 +380,7 @@ describe('Installer', () => { it('should install new version', async () => { tmpfolder = path.join(os.tmpdir(), uuid()) - let installer = new Installer(tmpfolder, 'npm', 'coc-omni') + let installer = new Installer(createSession(tmpfolder), 'coc-omni') let f = path.join(tmpfolder, 'coc-omni') fs.mkdirSync(f, { recursive: true }) fs.writeFileSync(path.join(f, 'package.json'), '{}', 'utf8') @@ -403,31 +402,15 @@ describe('Installer', () => { }) it('should install dependencies', async () => { - let npm = path.resolve(__dirname, '../npm') tmpfolder = path.join(os.tmpdir(), uuid()) fs.mkdirSync(tmpfolder) - let installer = new Installer(tmpfolder, npm, 'coc-omni') + let installer = new Installer(createSession(tmpfolder), 'coc-omni') let called = false installer.on('message', () => { called = true }) - await installer.installDependencies(tmpfolder, ['a', 'b']) + await installer.installDependencies(tmpfolder) expect(called).toBe(true) }) - - it('should reject on install error', async () => { - let npm = path.resolve(__dirname, '../npm') - tmpfolder = path.join(os.tmpdir(), uuid()) - fs.mkdirSync(tmpfolder) - let installer = new Installer(tmpfolder, npm, 'coc-omni') - let spy = jest.spyOn(installer, 'getInstallArguments').mockImplementation(() => { - return ['--error'] - }) - let fn = async () => { - await installer.installDependencies(tmpfolder, ['a', 'b']) - } - await expect(fn()).rejects.toThrow(Error) - spy.mockRestore() - }) }) }) diff --git a/src/__tests__/modules/util.test.ts b/src/__tests__/modules/util.test.ts index 56d90d8dc82..3860ec9d080 100644 --- a/src/__tests__/modules/util.test.ts +++ b/src/__tests__/modules/util.test.ts @@ -1,10 +1,12 @@ import style from 'ansi-styles' import * as assert from 'assert' import { spawn } from 'child_process' +import fs from 'fs' import os from 'os' import path from 'path' import vm from 'vm' import { CancellationTokenSource, Color, Position, Range, SymbolKind, TextDocumentEdit, TextEdit, WorkspaceEdit } from 'vscode-languageserver-protocol' +import which from 'which' import { LinesTextDocument } from '../../model/textdocument' import { ConfigurationScope } from '../../types' import { concurrent, delay, disposeAll, wait } from '../../util' @@ -24,6 +26,7 @@ import { Extensions, IJSONContributionRegistry } from '../../util/jsonRegistry' import * as lodash from '../../util/lodash' import { Mutex } from '../../util/mutex' import * as objects from '../../util/object' +import * as ping from '../../util/ping' import * as platform from '../../util/platform' import * as positions from '../../util/position' import { executable, isRunning, runCommand, terminate } from '../../util/processes' @@ -1347,4 +1350,36 @@ describe('diff', () => { }, token) }) }) + + describe('ping', () => { + it('should get ping config', async () => { + let check = (platform: NodeJS.Platform) => { + let res = ping.getPing(platform) + if (res) { + expect(fs.existsSync(res.bin)).toBe(true) + } + } + check('darwin') + check('linux') + check('win32') + check('android') + check('freebsd') + let spy = jest.spyOn(which, 'sync').mockImplementation(() => { + throw Error('not found') + }) + check('freebsd') + spy.mockRestore() + }) + + it('should find best host', async () => { + let res = await ping.findBestHost([], 500) + expect(res).toBeUndefined() + res = await ping.findBestHost(['www.baidu.com', 'www.google.com'], 1) + expect(res).toBeUndefined() + res = await ping.findBestHost(['www.baidu.com', 'www.google.com'], 500, 'not_exists_bin') + expect(res).toBeUndefined() + res = await ping.findBestHost(['127.0.0.1', 'registry.npmjs.org', 'registry.yarnpkg.com', 'registry.npmmirror.com'], 500) + expect(res).toBeDefined() + }) + }) }) diff --git a/src/__tests__/tree/treeView.test.ts b/src/__tests__/tree/treeView.test.ts index b1965ca41cb..842eb5e3941 100644 --- a/src/__tests__/tree/treeView.test.ts +++ b/src/__tests__/tree/treeView.test.ts @@ -295,7 +295,7 @@ describe('TreeView', () => { createTreeView(defaultDef, { canSelectMany: true }) await treeView.show() await events.race(['TextChanged']) - let selection: TreeNode[] + let selection: TreeNode[] = [] treeView.onDidChangeSelection(e => { selection = e.selection }) diff --git a/src/extension/dependency.ts b/src/extension/dependency.ts new file mode 100644 index 00000000000..97bdf249aac --- /dev/null +++ b/src/extension/dependency.ts @@ -0,0 +1,448 @@ +import bytes from 'bytes' +import { createHash } from 'crypto' +import fs, { createReadStream } from 'fs' +import path from 'path' +import semver from 'semver' +import tar from 'tar' +import { URL } from 'url' +import { promisify } from 'util' +import { CancellationToken, CancellationTokenSource } from 'vscode-languageserver-protocol' +import download from '../model/download' +import fetch, { FetchOptions } from '../model/fetch' +import { CancellationError } from '../util/errors' +import { loadJson, writeJson } from '../util/fs' +import { objectLiteral } from '../util/is' +import { Mutex } from '../util/mutex' +import { toObject } from '../util/object' +const logger = require('../util/logger')('extension-dependency') + +export interface Dependencies { [key: string]: string } + +// The information we cares about +export interface VersionInfo { + name: string + version: string + dependencies?: Dependencies + optionalDependencies?: Dependencies + dist: { + integrity: string // starts with sha512-, base64 string + shasum: string // sha1 hash, hex string + tarball: string // download url + } +} + +interface ModuleInfo { + name: string + latest?: string + versions: { + [version: string]: VersionInfo + } +} + +export interface DependencyItem { + name: string + version: string + resolved: string // download url + shasum: string + integrity: string + satisfiedVersions: string[] + dependencies?: { + [key: string]: string + } +} + +const DEV_DEPENDENCIES = ['coc.nvim', 'webpack', 'esbuild'] +const INFO_TIMEOUT = global.__TEST__ ? 100 : 20000 +const DOWNLOAD_TIMEOUT = global.__TEST__ ? 500 : 3 * 60 * 1000 + +function toFilename(item: DependencyItem): string { + return `${item.name}.${item.version}.tgz` +} + +export function findItem(name: string, requirement: string, items: ReadonlyArray): DependencyItem { + let item = items.find(o => o.name === name && o.satisfiedVersions.includes(requirement)) + if (!item) throw new Error(`item not found for: ${name} ${requirement}`) + return item +} + +export function validVersionInfo(info: any): info is VersionInfo { + if (!info) return false + if (typeof info.name !== 'string' || typeof info.version !== 'string' || !info.dist) return false + let { tarball, integrity, shasum } = info.dist + if (typeof tarball !== 'string' || typeof integrity !== 'string' || typeof shasum !== 'string') return false + return true +} + +/** + * Get required info form json text. + */ +export function getModuleInfo(text: string): ModuleInfo { + let obj + try { + obj = JSON.parse(text) as any + } catch (e) { + throw new Error(`Invalid JSON data, ${e}`) + } + if (typeof obj.name !== 'string' || !objectLiteral(obj.versions)) throw new Error(`Invalid JSON data, name or versions not found`) + let versions = Object.keys(obj.versions) + let latest = obj['dist-tags']?.latest + if (latest && versions[latest] == null) latest = semver.rsort(versions)[0] + return { + name: obj.name, + latest, + versions: obj.versions + } as ModuleInfo +} + +export function shouldRetry(error: any): boolean { + let message = error.message + if (typeof message !== 'string') return false + if (message.includes('timeout') || + message.includes('Invalid JSON') || + message.includes('Bad shasum') || + message.includes('ECONNRESET')) return true + return false +} + +function checkDeps(obj: any): void { + if (obj.optionalDependencies) { + console.log(`${obj.name} have optionalDependencies: ${JSON.stringify(obj.optionalDependencies)}`) + } + let dependencies = toObject(obj.dependencies) as { [key: string]: string } + let keys: string[] = [] + for (let [key, value] of Object.entries(dependencies)) { + if (!semver.validRange(value)) { + console.log(`${obj.name} have invalid dependency version: ${key}:${value}`) + keys.push(key) + } + } + keys.forEach(key => { + delete dependencies[key] + }) +} + +/** + * Production dependencies in directory + */ +export function readDependencies(directory: string): { name: string, version: string, dependencies: Dependencies } { + let jsonfile = path.join(directory, 'package.json') + let obj = loadJson(jsonfile) as any + let dependencies = obj.dependencies as { [key: string]: string } + for (let key of Object.keys(toObject(dependencies))) { + if (DEV_DEPENDENCIES.includes(key) || key.startsWith('@types/')) delete dependencies[key] + } + return { name: obj.name, version: obj.version, dependencies } +} + +export function getVersion(requirement: string, versions: string[], latest?: string): string | undefined { + if (latest && semver.satisfies(latest, requirement)) return latest + let sorted = semver.rsort(versions.filter(v => semver.valid(v, { includePrerelease: false }))) + for (let v of sorted) { + if (semver.satisfies(v, requirement)) return v + } +} + +/** + * Extract tarfile to dest folder, with strip option. + */ +export async function untar(dest: string, tarfile: string, strip = 1): Promise { + if (!fs.existsSync(tarfile)) throw new Error(`${tarfile} not exists`) + await promisify(fs.rm)(dest, { recursive: true, force: true }) + await promisify(fs.mkdir)(dest, { recursive: true }) + await new Promise((resolve, reject) => { + const input = createReadStream(tarfile) + input.on('error', reject) + let stream = tar.x({ strip, C: dest }) + input.pipe(stream) + stream.on('error', reject) + stream.on('finish', () => { + resolve() + }) + }) +} + +export async function checkFileSha1(filepath: string, shasum: string): Promise { + const hash = createHash('sha1') + if (!fs.existsSync(filepath)) return Promise.resolve(false) + return new Promise(resolve => { + const input = createReadStream(filepath) + input.on('error', e => { + resolve(false) + }) + input.on('readable', () => { + // Only one element is going to be produced by the hash stream. + const data = input.read() + if (data) + hash.update(data) + else { + resolve(hash.digest('hex') == shasum) + } + }) + }) +} + +const mutex = new Mutex() + +export class DependenciesInstaller { + private tokenSource: CancellationTokenSource = new CancellationTokenSource() + private fetched: string[] = [] + constructor( + private registry: URL, + public readonly resolvedInfos: Map, + public readonly modulesRoot: string, + private directory: string, + private onMessage: (msg: string) => void + ) { + } + + public get token(): CancellationToken { + return this.tokenSource.token + } + + private get dest(): string { + return path.join(this.modulesRoot, '.cache') + } + + public async installDependencies(): Promise { + let { directory, tokenSource } = this + let { dependencies, name, version } = readDependencies(directory) + // no need to install + if (!dependencies || Object.keys(dependencies).length == 0) { + this.onMessage(`No dependencies`) + return + } + this.onMessage('Waiting for install dependencies.') + let token = tokenSource.token + // TODO reuse resolved.json + await mutex.use(async () => { + if (token.isCancellationRequested) throw new CancellationError() + this.onMessage('Resolving dependencies.') + await this.fetchInfos(name, version, dependencies) + this.onMessage('Linking dependencies.') + // create DependencyItems + let items: DependencyItem[] = [] + this.linkDependencies(dependencies, items) + let filepath = path.join(directory, 'resolved.json') + writeJson(filepath, items) + this.onMessage(`Downloading dependencies to ${this.dest}.`) + await this.downloadItems(items, 3) + this.onMessage('Extract modules.') + await this.extractDependencies(items, dependencies, directory) + this.onMessage('Done') + }) + } + + public async extractDependencies(items: DependencyItem[], dependencies: Dependencies, directory: string): Promise { + items.sort((a, b) => b.satisfiedVersions.length - a.satisfiedVersions.length) + let rootPackages: Set = new Set() + let rootItems: DependencyItem[] = [] + const addToRoot = (item: DependencyItem) => { + if (!rootPackages.has(item.name)) { + rootPackages.add(item.name) + rootItems.push(item) + } + } + // Top level dependencies + for (let [key, requirement] of Object.entries(dependencies)) { + let item = findItem(key, requirement, items) + addToRoot(item) + } + items.forEach(item => { + addToRoot(item) + }) + rootPackages.clear() + for (let item of rootItems) { + let filename = toFilename(item) + let tarfile = path.join(this.dest, filename) + let dest = path.join(directory, 'node_modules', item.name) + await untar(dest, tarfile) + } + for (let item of rootItems) { + let folder = path.join(directory, 'node_modules', item.name) + await this.extractFor(item, items, rootItems, folder) + } + } + + /** + * Recursive extract dependencies for item in folder + */ + public async extractFor(item: DependencyItem, items: ReadonlyArray, rootItems: ReadonlyArray, folder: string): Promise { + // Deps to install, name to item + let deps: Map = new Map() + for (let [name, requirement] of Object.entries(item.dependencies ?? {})) { + let idx = rootItems.findIndex(o => o.name == name && o.satisfiedVersions.includes(requirement)) + if (idx == -1) deps.set(name, findItem(name, requirement, items)) + } + if (deps.size === 0) return + let newRoot: DependencyItem[] = [] + await Promise.all(Array.from(deps.values()).map(item => { + let tarfile = path.join(this.dest, toFilename(item)) + let dest = path.join(folder, 'node_modules', item.name) + newRoot.push(item) + return untar(dest, tarfile) + })) + newRoot.push(...rootItems) + for (let item of deps.values()) { + let dest = path.join(folder, 'node_modules', item.name) + await this.extractFor(item, items, newRoot, dest) + } + } + + public linkDependencies(dependencies: Dependencies | undefined, items: DependencyItem[]): void { + if (!dependencies) return + for (let [name, requirement] of Object.entries(dependencies)) { + let versionInfo = this.resolveVersion(name, requirement) + let item = items.find(o => o.name === name && o.version === versionInfo.version) + if (item) { + if (!item.satisfiedVersions.includes(requirement)) item.satisfiedVersions.push(requirement) + } else { + let { dist, version } = versionInfo + items.push({ + name, + version, + resolved: dist.tarball, + shasum: dist.shasum, + integrity: dist.integrity, + satisfiedVersions: [requirement], + dependencies: versionInfo.dependencies + }) + this.linkDependencies(versionInfo.dependencies, items) + } + } + } + + public resolveVersion(name: string, requirement: string): VersionInfo { + if (!semver.validRange(requirement)) throw new Error(`Unsupported version range "${requirement}" of "${name}"`) + let info = this.resolvedInfos.get(name) + if (info) { + let version = getVersion(requirement, Object.keys(info.versions), info.latest) + if (version) { + let versionInfo = info.versions[version] + if (validVersionInfo(versionInfo)) return versionInfo + } + } + throw new Error(`No valid version found for "${name}" ${requirement}`) + } + + /** + * Recursive fetch module info + */ + public async fetchInfos(name: string, version: string, dependencies: Dependencies | undefined): Promise { + let keys = Object.keys(dependencies ?? {}) + if (keys.length === 0) return + let id = `${name}-${version}` + if (this.fetched.includes(id)) return + this.fetched.push(id) + await Promise.all(keys.map(key => { + if (this.resolvedInfos.has(key)) return Promise.resolve() + return this.loadInfo(this.registry, key, INFO_TIMEOUT).then(info => { + this.resolvedInfos.set(key, info) + }) + })) + for (let key of keys) { + let versionInfo = this.resolveVersion(key, dependencies[key]) + await this.fetchInfos(versionInfo.name, versionInfo.version, versionInfo.dependencies) + } + } + + /** + * Concurrent download necessary dependencies + */ + public async downloadItems(items: DependencyItem[], retry: number): Promise> { + let res: Map = new Map() + let total = items.length + let finished = 0 + for (let item of items) { + let ts = Date.now() + let filename = toFilename(item) + let filepath = path.join(this.dest, filename) + let checked = await checkFileSha1(filepath, item.shasum) + let onFinish = () => { + res.set(filename, filepath) + finished++ + if (checked) { + this.onMessage(`File ${filename} already exists [${finished}/${total}]`) + } else { + let stat = fs.statSync(filepath) + this.onMessage(`Downloaded ${filename} (${bytes(stat.size)}) cost ${Date.now() - ts}ms [${finished}/${total}]`) + } + } + if (checked) { + onFinish() + } else { + await this.download(new URL(item.resolved), filename, item.shasum, retry, DOWNLOAD_TIMEOUT) + onFinish() + } + } + return res + } + + /** + * Fetch module info + */ + public async loadInfo(registry: URL, name: string, timeout = 100): Promise { + let buffer = await this.fetch(new URL(name, registry), { timeout, buffer: true }, 3) as Buffer + return getModuleInfo(buffer.toString()) + } + + public async fetch(url: string | URL, options: FetchOptions, retry = 1): Promise { + for (let i = 0; i < retry; i++) { + if (this.tokenSource.token.isCancellationRequested) throw new CancellationError() + try { + return await fetch(url, options, this.tokenSource.token) + } catch (e) { + if (i == retry - 1 || !shouldRetry(e)) { + throw e + } else { + this.onMessage(`Network issue, retry fetch for ${url}`) + } + } + } + } + + /** + * Download tgz file with sha1 check. + */ + public async download(url: string | URL, filename: string, shasum: string, retry = 1, timeout?: number): Promise { + if (this.tokenSource.token.isCancellationRequested) throw new CancellationError() + for (let i = 0; i < retry; i++) { + try { + let fullpath = path.join(this.dest, filename) + await download(url, { + dest: this.dest, + filename, + extract: false, + timeout + }, this.tokenSource.token) + if (shasum) { + let checked = await checkFileSha1(fullpath, shasum) + if (!checked) throw new Error(`Bad shasum for ${filename}`) + } + return fullpath + } catch (e) { + if (i == retry - 1 || !shouldRetry(e)) { + throw e + } else { + this.onMessage(`Network issue, retry download for ${url}`) + } + } + } + } + + public cancel(): void { + this.tokenSource.cancel() + } +} + +export class DependencySession { + private resolvedInfos: Map = new Map() + constructor( + public readonly registry: URL, + public readonly modulesRoot: string + ) { + } + + public createInstaller(directory: string, onMessage: (msg: string) => void): DependenciesInstaller { + return new DependenciesInstaller(this.registry, this.resolvedInfos, this.modulesRoot, directory, onMessage) + } +} diff --git a/src/extension/index.ts b/src/extension/index.ts index 9514f8576ed..becc121492f 100644 --- a/src/extension/index.ts +++ b/src/extension/index.ts @@ -1,18 +1,20 @@ 'use strict' import fs from 'fs' import path from 'path' -import { Event } from 'vscode-languageserver-protocol' -import which from 'which' +import { URL } from 'url' +import { CancellationTokenSource, Disposable, Event } from 'vscode-languageserver-protocol' import commandManager from '../commands' +import events from '../events' import type { OutputChannel } from '../types' -import { concurrent } from '../util' +import { concurrent, disposeAll } from '../util' import { distinct } from '../util/array' +import { isCancellationError } from '../util/errors' import '../util/extensions' import { isUrl } from '../util/is' -import { executable } from '../util/processes' import window from '../window' import workspace from '../workspace' -import { IInstaller, Installer } from './installer' +import { DependencySession } from './dependency' +import { IInstaller, Installer, registryUrl } from './installer' import { API, Extension, ExtensionInfo, ExtensionItem, ExtensionManager, ExtensionState } from './manager' import { checkExtensionRoot, ExtensionStat, loadExtensionJson } from './stat' import { InstallBuffer, InstallChannel, InstallUI } from './ui' @@ -37,10 +39,18 @@ export class Extensions { public readonly manager: ExtensionManager public readonly states: ExtensionStat public readonly modulesFolder = path.join(EXTENSIONS_FOLDER, 'node_modules') + private disposables: Disposable[] = [] constructor() { checkExtensionRoot(EXTENSIONS_FOLDER) this.states = new ExtensionStat(EXTENSIONS_FOLDER) this.manager = new ExtensionManager(this.states, EXTENSIONS_FOLDER) + events.on('VimLeavePre', () => { + this.cancelInstallers() + }) + } + + public cancelInstallers() { + disposeAll(this.disposables) } public async init(): Promise { @@ -139,40 +149,52 @@ export class Extensions { return await this.manager.call(id, method, args) } - public get npm(): string { - let npm = workspace.getConfiguration('npm', null).get('binPath', 'npm') - npm = workspace.expand(npm) - for (let exe of [npm, 'yarnpkg', 'yarn', 'npm']) { - if (executable(exe)) return which.sync(exe) - } - void window.showErrorMessage(`Can't find npm or yarn in your $PATH`) - return null - } - - private createInstallerUI(isUpdate: boolean, silent: boolean): InstallUI { - return silent ? new InstallChannel(isUpdate, this.outputChannel) : new InstallBuffer(isUpdate) + private createInstallerUI(isUpdate: boolean, silent: boolean, disposables: Disposable[]): InstallUI { + return silent ? new InstallChannel(isUpdate, this.outputChannel) : new InstallBuffer(isUpdate, async () => { + if (disposables.length > 0) { + disposeAll(disposables) + void window.showWarningMessage(`Extension install process canceled`) + } + }) } - public creteInstaller(npm: string, def: string): IInstaller { - return new Installer(this.modulesFolder, npm, def) + public createInstaller(registry: URL, def: string): IInstaller { + return new Installer(new DependencySession(registry, this.modulesFolder), def) } /** * Install extensions, can be called without initialize. */ public async installExtensions(list: string[]): Promise { - let { npm } = this - if (!npm || list.length == 0) return + if (list.length == 0) return + this.cancelInstallers() list = distinct(list) - let installBuffer = this.createInstallerUI(false, false) + let disposables: Disposable[] = this.disposables = [] + let installBuffer = this.createInstallerUI(false, false, disposables) + let tokenSource = new CancellationTokenSource() + let installers: Map = new Map() + installBuffer.onDidCancel(key => { + let item = installers.get(key) + if (item) item.dispose() + }) + disposables.push(Disposable.create(() => { + tokenSource.cancel() + for (let item of installers.values()) { + item.dispose() + } + })) await Promise.resolve(installBuffer.start(list)) + let registry = await registryUrl() let fn = async (key: string): Promise => { + let installer: IInstaller try { installBuffer.startProgress(key) - let installer = this.creteInstaller(npm, key) + installer = this.createInstaller(registry, key) + installers.set(key, installer) installer.on('message', (msg, isProgress) => { installBuffer.addMessage(key, msg, isProgress) }) + logger.debug('install:', key) let result = await installer.install() installBuffer.finishProgress(key, true) this.states.addExtension(result.name, result.url ? result.url : `>=${result.version}`) @@ -180,21 +202,19 @@ export class Extensions { if (ms != null) this.states.setLocked(result.name, true) await this.manager.loadExtension(result.folder) } catch (err: any) { - installBuffer.addMessage(key, err.message) - installBuffer.finishProgress(key, false) - void window.showErrorMessage(`Error on install ${key}: ${err}`) - logger.error(`Error on install ${key}`, err) + this.onInstallError(key, installBuffer, err) } } - await concurrent(list, fn) + await concurrent(list, fn, 3, tokenSource.token) + let len = disposables.length + disposables.splice(0, len) } /** * Update global extensions */ public async updateExtensions(silent = false): Promise { - let { npm } = this - if (!npm) return + this.cancelInstallers() let stats = this.globalExtensionStats() stats = stats.filter(s => { if (s.isLocked || s.state === 'disabled') { @@ -205,14 +225,30 @@ export class Extensions { }) this.states.setLastUpdate() this.cleanModulesFolder() - let installBuffer = this.createInstallerUI(true, silent) + let registry = await registryUrl() + let disposables: Disposable[] = this.disposables = [] + let installers: Map = new Map() + let installBuffer = this.createInstallerUI(true, silent, disposables) + let tokenSource = new CancellationTokenSource() + disposables.push(Disposable.create(() => { + tokenSource.cancel() + for (let item of installers.values()) { + item.dispose() + } + })) + installBuffer.onDidCancel(key => { + let item = installers.get(key) + if (item) item.dispose() + }) await Promise.resolve(installBuffer.start(stats.map(o => o.id))) let fn = async (stat: ExtensionInfo): Promise => { let { id } = stat + let installer: IInstaller try { installBuffer.startProgress(id) let url = stat.exotic ? stat.uri : null - let installer = this.creteInstaller(npm, id) + installer = this.createInstaller(registry, id) + installers.set(id, installer) installer.on('message', (msg, isProgress) => { installBuffer.addMessage(id, msg, isProgress) }) @@ -220,13 +256,20 @@ export class Extensions { installBuffer.finishProgress(id, true) if (directory) await this.manager.loadExtension(directory) } catch (err: any) { - installBuffer.addMessage(id, err.message) - installBuffer.finishProgress(id, false) - void window.showErrorMessage(`Error on update ${id}: ${err}`) - logger.error(`Error on update ${id}`, err) + this.onInstallError(id, installBuffer, err) } } - await concurrent(stats, fn, silent ? 1 : 3) + await concurrent(stats, fn, silent ? 1 : 3, tokenSource.token) + disposables.splice(0, disposables.length) + } + + private onInstallError(id: string, installBuffer: InstallUI, err: Error): void { + installBuffer.addMessage(id, err.message) + installBuffer.finishProgress(id, false) + if (!isCancellationError(err)) { + void window.showErrorMessage(`Error on install ${id}: ${err}`) + logger.error(`Error on update ${id}`, err) + } } /** @@ -305,18 +348,14 @@ export class Extensions { if (!fs.existsSync(this.modulesFolder)) return let files = fs.readdirSync(this.modulesFolder) for (let file of files) { - if (folders.includes(file)) continue + if (folders.includes(file) || file === '.cache') continue let p = path.join(this.modulesFolder, file) - let stat = fs.lstatSync(p) - if (stat.isSymbolicLink()) { - fs.unlinkSync(p) - } else if (stat.isDirectory()) { - fs.rmSync(p, { recursive: true, force: true }) - } + fs.rmSync(p, { recursive: true, force: true }) } } public dispose(): void { + this.cancelInstallers() this.manager.dispose() } } diff --git a/src/extension/installer.ts b/src/extension/installer.ts index 782671b1805..e2a6c21569b 100644 --- a/src/extension/installer.ts +++ b/src/extension/installer.ts @@ -1,19 +1,20 @@ 'use strict' -import { spawn } from 'child_process' import { EventEmitter } from 'events' import fs from 'fs' import os from 'os' import path from 'path' -import readline from 'readline' import semver from 'semver' -import { v4 as uuid } from 'uuid' import { URL } from 'url' +import { v4 as uuid } from 'uuid' +import { CancellationTokenSource } from 'vscode-languageserver-protocol' import download, { DownloadOptions } from '../model/download' import fetch, { FetchOptions } from '../model/fetch' -import workspace from '../workspace' +import { isFalsyOrEmpty } from '../util/array' import { loadJson } from '../util/fs' +import { findBestHost } from '../util/ping' +import workspace from '../workspace' +import { DependencySession } from './dependency' const logger = require('../util/logger')('extension-installer') -const local_dependencies = ['coc.nvim', 'esbuild', 'webpack', '@types/node'] export interface Info { 'dist.tarball'?: string @@ -32,11 +33,18 @@ export interface InstallResult { url?: string } -export function registryUrl(home = os.homedir()): URL { - let res: URL - let filepath = path.join(home, '.npmrc') - if (fs.existsSync(filepath)) { - try { +const TAOBAO_REGISTRY = new URL('https://registry.npmmirror.com') +const NPM_REGISTRY = new URL('https://registry.npmjs.org') +const YARN_REGISTRY = new URL('https://registry.yarnpkg.com') +const PINGTIMEOUT = global.__TEST__ ? 50 : 500 + +/** + * Find the user configured registry or the best one + */ +export async function registryUrl(home = os.homedir(), registries?: URL[], timeout = PINGTIMEOUT): Promise { + try { + let filepath = path.join(home, '.npmrc') + if (fs.existsSync(filepath)) { let content = fs.readFileSync(filepath, 'utf8') let uri: string for (let line of content.split(/\r?\n/)) { @@ -46,27 +54,16 @@ export function registryUrl(home = os.homedir()): URL { uri = ms[2] } } - if (uri) res = new URL(uri) - } catch (e) { - logger.debug('Error on parse .npmrc:', e) + if (uri) return new URL(uri) } + registries = isFalsyOrEmpty(registries) ? [TAOBAO_REGISTRY, NPM_REGISTRY, YARN_REGISTRY] : registries + const hosts = registries.map(o => o.hostname) + let host = await findBestHost(hosts, timeout) + return host == null ? NPM_REGISTRY : registries[hosts.indexOf(host)] + } catch (e) { + logger.debug('Error on get registry', e) + return NPM_REGISTRY } - return res ?? new URL('https://registry.npmjs.org') -} - -export function isNpmCommand(exePath: string): boolean { - let name = path.basename(exePath) - return name === 'npm' || name === 'npm.CMD' -} - -export function isYarn(exePath: string) { - let name = path.basename(exePath) - return ['yarn', 'yarn.CMD', 'yarnpkg', 'yarnpkg.CMD'].includes(name) -} - -function isPnpm(exePath: string) { - let name = path.basename(exePath) - return name === 'pnpm' || name === 'pnpm.CMD' } function isSymbolicLink(folder: string): boolean { @@ -83,15 +80,16 @@ export interface IInstaller { on(event: 'message', cb: (msg: string, isProgress: boolean) => void): void install(): Promise update(url?: string): Promise + dispose(): void } export class Installer extends EventEmitter implements IInstaller { private name: string private url: string private version: string + private tokenSource = new CancellationTokenSource() constructor( - private root: string, - private npm: string, + private dependencySession: DependencySession, // could be url or name@version or name private def: string ) { @@ -109,13 +107,17 @@ export class Installer extends EventEmitter implements IInstaller { } } + private get root(): string { + return this.dependencySession.modulesRoot + } + public get info() { return { name: this.name, version: this.version } } public async getInfo(): Promise { if (this.url) return await this.getInfoFromUri() - let registry = registryUrl() + let registry = this.dependencySession.registry this.log(`Get info from ${registry}`) let buffer = await this.fetch(new URL(this.name, registry), { timeout: 10000, buffer: true }) let res = JSON.parse(buffer.toString()) @@ -163,7 +165,7 @@ export class Installer extends EventEmitter implements IInstaller { } public async install(): Promise { - this.log(`Using npm from: ${this.npm}`) + this.emit('message', `fetch info of ${this.def}`, false) let info = await this.getInfo() logger.info(`Fetched info of ${this.def}`, info) let { name, version } = info @@ -187,7 +189,6 @@ export class Installer extends EventEmitter implements IInstaller { let obj = loadJson(path.join(folder, 'package.json')) as any version = obj.version } - this.log(`Using npm from: ${this.npm}`) let info = await this.getInfo() if (version && info.version && semver.gte(version, info.version)) { this.log(`Current version ${version} is up to date.`) @@ -204,58 +205,15 @@ export class Installer extends EventEmitter implements IInstaller { return path.dirname(jsonFile) } - public getInstallArguments(exePath: string, url: string | undefined): string[] { - let args = ['install', '--ignore-scripts', '--no-lockfile'] - if (url && url.startsWith('https://github.com')) { - args = ['install'] - } - if (isNpmCommand(exePath)) { - args.push('--omit=dev') - args.push('--legacy-peer-deps') - args.push('--no-global') - } - if (isYarn(exePath)) { - args.push('--production') - args.push('--ignore-engines') - } - if (isPnpm(exePath)) { - args.push('--production') - args.push('--config.strict-peer-dependencies=false') - } - return args - } - - private readLines(key: string, stream: NodeJS.ReadableStream): void { - const rl = readline.createInterface({ - input: stream - }) - rl.on('line', line => { - this.log(`${key} ${line}`, true) + public async installDependencies(folder: string): Promise { + let { dependencySession, tokenSource } = this + let installer = dependencySession.createInstaller(folder, msg => { + this.log(msg) }) - } - - public installDependencies(folder: string, dependencies: string[]): Promise { - if (dependencies.length == 0) return Promise.resolve() - return new Promise((resolve, reject) => { - let args = this.getInstallArguments(this.npm, this.url) - this.log(`Installing dependencies by: ${this.npm} ${args.join(' ')}.`) - const child = spawn(this.npm, args, { - cwd: folder, - env: Object.assign(process.env, { NODE_ENV: 'production' }) - }) - this.readLines('[npm stdout]', child.stdout) - this.readLines('[npm stderr]', child.stderr) - child.stderr.setEncoding('utf8') - child.stdout.setEncoding('utf8') - child.on('error', reject) - child.on('exit', code => { - if (code) { - reject(new Error(`${this.npm} install exited with ${code}`)) - return - } - resolve() - }) + tokenSource.token.onCancellationRequested(() => { + installer.cancel() }) + await installer.installDependencies() } public async doInstall(info: Info): Promise { @@ -274,8 +232,7 @@ export class Installer extends EventEmitter implements IInstaller { onProgress: p => this.log(`Download progress ${p}%`, true), }) this.log(`Extension download at ${downloadFolder}`) - let obj = loadJson(path.join(downloadFolder, 'package.json')) as any - await this.installDependencies(downloadFolder, getDependencies(obj)) + await this.installDependencies(downloadFolder) } catch (e) { fs.rmSync(downloadFolder, { recursive: true, force: true }) throw e @@ -289,14 +246,14 @@ export class Installer extends EventEmitter implements IInstaller { } public async download(url: string, options: DownloadOptions): Promise { - return await download(url, options) + return await download(url, options, this.tokenSource.token) } public async fetch(url: string | URL, options: FetchOptions = {}): Promise { - return await fetch(url, options) + return await fetch(url, options, this.tokenSource.token) } -} -export function getDependencies(obj: { dependencies?: { [key: string]: string } }): string[] { - return Object.keys(obj.dependencies ?? {}).filter(id => !local_dependencies.includes(id)) + public dispose(): void { + this.tokenSource.cancel() + } } diff --git a/src/extension/ui.ts b/src/extension/ui.ts index 23be439e411..5ba86951827 100644 --- a/src/extension/ui.ts +++ b/src/extension/ui.ts @@ -1,6 +1,6 @@ 'use strict' import { debounce } from 'debounce' -import { Disposable } from 'vscode-languageserver-protocol' +import { Disposable, Emitter, Event } from 'vscode-languageserver-protocol' import events from '../events' import { frames } from '../model/status' import { HighlightItem, OutputChannel } from '../types' @@ -18,6 +18,7 @@ export enum State { } export interface InstallUI { + onDidCancel: Event start(names: string[]): void | Promise addMessage(name: string, msg: string, isProgress?: boolean): void startProgress(name: string): void @@ -25,6 +26,8 @@ export interface InstallUI { } export class InstallChannel implements InstallUI { + private readonly _onDidCancel = new Emitter() + public readonly onDidCancel: Event = this._onDidCancel.event constructor(private isUpdate: boolean, private channel: OutputChannel) { } @@ -59,8 +62,10 @@ export class InstallBuffer implements InstallUI { private names: string[] = [] private interval: NodeJS.Timer public bufnr: number + private readonly _onDidCancel = new Emitter() + public readonly onDidCancel: Event = this._onDidCancel.event - constructor(private isUpdate: boolean) { + constructor(private isUpdate: boolean, onClose = () => {}) { let floatFactory = window.createFloatFactory({ modes: ['n'] }) this.disposables.push(floatFactory) let fn = debounce(async (bufnr, cursor) => { @@ -77,6 +82,7 @@ export class InstallBuffer implements InstallUI { events.on('BufUnload', bufnr => { if (bufnr === this.bufnr) { this.dispose() + onClose() } }, null, this.disposables) } @@ -162,6 +168,21 @@ export class InstallBuffer implements InstallUI { return this.interval == null } + public async onTab(): Promise { + let line = await workspace.nvim.eval(`line(".")-1`) as number + let name = this.names[line - 2] + if (!name) return + let state = this.statMap.get(name) + let couldCancel = state === State.Progressing + let actions: string[] = [] + if (couldCancel) actions.push('Cancel') + if (actions.length === 0) return + let idx = await window.showMenuPicker(['Cancel']) + if (idx === 0) { + this._onDidCancel.fire(name) + } + } + // draw frame public draw(): void { let { remains, bufnr } = this @@ -197,6 +218,7 @@ export class InstallBuffer implements InstallUI { if (!isSync) nvim.command('nnoremap q :q', true) this.highlight() let res = await nvim.resumeNotification() + workspace.registerLocalKeymap('n', '', this.onTab.bind(this)) this.bufnr = res[0][1] as number this.interval = setInterval(() => { this.draw() diff --git a/src/index.ts b/src/index.ts index 0b7d5de8c5c..c5a7d86cd08 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,6 +23,7 @@ import BasicList from './list/basic' import { Mutex } from './util/mutex' import { URI } from 'vscode-uri' import { + CodeAction, CodeActionKind, CodeActionTriggerKind, Disposable, diff --git a/src/list/source/extensions.ts b/src/list/source/extensions.ts index 73429665a08..e993912479d 100644 --- a/src/list/source/extensions.ts +++ b/src/list/source/extensions.ts @@ -77,25 +77,6 @@ export default class ExtensionList extends BasicList { await extensions.manager.reloadExtension(id) }, { persist: true, reload: true }) - this.addAction('fix', async item => { - let { root, isLocal } = item.data - let { npm } = extensions - if (isLocal) { - void window.showWarningMessage(`Can't fix for local extension.`) - return - } - if (!npm) return - let folder = path.join(root, 'node_modules') - fs.rmSync(folder, { recursive: true, force: true }) - let terminal = await window.createTerminal({ - cwd: root - }) - let shown = await terminal.show(false) - if (!shown) return - workspace.nvim.command(`startinsert`, true) - terminal.sendText(`${npm} install --production --ignore-scripts --no-lockfile`, true) - }) - this.addMultipleAction('uninstall', async items => { let ids = [] for (let item of items) { diff --git a/src/model/download.ts b/src/model/download.ts index 79f677e3316..3d296b3ab3b 100644 --- a/src/model/download.ts +++ b/src/model/download.ts @@ -7,7 +7,7 @@ import path from 'path' import tar from 'tar' import unzip from 'unzip-stream' import { URL } from 'url' -import { v1 as uuidv1 } from 'uuid' +import { v4 as uuidv4 } from 'uuid' import { CancellationToken } from 'vscode-languageserver-protocol' import { FetchOptions, getRequestModule, resolveRequestOptions, toURL } from './fetch' const logger = require('../util/logger')('model-download') @@ -17,6 +17,10 @@ export interface DownloadOptions extends Omit { * Folder that contains downloaded file or extracted files by untar or unzip */ dest: string + /** + * filename in dest, Could contains path separator, will be removed when exists. + */ + filename?: string /** * algorithm for check etag. */ @@ -50,7 +54,7 @@ export function getEtag(headers: IncomingHttpHeaders): string | undefined { export default function download(urlInput: string | URL, options: DownloadOptions, token?: CancellationToken): Promise { let url = toURL(urlInput) let { etagAlgorithm } = options - let { dest, onProgress, extract } = options + let { dest, onProgress, extract, filename } = options if (!dest || !path.isAbsolute(dest)) { throw new Error(`Expect absolute file path for dest option.`) } @@ -62,6 +66,14 @@ export default function download(urlInput: string | URL, options: DownloadOption throw new Error(`${dest} exists, but not directory!`) } } + if (filename) { + let fullpath = path.join(dest, filename) + if (fs.existsSync(fullpath)) { + fs.rmSync(fullpath, { force: true, recursive: true }) + } else { + fs.mkdirSync(path.dirname(fullpath), { recursive: true }) + } + } let mod = getRequestModule(url) let opts = resolveRequestOptions(url, options) if (!opts.agent && options.agent) opts.agent = options.agent @@ -109,7 +121,7 @@ export default function download(urlInput: string | URL, options: DownloadOption if (hash) hash.update(chunk) if (hasTotal) { let percent = (cur / total * 100).toFixed(1) - typeof onProgress === 'function' ? onProgress(percent) : logger.info(`Download ${url} progress ${percent}%`) + if (typeof onProgress === 'function') onProgress(percent) } }) res.on('end', () => { @@ -124,7 +136,7 @@ export default function download(urlInput: string | URL, options: DownloadOption } else if (extract === 'unzip') { stream = res.pipe(unzip.Extract({ path: dest })) } else { - dest = path.join(dest, `${uuidv1()}${extname}`) + dest = path.join(dest, filename ?? `${uuidv4()}${extname}`) stream = res.pipe(fs.createWriteStream(dest)) } stream.on('finish', () => { @@ -138,7 +150,7 @@ export default function download(urlInput: string | URL, options: DownloadOption logger.info(`Downloaded ${url} => ${dest}`) setTimeout(() => { resolve(dest) - }, 100) + }, global.__TEST__ ? 20 : 100) }) stream.on('error', reject) } else { diff --git a/src/model/fetch.ts b/src/model/fetch.ts index 45949f71ba9..9095d933cf7 100644 --- a/src/model/fetch.ts +++ b/src/model/fetch.ts @@ -8,7 +8,7 @@ import { ParsedUrlQueryInput, stringify } from 'querystring' import { Readable } from 'stream' import { URL } from 'url' import { CancellationToken } from 'vscode-languageserver-protocol' -import { CancellationError } from '../util/errors' +import { CancellationError, isCancellationError } from '../util/errors' import { objectLiteral } from '../util/is' import workspace from '../workspace' const logger = require('../util/logger')('model-fetch') @@ -244,7 +244,7 @@ export function request(url: URL, data: any, opts: any, token?: CancellationToke } }) req.on('timeout', () => { - req.destroy(new Error(`Request timeout after ${opts.timeout}ms`)) + req.destroy(new Error(`Request ${url} timeout after ${opts.timeout}ms`)) }) if (data) req.write(getText(data)) if (opts.timeout) req.setTimeout(opts.timeout) @@ -267,7 +267,7 @@ export default function fetch(urlInput: string | URL, options: FetchOptions = {} let url = toURL(urlInput) let opts = resolveRequestOptions(url, options) return request(url, options.data, opts, token).catch(err => { - logger.error(`Fetch error for ${url}:`, opts, err) + if (!isCancellationError(err)) logger.error(`Fetch error for ${url}:`, opts, err) if (opts.agent && opts.agent.proxy) { let { proxy } = opts.agent throw new Error(`Request failed using proxy ${proxy.host}: ${err.message}`) diff --git a/src/util/index.ts b/src/util/index.ts index bdd68b6eb1f..9a1a6f4f8b7 100644 --- a/src/util/index.ts +++ b/src/util/index.ts @@ -1,7 +1,5 @@ 'use strict' -interface Disposable { - dispose(): void -} +import { CancellationToken, Disposable } from 'vscode-languageserver-protocol' export const CONFIG_FILE_NAME = 'coc-settings.json' @@ -48,27 +46,28 @@ export function delay(func: () => void, defaultDelay: number): ((ms?: number) => return fn as any } -export function concurrent(arr: T[], fn: (val: T) => Promise, limit = 3): Promise { +export function concurrent(arr: ReadonlyArray, fn: (val: T) => Promise, limit = 3, token?: CancellationToken): Promise { if (arr.length == 0) return Promise.resolve() let finished = 0 let total = arr.length - let remain = arr.slice() + let curr = 0 return new Promise(resolve => { let run = (val): void => { + if (token && token.isCancellationRequested) return resolve() let cb = () => { finished = finished + 1 if (finished == total) { resolve() - } else if (remain.length) { - let next = remain.shift() - run(next) + } else if (curr < total - 1) { + curr++ + run(arr[curr]) } } fn(val).then(cb, cb) } - for (let i = 0; i < Math.min(limit, remain.length); i++) { - let val = remain.shift() - run(val) + curr = Math.min(limit, total) - 1 + for (let i = 0; i <= curr; i++) { + run(arr[i]) } }) } diff --git a/src/util/ping.ts b/src/util/ping.ts new file mode 100644 index 00000000000..fc1e9002ea6 --- /dev/null +++ b/src/util/ping.ts @@ -0,0 +1,101 @@ +import { spawn } from 'child_process' +import fs from 'fs' +import which from 'which' + +interface PingConfig { + bin: string + args: string[] + regmatch: RegExp +} + +let pingConfig: PingConfig | undefined + +export function getPing(platform = process.platform): PingConfig | undefined { + let config: PingConfig + if (/^win/.test(platform)) { + config = { + bin: 'c:/windows/system32/ping.exe', + args: ['-n', '1', '-w', '5000'], + regmatch: /[><=]([0-9.]+?)\s?ms/ + } + } else if (/^linux/.test(platform)) { + config = { + bin: '/bin/ping', + args: ['-n', '-w', '2', '-c', '1'], + regmatch: /=([0-9.]+?) ms/ + } + } else if (/^darwin/.test(platform)) { + config = { + bin: '/sbin/ping', + args: ['-n', '-t', '2', '-c', '1'], + regmatch: /=([0-9.]+?) ms/ + } + } else { + try { + let bin = which.sync('ping') + config = { + bin, + args: ['-n', '-w', '2', '-c', '1'], + regmatch: /=([0-9.]+?) ms/ + } + } catch (_e) { + // not found + } + } + if (!config || !fs.existsSync(config.bin)) return undefined + return config +} + +export async function findBestHost(hosts: string[], timeout: number, bin?: string): Promise { + let config = pingConfig ?? getPing() + pingConfig = config + if (hosts.length === 0 || !config) return undefined + if (hosts.length === 1) return hosts[0] + const check = (host: string) => { + return new Promise(resolve => { + let args = pingConfig.args.concat([host]) + let stdout = '' + let cp = spawn(bin ?? pingConfig.bin, args) + let finished = false + let timer = setTimeout(() => { + finished = true + cp.kill() + resolve(undefined) + }, timeout) + cp.on('error', _e => { + finished = true + clearTimeout(timer) + resolve(undefined) + }) + let onEnd = () => { + clearTimeout(timer) + if (finished) return + finished = true + let ms = stdout.match(config.regmatch) + let milliseconds = (ms && ms[1]) ? Number(ms[1]) : undefined + resolve(milliseconds) + } + cp.stdout.on('data', data => { + stdout = stdout + data.toString() + }) + cp.stdout.on('end', () => { + onEnd() + }) + cp.on('exit', () => { + onEnd() + }) + }) + } + let minIndex = -1 + let minValue: undefined | number + await Promise.all(hosts.map((host, idx) => { + return check(host).then(val => { + if (val === undefined) return + if (!minValue || val < minValue) { + minValue = val + minIndex = idx + } + }) + })) + return minIndex == -1 ? undefined : hosts[minIndex] +} diff --git a/typings/index.d.ts b/typings/index.d.ts index 6b240302dfa..61bad8a15eb 100644 --- a/typings/index.d.ts +++ b/typings/index.d.ts @@ -5261,7 +5261,7 @@ declare module 'coc.nvim' { /** * Concurrent run async functions with limit support. */ - export function concurrent(arr: T[], fn: (val: T) => Promise, limit?: number): Promise + export function concurrent(arr: T[], fn: (val: T) => Promise, limit?: number, token?: CancellationToken): Promise /** * Create promise resolved after ms milliseconds. diff --git a/yarn.lock b/yarn.lock index a7e01155cda..5ffdd9c9985 100644 --- a/yarn.lock +++ b/yarn.lock @@ -844,13 +844,6 @@ dependencies: "@types/node" "*" -"@types/mkdirp@^1.0.1": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-1.0.2.tgz#8d0bad7aa793abe551860be1f7ae7f3198c16666" - integrity sha512-o0K1tSO0Dx5X6xlU5F1D6625FawhC3dU3iqr25lluNv/+/QIVH8RLNEiVokgIZo+mz+87w/3Mkg/VvQS+J51fQ== - dependencies: - "@types/node" "*" - "@types/node-int64@*": version "0.4.29" resolved "https://registry.yarnpkg.com/@types/node-int64/-/node-int64-0.4.29.tgz#8c7c16a7c1195ae4f8beaa903b0018ac66291d16"