From 73ba662500e21c1457cb14499deb8d64392edf35 Mon Sep 17 00:00:00 2001 From: Christof Marti Date: Fri, 7 Oct 2016 17:20:17 -0700 Subject: [PATCH] Incrementally decode find result (fixes #11842) --- .../services/search/node/fileSearch.ts | 192 ++++++++++-------- 1 file changed, 103 insertions(+), 89 deletions(-) diff --git a/src/vs/workbench/services/search/node/fileSearch.ts b/src/vs/workbench/services/search/node/fileSearch.ts index 6e9472386d155..6b4934ca0b142 100644 --- a/src/vs/workbench/services/search/node/fileSearch.ts +++ b/src/vs/workbench/services/search/node/fileSearch.ts @@ -6,7 +6,7 @@ 'use strict'; import * as childProcess from 'child_process'; -import {StringDecoder} from 'string_decoder'; +import {StringDecoder, NodeStringDecoder} from 'string_decoder'; import {toErrorMessage} from 'vs/base/common/errorMessage'; import fs = require('fs'); import paths = require('path'); @@ -129,15 +129,14 @@ export class FileWalker { if (!this.maxFilesize) { if (platform.isMacintosh) { this.traversal = Traversal.MacFind; - traverse = this.macFindTraversal; + traverse = this.findTraversal; // Disable 'dir' for now (#11181, #11179, #11183, #11182). - // TS (2.0.2) warns about unreachable code. Using comments. - } /* else if (false && platform.isWindows) { + } /* else if (platform.isWindows) { this.traversal = Traversal.WindowsDir; traverse = this.windowsDirTraversal; } */ else if (platform.isLinux) { this.traversal = Traversal.LinuxFind; - traverse = this.linuxFindTraversal; + traverse = this.findTraversal; } } @@ -169,48 +168,38 @@ export class FileWalker { }); } - private macFindTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void { + private findTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void { + const isMac = platform.isMacintosh; + let done = (err?: Error) => { + done = () => {}; + cb(err); + }; + let leftover = ''; + let first = true; + const tree = this.initDirectoryTree(); const cmd = this.spawnFindCmd(rootFolder, this.excludePattern); - this.readStdout(cmd, 'utf8', (err: Error, stdout?: string) => { + this.collectStdout(cmd, 'utf8', (err: Error, stdout?: string, last?: boolean) => { if (err) { done(err); return; } // Mac: uses NFD unicode form on disk, but we want NFC - const relativeFiles = strings.normalizeNFC(stdout).split('\n./'); - relativeFiles[0] = relativeFiles[0].trim().substr(2); - const n = relativeFiles.length; - relativeFiles[n - 1] = relativeFiles[n - 1].trim(); - if (!relativeFiles[n - 1]) { - relativeFiles.pop(); - } - - if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) { - done(new Error('Splitting up files failed')); - return; - } - - this.matchFiles(rootFolder, relativeFiles, onResult); - - done(); - }); - } - - protected windowsDirTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void { - const cmd = childProcess.spawn('cmd', ['/U', '/c', 'dir', '/s', '/b', '/a-d', rootFolder]); - this.readStdout(cmd, 'ucs2', (err: Error, stdout?: string) => { - if (err) { - done(err); - return; + const normalized = leftover + (isMac ? strings.normalizeNFC(stdout) : stdout); + const relativeFiles = normalized.split('\n./'); + if (first && normalized.length >= 2) { + first = false; + relativeFiles[0] = relativeFiles[0].trim().substr(2); } - const relativeFiles = stdout.split(`\r\n${rootFolder}\\`); - relativeFiles[0] = relativeFiles[0].trim().substr(rootFolder.length + 1); - const n = relativeFiles.length; - relativeFiles[n - 1] = relativeFiles[n - 1].trim(); - if (!relativeFiles[n - 1]) { - relativeFiles.pop(); + if (last) { + const n = relativeFiles.length; + relativeFiles[n - 1] = relativeFiles[n - 1].trim(); + if (!relativeFiles[n - 1]) { + relativeFiles.pop(); + } + } else { + leftover = relativeFiles.pop(); } if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) { @@ -218,39 +207,42 @@ export class FileWalker { return; } - this.matchFiles(rootFolder, relativeFiles, onResult); - - done(); - }); - } - - private linuxFindTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void { - const cmd = this.spawnFindCmd(rootFolder, this.excludePattern); - this.readStdout(cmd, 'utf8', (err: Error, stdout?: string) => { - if (err) { - done(err); - return; - } - - const relativeFiles = stdout.split('\n./'); - relativeFiles[0] = relativeFiles[0].trim().substr(2); - const n = relativeFiles.length; - relativeFiles[n - 1] = relativeFiles[n - 1].trim(); - if (!relativeFiles[n - 1]) { - relativeFiles.pop(); - } + this.addDirectoryEntries(tree, rootFolder, relativeFiles, onResult); - if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) { - done(new Error('Splitting up files failed')); - return; + if (last) { + this.matchDirectoryTree(tree, rootFolder, onResult); + done(); } - - this.matchFiles(rootFolder, relativeFiles, onResult); - - done(); }); } + // protected windowsDirTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void { + // const cmd = childProcess.spawn('cmd', ['/U', '/c', 'dir', '/s', '/b', '/a-d', rootFolder]); + // this.readStdout(cmd, 'ucs2', (err: Error, stdout?: string) => { + // if (err) { + // done(err); + // return; + // } + + // const relativeFiles = stdout.split(`\r\n${rootFolder}\\`); + // relativeFiles[0] = relativeFiles[0].trim().substr(rootFolder.length + 1); + // const n = relativeFiles.length; + // relativeFiles[n - 1] = relativeFiles[n - 1].trim(); + // if (!relativeFiles[n - 1]) { + // relativeFiles.pop(); + // } + + // if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) { + // done(new Error('Splitting up files failed')); + // return; + // } + + // this.matchFiles(rootFolder, relativeFiles, onResult); + + // done(); + // }); + // } + /** * Public for testing. */ @@ -279,13 +271,30 @@ export class FileWalker { * Public for testing. */ public readStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string) => void): void { - let done = (err: Error, stdout?: string) => { - done = () => {}; - this.cmdForkResultTime = Date.now(); - cb(err, stdout); + let all = ''; + this.collectStdout(cmd, encoding, (err: Error, stdout?: string, last?: boolean) => { + if (err) { + cb(err); + return; + } + + all += stdout; + if (last) { + cb(null, all); + } + }); + } + + private collectStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string, last?: boolean) => void): void { + let done = (err: Error, stdout?: string, last?: boolean) => { + if (err || last) { + done = () => {}; + this.cmdForkResultTime = Date.now(); + } + cb(err, stdout, last); }; - const stdout = this.collectData(cmd.stdout); + this.forwardData(cmd.stdout, encoding, done); const stderr = this.collectData(cmd.stderr); cmd.on('error', (err: Error) => { @@ -296,11 +305,19 @@ export class FileWalker { if (code !== 0) { done(new Error(`find failed with error code ${code}: ${this.decodeData(stderr, encoding)}`)); } else { - done(null, this.decodeData(stdout, encoding)); + done(null, '', true); } }); } + private forwardData(stream: Readable, encoding: string, cb: (err: Error, stdout?: string) => void): NodeStringDecoder { + const decoder = new StringDecoder(encoding); + stream.on('data', (data: Buffer) => { + cb(null, decoder.write(data)); + }); + return decoder; + } + private collectData(stream: Readable): Buffer[] { const buffers: Buffer[] = []; stream.on('data', (data: Buffer) => { @@ -314,27 +331,25 @@ export class FileWalker { return buffers.map(buffer => decoder.write(buffer)).join(''); } - private matchFiles(rootFolder: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) { - this.cmdResultCount = relativeFiles.length; + private initDirectoryTree(): IDirectoryTree { + const tree: IDirectoryTree = { + rootEntries: [], + pathToEntries: Object.create(null) + }; + tree.pathToEntries['.'] = tree.rootEntries; + return tree; + } + + private addDirectoryEntries({pathToEntries}: IDirectoryTree, base: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) { + this.cmdResultCount += relativeFiles.length; // Support relative paths to files from a root resource (ignores excludes) if (relativeFiles.indexOf(this.filePattern) !== -1) { const basename = paths.basename(this.filePattern); - this.matchFile(onResult, { base: rootFolder, relativePath: this.filePattern, basename }); + this.matchFile(onResult, { base: base, relativePath: this.filePattern, basename }); } - const tree = this.buildDirectoryTree(rootFolder, relativeFiles); - this.matchDirectoryTree(rootFolder, tree, onResult); - } - - private buildDirectoryTree(base: string, relativeFilePaths: string[]): IDirectoryTree { - const tree: IDirectoryTree = { - rootEntries: [], - pathToEntries: Object.create(null) - }; - const {pathToEntries} = tree; - pathToEntries['.'] = tree.rootEntries; - relativeFilePaths.forEach(function add(relativePath: string) { + relativeFiles.forEach(function add(relativePath: string) { const basename = paths.basename(relativePath); const dirname = paths.dirname(relativePath); let entries = pathToEntries[dirname]; @@ -348,10 +363,9 @@ export class FileWalker { basename }); }); - return tree; } - private matchDirectoryTree(rootFolder: string, { rootEntries, pathToEntries }: IDirectoryTree, onResult: (result: IRawFileMatch) => void) { + private matchDirectoryTree({ rootEntries, pathToEntries }: IDirectoryTree, rootFolder: string, onResult: (result: IRawFileMatch) => void) { const self = this; const excludePattern = this.excludePattern; const filePattern = this.filePattern;