Skip to content

Commit

Permalink
Incrementally decode find result (fixes #11842)
Browse files Browse the repository at this point in the history
  • Loading branch information
@chrmarti
chrmarti committed Oct 8, 2016
1 parent 4315160 commit 73ba662
Showing 1 changed file with 103 additions and 89 deletions.
192 changes: 103 additions & 89 deletions src/vs/workbench/services/search/node/fileSearch.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
'use strict';

import * as childProcess from 'child_process';
import {StringDecoder} from 'string_decoder';
import {StringDecoder, NodeStringDecoder} from 'string_decoder';
import {toErrorMessage} from 'vs/base/common/errorMessage';
import fs = require('fs');
import paths = require('path');
Expand Down Expand Up @@ -129,15 +129,14 @@ export class FileWalker {
if (!this.maxFilesize) {
if (platform.isMacintosh) {
this.traversal = Traversal.MacFind;
traverse = this.macFindTraversal;
traverse = this.findTraversal;
// Disable 'dir' for now (#11181, #11179, #11183, #11182).
// TS (2.0.2) warns about unreachable code. Using comments.
} /* else if (false && platform.isWindows) {
} /* else if (platform.isWindows) {
this.traversal = Traversal.WindowsDir;
traverse = this.windowsDirTraversal;
} */ else if (platform.isLinux) {
this.traversal = Traversal.LinuxFind;
traverse = this.linuxFindTraversal;
traverse = this.findTraversal;
}
}

Expand Down Expand Up @@ -169,88 +168,81 @@ export class FileWalker {
});
}

private macFindTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
private findTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, cb: (err?: Error) => void): void {
const isMac = platform.isMacintosh;
let done = (err?: Error) => {
done = () => {};
cb(err);
};
let leftover = '';
let first = true;
const tree = this.initDirectoryTree();
const cmd = this.spawnFindCmd(rootFolder, this.excludePattern);
this.readStdout(cmd, 'utf8', (err: Error, stdout?: string) => {
this.collectStdout(cmd, 'utf8', (err: Error, stdout?: string, last?: boolean) => {
if (err) {
done(err);
return;
}

// Mac: uses NFD unicode form on disk, but we want NFC
const relativeFiles = strings.normalizeNFC(stdout).split('\n./');
relativeFiles[0] = relativeFiles[0].trim().substr(2);
const n = relativeFiles.length;
relativeFiles[n - 1] = relativeFiles[n - 1].trim();
if (!relativeFiles[n - 1]) {
relativeFiles.pop();
}

if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
done(new Error('Splitting up files failed'));
return;
}

this.matchFiles(rootFolder, relativeFiles, onResult);

done();
});
}

protected windowsDirTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
const cmd = childProcess.spawn('cmd', ['/U', '/c', 'dir', '/s', '/b', '/a-d', rootFolder]);
this.readStdout(cmd, 'ucs2', (err: Error, stdout?: string) => {
if (err) {
done(err);
return;
const normalized = leftover + (isMac ? strings.normalizeNFC(stdout) : stdout);
const relativeFiles = normalized.split('\n./');
if (first && normalized.length >= 2) {
first = false;
relativeFiles[0] = relativeFiles[0].trim().substr(2);
}

const relativeFiles = stdout.split(`\r\n${rootFolder}\\`);
relativeFiles[0] = relativeFiles[0].trim().substr(rootFolder.length + 1);
const n = relativeFiles.length;
relativeFiles[n - 1] = relativeFiles[n - 1].trim();
if (!relativeFiles[n - 1]) {
relativeFiles.pop();
if (last) {
const n = relativeFiles.length;
relativeFiles[n - 1] = relativeFiles[n - 1].trim();
if (!relativeFiles[n - 1]) {
relativeFiles.pop();
}
} else {
leftover = relativeFiles.pop();
}

if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
done(new Error('Splitting up files failed'));
return;
}

this.matchFiles(rootFolder, relativeFiles, onResult);

done();
});
}

private linuxFindTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
const cmd = this.spawnFindCmd(rootFolder, this.excludePattern);
this.readStdout(cmd, 'utf8', (err: Error, stdout?: string) => {
if (err) {
done(err);
return;
}

const relativeFiles = stdout.split('\n./');
relativeFiles[0] = relativeFiles[0].trim().substr(2);
const n = relativeFiles.length;
relativeFiles[n - 1] = relativeFiles[n - 1].trim();
if (!relativeFiles[n - 1]) {
relativeFiles.pop();
}
this.addDirectoryEntries(tree, rootFolder, relativeFiles, onResult);

if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
done(new Error('Splitting up files failed'));
return;
if (last) {
this.matchDirectoryTree(tree, rootFolder, onResult);
done();
}

this.matchFiles(rootFolder, relativeFiles, onResult);

done();
});
}

// protected windowsDirTraversal(rootFolder: string, onResult: (result: IRawFileMatch) => void, done: (err?: Error) => void): void {
// const cmd = childProcess.spawn('cmd', ['/U', '/c', 'dir', '/s', '/b', '/a-d', rootFolder]);
// this.readStdout(cmd, 'ucs2', (err: Error, stdout?: string) => {
// if (err) {
// done(err);
// return;
// }

// const relativeFiles = stdout.split(`\r\n${rootFolder}\\`);
// relativeFiles[0] = relativeFiles[0].trim().substr(rootFolder.length + 1);
// const n = relativeFiles.length;
// relativeFiles[n - 1] = relativeFiles[n - 1].trim();
// if (!relativeFiles[n - 1]) {
// relativeFiles.pop();
// }

// if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
// done(new Error('Splitting up files failed'));
// return;
// }

// this.matchFiles(rootFolder, relativeFiles, onResult);

// done();
// });
// }

/**
* Public for testing.
*/
Expand Down Expand Up @@ -279,13 +271,30 @@ export class FileWalker {
* Public for testing.
*/
public readStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string) => void): void {
let done = (err: Error, stdout?: string) => {
done = () => {};
this.cmdForkResultTime = Date.now();
cb(err, stdout);
let all = '';
this.collectStdout(cmd, encoding, (err: Error, stdout?: string, last?: boolean) => {
if (err) {
cb(err);
return;
}

all += stdout;
if (last) {
cb(null, all);
}
});
}

private collectStdout(cmd: childProcess.ChildProcess, encoding: string, cb: (err: Error, stdout?: string, last?: boolean) => void): void {
let done = (err: Error, stdout?: string, last?: boolean) => {
if (err || last) {
done = () => {};
this.cmdForkResultTime = Date.now();
}
cb(err, stdout, last);
};

const stdout = this.collectData(cmd.stdout);
this.forwardData(cmd.stdout, encoding, done);
const stderr = this.collectData(cmd.stderr);

cmd.on('error', (err: Error) => {
Expand All @@ -296,11 +305,19 @@ export class FileWalker {
if (code !== 0) {
done(new Error(`find failed with error code ${code}: ${this.decodeData(stderr, encoding)}`));
} else {
done(null, this.decodeData(stdout, encoding));
done(null, '', true);
}
});
}

private forwardData(stream: Readable, encoding: string, cb: (err: Error, stdout?: string) => void): NodeStringDecoder {
const decoder = new StringDecoder(encoding);
stream.on('data', (data: Buffer) => {
cb(null, decoder.write(data));
});
return decoder;
}

private collectData(stream: Readable): Buffer[] {
const buffers: Buffer[] = [];
stream.on('data', (data: Buffer) => {
Expand All @@ -314,27 +331,25 @@ export class FileWalker {
return buffers.map(buffer => decoder.write(buffer)).join('');
}

private matchFiles(rootFolder: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) {
this.cmdResultCount = relativeFiles.length;
private initDirectoryTree(): IDirectoryTree {
const tree: IDirectoryTree = {
rootEntries: [],
pathToEntries: Object.create(null)
};
tree.pathToEntries['.'] = tree.rootEntries;
return tree;
}

private addDirectoryEntries({pathToEntries}: IDirectoryTree, base: string, relativeFiles: string[], onResult: (result: IRawFileMatch) => void) {
this.cmdResultCount += relativeFiles.length;

// Support relative paths to files from a root resource (ignores excludes)
if (relativeFiles.indexOf(this.filePattern) !== -1) {
const basename = paths.basename(this.filePattern);
this.matchFile(onResult, { base: rootFolder, relativePath: this.filePattern, basename });
this.matchFile(onResult, { base: base, relativePath: this.filePattern, basename });
}

const tree = this.buildDirectoryTree(rootFolder, relativeFiles);
this.matchDirectoryTree(rootFolder, tree, onResult);
}

private buildDirectoryTree(base: string, relativeFilePaths: string[]): IDirectoryTree {
const tree: IDirectoryTree = {
rootEntries: [],
pathToEntries: Object.create(null)
};
const {pathToEntries} = tree;
pathToEntries['.'] = tree.rootEntries;
relativeFilePaths.forEach(function add(relativePath: string) {
relativeFiles.forEach(function add(relativePath: string) {
const basename = paths.basename(relativePath);
const dirname = paths.dirname(relativePath);
let entries = pathToEntries[dirname];
Expand All @@ -348,10 +363,9 @@ export class FileWalker {
basename
});
});
return tree;
}

private matchDirectoryTree(rootFolder: string, { rootEntries, pathToEntries }: IDirectoryTree, onResult: (result: IRawFileMatch) => void) {
private matchDirectoryTree({ rootEntries, pathToEntries }: IDirectoryTree, rootFolder: string, onResult: (result: IRawFileMatch) => void) {
const self = this;
const excludePattern = this.excludePattern;
const filePattern = this.filePattern;
Expand Down

0 comments on commit 73ba662

Please sign in to comment.