Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Optimize the size for tsbuildinfo #43155

Merged
merged 7 commits into from Mar 10, 2021
Merged

Optimize the size for tsbuildinfo #43155

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
cd71acc
Baseline readable buildinfo
sheetalkamat Mar 9, 2021
bda4556
Use file names as index in file name list
sheetalkamat Mar 9, 2021
2a0224e
Deduplicate reference map lists and use file name index to sort them
sheetalkamat Mar 9, 2021
41026d7
Minimal json.stringify for the tsbuildinfo
sheetalkamat Mar 9, 2021
497bc4e
Update src/testRunner/unittests/tsbuild/helpers.ts
sheetalkamat Mar 9, 2021
9fcace3
Readable version of buildinfo all the time
sheetalkamat Mar 9, 2021
c597a4d
Some renames for readability as per feedback
sheetalkamat Mar 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
123 changes: 69 additions & 54 deletions src/compiler/builder.ts
View file
Open in desktop
Expand Up @@ -690,13 +690,16 @@ namespace ts {
return filterSemanticDiagnotics(diagnostics, state.compilerOptions);
}

export type ProgramBuildInfoDiagnostic = string | [string, readonly ReusableDiagnostic[]];
export type ProgramBuilderInfoFilePendingEmit = [string, BuilderFileEmit];
export type ProgramBuildInfoDiagnostic = number | [number, readonly ReusableDiagnostic[]];
sheetalkamat marked this conversation as resolved.
Show resolved Hide resolved
export type ProgramBuilderInfoFilePendingEmit = [number, BuilderFileEmit];
export type ProgramBuildInfoReferencedMap = [FileName: number, FileNameList: number][];
sheetalkamat marked this conversation as resolved.
Show resolved Hide resolved
export interface ProgramBuildInfo {
fileInfos: MapLike<BuilderState.FileInfo>;
fileNames: readonly string[];
fileInfos: readonly BuilderState.FileInfo[];
options: CompilerOptions;
referencedMap?: MapLike<string[]>;
exportedModulesMap?: MapLike<string[]>;
fileNamesList?: readonly (readonly number[])[];
referencedMap?: ProgramBuildInfoReferencedMap;
exportedModulesMap?: ProgramBuildInfoReferencedMap;
semanticDiagnosticsPerFile?: ProgramBuildInfoDiagnostic[];
affectedFilesPendingEmit?: ProgramBuilderInfoFilePendingEmit[];
}
Expand All @@ -708,66 +711,74 @@ namespace ts {
if (outFile(state.compilerOptions)) return undefined;
const currentDirectory = Debug.checkDefined(state.program).getCurrentDirectory();
const buildInfoDirectory = getDirectoryPath(getNormalizedAbsolutePath(getTsBuildInfoEmitOutputFilePath(state.compilerOptions)!, currentDirectory));
const fileInfos: MapLike<BuilderState.FileInfo> = {};
state.fileInfos.forEach((value, key) => {
const fileNames: string[] = [];
const fileNamesMap = new Map<string, number>();
let fileNamesList: (readonly number[])[] | undefined;
let fileNamesListMap: ESMap<string, number> | undefined;
const fileInfos = arrayFrom(state.fileInfos.entries(), ([key, value]) => {
// Ensure file name index
const index = toFileNameIndex(key);
Debug.assert(fileNames[index] === relativeToBuildInfo(key));
const signature = state.currentAffectedFilesSignatures && state.currentAffectedFilesSignatures.get(key);
fileInfos[relativeToBuildInfo(key)] = signature === undefined ? value : { version: value.version, signature, affectsGlobalScope: value.affectsGlobalScope };
return signature === undefined ? value : { version: value.version, signature, affectsGlobalScope: value.affectsGlobalScope };
});

const result: ProgramBuildInfo = {
fileInfos,
options: convertToReusableCompilerOptions(state.compilerOptions, relativeToBuildInfoEnsuringAbsolutePath)
};
let referencedMap: ProgramBuildInfoReferencedMap | undefined;
if (state.referencedMap) {
const referencedMap: MapLike<string[]> = {};
for (const key of arrayFrom(state.referencedMap.keys()).sort(compareStringsCaseSensitive)) {
referencedMap[relativeToBuildInfo(key)] = arrayFrom(state.referencedMap.get(key)!.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
}
result.referencedMap = referencedMap;
referencedMap = arrayFrom(state.referencedMap.keys()).sort(compareStringsCaseSensitive).map(key => [
toFileNameIndex(key),
toFileNamesListIndex(state.referencedMap!.get(key)!)
]);
}

let exportedModulesMap: ProgramBuildInfoReferencedMap | undefined;
if (state.exportedModulesMap) {
const exportedModulesMap: MapLike<string[]> = {};
for (const key of arrayFrom(state.exportedModulesMap.keys()).sort(compareStringsCaseSensitive)) {
exportedModulesMap = mapDefined(arrayFrom(state.exportedModulesMap.keys()).sort(compareStringsCaseSensitive), key => {
const newValue = state.currentAffectedFilesExportedModulesMap && state.currentAffectedFilesExportedModulesMap.get(key);
// Not in temporary cache, use existing value
if (newValue === undefined) exportedModulesMap[relativeToBuildInfo(key)] = arrayFrom(state.exportedModulesMap.get(key)!.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
if (newValue === undefined) return [toFileNameIndex(key), toFileNamesListIndex(state.exportedModulesMap!.get(key)!)];
// Value in cache and has updated value map, use that
else if (newValue) exportedModulesMap[relativeToBuildInfo(key)] = arrayFrom(newValue.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
}
result.exportedModulesMap = exportedModulesMap;
else if (newValue) return [toFileNameIndex(key), toFileNamesListIndex(newValue)];
});
}

let semanticDiagnosticsPerFile: ProgramBuildInfoDiagnostic[] | undefined;
if (state.semanticDiagnosticsPerFile) {
const semanticDiagnosticsPerFile: ProgramBuildInfoDiagnostic[] = [];
for (const key of arrayFrom(state.semanticDiagnosticsPerFile.keys()).sort(compareStringsCaseSensitive)) {
const value = state.semanticDiagnosticsPerFile.get(key)!;
semanticDiagnosticsPerFile.push(
(semanticDiagnosticsPerFile ||= []).push(
value.length ?
[
relativeToBuildInfo(key),
toFileNameIndex(key),
state.hasReusableDiagnostic ?
value as readonly ReusableDiagnostic[] :
convertToReusableDiagnostics(value as readonly Diagnostic[], relativeToBuildInfo)
] :
relativeToBuildInfo(key)
toFileNameIndex(key)
);
}
result.semanticDiagnosticsPerFile = semanticDiagnosticsPerFile;
}

let affectedFilesPendingEmit: ProgramBuilderInfoFilePendingEmit[] | undefined;
if (state.affectedFilesPendingEmit) {
const affectedFilesPendingEmit: ProgramBuilderInfoFilePendingEmit[] = [];
const seenFiles = new Set<Path>();
for (const path of state.affectedFilesPendingEmit.slice(state.affectedFilesPendingEmitIndex).sort(compareStringsCaseSensitive)) {
if (tryAddToSet(seenFiles, path)) {
affectedFilesPendingEmit.push([relativeToBuildInfo(path), state.affectedFilesPendingEmitKind!.get(path)!]);
(affectedFilesPendingEmit ||= []).push([toFileNameIndex(path), state.affectedFilesPendingEmitKind!.get(path)!]);
}
}
result.affectedFilesPendingEmit = affectedFilesPendingEmit;
}

return result;
return {
fileNames,
fileInfos,
options: convertToReusableCompilerOptions(state.compilerOptions, relativeToBuildInfoEnsuringAbsolutePath),
fileNamesList,
referencedMap,
exportedModulesMap,
semanticDiagnosticsPerFile,
affectedFilesPendingEmit,
};

function relativeToBuildInfoEnsuringAbsolutePath(path: string) {
return relativeToBuildInfo(getNormalizedAbsolutePath(path, currentDirectory));
Expand All @@ -776,6 +787,22 @@ namespace ts {
function relativeToBuildInfo(path: string) {
return ensurePathIsNonModuleName(getRelativePathFromDirectory(buildInfoDirectory, path, getCanonicalFileName));
}

function toFileNameIndex(path: Path): number {
const existing = fileNamesMap.get(path);
if (existing !== undefined) return existing;
fileNamesMap.set(path, fileNames.length);
return fileNames.push(relativeToBuildInfo(path)) - 1;
}

function toFileNamesListIndex(set: ReadonlySet<Path>): number {
const paths = arrayFrom(set.keys(), toFileNameIndex).sort(compareValues);
const key = paths.join();
const existing = fileNamesListMap?.get(key);
if (existing !== undefined) return existing;
(fileNamesListMap ||= new Map()).set(key, fileNamesList?.length || 0);
return (fileNamesList ||= []).push(paths) - 1;
}
}

function convertToReusableCompilerOptions(options: CompilerOptions, relativeToBuildInfo: (path: string) => string) {
Expand Down Expand Up @@ -1167,39 +1194,27 @@ namespace ts {
}
}

function getMapOfReferencedSet(mapLike: MapLike<readonly string[]> | undefined, toPath: (path: string) => Path): ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined {
if (!mapLike) return undefined;
const map = new Map<Path, BuilderState.ReferencedSet>();
// Copies keys/values from template. Note that for..in will not throw if
// template is undefined, and instead will just exit the loop.
for (const key in mapLike) {
if (hasProperty(mapLike, key)) {
map.set(toPath(key), new Set(mapLike[key].map(toPath)));
}
}
return map;
function getMapOfReferencedSet(referenceMap: ProgramBuildInfoReferencedMap | undefined, fileNames: readonly Path[], fileNamesList: readonly ReadonlySet<Path>[] | undefined): ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined {
return referenceMap && arrayToMap(referenceMap, value => fileNames[value[0]], value => fileNamesList![value[1]]);
}

export function createBuildProgramUsingProgramBuildInfo(program: ProgramBuildInfo, buildInfoPath: string, host: ReadBuildProgramHost): EmitAndSemanticDiagnosticsBuilderProgram {
const buildInfoDirectory = getDirectoryPath(getNormalizedAbsolutePath(buildInfoPath, host.getCurrentDirectory()));
const getCanonicalFileName = createGetCanonicalFileName(host.useCaseSensitiveFileNames());

const fileNames = program.fileNames.map(toPath);
const fileNamesList = program.fileNamesList?.map(list => new Set(list.map(index => fileNames[index])));
const fileInfos = new Map<Path, BuilderState.FileInfo>();
for (const key in program.fileInfos) {
if (hasProperty(program.fileInfos, key)) {
fileInfos.set(toPath(key), program.fileInfos[key]);
}
}

program.fileInfos.forEach((fileInfo, index) => fileInfos.set(fileNames[index], fileInfo));
const state: ReusableBuilderProgramState = {
fileInfos,
compilerOptions: convertToOptionsWithAbsolutePaths(program.options, toAbsolutePath),
referencedMap: getMapOfReferencedSet(program.referencedMap, toPath),
exportedModulesMap: getMapOfReferencedSet(program.exportedModulesMap, toPath),
semanticDiagnosticsPerFile: program.semanticDiagnosticsPerFile && arrayToMap(program.semanticDiagnosticsPerFile, value => toPath(isString(value) ? value : value[0]), value => isString(value) ? emptyArray : value[1]),
referencedMap: getMapOfReferencedSet(program.referencedMap, fileNames, fileNamesList),
exportedModulesMap: getMapOfReferencedSet(program.exportedModulesMap, fileNames, fileNamesList),
semanticDiagnosticsPerFile: program.semanticDiagnosticsPerFile && arrayToMap(program.semanticDiagnosticsPerFile, value => fileNames[isNumber(value) ? value : value[0]], value => isNumber(value) ? emptyArray : value[1]),
hasReusableDiagnostic: true,
affectedFilesPendingEmit: map(program.affectedFilesPendingEmit, value => toPath(value[0])),
affectedFilesPendingEmitKind: program.affectedFilesPendingEmit && arrayToMap(program.affectedFilesPendingEmit, value => toPath(value[0]), value => value[1]),
affectedFilesPendingEmit: map(program.affectedFilesPendingEmit, value => fileNames[value[0]]),
affectedFilesPendingEmitKind: program.affectedFilesPendingEmit && arrayToMap(program.affectedFilesPendingEmit, value => fileNames[value[0]], value => value[1]),
affectedFilesPendingEmitIndex: program.affectedFilesPendingEmit && 0,
};
return {
Expand Down
2 changes: 1 addition & 1 deletion src/compiler/emitter.ts
View file
Open in desktop
Expand Up @@ -663,7 +663,7 @@ namespace ts {

/*@internal*/
export function getBuildInfoText(buildInfo: BuildInfo) {
return JSON.stringify(buildInfo, undefined, 2);
return JSON.stringify(buildInfo);
}

/*@internal*/
Expand Down
65 changes: 58 additions & 7 deletions src/testRunner/unittests/tsbuild/helpers.ts
View file
Open in desktop
Expand Up @@ -236,18 +236,70 @@ interface Symbol {
}
}

function generateBuildInfoProgramBaseline(sys: System, originalWriteFile: System["writeFile"], buildInfoPath: string, buildInfo: BuildInfo) {
type ProgramBuildInfoDiagnostic = string | [string, readonly ReusableDiagnostic[]];
type ProgramBuilderInfoFilePendingEmit = [string, BuilderFileEmit];
interface ProgramBuildInfo {
fileInfos: MapLike<BuilderState.FileInfo>;
options: CompilerOptions;
referencedMap?: MapLike<string[]>;
exportedModulesMap?: MapLike<string[]>;
semanticDiagnosticsPerFile?: ProgramBuildInfoDiagnostic[];
affectedFilesPendingEmit?: ProgramBuilderInfoFilePendingEmit[];
}
const fileInfos: ProgramBuildInfo["fileInfos"] = {};
buildInfo.program?.fileInfos.forEach((fileInfo, index) => {
fileInfos[buildInfo.program!.fileNames[index]] = fileInfo;
});
const fileNamesList = buildInfo.program?.fileNamesList?.map(list => list.map(index => buildInfo.program!.fileNames[index]));
const program: ProgramBuildInfo | undefined = buildInfo.program && {
fileInfos,
options: buildInfo.program.options,
referencedMap: getMapOfReferencedSet(buildInfo.program.referencedMap),
exportedModulesMap: getMapOfReferencedSet(buildInfo.program.exportedModulesMap),
semanticDiagnosticsPerFile: buildInfo.program.semanticDiagnosticsPerFile?.map(d =>
isNumber(d) ?
buildInfo.program!.fileNames[d] :
[buildInfo.program!.fileNames[d[0]], d[1]]
),
affectedFilesPendingEmit: buildInfo.program.affectedFilesPendingEmit?.map(([file, kind]) => [
buildInfo.program!.fileNames[file],
kind
]),
};
const result: Omit<BuildInfo, "program"> & { program: ProgramBuildInfo | undefined; } = {
bundle: buildInfo.bundle,
program,
version: buildInfo.version === version ? fakes.version : buildInfo.version,
};
// For now its just JSON.stringify
originalWriteFile.call(sys, `${buildInfoPath}.readable.baseline.txt`, JSON.stringify(result, /*replacer*/ undefined, 2));

function getMapOfReferencedSet(referenceMap: ProgramBuildInfoReferencedMap | undefined): MapLike<string[]> | undefined {
if (!referenceMap) return undefined;
const result: MapLike<string[]> = {};
for (const [fileNamesKey, fileNamesListKey] of referenceMap) {
result[buildInfo.program!.fileNames[fileNamesKey]] = fileNamesList![fileNamesListKey];
}
return result;
}
}

export function baselineBuildInfo(
options: CompilerOptions,
sys: System & { writtenFiles: ReadonlyCollection<string>; },
originalReadCall?: System["readFile"]
originalReadCall?: System["readFile"],
originalWriteFile?: System["writeFile"],
) {
const out = outFile(options);
if (!out) return;
const { buildInfoPath, jsFilePath, declarationFilePath } = getOutputPathsForBundle(options, /*forceDts*/ false);
const buildInfoPath = getTsBuildInfoEmitOutputFilePath(options);
if (!buildInfoPath || !sys.writtenFiles.has(buildInfoPath)) return;
if (!sys.fileExists(buildInfoPath)) return;

const buildInfo = getBuildInfo((originalReadCall || sys.readFile).call(sys, buildInfoPath, "utf8")!);
generateBuildInfoProgramBaseline(sys, originalWriteFile || sys.writeFile, buildInfoPath, buildInfo);

if (!outFile(options)) return;
const { jsFilePath, declarationFilePath } = getOutputPathsForBundle(options, /*forceDts*/ false);
const bundle = buildInfo.bundle;
if (!bundle || (!length(bundle.js && bundle.js.sections) && !length(bundle.dts && bundle.dts.sections))) return;

Expand All @@ -256,9 +308,8 @@ interface Symbol {
generateBundleFileSectionInfo(sys, originalReadCall || sys.readFile, baselineRecorder, bundle.js, jsFilePath);
generateBundleFileSectionInfo(sys, originalReadCall || sys.readFile, baselineRecorder, bundle.dts, declarationFilePath);
baselineRecorder.Close();

const text = baselineRecorder.lines.join("\r\n");
sys.writeFile(`${buildInfoPath}.baseline.txt`, text);
(originalWriteFile || sys.writeFile).call(sys, `${buildInfoPath}.baseline.txt`, text);
}

interface VerifyIncrementalCorrectness {
Expand Down Expand Up @@ -295,7 +346,7 @@ interface Symbol {
const cleanBuildText = sys.readFile(outputFile);
const incrementalBuildText = newSys.readFile(outputFile);
const descrepancyInClean = discrepancies?.get(outputFile);
if (!isBuildInfoFile(outputFile)) {
if (!isBuildInfoFile(outputFile) && !fileExtensionIs(outputFile, ".tsbuildinfo.readable.baseline.txt")) {
verifyTextEqual(incrementalBuildText, cleanBuildText, descrepancyInClean, `File: ${outputFile}`);
}
else if (incrementalBuildText !== cleanBuildText) {
Expand Down
9 changes: 4 additions & 5 deletions src/testRunner/unittests/tsbuild/outputPaths.ts
View file
Open in desktop
Expand Up @@ -60,11 +60,10 @@ namespace ts {
noChangeRun,
{
...noChangeProject,
cleanBuildDiscrepancies: () => {
const map = new Map<string, CleanBuildDescrepancy>();
map.set("/src/dist/tsconfig.tsbuildinfo", CleanBuildDescrepancy.CleanFileTextDifferent); // tsbuildinfo will have -p setting when built using -p vs no build happens incrementally because of no change.
return map;
}
cleanBuildDiscrepancies: () => new Map([
["/src/dist/tsconfig.tsbuildinfo", CleanBuildDescrepancy.CleanFileTextDifferent], // tsbuildinfo will have -p setting when built using -p vs no build happens incrementally because of no change.
["/src/dist/tsconfig.tsbuildinfo.readable.baseline.txt", CleanBuildDescrepancy.CleanFileTextDifferent] // tsbuildinfo will have -p setting when built using -p vs no build happens incrementally because of no change.
]),
}
],
}, ["/src/dist/src/index.js", "/src/dist/src/index.d.ts"]);
Expand Down