Skip to content

Commit

Permalink
Refactored old server errors
Browse files Browse the repository at this point in the history
  • Loading branch information
@armanio123
armanio123 committed Jul 2, 2024
1 parent 9e0ad34 commit bf96b26
Show file tree
Hide file tree
Showing 3 changed files with 291 additions and 21 deletions.
69 changes: 59 additions & 10 deletions src/main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,10 +335,7 @@ async function getTsServerRepoResult(
installCommands,
};

if (oldServerFailed && !newServerFailed) {
return { status: "Detected interesting changes", tsServerResult }
}
if (!newServerFailed) {
if (!oldServerFailed && !newServerFailed) {
return { status: "Detected no interesting changes" };
}

Expand Down Expand Up @@ -404,25 +401,77 @@ function createOldErrorSummary(summaries: Summary[]): string {

let text = `
<details>
<summary>${errorMessage}</summary>
<summary>New server no longer reports this error: ${errorMessage}</summary>
\`\`\`
${oldServerError}
\`\`\`
<h4>Repos no longer reporting the error</h4>
<ul>
`;
<h4>Affected repos</h4>`;

for (const summary of summaries) {
const owner = summary.repo.owner ? `${mdEscape(summary.repo.owner)}/` : "";
const url = summary.repo.url ?? "";

text += `<li><a href="${url}">${owner + mdEscape(summary.repo.name)}</a></li>\n`
text += `
<details>
<summary><a href="${url}">${owner + mdEscape(summary.repo.name)}</a></summary>
Raw error text: <code>${summary.rawErrorArtifactPath}</code> in the <a href="${artifactFolderUrlPlaceholder}">artifact folder</a> <br />
Replay commands: <code>${summary.replayScriptArtifactPath}</code> in the <a href="${artifactFolderUrlPlaceholder}">artifact folder</a>
<h4>Last few requests</h4>
\`\`\`json
${summary.replayScript}
\`\`\`
<h4>Repro steps</h4>
\`\`\`bash
#!/bin/bash
`;
// No url means is user test repo
if (!summary.repo.url) {
text += `# Manually download user test \`${summary.repo.name}\`\n`;
}
else {
text += `git clone ${summary.repo.url} --recurse-submodules\n`;

if (summary.commit) {
text += `git -C "./${summary.repo.name}" reset --hard ${summary.commit}\n`;
}
}

if (summary.tsServerResult.installCommands.length > 1) {
text += "# Install packages (exact steps are below, but it might be easier to follow the repo readme)\n";
}
for (const command of summary.tsServerResult.installCommands) {
const workingDirFlag = command.tool === ip.InstallTool.Npm
? "--prefix"
: command.tool === ip.InstallTool.Yarn
? "--cwd"
: "--dir"; // pnpm

text += `${command.tool} ${workingDirFlag} "./${command.prettyDirectory}" ${command.arguments.join(" ")}\n`;
}

text += `downloadUrl=$(curl -s "${getArtifactsApiUrlPlaceholder}?artifactName=${summary.resultDirName}&api-version=7.0" | jq -r ".resource.downloadUrl")
wget -O ${summary.resultDirName}.zip "$downloadUrl"
unzip -p ${summary.resultDirName}.zip ${summary.resultDirName}/${summary.replayScriptName} > ${summary.replayScriptName}
npm install --no-save @typescript/server-replay
\`\`\`
To run the repro:
\`\`\`bash
# \`npx tsreplay --help\` to learn about helpful switches for debugging, logging, etc.
npx tsreplay ./${summary.repo.name} ./${summary.replayScriptName} <PATH_TO_tsserver.js>
\`\`\`
</details>
`;
}

text += `
</ul>
</details>
`;

Expand Down
170 changes: 170 additions & 0 deletions test/__snapshots__/main.test.ts.snap
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,175 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`main outputs old server errors 1`] = `
[MockFunction] {
"calls": [
[
"<BASE_PATH>/ts_downloads/base/MockRepoOwner.MockRepoName.rawError.txt",
"Req #123 - cursedCommand
Some error. Could not do something.
Maybe a Debug fail.
at a (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:1:1)
at b (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:2:2)
at c (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:3:3)
at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4)
at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5)",
],
[
"<BASE_PATH>/RepoResults123/718e48b799650d4b66e5d80ad6bac7b9.results.txt",
"<h2>Maybe a Debug fail.</h2>
\`\`\`
Req #123 - cursedCommand
at a (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:1:1)
at b (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:2:2)
at c (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:3:3)
at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4)
at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5)
\`\`\`
<h4>Affected repos</h4>
<details>
<summary><a href="https://github.com/MockRepoOwner/MockRepoName">MockRepoOwner/MockRepoName</a></summary>
Raw error text: <code>RepoResults123/MockRepoOwner.MockRepoName.rawError.txt</code> in the <a href="PLACEHOLDER_ARTIFACT_FOLDER">artifact folder</a> <br />
Replay commands: <code>RepoResults123/MockRepoOwner.MockRepoName.replay.txt</code> in the <a href="PLACEHOLDER_ARTIFACT_FOLDER">artifact folder</a>
<h4>Last few requests</h4>
\`\`\`json
{"rootDirPlaceholder":"@PROJECT_ROOT@","serverArgs":["--disableAutomaticTypingAcquisition"]}
{"seq":1,"type":"request","command":"configure","arguments":{"preferences":{"disableLineTextInReferences":true,"includePackageJsonAutoImports":"auto","includeCompletionsForImportStatements":true,"includeCompletionsWithSnippetText":true,"includeAutomaticOptionalChainCompletions":true,"includeCompletionsWithInsertText":true,"includeCompletionsWithClassMemberSnippets":true,"allowIncompleteCompletions":true,"includeCompletionsForModuleExports":false},"watchOptions":{"excludeDirectories":["**/node_modules"]}}}
{"seq":2,"type":"request","command":"updateOpen","arguments":{"changedFiles":[],"closedFiles":[],"openFiles":[{"file":"@PROJECT_ROOT@/sample_repoName.config.js","projectRootPath":"@PROJECT_ROOT@"}]}}
{"seq":3,"type":"request","command":"cursedCommand","arguments":{"file":"@PROJECT_ROOT@/src/sampleTsFile.ts","line":1,"offset":1,"includeExternalModuleExports":false,"triggerKind":1}}
\`\`\`
<h4>Repro steps</h4>
\`\`\`bash
#!/bin/bash
git clone https://github.com/MockRepoOwner/MockRepoName --recurse-submodules
git -C "./MockRepoName" reset --hard 57b462387e88aa7e363af0daf867a5dc1e83a935
# Install packages (exact steps are below, but it might be easier to follow the repo readme)
npm --prefix "./dirA" install --prefer-offline --no-audit --no-progress --legacy-peer-deps --ignore-scripts -q
npm --prefix "./dirB/dirC" install --prefer-offline --no-audit --no-progress --legacy-peer-deps --ignore-scripts -q
npm --prefix "./dirD/dirE/dirF" install --prefer-offline --no-audit --no-progress --legacy-peer-deps --ignore-scripts -q
downloadUrl=$(curl -s "PLACEHOLDER_GETARTIFACTS_API?artifactName=RepoResults123&api-version=7.0" | jq -r ".resource.downloadUrl")
wget -O RepoResults123.zip "$downloadUrl"
unzip -p RepoResults123.zip RepoResults123/MockRepoOwner.MockRepoName.replay.txt > MockRepoOwner.MockRepoName.replay.txt
npm install --no-save @typescript/server-replay
\`\`\`
To run the repro:
\`\`\`bash
# \`npx tsreplay --help\` to learn about helpful switches for debugging, logging, etc.
npx tsreplay ./MockRepoName ./MockRepoOwner.MockRepoName.replay.txt <PATH_TO_tsserver.js>
\`\`\`
</details>
",
{
"encoding": "utf-8",
},
],
[
"<BASE_PATH>/RepoResults123/metadata.json",
"{"newTsResolvedVersion":"1.1.1","oldTsResolvedVersion":"0.0.0","statusCounts":{"Detected interesting changes":1}}",
{
"encoding": "utf-8",
},
],
[
"<BASE_PATH>/RepoResults123/!718e48b799650d4b66e5d80ad6bac7b9.results.txt",
"
<details>
<summary>New server no longer reports this error: Maybe a Debug fail.</summary>
\`\`\`
Req #123 - cursedCommand
at a (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:1:1)
at b (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:2:2)
at c (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:3:3)
at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4)
at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5)
\`\`\`
<h4>Affected repos</h4>
<details>
<summary><a href="https://github.com/MockRepoOwner/MockRepoName">MockRepoOwner/MockRepoName</a></summary>
Raw error text: <code>RepoResults123/MockRepoOwner.MockRepoName.rawError.txt</code> in the <a href="PLACEHOLDER_ARTIFACT_FOLDER">artifact folder</a> <br />
Replay commands: <code>RepoResults123/MockRepoOwner.MockRepoName.replay.txt</code> in the <a href="PLACEHOLDER_ARTIFACT_FOLDER">artifact folder</a>
<h4>Last few requests</h4>
\`\`\`json
{"rootDirPlaceholder":"@PROJECT_ROOT@","serverArgs":["--disableAutomaticTypingAcquisition"]}
{"seq":1,"type":"request","command":"configure","arguments":{"preferences":{"disableLineTextInReferences":true,"includePackageJsonAutoImports":"auto","includeCompletionsForImportStatements":true,"includeCompletionsWithSnippetText":true,"includeAutomaticOptionalChainCompletions":true,"includeCompletionsWithInsertText":true,"includeCompletionsWithClassMemberSnippets":true,"allowIncompleteCompletions":true,"includeCompletionsForModuleExports":false},"watchOptions":{"excludeDirectories":["**/node_modules"]}}}
{"seq":2,"type":"request","command":"updateOpen","arguments":{"changedFiles":[],"closedFiles":[],"openFiles":[{"file":"@PROJECT_ROOT@/sample_repoName.config.js","projectRootPath":"@PROJECT_ROOT@"}]}}
{"seq":3,"type":"request","command":"cursedCommand","arguments":{"file":"@PROJECT_ROOT@/src/sampleTsFile.ts","line":1,"offset":1,"includeExternalModuleExports":false,"triggerKind":1}}
\`\`\`
<h4>Repro steps</h4>
\`\`\`bash
#!/bin/bash
git clone https://github.com/MockRepoOwner/MockRepoName --recurse-submodules
git -C "./MockRepoName" reset --hard 57b462387e88aa7e363af0daf867a5dc1e83a935
# Install packages (exact steps are below, but it might be easier to follow the repo readme)
npm --prefix "./dirA" install --prefer-offline --no-audit --no-progress --legacy-peer-deps --ignore-scripts -q
npm --prefix "./dirB/dirC" install --prefer-offline --no-audit --no-progress --legacy-peer-deps --ignore-scripts -q
npm --prefix "./dirD/dirE/dirF" install --prefer-offline --no-audit --no-progress --legacy-peer-deps --ignore-scripts -q
downloadUrl=$(curl -s "PLACEHOLDER_GETARTIFACTS_API?artifactName=RepoResults123&api-version=7.0" | jq -r ".resource.downloadUrl")
wget -O RepoResults123.zip "$downloadUrl"
unzip -p RepoResults123.zip RepoResults123/MockRepoOwner.MockRepoName.replay.txt > MockRepoOwner.MockRepoName.replay.txt
npm install --no-save @typescript/server-replay
\`\`\`
To run the repro:
\`\`\`bash
# \`npx tsreplay --help\` to learn about helpful switches for debugging, logging, etc.
npx tsreplay ./MockRepoName ./MockRepoOwner.MockRepoName.replay.txt <PATH_TO_tsserver.js>
\`\`\`
</details>
</details>
",
{
"encoding": "utf-8",
},
],
[
"<BASE_PATH>/RepoResults123/metadata.json",
"{"newTsResolvedVersion":"1.1.1","oldTsResolvedVersion":"0.0.0","statusCounts":{"Detected interesting changes":1}}",
{
"encoding": "utf-8",
},
],
],
"results": [
{
"type": "return",
"value": undefined,
},
{
"type": "return",
"value": undefined,
},
{
"type": "return",
"value": undefined,
},
{
"type": "return",
"value": undefined,
},
{
"type": "return",
"value": undefined,
},
],
}
`;
exports[`main outputs server errors 1`] = `
[MockFunction] {
"calls": [
Expand Down
73 changes: 62 additions & 11 deletions test/main.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { getTscRepoResult, downloadTsRepoAsync, mainAsync } from '../src/main'
import { execSync } from "child_process"
import path = require("path")
import { createCopyingOverlayFS } from '../src/utils/overlayFS'
import { SpawnResult } from '../src/utils/execUtils';

jest.mock('random-seed', () => ({
create: () => {
Expand All @@ -27,17 +28,7 @@ jest.mock("../src/utils/execUtils", () => ({
return {};
}

return {
stdout: JSON.stringify({
"request_seq": "123",
"command": "cursedCommand",
"message": "Some error. Could not do something. \nMaybe a Debug fail.\n at a (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:1:1)\n at b (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:2:2)\n at c (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:3:3)\n at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4)\n at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5)"
}),
stderr: '',
code: 5,
signal: null,

}
return typeScriptSpawnResult(args);
}),
execAsync: async (cwd: string, command: string) => {
if (command.startsWith('npm pack typescript@latest')) {
Expand Down Expand Up @@ -110,6 +101,14 @@ jest.mock('../src/utils/installPackages', () => {
}
});

let typeScriptSpawnResult: (args: readonly string[]) => SpawnResult;

const errorStdout = JSON.stringify({
"request_seq": "123",
"command": "cursedCommand",
"message": "Some error. Could not do something. \nMaybe a Debug fail.\n at a (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:1:1)\n at b (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:2:2)\n at c (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:3:3)\n at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4)\n at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5)"
});

describe("main", () => {
jest.setTimeout(10 * 60 * 1000);

Expand Down Expand Up @@ -146,6 +145,58 @@ describe("main", () => {
it("outputs server errors", async () => {
const mockedFs = require('fs');

typeScriptSpawnResult = () => ({
stdout: errorStdout,
stderr: '',
code: 5,
signal: null,

});

await mainAsync({
testType: "github",
tmpfs: false,
entrypoint: 'tsserver',
diagnosticOutput: false,
buildWithNewWhenOldFails: false,
repoListPath: "./artifacts/repos.json",
workerCount: 1,
workerNumber: 1,
oldTsNpmVersion: 'latest',
newTsNpmVersion: 'next',
resultDirName: 'RepoResults123',
prngSeed: 'testSeed',
});

// Remove all references to the base path so that snapshot pass successfully.
mockedFs.promises.writeFile.mock.calls.forEach((e: [string, string]) => {
e[0] = e[0].replace(process.cwd(), "<BASE_PATH>");
});

expect(mockedFs.promises.writeFile).toMatchSnapshot();
});

it("outputs old server errors", async () => {
const mockedFs = require('fs');

typeScriptSpawnResult = args => {
let isOldServer = args.some(x => x.includes('0.0.0'));

// Only "old" reports an error.
return isOldServer ? {
stdout: errorStdout,
stderr: '',
code: 5,
signal: null,
} : {
stdout: '',
stderr: '',
code: 0,
signal: null,
};

};

await mainAsync({
testType: "github",
tmpfs: false,
Expand Down

0 comments on commit bf96b26

Please sign in to comment.