Skip to content

Commit b1be37c

Browse files
koesie10koesie10
authored
Merge pull request github#3763 from github/koesie10/cleanup-distributions
Clean up old distributions
2 parents f19b0df + a488877 commit b1be37c

File tree

5 files changed

+298
-0
lines changed

5 files changed

+298
-0
lines changed

‎extensions/ql-vscode/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
- Support result columns of type `QlBuiltins::BigInt` in quick evaluations. [#3647](https://github.com/github/vscode-codeql/pull/3647)
66
- Fix a bug where the CodeQL CLI would be re-downloaded if you switched to a different filesystem (for example Codespaces or a remote SSH host). [#3762](https://github.com/github/vscode-codeql/pull/3762)
7+
- Clean up old extension-managed CodeQL CLI distributions. [#3763](https://github.com/github/vscode-codeql/pull/3763)
78

89
## 1.16.0 - 10 October 2024
910

‎extensions/ql-vscode/src/codeql-cli/distribution.ts

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import { asError, getErrorMessage } from "../common/helpers-pure";
4242
import { isIOError } from "../common/files";
4343
import { telemetryListener } from "../common/vscode/telemetry";
4444
import { redactableError } from "../common/errors";
45+
import { ExtensionManagedDistributionCleaner } from "./distribution/cleaner";
4546

4647
/**
4748
* distribution.ts
@@ -99,6 +100,12 @@ export class DistributionManager implements DistributionProvider {
99100
() =>
100101
this.extensionSpecificDistributionManager.checkForUpdatesToDistribution(),
101102
);
103+
this.extensionManagedDistributionCleaner =
104+
new ExtensionManagedDistributionCleaner(
105+
extensionContext,
106+
logger,
107+
this.extensionSpecificDistributionManager,
108+
);
102109
}
103110

104111
public async initialize(): Promise<void> {
@@ -280,6 +287,10 @@ export class DistributionManager implements DistributionProvider {
280287
);
281288
}
282289

290+
public startCleanup() {
291+
this.extensionManagedDistributionCleaner.start();
292+
}
293+
283294
public get onDidChangeDistribution(): Event<void> | undefined {
284295
return this._onDidChangeDistribution;
285296
}
@@ -301,6 +312,7 @@ export class DistributionManager implements DistributionProvider {
301312

302313
private readonly extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
303314
private readonly updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
315+
private readonly extensionManagedDistributionCleaner: ExtensionManagedDistributionCleaner;
304316
private readonly _onDidChangeDistribution: Event<void> | undefined;
305317
}
306318

@@ -718,6 +730,16 @@ class ExtensionSpecificDistributionManager {
718730
await outputJson(distributionStatePath, newState);
719731
}
720732

733+
public get folderIndex() {
734+
const distributionState = this.getDistributionState();
735+
736+
return distributionState.folderIndex;
737+
}
738+
739+
public get distributionFolderPrefix() {
740+
return ExtensionSpecificDistributionManager._currentDistributionFolderBaseName;
741+
}
742+
721743
private static readonly _currentDistributionFolderBaseName = "distribution";
722744
private static readonly _codeQlExtractedFolderName = "codeql";
723745
private static readonly _distributionStateFilename = "distribution.json";

‎extensions/ql-vscode/src/codeql-cli/distribution/cleaner.ts

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
import type { ExtensionContext } from "vscode";
2+
import { getDirectoryNamesInsidePath, isIOError } from "../../common/files";
3+
import { sleep } from "../../common/time";
4+
import type { BaseLogger } from "../../common/logging";
5+
import { join } from "path";
6+
import { getErrorMessage } from "../../common/helpers-pure";
7+
import { pathExists, remove } from "fs-extra";
8+
9+
interface ExtensionManagedDistributionManager {
10+
folderIndex: number;
11+
distributionFolderPrefix: string;
12+
}
13+
14+
interface DistributionDirectory {
15+
directoryName: string;
16+
folderIndex: number;
17+
}
18+
19+
/**
20+
* This class is responsible for cleaning up old distributions that are no longer needed. In normal operation, this
21+
* should not be necessary as the old distribution is deleted when the distribution is updated. However, in some cases
22+
* the extension may leave behind old distribution which can result in a significant amount of space (> 100 GB) being
23+
* taking up by unused distributions.
24+
*/
25+
export class ExtensionManagedDistributionCleaner {
26+
constructor(
27+
private readonly extensionContext: ExtensionContext,
28+
private readonly logger: BaseLogger,
29+
private readonly manager: ExtensionManagedDistributionManager,
30+
) {}
31+
32+
public start() {
33+
// Intentionally starting this without waiting for it
34+
void this.cleanup().catch((e: unknown) => {
35+
void this.logger.log(
36+
`Failed to clean up old versions of the CLI: ${getErrorMessage(e)}`,
37+
);
38+
});
39+
}
40+
41+
public async cleanup() {
42+
if (!(await pathExists(this.extensionContext.globalStorageUri.fsPath))) {
43+
return;
44+
}
45+
46+
const currentFolderIndex = this.manager.folderIndex;
47+
48+
const distributionDirectoryRegex = new RegExp(
49+
`^${this.manager.distributionFolderPrefix}(\\d+)$`,
50+
);
51+
52+
const existingDirectories = await getDirectoryNamesInsidePath(
53+
this.extensionContext.globalStorageUri.fsPath,
54+
);
55+
const distributionDirectories = existingDirectories
56+
.map((dir): DistributionDirectory | null => {
57+
const match = dir.match(distributionDirectoryRegex);
58+
if (!match) {
59+
// When the folderIndex is 0, the distributionFolderPrefix is used as the directory name
60+
if (dir === this.manager.distributionFolderPrefix) {
61+
return {
62+
directoryName: dir,
63+
folderIndex: 0,
64+
};
65+
}
66+
67+
return null;
68+
}
69+
70+
return {
71+
directoryName: dir,
72+
folderIndex: parseInt(match[1]),
73+
};
74+
})
75+
.filter((dir) => dir !== null);
76+
77+
// Clean up all directories that are older than the current one
78+
const cleanableDirectories = distributionDirectories.filter(
79+
(dir) => dir.folderIndex < currentFolderIndex,
80+
);
81+
82+
if (cleanableDirectories.length === 0) {
83+
return;
84+
}
85+
86+
// Shuffle the array so that multiple VS Code processes don't all try to clean up the same directory at the same time
87+
for (let i = cleanableDirectories.length - 1; i > 0; i--) {
88+
const j = Math.floor(Math.random() * (i + 1));
89+
[cleanableDirectories[i], cleanableDirectories[j]] = [
90+
cleanableDirectories[j],
91+
cleanableDirectories[i],
92+
];
93+
}
94+
95+
void this.logger.log(
96+
`Cleaning up ${cleanableDirectories.length} old versions of the CLI.`,
97+
);
98+
99+
for (const cleanableDirectory of cleanableDirectories) {
100+
// Wait 10 seconds between each cleanup to avoid overloading the system (even though the remove call should be async)
101+
await sleep(10_000);
102+
103+
const path = join(
104+
this.extensionContext.globalStorageUri.fsPath,
105+
cleanableDirectory.directoryName,
106+
);
107+
108+
// Delete this directory
109+
try {
110+
await remove(path);
111+
} catch (e) {
112+
if (isIOError(e) && e.code === "ENOENT") {
113+
// If the directory doesn't exist, that's fine
114+
continue;
115+
}
116+
117+
void this.logger.log(
118+
`Tried to clean up an old version of the CLI at ${path} but encountered an error: ${getErrorMessage(e)}.`,
119+
);
120+
}
121+
}
122+
123+
void this.logger.log(
124+
`Cleaned up ${cleanableDirectories.length} old versions of the CLI.`,
125+
);
126+
}
127+
}

‎extensions/ql-vscode/src/extension.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1125,6 +1125,8 @@ async function activateWithInstalledDistribution(
11251125
void extLogger.log("Reading query history");
11261126
await qhm.readQueryHistory();
11271127

1128+
distributionManager.startCleanup();
1129+
11281130
void extLogger.log("Successfully finished extension initialization.");
11291131

11301132
return {

‎extensions/ql-vscode/test/vscode-tests/no-workspace/codeql-cli/distribution/cleaner.test.ts

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import { ExtensionManagedDistributionCleaner } from "../../../../../src/codeql-cli/distribution/cleaner";
2+
import { mockedObject } from "../../../../mocked-object";
3+
import type { ExtensionContext } from "vscode";
4+
import { Uri } from "vscode";
5+
import { createMockLogger } from "../../../../__mocks__/loggerMock";
6+
import type { DirectoryResult } from "tmp-promise";
7+
import { dir } from "tmp-promise";
8+
import { outputFile, pathExists } from "fs-extra";
9+
import { join } from "path";
10+
import { codeQlLauncherName } from "../../../../../src/common/distribution";
11+
import { getDirectoryNamesInsidePath } from "../../../../../src/common/files";
12+
13+
describe("ExtensionManagedDistributionCleaner", () => {
14+
let globalStorageDirectory: DirectoryResult;
15+
16+
let manager: ExtensionManagedDistributionCleaner;
17+
18+
beforeEach(async () => {
19+
globalStorageDirectory = await dir({
20+
unsafeCleanup: true,
21+
});
22+
23+
manager = new ExtensionManagedDistributionCleaner(
24+
mockedObject<ExtensionContext>({
25+
globalStorageUri: Uri.file(globalStorageDirectory.path),
26+
}),
27+
createMockLogger(),
28+
{
29+
folderIndex: 768,
30+
distributionFolderPrefix: "distribution",
31+
},
32+
);
33+
34+
// Mock setTimeout to call the callback immediately
35+
jest.spyOn(global, "setTimeout").mockImplementation((callback) => {
36+
callback();
37+
return 0 as unknown as ReturnType<typeof setTimeout>;
38+
});
39+
});
40+
41+
afterEach(async () => {
42+
await globalStorageDirectory.cleanup();
43+
});
44+
45+
it("does nothing when no distributions exist", async () => {
46+
await manager.cleanup();
47+
});
48+
49+
it("does nothing when only the current distribution exists", async () => {
50+
await outputFile(
51+
join(
52+
globalStorageDirectory.path,
53+
"distribution768",
54+
"codeql",
55+
"bin",
56+
codeQlLauncherName(),
57+
),
58+
"launcher!",
59+
);
60+
61+
await manager.cleanup();
62+
63+
expect(
64+
await pathExists(
65+
join(
66+
globalStorageDirectory.path,
67+
"distribution768",
68+
"codeql",
69+
"bin",
70+
codeQlLauncherName(),
71+
),
72+
),
73+
).toBe(true);
74+
});
75+
76+
it("removes old distributions", async () => {
77+
await outputFile(
78+
join(
79+
globalStorageDirectory.path,
80+
"distribution",
81+
"codeql",
82+
"bin",
83+
codeQlLauncherName(),
84+
),
85+
"launcher!",
86+
);
87+
await outputFile(
88+
join(
89+
globalStorageDirectory.path,
90+
"distribution12",
91+
"codeql",
92+
"bin",
93+
codeQlLauncherName(),
94+
),
95+
"launcher!",
96+
);
97+
await outputFile(
98+
join(
99+
globalStorageDirectory.path,
100+
"distribution244",
101+
"codeql",
102+
"bin",
103+
codeQlLauncherName(),
104+
),
105+
"launcher!",
106+
);
107+
await outputFile(
108+
join(
109+
globalStorageDirectory.path,
110+
"distribution637",
111+
"codeql",
112+
"bin",
113+
codeQlLauncherName(),
114+
),
115+
"launcher!",
116+
);
117+
await outputFile(
118+
join(
119+
globalStorageDirectory.path,
120+
"distribution768",
121+
"codeql",
122+
"bin",
123+
codeQlLauncherName(),
124+
),
125+
"launcher!",
126+
);
127+
await outputFile(
128+
join(
129+
globalStorageDirectory.path,
130+
"distribution890",
131+
"codeql",
132+
"bin",
133+
codeQlLauncherName(),
134+
),
135+
"launcher!",
136+
);
137+
138+
const promise = manager.cleanup();
139+
140+
await promise;
141+
142+
expect(
143+
(await getDirectoryNamesInsidePath(globalStorageDirectory.path)).sort(),
144+
).toEqual(["distribution768", "distribution890"]);
145+
});
146+
});

0 commit comments

Comments
 (0)