Add logging to download:plugins script (#13905)

Attempted mitigation for #13902 - do not eat exceptions and properly log errors - reduces request rate to 3/sec. Contributed on behalf of STMicroelectronics Signed-off-by: Thomas Mäder <t.s.maeder@gmail.com>
eclipse-theia · Jul 12, 2024 · 54c7927 · 54c7927
1 parent f12a854
commit 54c7927
Show file tree

Hide file tree

Showing 8 changed files with 99 additions and 94 deletions.
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
@@ -86,7 +86,7 @@ jobs:
         if: runner.os == 'Linux'
         shell: bash
         run: |
-          yarn -s download:plugins
+          yarn -s download:plugins --rate-limit 3
 
       - name: Build
         shell: bash

diff --git a/dev-packages/cli/src/download-plugins.ts b/dev-packages/cli/src/download-plugins.ts
@@ -143,6 +143,7 @@ export default async function downloadPlugins(ovsxClient: OVSXClient, requestSer
                         failures.push(`No download url for extension pack ${id} (${version})`);
                     }
                 } catch (err) {
+                    console.error(err);
                     failures.push(err.message);
                 }
             }));

diff --git a/dev-packages/ovsx-client/src/ovsx-api-filter.ts b/dev-packages/ovsx-client/src/ovsx-api-filter.ts
@@ -69,9 +69,10 @@ export class OVSXApiFilterImpl implements OVSXApiFilter {
         let offset = 0;
         let loop = true;
         while (loop) {
-            const queryOptions = {
+            const queryOptions: VSXQueryOptions = {
                 ...query,
-                offset
+                offset,
+                size: 5 // there is a great chance that the newest version will work
             };
             const results = await this.client.query(queryOptions);
             const compatibleExtension = this.getLatestCompatibleExtension(results.extensions);

diff --git a/dev-packages/ovsx-client/src/ovsx-http-client.ts b/dev-packages/ovsx-client/src/ovsx-http-client.ts
@@ -34,28 +34,12 @@ export class OVSXHttpClient implements OVSXClient {
         protected requestService: RequestService
     ) { }
 
-    async search(searchOptions?: VSXSearchOptions): Promise<VSXSearchResult> {
-        try {
-            return await this.requestJson(this.buildUrl('api/-/search', searchOptions));
-        } catch (err) {
-            return {
-                error: err?.message || String(err),
-                offset: -1,
-                extensions: []
-            };
-        }
+    search(searchOptions?: VSXSearchOptions): Promise<VSXSearchResult> {
+        return this.requestJson(this.buildUrl('api/-/search', searchOptions));
     }
 
-    async query(queryOptions?: VSXQueryOptions): Promise<VSXQueryResult> {
-        try {
-            return await this.requestJson(this.buildUrl('api/v2/-/query', queryOptions));
-        } catch (error) {
-            return {
-                offset: 0,
-                totalSize: 0,
-                extensions: []
-            };
-        }
+    query(queryOptions?: VSXQueryOptions): Promise<VSXQueryResult> {
+        return this.requestJson(this.buildUrl('api/v2/-/query', queryOptions));
     }
 
     protected async requestJson<R>(url: string): Promise<R> {

diff --git a/dev-packages/ovsx-client/src/ovsx-types.ts b/dev-packages/ovsx-client/src/ovsx-types.ts
@@ -105,7 +105,6 @@ export interface VSXSearchOptions {
  * Should be aligned with https://github.com/eclipse/openvsx/blob/e8f64fe145fc05d2de1469735d50a7a90e400bc4/server/src/main/java/org/eclipse/openvsx/json/SearchResultJson.java
  */
 export interface VSXSearchResult {
-    error?: string;
     offset: number;
     extensions: VSXSearchEntry[];
 }

diff --git a/dev-packages/request/src/node-request-service.ts b/dev-packages/request/src/node-request-service.ts
@@ -136,11 +136,19 @@ export class NodeRequestService implements RequestService {
                         });
                     });
 
-                    stream.on('error', reject);
+                    stream.on('error', err => {
+                        reject(err);
+                    });
                 }
             });
 
-            req.on('error', reject);
+            req.on('error', err => {
+                reject(err);
+            });
+
+            req.on('timeout', () => {
+                reject('timeout');
+            });
 
             if (options.timeout) {
                 req.setTimeout(options.timeout);
@@ -153,7 +161,7 @@ export class NodeRequestService implements RequestService {
             req.end();
 
             token?.onCancellationRequested(() => {
-                req.abort();
+                req.destroy();
                 reject();
             });
         });

diff --git a/packages/vsx-registry/src/browser/vsx-extensions-model.ts b/packages/vsx-registry/src/browser/vsx-extensions-model.ts
@@ -225,49 +225,59 @@ export class VSXExtensionsModel {
             }
             const client = await this.clientProvider();
             const filter = await this.vsxApiFilter();
-            const result = await client.search(param);
-            this._searchError = result.error;
-            if (token.isCancellationRequested) {
-                return;
-            }
-            for (const data of result.extensions) {
-                const id = data.namespace.toLowerCase() + '.' + data.name.toLowerCase();
-                const allVersions = filter.getLatestCompatibleVersion(data);
-                if (!allVersions) {
-                    continue;
+            try {
+                const result = await client.search(param);
+
+                if (token.isCancellationRequested) {
+                    return;
                 }
-                if (this.preferences.get('extensions.onlyShowVerifiedExtensions')) {
-                    this.fetchVerifiedStatus(id, client, allVersions).then(verified => {
-                        this.doChange(() => {
-                            this.addExtensions(data, id, allVersions, !!verified);
-                            return Promise.resolve();
+                for (const data of result.extensions) {
+                    const id = data.namespace.toLowerCase() + '.' + data.name.toLowerCase();
+                    const allVersions = filter.getLatestCompatibleVersion(data);
+                    if (!allVersions) {
+                        continue;
+                    }
+                    if (this.preferences.get('extensions.onlyShowVerifiedExtensions')) {
+                        this.fetchVerifiedStatus(id, client, allVersions).then(verified => {
+                            this.doChange(() => {
+                                this.addExtensions(data, id, allVersions, !!verified);
+                                return Promise.resolve();
+                            });
                         });
-                    });
-                } else {
-                    this.addExtensions(data, id, allVersions);
-                    this.fetchVerifiedStatus(id, client, allVersions).then(verified => {
-                        this.doChange(() => {
-                            let extension = this.getExtension(id);
-                            extension = this.setExtension(id);
-                            extension.update(Object.assign({
-                                verified: verified
-                            }));
-                            return Promise.resolve();
+                    } else {
+                        this.addExtensions(data, id, allVersions);
+                        this.fetchVerifiedStatus(id, client, allVersions).then(verified => {
+                            this.doChange(() => {
+                                let extension = this.getExtension(id);
+                                extension = this.setExtension(id);
+                                extension.update(Object.assign({
+                                    verified: verified
+                                }));
+                                return Promise.resolve();
+                            });
                         });
-                    });
+                    }
                 }
+            } catch (error) {
+                this._searchError = error?.message || String(error);
             }
+
         }, token);
     }
 
     protected async fetchVerifiedStatus(id: string, client: OVSXClient, allVersions: VSXAllVersions): Promise<boolean | undefined> {
-        const res = await client.query({ extensionId: id, extensionVersion: allVersions.version, includeAllVersions: true });
-        const extension = res.extensions?.[0];
-        let verified = extension?.verified;
-        if (!verified && extension?.publishedBy.loginName === 'open-vsx') {
-            verified = true;
+        try {
+            const res = await client.query({ extensionId: id, extensionVersion: allVersions.version, includeAllVersions: true });
+            const extension = res.extensions?.[0];
+            let verified = extension?.verified;
+            if (!verified && extension?.publishedBy.loginName === 'open-vsx') {
+                verified = true;
+            }
+            return verified;
+        } catch (error) {
+            console.error(error);
+            return false;
         }
-        return verified;
     }
 
     protected addExtensions(data: VSXSearchEntry, id: string, allVersions: VSXAllVersions, verified?: boolean): void {

diff --git a/packages/vsx-registry/src/browser/vsx-language-quick-pick-service.ts b/packages/vsx-registry/src/browser/vsx-language-quick-pick-service.ts
@@ -42,47 +42,49 @@ export class VSXLanguageQuickPickService extends LanguageQuickPickService {
 
     protected override async getAvailableLanguages(): Promise<LanguageQuickPickItem[]> {
         const client = await this.clientProvider();
-        const searchResult = await client.search({
-            category: 'Language Packs',
-            sortBy: 'downloadCount',
-            sortOrder: 'desc',
-            size: 20
-        });
-        if (searchResult.error) {
-            throw new Error('Error while loading available languages: ' + searchResult.error);
-        }
+        try {
+            const searchResult = await client.search({
+                category: 'Language Packs',
+                sortBy: 'downloadCount',
+                sortOrder: 'desc',
+                size: 20
+            });
 
-        const extensionLanguages = await Promise.all(
-            searchResult.extensions.map(async extension => ({
-                extension,
-                languages: await this.loadExtensionLanguages(extension)
-            }))
-        );
+            const extensionLanguages = await Promise.all(
+                searchResult.extensions.map(async extension => ({
+                    extension,
+                    languages: await this.loadExtensionLanguages(extension)
+                }))
+            );
 
-        const languages = new Map<string, LanguageQuickPickItem>();
+            const languages = new Map<string, LanguageQuickPickItem>();
 
-        for (const extension of extensionLanguages) {
-            for (const localizationContribution of extension.languages) {
-                if (!languages.has(localizationContribution.languageId)) {
-                    languages.set(localizationContribution.languageId, {
-                        ...this.createLanguageQuickPickItem(localizationContribution),
-                        execute: async () => {
-                            const progress = await this.messageService.showProgress({
-                                text: nls.localizeByDefault('Installing {0} language support...',
-                                    localizationContribution.localizedLanguageName ?? localizationContribution.languageName ?? localizationContribution.languageId),
-                            });
-                            try {
-                                const extensionUri = VSCodeExtensionUri.fromId(`${extension.extension.namespace}.${extension.extension.name}`).toString();
-                                await this.pluginServer.deploy(extensionUri);
-                            } finally {
-                                progress.cancel();
+            for (const extension of extensionLanguages) {
+                for (const localizationContribution of extension.languages) {
+                    if (!languages.has(localizationContribution.languageId)) {
+                        languages.set(localizationContribution.languageId, {
+                            ...this.createLanguageQuickPickItem(localizationContribution),
+                            execute: async () => {
+                                const progress = await this.messageService.showProgress({
+                                    text: nls.localizeByDefault('Installing {0} language support...',
+                                        localizationContribution.localizedLanguageName ?? localizationContribution.languageName ?? localizationContribution.languageId),
+                                });
+                                try {
+                                    const extensionUri = VSCodeExtensionUri.fromId(`${extension.extension.namespace}.${extension.extension.name}`).toString();
+                                    await this.pluginServer.deploy(extensionUri);
+                                } finally {
+                                    progress.cancel();
+                                }
                             }
-                        }
-                    });
+                        });
+                    }
                 }
             }
+            return Array.from(languages.values());
+        } catch (error) {
+            console.error(error);
+            return [];
         }
-        return Array.from(languages.values());
     }
 
     protected async loadExtensionLanguages(extension: VSXSearchEntry): Promise<LanguageInfo[]> {