From 025b4e2e3344ea30cb3b7ea5c1f337976f990d79 Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Fri, 8 Nov 2024 15:22:45 +0100 Subject: [PATCH 1/2] Increase concurrency and handle errors better --- app/lib/package/api_export/api_exporter.dart | 12 +++++++++++- app/lib/package/api_export/exported_api.dart | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/app/lib/package/api_export/api_exporter.dart b/app/lib/package/api_export/api_exporter.dart index 82e72a4e25..064fb1e9fa 100644 --- a/app/lib/package/api_export/api_exporter.dart +++ b/app/lib/package/api_export/api_exporter.dart @@ -30,7 +30,7 @@ void registerApiExporter(ApiExporter value) => /// The active API Exporter service or null if it hasn't been initialized. ApiExporter? get apiExporter => ss.lookup(#_apiExporter) as ApiExporter?; -const _concurrency = 30; +const _concurrency = 50; class ApiExporter { final ExportedApi _api; @@ -134,6 +134,7 @@ class ApiExporter { Future synchronizeExportedApi() async { final allPackageNames = {}; final packageQuery = dbService.query(); + var errCount = 0; await packageQuery.run().parallelForEach(_concurrency, (pkg) async { final name = pkg.name!; if (pkg.isNotVisible) { @@ -143,6 +144,9 @@ class ApiExporter { // TODO: Consider retries around all this logic await synchronizePackage(name); + }, onError: (e, st) { + _log.warning('synchronizePackage() failed', e, st); + errCount++; }); await synchronizePackageNameCompletionData(); @@ -157,6 +161,12 @@ class ApiExporter { }); await _api.garbageCollect(allPackageNames); + + if (errCount > 0) { + throw Exception( + '$errCount exceptions happened while calling synchronizePackage', + ); + } } /// Sync package and into [ExportedApi], this will synchronize package into diff --git a/app/lib/package/api_export/exported_api.dart b/app/lib/package/api_export/exported_api.dart index e7c5dd51ee..a97ffca810 100644 --- a/app/lib/package/api_export/exported_api.dart +++ b/app/lib/package/api_export/exported_api.dart @@ -44,7 +44,7 @@ const _minGarbageAge = Duration(hours: 3); /// All writes to the bucket containing the exported API should go through this /// interface. final class ExportedApi { - final Pool _pool = Pool(50); + final Pool _pool = Pool(80); final Storage _storage; final Bucket _bucket; final List _prefixes = [ From 286bd4e27e66514c5b036792f7eb17ea9bd4ff71 Mon Sep 17 00:00:00 2001 From: Jonas Finnemann Jensen Date: Fri, 8 Nov 2024 15:39:21 +0100 Subject: [PATCH 2/2] Update app/lib/package/api_export/api_exporter.dart Co-authored-by: Sigurd Meldgaard --- app/lib/package/api_export/api_exporter.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/package/api_export/api_exporter.dart b/app/lib/package/api_export/api_exporter.dart index 064fb1e9fa..4cdee8fabf 100644 --- a/app/lib/package/api_export/api_exporter.dart +++ b/app/lib/package/api_export/api_exporter.dart @@ -164,7 +164,7 @@ class ApiExporter { if (errCount > 0) { throw Exception( - '$errCount exceptions happened while calling synchronizePackage', + '$errCount exceptions happened while calling synchronizeExportedApi', ); } }