From d24e0d36fe571d4fb6c886217585f1ef131badb8 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Thu, 20 Nov 2025 12:31:19 +0100 Subject: [PATCH 1/2] Reuse temporary database in local postgres in fake environment (between specific test runs). --- app/lib/database/database.dart | 58 +++++++++++++++++++---------- app/lib/service/services.dart | 4 ++ app/test/shared/test_services.dart | 42 ++++++++++++++++----- app/test/task/fallback_test.dart | 59 +++++++++++++++++------------- 4 files changed, 109 insertions(+), 54 deletions(-) diff --git a/app/lib/database/database.dart b/app/lib/database/database.dart index ae8fa42f31..eab6667014 100644 --- a/app/lib/database/database.dart +++ b/app/lib/database/database.dart @@ -24,7 +24,9 @@ void registerPrimaryDatabase(PrimaryDatabase database) => ss.register(#_primaryDatabase, database); /// The active primary database service. -PrimaryDatabase? get primaryDatabase => +PrimaryDatabase? get primaryDatabase => _lookupPrimaryDatabase(); + +PrimaryDatabase? _lookupPrimaryDatabase() => ss.lookup(#_primaryDatabase) as PrimaryDatabase?; /// Access to the primary database connection and object mapping. @@ -32,8 +34,9 @@ class PrimaryDatabase { final Pool _pg; final DatabaseAdapter _adapter; final Database db; + final Future Function()? _closeFn; - PrimaryDatabase._(this._pg, this._adapter, this.db); + PrimaryDatabase._(this._pg, this._adapter, this.db, this._closeFn); /// Gets the connection string either from the environment variable or from /// the secret backend, connects to it and registers the primary database @@ -43,54 +46,71 @@ class PrimaryDatabase { // Production is not configured for postgresql yet. return; } - var connectionString = + if (_lookupPrimaryDatabase() != null) { + // Already initialized, must be in a local test environment. + assert(activeConfiguration.isFakeOrTest); + return; + } + final connectionString = envConfig.pubPostgresUrl ?? (await secretBackend.lookup(SecretKey.postgresConnectionString)); if (connectionString == null && activeConfiguration.isStaging) { // Staging may not have the connection string set yet. return; } + final database = connectionString != null + ? await _fromConnectionString(connectionString) + : await startOrUseLocalDatabase(); + registerPrimaryDatabase(database); + ss.registerScopeExitCallback(database.close); + } + + static Future startOrUseLocalDatabase() async { + late String url; // The scope-specific custom database. We are creating a custom database for // each test run, in order to provide full isolation, however, this must not // be used in Appengine. String? customDb; - if (connectionString == null) { - (connectionString, customDb) = await _startOrUseLocalPostgresInDocker(); - } + (url, customDb) = await _startOrUseLocalPostgresInDocker(); if (customDb == null && !envConfig.isRunningInAppengine) { - customDb = await _createCustomDatabase(connectionString); + customDb = await _createCustomDatabase(url); } + final originalUrl = url; if (customDb != null) { if (envConfig.isRunningInAppengine) { throw StateError('Should not use custom database inside AppEngine.'); } - final originalUrl = connectionString; - connectionString = Uri.parse( - connectionString, - ).replace(path: customDb).toString(); - ss.registerScopeExitCallback(() async { - await _dropCustomDatabase(originalUrl, customDb!); - }); + url = Uri.parse(url).replace(path: customDb).toString(); } - final database = await _fromConnectionString(connectionString); - registerPrimaryDatabase(database); - ss.registerScopeExitCallback(database.close); + Future closeFn() async { + if (customDb != null) { + await _dropCustomDatabase(originalUrl, customDb); + } + } + + return await _fromConnectionString(url, closeFn: closeFn); } - static Future _fromConnectionString(String value) async { + static Future _fromConnectionString( + String value, { + Future Function()? closeFn, + }) async { final pg = Pool.withUrl(value); final adapter = DatabaseAdapter.postgres(pg); final db = Database(adapter, SqlDialect.postgres()); await db.createTables(); - return PrimaryDatabase._(pg, adapter, db); + return PrimaryDatabase._(pg, adapter, db, closeFn); } Future close() async { await _adapter.close(); await _pg.close(); + if (_closeFn != null) { + await _closeFn(); + } } @visibleForTesting diff --git a/app/lib/service/services.dart b/app/lib/service/services.dart index 7e8c94a64d..868781f833 100644 --- a/app/lib/service/services.dart +++ b/app/lib/service/services.dart @@ -145,6 +145,7 @@ Future withFakeServices({ MemDatastore? datastore, MemStorage? storage, FakeCloudCompute? cloudCompute, + PrimaryDatabase? primaryDatabase, }) async { if (!envConfig.isRunningLocally) { throw StateError("Mustn't use fake services inside AppEngine."); @@ -156,6 +157,9 @@ Future withFakeServices({ register(#appengine.context, FakeClientContext()); registerDbService(DatastoreDB(datastore!)); registerStorageService(RetryEnforcerStorage(storage!)); + if (primaryDatabase != null) { + registerPrimaryDatabase(primaryDatabase); + } IOServer? frontendServer; IOServer? searchServer; if (configuration == null) { diff --git a/app/test/shared/test_services.dart b/app/test/shared/test_services.dart index 596dc533f4..fe3f7bcdb1 100644 --- a/app/test/shared/test_services.dart +++ b/app/test/shared/test_services.dart @@ -11,6 +11,7 @@ import 'package:gcloud/db.dart'; import 'package:gcloud/service_scope.dart'; import 'package:meta/meta.dart'; import 'package:pub_dev/account/models.dart'; +import 'package:pub_dev/database/database.dart'; import 'package:pub_dev/fake/backend/fake_auth_provider.dart'; import 'package:pub_dev/fake/backend/fake_download_counts.dart'; import 'package:pub_dev/fake/backend/fake_email_sender.dart'; @@ -77,6 +78,28 @@ final class FakeAppengineEnv { final _storage = MemStorage(); final _datastore = MemDatastore(); final _cloudCompute = FakeCloudCompute(); + final PrimaryDatabase _primaryDatabase; + + FakeAppengineEnv._(this._primaryDatabase); + + /// Initializes, provides and then disposes a fake environment, preserving + /// the backing databases, allowing the use of new runtimes and other dynamic + /// features. + static Future withEnv( + Future Function(FakeAppengineEnv env) fn, + ) async { + final database = await PrimaryDatabase.startOrUseLocalDatabase(); + final env = FakeAppengineEnv._(database); + try { + return await fn(env); + } finally { + await env._dispose(); + } + } + + Future _dispose() async { + await _primaryDatabase.close(); + } /// Create a service scope with fake services and run [fn] in it. /// @@ -102,6 +125,7 @@ final class FakeAppengineEnv { datastore: _datastore, storage: _storage, cloudCompute: _cloudCompute, + primaryDatabase: _primaryDatabase, fn: () async { registerStaticFileCacheForTest(_staticFileCacheForTesting); @@ -171,19 +195,19 @@ void testWithProfile( Iterable? expectedLogMessages, dynamic skip, }) { - final env = FakeAppengineEnv(); - scopedTest( name, () async { setupDebugEnvBasedLogging(); - await env.run( - fn, - testProfile: testProfile ?? defaultTestProfile, - importSource: importSource, - processJobsWithFakeRunners: processJobsWithFakeRunners, - integrityProblem: integrityProblem, - ); + await FakeAppengineEnv.withEnv((env) async { + await env.run( + fn, + testProfile: testProfile ?? defaultTestProfile, + importSource: importSource, + processJobsWithFakeRunners: processJobsWithFakeRunners, + integrityProblem: integrityProblem, + ); + }); }, expectedLogMessages: expectedLogMessages, timeout: timeout, diff --git a/app/test/task/fallback_test.dart b/app/test/task/fallback_test.dart index 4c65321641..3022c3adcf 100644 --- a/app/test/task/fallback_test.dart +++ b/app/test/task/fallback_test.dart @@ -16,38 +16,45 @@ import '../shared/test_services.dart'; void main() { group('task fallback test', () { test('analysis fallback', () async { - final env = FakeAppengineEnv(); - await env.run( - testProfile: TestProfile( - generatedPackages: [ - GeneratedTestPackage( - name: 'oxygen', - versions: [GeneratedTestVersion(version: '1.0.0')], - ), - ], - defaultUser: adminAtPubDevEmail, - ), - processJobsWithFakeRunners: true, - runtimeVersions: ['2023.08.24'], - () async { + await FakeAppengineEnv.withEnv((env) async { + await env.run( + testProfile: TestProfile( + generatedPackages: [ + GeneratedTestPackage( + name: 'oxygen', + versions: [GeneratedTestVersion(version: '1.0.0')], + ), + ], + defaultUser: adminAtPubDevEmail, + ), + processJobsWithFakeRunners: true, + runtimeVersions: ['2023.08.24'], + () async { + final card = await scoreCardBackend.getScoreCardData( + 'oxygen', + '1.0.0', + ); + expect(card.runtimeVersion, '2023.08.24'); + }, + ); + + await env.run(runtimeVersions: ['2023.08.25', '2023.08.24'], () async { + // fallback into accepted runtime works final card = await scoreCardBackend.getScoreCardData( 'oxygen', '1.0.0', ); expect(card.runtimeVersion, '2023.08.24'); - }, - ); - - await env.run(runtimeVersions: ['2023.08.25', '2023.08.24'], () async { - // fallback into accepted runtime works - final card = await scoreCardBackend.getScoreCardData('oxygen', '1.0.0'); - expect(card.runtimeVersion, '2023.08.24'); - }); + }); - await env.run(runtimeVersions: ['2023.08.26', '2023.08.23'], () async { - // fallback into non-accepted runtime doesn't work - final card = await scoreCardBackend.getScoreCardData('oxygen', '1.0.0'); - expect(card.runtimeVersion, '2023.08.26'); + await env.run(runtimeVersions: ['2023.08.26', '2023.08.23'], () async { + // fallback into non-accepted runtime doesn't work + final card = await scoreCardBackend.getScoreCardData( + 'oxygen', + '1.0.0', + ); + expect(card.runtimeVersion, '2023.08.26'); + }); }); }); }); From 8a0564ece354728c4323be5fe087dd3b1ab4fc29 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Fri, 21 Nov 2025 04:27:52 +0100 Subject: [PATCH 2/2] Fix initialization logic. --- app/lib/database/database.dart | 28 ++++++++++++++-------------- app/test/shared/test_services.dart | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/app/lib/database/database.dart b/app/lib/database/database.dart index eab6667014..9fb0a36f4d 100644 --- a/app/lib/database/database.dart +++ b/app/lib/database/database.dart @@ -58,20 +58,27 @@ class PrimaryDatabase { // Staging may not have the connection string set yet. return; } - final database = connectionString != null - ? await _fromConnectionString(connectionString) - : await startOrUseLocalDatabase(); + final database = await createAndInit(url: connectionString); registerPrimaryDatabase(database); ss.registerScopeExitCallback(database.close); } - static Future startOrUseLocalDatabase() async { - late String url; + /// Creates and initializes a [PrimaryDatabase] instance. + /// + /// When [url] is not provided, it will start a new local postgresql instance, or + /// if it detects an existing one, connects to it. + /// + /// When NOT running in the AppEngine environment (e.g. testing or local fake), + /// the initilization will create a new database, which will be dropped when the + /// [close] method is called. + static Future createAndInit({String? url}) async { // The scope-specific custom database. We are creating a custom database for // each test run, in order to provide full isolation, however, this must not // be used in Appengine. String? customDb; - (url, customDb) = await _startOrUseLocalPostgresInDocker(); + if (url == null) { + (url, customDb) = await _startOrUseLocalPostgresInDocker(); + } if (customDb == null && !envConfig.isRunningInAppengine) { customDb = await _createCustomDatabase(url); } @@ -91,14 +98,7 @@ class PrimaryDatabase { } } - return await _fromConnectionString(url, closeFn: closeFn); - } - - static Future _fromConnectionString( - String value, { - Future Function()? closeFn, - }) async { - final pg = Pool.withUrl(value); + final pg = Pool.withUrl(url); final adapter = DatabaseAdapter.postgres(pg); final db = Database(adapter, SqlDialect.postgres()); await db.createTables(); diff --git a/app/test/shared/test_services.dart b/app/test/shared/test_services.dart index fe3f7bcdb1..64afd6a1d4 100644 --- a/app/test/shared/test_services.dart +++ b/app/test/shared/test_services.dart @@ -88,7 +88,7 @@ final class FakeAppengineEnv { static Future withEnv( Future Function(FakeAppengineEnv env) fn, ) async { - final database = await PrimaryDatabase.startOrUseLocalDatabase(); + final database = await PrimaryDatabase.createAndInit(); final env = FakeAppengineEnv._(database); try { return await fn(env);