diff --git a/.github/workflows/all-test.yml b/.github/workflows/all-test.yml index 66fdc9ea0b..afe5daf885 100644 --- a/.github/workflows/all-test.yml +++ b/.github/workflows/all-test.yml @@ -97,6 +97,19 @@ jobs: name: pkg/${{matrix.package}}/ runs-on: ubuntu-latest needs: define_pkg_list + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 strategy: fail-fast: false matrix: @@ -137,3 +150,5 @@ jobs: fi dart test --run-skipped working-directory: pkg/${{matrix.package}} + env: + PUB_POSTGRES_URL: postgresql://postgres:postgres@localhost:5432/postgres?sslmode=disable diff --git a/app/lib/database/database.dart b/app/lib/database/database.dart index 358c96600f..c947d4f6f9 100644 --- a/app/lib/database/database.dart +++ b/app/lib/database/database.dart @@ -2,12 +2,19 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:async'; +import 'dart:io'; +import 'dart:math'; + +import 'package:clock/clock.dart'; import 'package:gcloud/service_scope.dart' as ss; import 'package:meta/meta.dart'; import 'package:postgres/postgres.dart'; import 'package:pub_dev/service/secret/backend.dart'; import 'package:pub_dev/shared/env_config.dart'; +final _random = Random.secure(); + /// Sets the primary database service. void registerPrimaryDatabase(PrimaryDatabase database) => ss.register(#_primaryDatabase, database); @@ -26,13 +33,38 @@ class PrimaryDatabase { /// the secret backend, connects to it and registers the primary database /// service in the current scope. static Future tryRegisterInScope() async { - final connectionString = + var connectionString = envConfig.pubPostgresUrl ?? (await secretBackend.lookup(SecretKey.postgresConnectionString)); - if (connectionString == null) { + if (connectionString == null && envConfig.isRunningInAppengine) { // ignore for now, must throw once we have the environment setup ready return; } + // The scope-specific custom database. We are creating a custom database for + // each test run, in order to provide full isolation, however, this must not + // be used in Appengine. + String? customDb; + if (connectionString == null) { + (connectionString, customDb) = await _startOrUseLocalPostgresInDocker(); + } + if (customDb == null && !envConfig.isRunningInAppengine) { + customDb = await _createCustomDatabase(connectionString); + } + + if (customDb != null) { + if (envConfig.isRunningInAppengine) { + throw StateError('Should not use custom database inside AppEngine.'); + } + + final originalUrl = connectionString; + connectionString = Uri.parse( + connectionString, + ).replace(path: customDb).toString(); + ss.registerScopeExitCallback(() async { + await _dropCustomDatabase(originalUrl, customDb!); + }); + } + final database = await _fromConnectionString(connectionString); registerPrimaryDatabase(database); ss.registerScopeExitCallback(database.close); @@ -48,10 +80,64 @@ class PrimaryDatabase { } @visibleForTesting - Future verifyConnection() async { - final rs = await _pg.execute('SELECT 1'); + Future verifyConnection() async { + final rs = await _pg.execute('SELECT current_database();'); if (rs.length != 1) { throw StateError('Connection is not returning expected rows.'); } + return rs.single.single as String; + } +} + +Future<(String, String?)> _startOrUseLocalPostgresInDocker() async { + // sanity check + if (envConfig.isRunningInAppengine) { + throw StateError('Missing connection URL in Appengine environment.'); + } + + // the default connection URL for local server + final url = Uri( + scheme: 'postgresql', + host: 'localhost', + port: 55432, + path: 'postgres', + userInfo: 'postgres:postgres', + queryParameters: {'sslmode': 'disable'}, + ).toString(); + + try { + // try opening the connection + final customDb = await _createCustomDatabase(url); + return (url, customDb); + } catch (_) { + // on failure start the local server + final pr = await Process.run('tool/start-local-postgres.sh', []); + if (pr.exitCode != 0) { + throw StateError( + 'Unexpect exit code from tool/start-local-postgres.sh\n${pr.stderr}', + ); + } } + return (url, null); +} + +int _customDbCount = 0; + +Future _createCustomDatabase(String url) async { + _customDbCount++; + final dbName = + 'fake_pub_${pid.toRadixString(36)}' + '${_customDbCount.toRadixString(36)}' + '${clock.now().millisecondsSinceEpoch.toRadixString(36)}' + '${_random.nextInt(1 << 32).toRadixString(36)}'; + final conn = await Connection.openFromUrl(url); + await conn.execute('CREATE DATABASE "$dbName";'); + await conn.close(force: true); + return dbName; +} + +Future _dropCustomDatabase(String url, String dbName) async { + final conn = await Connection.openFromUrl(url); + await conn.execute('DROP DATABASE "$dbName";'); + await conn.close(force: true); } diff --git a/app/test/database/postgresql_ci_test.dart b/app/test/database/postgresql_ci_test.dart index f8e5f03419..e8dcc4ddd4 100644 --- a/app/test/database/postgresql_ci_test.dart +++ b/app/test/database/postgresql_ci_test.dart @@ -39,12 +39,8 @@ void main() { testWithProfile( 'registered database scope', fn: () async { - final pubPostgresUrl = envConfig.pubPostgresUrl; - if (pubPostgresUrl == null) { - markTestSkipped('PUB_POSTGRES_URL was not specified.'); - return; - } - await primaryDatabase!.verifyConnection(); + final name = await primaryDatabase!.verifyConnection(); + expect(name, contains('fake_pub_')); }, ); }); diff --git a/app/tool/docker-postgres-timeout-entrypoint.sh b/app/tool/docker-postgres-timeout-entrypoint.sh new file mode 100755 index 0000000000..2f856bc73f --- /dev/null +++ b/app/tool/docker-postgres-timeout-entrypoint.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +# Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +set -e + +/usr/bin/timeout 6h /usr/local/bin/docker-entrypoint.sh $* diff --git a/app/tool/start-local-postgres.sh b/app/tool/start-local-postgres.sh new file mode 100755 index 0000000000..91cd537c90 --- /dev/null +++ b/app/tool/start-local-postgres.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +PG_TEMP_DIR="${TMPDIR:-/tmp}/pub_dev_postgres/" +mkdir -p "${PG_TEMP_DIR}" + +# Create directory for exposing sockets +SOCKET_DIR="${PG_TEMP_DIR}/run/" +mkdir -p "$SOCKET_DIR" + +# Use an extra lock file to avoid every creating more than one docker container +LOCKFILE="${PG_TEMP_DIR}/.docker.lock" +touch "${LOCKFILE}" + +CONTAINER_ID=$(( + flock -ox 200 + if ! docker inspect 'pub_dev_postgres' > /dev/null 2>&1; then + docker run \ + --detach \ + --rm \ + --name pub_dev_postgres \ + -e POSTGRES_PASSWORD=postgres \ + -v "${SCRIPT_DIR}/docker-postgres-timeout-entrypoint.sh":/pub-entrypoint.sh \ + -v "${SOCKET_DIR}":/var/run/postgresql/ \ + -p 55432:5432 \ + --mount type=tmpfs,destination=/var/lib/postgresql/data \ + --entrypoint /pub-entrypoint.sh \ + postgres:17 \ + postgres \ + -c fsync=off \ + -c synchronous_commit=off \ + -c full_page_writes=off \ + -c wal_level=minimal \ + -c max_wal_senders=0 \ + -c archive_mode=off + fi +) 200>"$LOCKFILE") + +if [ -n "$CONTAINER_ID" ]; then + if [ "$1" != '--quiet' ]; then + echo 'Started postgres test database. Will auto-terminate in 6 hours.' + fi +else + if [ "$1" != '--quiet' ]; then + echo 'Found postgres test database already running!' + echo 'If you want to restart it, you can use with:' + echo 'docker kill pub_dev_postgres' + fi +fi