diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index c226616f0b02c..760e15c634d75 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -121,12 +121,12 @@ public ConfigRepository(final Database database) { * @return true if read succeeds, even if the table is empty, and false if any error happens. */ public boolean healthCheck() { - // try { - // database.query(ctx -> ctx.select(WORKSPACE.ID).from(WORKSPACE).limit(1).fetch()); - // } catch (final Exception e) { - // LOGGER.error("Health check error: ", e); - // return false; - // } + try { + database.query(ctx -> ctx.select(WORKSPACE.ID).from(WORKSPACE).limit(1).fetch()); + } catch (final Exception e) { + LOGGER.error("Health check error: ", e); + return false; + } return true; } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java index 2ddadeb65479c..6acb5cd39cd2f 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/HealthApiController.java @@ -10,10 +10,8 @@ import io.micronaut.http.MediaType; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; -import jakarta.inject.Singleton; @Controller("/api/v1/health") -@Singleton public class HealthApiController implements HealthApi { private final HealthCheckHandler healthCheckHandler; diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java index 3684d242edfdc..5c79b6704b55e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java @@ -11,8 +11,6 @@ import io.airbyte.db.check.impl.JobsDatabaseAvailabilityCheck; import io.airbyte.db.factory.DatabaseCheckFactory; import io.airbyte.db.instance.DatabaseConstants; -import io.airbyte.persistence.job.DefaultJobPersistence; -import io.airbyte.persistence.job.JobPersistence; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; import io.micronaut.context.annotation.Value; @@ -35,7 +33,7 @@ public class DatabaseBeanFactory { private static final String BASELINE_DESCRIPTION = "Baseline from file-based migration v1"; private static final Boolean BASELINE_ON_MIGRATION = true; - private static final String INSTALLED_BY = "ServerApp"; + private static final String INSTALLED_BY = "WorkerApp"; @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) @@ -44,6 +42,13 @@ public Database configDatabase(@Named("config") final DSLContext dslContext) thr return new Database(dslContext); } + @Singleton + @Requires(env = WorkerMode.CONTROL_PLANE) + @Named("jobsDatabase") + public Database jobsDatabase(@Named("jobs") final DSLContext dslContext) throws IOException { + return new Database(dslContext); + } + @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) @Named("configFlyway") @@ -64,7 +69,7 @@ public Flyway configFlyway(@Named("config") final FlywayConfigurationProperties @Requires(env = WorkerMode.CONTROL_PLANE) @Named("jobsFlyway") public Flyway jobsFlyway(@Named("jobs") final FlywayConfigurationProperties jobsFlywayConfigurationProperties, - @Named("config") final DataSource jobsDataSource, + @Named("jobs") final DataSource jobsDataSource, @Value("${airbyte.flyway.jobs.minimum-migration-version}") final String baselineVersion) { return jobsFlywayConfigurationProperties.getFluentConfiguration() .dataSource(jobsDataSource) @@ -82,12 +87,6 @@ public ConfigRepository configRepository(@Named("configDatabase") final Database return new ConfigRepository(configDatabase); } - @Singleton - @Requires(env = WorkerMode.CONTROL_PLANE) - public JobPersistence jobPersistence(@Named("configDatabase") final Database jobDatabase) { - return new DefaultJobPersistence(jobDatabase); - } - @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) @Named("configsDatabaseMigrationCheck") @@ -104,7 +103,7 @@ public DatabaseMigrationCheck configsDatabaseMigrationCheck(@Named("config") fin @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) @Named("jobsDatabaseMigrationCheck") - public DatabaseMigrationCheck jobsDatabaseMigrationCheck(@Named("config") final DSLContext dslContext, + public DatabaseMigrationCheck jobsDatabaseMigrationCheck(@Named("jobs") final DSLContext dslContext, @Named("jobsFlyway") final Flyway jobsFlyway, @Value("${airbyte.flyway.jobs.minimum-migration-version}") final String jobsDatabaseMinimumFlywayMigrationVersion, @Value("${airbyte.flyway.jobs.initialization-timeout-ms}") final Long jobsDatabaseInitializationTimeoutMs) { @@ -116,7 +115,7 @@ public DatabaseMigrationCheck jobsDatabaseMigrationCheck(@Named("config") final @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) @Named("jobsDatabaseAvailabilityCheck") - public JobsDatabaseAvailabilityCheck jobsDatabaseAvailabilityCheck(@Named("config") final DSLContext dslContext) { + public JobsDatabaseAvailabilityCheck jobsDatabaseAvailabilityCheck(@Named("jobs") final DSLContext dslContext) { return new JobsDatabaseAvailabilityCheck(dslContext, DatabaseConstants.DEFAULT_ASSERT_DATABASE_TIMEOUT_MS); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/HandlerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/HandlerFactory.java index 7bda5ef46c7f9..bf9a3fe5ddb42 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/config/HandlerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/config/HandlerFactory.java @@ -9,7 +9,6 @@ import io.airbyte.server.handlers.HealthCheckHandler; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; -import jakarta.inject.Named; import jakarta.inject.Singleton; @Factory @@ -17,7 +16,7 @@ public class HandlerFactory { @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) - public HealthCheckHandler healthCheckHandler(@Named("configRepository") final ConfigRepository configRepository) { + public HealthCheckHandler healthCheckHandler(final ConfigRepository configRepository) { return new HealthCheckHandler(configRepository); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/SecretPersistenceBeanFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/SecretPersistenceBeanFactory.java deleted file mode 100644 index a7cb8e66eee17..0000000000000 --- a/airbyte-server/src/main/java/io/airbyte/server/config/SecretPersistenceBeanFactory.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.server.config; - -import io.airbyte.commons.temporal.config.WorkerMode; -import io.airbyte.config.persistence.split_secrets.GoogleSecretManagerPersistence; -import io.airbyte.config.persistence.split_secrets.LocalTestingSecretPersistence; -import io.airbyte.config.persistence.split_secrets.RealSecretsHydrator; -import io.airbyte.config.persistence.split_secrets.SecretPersistence; -import io.airbyte.config.persistence.split_secrets.SecretsHydrator; -import io.airbyte.config.persistence.split_secrets.VaultSecretPersistence; -import io.airbyte.db.Database; -import io.micronaut.context.annotation.Factory; -import io.micronaut.context.annotation.Requires; -import io.micronaut.context.annotation.Value; -import jakarta.inject.Named; -import jakarta.inject.Singleton; - -/** - * Micronaut bean factory for secret persistence-related singletons. - */ -@Factory -@SuppressWarnings("PMD.AvoidDuplicateLiterals") -public class SecretPersistenceBeanFactory { - - @Singleton - @Requires(property = "airbyte.secret.persistence", - pattern = "(?i)^(?!testing_config_db_table).*") - @Requires(property = "airbyte.secret.persistence", - pattern = "(?i)^(?!google_secret_manager).*") - @Requires(property = "airbyte.secret.persistence", - pattern = "(?i)^(?!vault).*") - @Requires(env = WorkerMode.CONTROL_PLANE) - @Named("longLivedSecretPersistence") - public SecretPersistence defaultSecretPersistence(@Named("configDatabase") final Database configDatabase) { - return localTestingSecretPersistence(configDatabase); - } - - @Singleton - @Requires(property = "airbyte.secret.persistence", - pattern = "(?i)^testing_config_db_table$") - @Requires(env = WorkerMode.CONTROL_PLANE) - @Named("longLivedSecretPersistence") - public SecretPersistence localTestingSecretPersistence(@Named("configDatabase") final Database configDatabase) { - return new LocalTestingSecretPersistence(configDatabase); - } - - @Singleton - @Requires(property = "airbyte.secret.persistence", - pattern = "(?i)^google_secret_manager$") - @Named("longLivedSecretPersistence") - public SecretPersistence googleSecretPersistence(@Value("${airbyte.secret.store.gcp.credentials}") final String credentials, - @Value("${airbyte.secret.store.gcp.project-id}") final String projectId) { - return GoogleSecretManagerPersistence.getLongLived(projectId, credentials); - } - - @Singleton - @Requires(property = "airbyte.secret.persistence", - pattern = "(?i)^vault$") - @Requires(env = WorkerMode.CONTROL_PLANE) - @Named("longLivedSecretPersistence") - public SecretPersistence vaultSecretPersistence(@Value("${airbyte.secret.store.vault.address}") final String address, - @Value("${airbyte.secret.store.vault.prefix}") final String prefix, - @Value("${airbyte.secret.store.vault.token}") final String token) { - return new VaultSecretPersistence(address, prefix, token); - } - - @Singleton - @Requires(property = "airbyte.secret.persistence", - pattern = "(?i)^(?!testing_config_db_table).*") - @Requires(property = "airbyte.secret.persistence", - pattern = "(?i)^(?!google_secret_manager).*") - @Requires(property = "airbyte.secret.persistence", - pattern = "(?i)^(?!vault).*") - @Requires(env = WorkerMode.CONTROL_PLANE) - @Named("ephemeralSecretPersistence") - public SecretPersistence defaultEphemeralSecretPersistence(@Named("configDatabase") final Database configDatabase) { - return localTestingSecretPersistence(configDatabase); - } - - @Singleton - @Requires(property = "airbyte.secret.persistence", - pattern = "(?i)^testing_config_db_table$") - @Requires(env = WorkerMode.CONTROL_PLANE) - @Named("ephemeralSecretPersistence") - public SecretPersistence localTestingEphemeralSecretPersistence(@Named("configDatabase") final Database configDatabase) { - return new LocalTestingSecretPersistence(configDatabase); - } - - @Singleton - @Requires(property = "airbyte.secret.persistence", - pattern = "(?i)^google_secret_manager$") - @Named("ephemeralSecretPersistence") - public SecretPersistence googleEphemeralSecretPersistence(@Value("${airbyte.secret.store.gcp.credentials}") final String credentials, - @Value("${airbyte.secret.store.gcp.project-id}") final String projectId) { - return GoogleSecretManagerPersistence.getEphemeral(projectId, credentials); - } - - @Singleton - @Requires(property = "airbyte.secret.persistence", - pattern = "(?i)^vault$") - @Requires(env = WorkerMode.CONTROL_PLANE) - @Named("ephemeralSecretPersistence") - public SecretPersistence vaultEphemeralSecretPersistence(@Value("${airbyte.secret.store.vault.address}") final String address, - @Value("${airbyte.secret.store.vault.prefix}") final String prefix, - @Value("${airbyte.secret.store.vault.token}") final String token) { - return new VaultSecretPersistence(address, prefix, token); - } - - @Singleton - public SecretsHydrator secretsHydrator(@Named("longLivedSecretPersistence") final SecretPersistence secretPersistence) { - return new RealSecretsHydrator(secretPersistence); - } - -} diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/ServerBeanFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/ServerBeanFactory.java index d248f94018d54..5672ddc61da7f 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/config/ServerBeanFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/config/ServerBeanFactory.java @@ -25,6 +25,7 @@ import io.airbyte.config.persistence.split_secrets.SecretsHydrator; import io.airbyte.db.Database; import io.airbyte.db.check.DatabaseCheckException; +import io.airbyte.persistence.job.DefaultJobPersistence; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.persistence.job.WebUrlHelper; import io.airbyte.persistence.job.WorkspaceHelper; @@ -145,13 +146,9 @@ public ServerRunnable serverRunnable( final Configs configs, @Named("config") final DSLContext configsDslContext, @Named("configFlyway") final Flyway configsFlyway, + @Named("jobs") final DSLContext jobsDslContext, @Named("jobsFlyway") final Flyway jobsFlyway, - @Named("configRepository") final ConfigRepository configRepository, - final SecretsHydrator secretsHydrator, - @Named("longLivedSecretPersistence") final SecretPersistence longLivedSecretPersistence, - @Named("ephemeralSecretPersistence") final SecretPersistence ephemeralSecretPersistence, - @Named("configDatabase") final Database configsDatabase, - final JobPersistence jobPersistence, + final ConfigRepository configRepository, final HealthApiController healthApiController) throws DatabaseCheckException, IOException { final Set> componentClasses = Set.of( @@ -204,17 +201,18 @@ public ServerRunnable serverRunnable( configs.getLogConfigs(), LogClientSingleton.getInstance().getServerLogsRoot(configs.getWorkspaceRoot())); - ServerApp.assertDatabasesReady(configs, configsDslContext, configsFlyway, configsDslContext, jobsFlyway); + ServerApp.assertDatabasesReady(configs, configsDslContext, configsFlyway, jobsDslContext, jobsFlyway); - /* - * final SecretsHydrator secretsHydrator = SecretPersistence.getSecretsHydrator(configsDslContext, - * configs); final Optional secretPersistence = - * SecretPersistence.getLongLived(configsDslContext, configs); final Optional - * ephemeralSecretPersistence = SecretPersistence.getEphemeral(configsDslContext, configs); - */ + final Database configsDatabase = new Database(configsDslContext); + final SecretsHydrator secretsHydrator = SecretPersistence.getSecretsHydrator(configsDslContext, configs); + final Optional secretPersistence = SecretPersistence.getLongLived(configsDslContext, configs); + final Optional ephemeralSecretPersistence = SecretPersistence.getEphemeral(configsDslContext, configs); final SecretsRepositoryReader secretsRepositoryReader = new SecretsRepositoryReader(configRepository, secretsHydrator); final SecretsRepositoryWriter secretsRepositoryWriter = - new SecretsRepositoryWriter(configRepository, Optional.of(longLivedSecretPersistence), Optional.of(ephemeralSecretPersistence)); + new SecretsRepositoryWriter(configRepository, secretPersistence, ephemeralSecretPersistence); + + final Database jobsDatabase = new Database(jobsDslContext); + final JobPersistence jobPersistence = new DefaultJobPersistence(jobsDatabase); TrackingClientSingleton.initialize( configs.getTrackingStrategy(), @@ -304,7 +302,7 @@ public ServerRunnable serverRunnable( connectionsHandler, featureFlags); - final DbMigrationHandler dbMigrationHandler = new DbMigrationHandler(configsDatabase, configsFlyway, configsDatabase, jobsFlyway); + final DbMigrationHandler dbMigrationHandler = new DbMigrationHandler(configsDatabase, configsFlyway, jobsDatabase, jobsFlyway); final DestinationDefinitionsHandler destinationDefinitionsHandler = new DestinationDefinitionsHandler(configRepository, syncSchedulerClient, destinationHandler); diff --git a/airbyte-server/src/main/resources/application-control-plane.yml b/airbyte-server/src/main/resources/application-control-plane.yml index 52eb26eac4473..002b3e79aec0e 100644 --- a/airbyte-server/src/main/resources/application-control-plane.yml +++ b/airbyte-server/src/main/resources/application-control-plane.yml @@ -2,10 +2,19 @@ datasources: config: connection-test-query: SELECT 1 connection-timeout: 30000 + idle-timeout: 600000 maximum-pool-size: 10 - minimum-pool-size: 0 - minimum-idle: 1 - maximum-idle: 1000 + minimum-idle: 0 + url: ${DATABASE_URL} + driverClassName: org.postgresql.Driver + username: ${DATABASE_USER} + password: ${DATABASE_PASSWORD} + jobs: + connection-test-query: SELECT 1 + connection-timeout: 30000 + idle-timeout: 600000 + maximum-pool-size: 10 + minimum-idle: 0 url: ${DATABASE_URL} driverClassName: org.postgresql.Driver username: ${DATABASE_USER} diff --git a/airbyte-server/src/main/resources/application.yml b/airbyte-server/src/main/resources/application.yml index 8f6562a168d34..6c0becd46dc08 100644 --- a/airbyte-server/src/main/resources/application.yml +++ b/airbyte-server/src/main/resources/application.yml @@ -19,13 +19,3 @@ airbyte: initialization-timeout-ms: ${JOBS_DATABASE_INITIALIZATION_TIMEOUT_MS:60000} minimum-migration-version: ${JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION} version: ${AIRBYTE_VERSION} - secret: - persistence: ${SECRET_PERSISTENCE:TESTING_CONFIG_DB_TABLE} - store: - gcp: - credentials: ${SECRET_STORE_GCP_CREDENTIALS:} - project-id: ${SECRET_STORE_GCP_PROJECT_ID:} - vault: - address: ${VAULT_ADDRESS:} - prefix: ${VAULT_PREFIX:} - token: ${VAULT_AUTH_TOKEN:} diff --git a/docker-compose.yaml b/docker-compose.yaml index 5afaaeac45499..111a6f7de91c0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -153,7 +153,7 @@ services: - AUTO_DETECT_SCHEMA=${AUTO_DETECT_SCHEMA} ports: - 8001 - - 8080:8080 + - 8080 volumes: - workspace:${WORKSPACE_ROOT} - data:${CONFIG_ROOT}