rm airbyte-scheduler; migrate existing client to airbyte-server (#17051)

* wip; move scheduler client to server package

* remove additional references to airbyte-scheduler

* removed more references to airbyte-scheduler

.github/labeler.yml

@@ -4,12 +4,12 @@ area/platform:
   - airbyte-webapp/**/*
   - airbyte-api/*
   - airbyte-api/**/*
-  - airbyte-server/*
-  - airbyte-server/**/*
-  - airbyte-scheduler/*
-  - airbyte-scheduler/**/*
+  - airbyte-persistence/*
+  - airbyte-persistence/**/*
   - airbyte-protocol/*
   - airbyte-protocol/**/*
+  - airbyte-server/*
+  - airbyte-server/**/*
   - airbyte-workers/*
   - airbyte-workers/**/*
   - kube/*
@@ -29,10 +29,6 @@ area/server:
   - airbyte-server/*
   - airbyte-server/**/*
 
-area/scheduler:
-  - airbyte-scheduler/*
-  - airbyte-scheduler/**/*
-
 area/protocol:
   - airbyte-protocol/*
   - airbyte-protocol/**/*

airbyte-config/config-persistence/readme.md

@@ -1,6 +1,6 @@
 # airbyte-config:config-persistence
 
-This module contains the logic for accessing the config database. This database is primarily used by the `airbyte-server` but is also accessed from `airbyte-scheduler` and `airbyte-workers`. It contains all configuration information for Airbyte.
+This module contains the logic for accessing the config database. This database is primarily used by the `airbyte-server` but is also accessed from `airbyte-workers`. It contains all configuration information for Airbyte.
 
 ## Key files
 * `ConfigPersistence.java` is the interface over "low-level" access to the db. The most commonly used implementation of it is `DatabaseConfigPersistence.java` The only other one that is used is the `YamlSeedConfigPersistence.java` which is used for loading configs that ship with the app.

airbyte-container-orchestrator/build.gradle

@@ -16,7 +16,7 @@ dependencies {
     implementation project(':airbyte-db:db-lib')
     implementation project(':airbyte-json-validation')
     implementation project(':airbyte-protocol:protocol-models')
-    implementation project(':airbyte-scheduler:scheduler-persistence')
+    implementation project(':airbyte-persistence:job-persistence')
     implementation project(':airbyte-workers')
     implementation project(':airbyte-metrics:metrics-lib')
 

airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricEmittingApps.java

@@ -12,7 +12,7 @@
  *
  * Application Name Conventions:
  * <p>
- * - Drop the airbyte prefix when naming applications, e.g airbyte-scheduler -> scheduler.
+ * - Drop the airbyte prefix when naming applications, e.g airbyte-server -> server.
  * <p>
  * - Use dashes to delimit application names with multiple words.
  * <p>

airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobPersistenceTest.java

@@ -551,7 +551,7 @@ void testYamlSchemas() throws IOException {
       final String tableName = tableSchema.name();
       final JsonNode schema = tableSchema.getTableDefinition();
       assertNotNull(schema,
-          "Json schema files should be created in airbyte-scheduler/src/main/resources/tables for every table in the Database to validate its content");
+          "Json schema files should be created in airbyte-persistence/job-persistence/src/main/resources/tables for every table in the Database to validate its content");
       tableStream.forEach(row -> {
         try {
           jsonSchemaValidator.ensure(schema, row);

airbyte-server/build.gradle

@@ -15,9 +15,6 @@ dependencies {
     implementation project(':airbyte-notification')
     implementation project(':airbyte-oauth')
     implementation project(':airbyte-protocol:protocol-models')
-    implementation(project(':airbyte-scheduler:client')) {
-        exclude module: 'airbyte-workers'
-    }
     implementation project(':airbyte-persistence:job-persistence')
     implementation(project(':airbyte-workers')) {
         // Temporary hack to avoid dependency conflicts

airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java

@@ -16,9 +16,9 @@
 import io.airbyte.config.persistence.StatePersistence;
 import io.airbyte.db.Database;
 import io.airbyte.persistence.job.JobPersistence;
-import io.airbyte.scheduler.client.EventRunner;
-import io.airbyte.scheduler.client.SynchronousSchedulerClient;
 import io.airbyte.server.apis.ConfigurationApi;
+import io.airbyte.server.scheduler.EventRunner;
+import io.airbyte.server.scheduler.SynchronousSchedulerClient;
 import java.net.http.HttpClient;
 import java.nio.file.Path;
 import java.util.Map;

airbyte-server/src/main/java/io/airbyte/server/ServerApp.java

@@ -44,16 +44,16 @@
 import io.airbyte.persistence.job.errorreporter.JobErrorReportingClientFactory;
 import io.airbyte.persistence.job.factory.OAuthConfigSupplier;
 import io.airbyte.persistence.job.tracker.JobTracker;
-import io.airbyte.scheduler.client.DefaultSynchronousSchedulerClient;
-import io.airbyte.scheduler.client.EventRunner;
-import io.airbyte.scheduler.client.TemporalEventRunner;
 import io.airbyte.server.errors.InvalidInputExceptionMapper;
 import io.airbyte.server.errors.InvalidJsonExceptionMapper;
 import io.airbyte.server.errors.InvalidJsonInputExceptionMapper;
 import io.airbyte.server.errors.KnownExceptionMapper;
 import io.airbyte.server.errors.NotFoundExceptionMapper;
 import io.airbyte.server.errors.UncaughtExceptionMapper;
 import io.airbyte.server.handlers.DbMigrationHandler;
+import io.airbyte.server.scheduler.DefaultSynchronousSchedulerClient;
+import io.airbyte.server.scheduler.EventRunner;
+import io.airbyte.server.scheduler.TemporalEventRunner;
 import io.airbyte.validation.json.JsonValidationException;
 import io.airbyte.workers.normalization.NormalizationRunnerFactory;
 import io.airbyte.workers.temporal.ConnectionManagerUtils;

airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java

@@ -16,9 +16,9 @@
 import io.airbyte.config.persistence.StatePersistence;
 import io.airbyte.db.Database;
 import io.airbyte.persistence.job.JobPersistence;
-import io.airbyte.scheduler.client.EventRunner;
-import io.airbyte.scheduler.client.SynchronousSchedulerClient;
 import io.airbyte.server.apis.ConfigurationApi;
+import io.airbyte.server.scheduler.EventRunner;
+import io.airbyte.server.scheduler.SynchronousSchedulerClient;
 import java.net.http.HttpClient;
 import java.nio.file.Path;
 import java.util.Set;

airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java

@@ -113,8 +113,6 @@
 import io.airbyte.db.Database;
 import io.airbyte.persistence.job.JobPersistence;
 import io.airbyte.persistence.job.WorkspaceHelper;
-import io.airbyte.scheduler.client.EventRunner;
-import io.airbyte.scheduler.client.SynchronousSchedulerClient;
 import io.airbyte.server.errors.BadObjectSchemaKnownException;
 import io.airbyte.server.errors.IdNotFoundKnownException;
 import io.airbyte.server.handlers.ArchiveHandler;
@@ -135,6 +133,8 @@
 import io.airbyte.server.handlers.StateHandler;
 import io.airbyte.server.handlers.WebBackendConnectionsHandler;
 import io.airbyte.server.handlers.WorkspacesHandler;
+import io.airbyte.server.scheduler.EventRunner;
+import io.airbyte.server.scheduler.SynchronousSchedulerClient;
 import io.airbyte.validation.json.JsonSchemaValidator;
 import io.airbyte.validation.json.JsonValidationException;
 import java.io.File;

airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java

@@ -39,8 +39,8 @@
 import io.airbyte.config.helpers.LogConfigs;
 import io.airbyte.persistence.job.models.Attempt;
 import io.airbyte.persistence.job.models.Job;
-import io.airbyte.scheduler.client.SynchronousJobMetadata;
-import io.airbyte.scheduler.client.SynchronousResponse;
+import io.airbyte.server.scheduler.SynchronousJobMetadata;
+import io.airbyte.server.scheduler.SynchronousResponse;
 import io.airbyte.workers.helper.ProtocolConverters;
 import java.io.IOException;
 import java.nio.file.Path;

airbyte-server/src/main/java/io/airbyte/server/converters/SpecFetcher.java

@@ -6,7 +6,7 @@
 
 import com.google.common.base.Preconditions;
 import io.airbyte.protocol.models.ConnectorSpecification;
-import io.airbyte.scheduler.client.SynchronousResponse;
+import io.airbyte.server.scheduler.SynchronousResponse;
 
 public class SpecFetcher {
 

airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java

@@ -43,14 +43,14 @@
 import io.airbyte.persistence.job.WorkspaceHelper;
 import io.airbyte.protocol.models.CatalogHelpers;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
-import io.airbyte.scheduler.client.EventRunner;
 import io.airbyte.server.converters.ApiPojoConverters;
 import io.airbyte.server.converters.CatalogDiffConverters;
 import io.airbyte.server.handlers.helpers.CatalogConverter;
 import io.airbyte.server.handlers.helpers.ConnectionMatcher;
 import io.airbyte.server.handlers.helpers.ConnectionScheduleHelper;
 import io.airbyte.server.handlers.helpers.DestinationMatcher;
 import io.airbyte.server.handlers.helpers.SourceMatcher;
+import io.airbyte.server.scheduler.EventRunner;
 import io.airbyte.validation.json.JsonValidationException;
 import io.airbyte.workers.helper.ConnectionHelper;
 import java.io.IOException;

airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java

@@ -30,12 +30,12 @@
 import io.airbyte.config.persistence.ConfigNotFoundException;
 import io.airbyte.config.persistence.ConfigRepository;
 import io.airbyte.protocol.models.ConnectorSpecification;
-import io.airbyte.scheduler.client.SynchronousResponse;
-import io.airbyte.scheduler.client.SynchronousSchedulerClient;
 import io.airbyte.server.converters.ApiPojoConverters;
 import io.airbyte.server.converters.SpecFetcher;
 import io.airbyte.server.errors.IdNotFoundKnownException;
 import io.airbyte.server.errors.InternalServerKnownException;
+import io.airbyte.server.scheduler.SynchronousResponse;
+import io.airbyte.server.scheduler.SynchronousSchedulerClient;
 import io.airbyte.server.services.AirbyteGithubStore;
 import io.airbyte.validation.json.JsonValidationException;
 import java.io.IOException;

airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java

@@ -52,15 +52,15 @@
 import io.airbyte.persistence.job.models.Job;
 import io.airbyte.protocol.models.AirbyteCatalog;
 import io.airbyte.protocol.models.ConnectorSpecification;
-import io.airbyte.scheduler.client.EventRunner;
-import io.airbyte.scheduler.client.SynchronousJobMetadata;
-import io.airbyte.scheduler.client.SynchronousResponse;
-import io.airbyte.scheduler.client.SynchronousSchedulerClient;
 import io.airbyte.server.converters.ConfigurationUpdate;
 import io.airbyte.server.converters.JobConverter;
 import io.airbyte.server.converters.OauthModelConverter;
 import io.airbyte.server.errors.ValueConflictKnownException;
 import io.airbyte.server.handlers.helpers.CatalogConverter;
+import io.airbyte.server.scheduler.EventRunner;
+import io.airbyte.server.scheduler.SynchronousJobMetadata;
+import io.airbyte.server.scheduler.SynchronousResponse;
+import io.airbyte.server.scheduler.SynchronousSchedulerClient;
 import io.airbyte.validation.json.JsonSchemaValidator;
 import io.airbyte.validation.json.JsonValidationException;
 import io.airbyte.workers.temporal.ErrorCode;

airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java

@@ -31,12 +31,12 @@
 import io.airbyte.config.persistence.ConfigNotFoundException;
 import io.airbyte.config.persistence.ConfigRepository;
 import io.airbyte.protocol.models.ConnectorSpecification;
-import io.airbyte.scheduler.client.SynchronousResponse;
-import io.airbyte.scheduler.client.SynchronousSchedulerClient;
 import io.airbyte.server.converters.ApiPojoConverters;
 import io.airbyte.server.converters.SpecFetcher;
 import io.airbyte.server.errors.IdNotFoundKnownException;
 import io.airbyte.server.errors.InternalServerKnownException;
+import io.airbyte.server.scheduler.SynchronousResponse;
+import io.airbyte.server.scheduler.SynchronousSchedulerClient;
 import io.airbyte.server.services.AirbyteGithubStore;
 import io.airbyte.validation.json.JsonValidationException;
 import java.io.IOException;

airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java

@@ -47,8 +47,8 @@
 import io.airbyte.config.persistence.ConfigNotFoundException;
 import io.airbyte.config.persistence.ConfigRepository;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
-import io.airbyte.scheduler.client.EventRunner;
 import io.airbyte.server.handlers.helpers.CatalogConverter;
+import io.airbyte.server.scheduler.EventRunner;
 import io.airbyte.validation.json.JsonValidationException;
 import io.airbyte.workers.helper.ProtocolConverters;
 import io.airbyte.workers.temporal.TemporalClient.ManualOperationResult;

airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClient.java → airbyte-server/src/main/java/io/airbyte/server/scheduler/DefaultSynchronousSchedulerClient.java

@@ -2,7 +2,7 @@
  * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
-package io.airbyte.scheduler.client;
+package io.airbyte.server.scheduler;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.annotations.VisibleForTesting;

airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/EventRunner.java → airbyte-server/src/main/java/io/airbyte/server/scheduler/EventRunner.java

@@ -2,7 +2,7 @@
  * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
-package io.airbyte.scheduler.client;
+package io.airbyte.server.scheduler;
 
 import io.airbyte.protocol.models.StreamDescriptor;
 import io.airbyte.workers.temporal.TemporalClient.ManualOperationResult;

airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/SynchronousJobMetadata.java → airbyte-server/src/main/java/io/airbyte/server/scheduler/SynchronousJobMetadata.java

@@ -2,7 +2,7 @@
  * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
-package io.airbyte.scheduler.client;
+package io.airbyte.server.scheduler;
 
 import io.airbyte.config.JobConfig.ConfigType;
 import io.airbyte.workers.temporal.JobMetadata;

airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/SynchronousResponse.java → airbyte-server/src/main/java/io/airbyte/server/scheduler/SynchronousResponse.java

@@ -2,7 +2,7 @@
  * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
-package io.airbyte.scheduler.client;
+package io.airbyte.server.scheduler;
 
 import io.airbyte.config.JobConfig.ConfigType;
 import io.airbyte.workers.temporal.TemporalResponse;

airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/SynchronousSchedulerClient.java → airbyte-server/src/main/java/io/airbyte/server/scheduler/SynchronousSchedulerClient.java

@@ -2,7 +2,7 @@
  * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
-package io.airbyte.scheduler.client;
+package io.airbyte.server.scheduler;
 
 import io.airbyte.config.DestinationConnection;
 import io.airbyte.config.SourceConnection;

airbyte-scheduler/client/src/main/java/io/airbyte/scheduler/client/TemporalEventRunner.java → airbyte-server/src/main/java/io/airbyte/server/scheduler/TemporalEventRunner.java

@@ -2,7 +2,7 @@
  * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
  */
 
-package io.airbyte.scheduler.client;
+package io.airbyte.server.scheduler;
 
 import io.airbyte.protocol.models.StreamDescriptor;
 import io.airbyte.workers.temporal.TemporalClient;

airbyte-server/src/test/java/io/airbyte/server/ServerAppTest.java

@@ -12,7 +12,7 @@
 import io.airbyte.config.StandardSync.Status;
 import io.airbyte.config.persistence.ConfigRepository;
 import io.airbyte.persistence.job.JobPersistence;
-import io.airbyte.scheduler.client.EventRunner;
+import io.airbyte.server.scheduler.EventRunner;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;

airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java

@@ -21,8 +21,8 @@
 import io.airbyte.config.persistence.StatePersistence;
 import io.airbyte.db.Database;
 import io.airbyte.persistence.job.JobPersistence;
-import io.airbyte.scheduler.client.EventRunner;
-import io.airbyte.scheduler.client.SynchronousSchedulerClient;
+import io.airbyte.server.scheduler.EventRunner;
+import io.airbyte.server.scheduler.SynchronousSchedulerClient;
 import java.net.http.HttpClient;
 import java.nio.file.Path;
 import org.flywaydb.core.Flyway;

airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java

@@ -59,10 +59,10 @@
 import io.airbyte.config.persistence.ConfigRepository;
 import io.airbyte.persistence.job.WorkspaceHelper;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
-import io.airbyte.scheduler.client.EventRunner;
 import io.airbyte.server.converters.ApiPojoConverters;
 import io.airbyte.server.handlers.helpers.CatalogConverter;
 import io.airbyte.server.helpers.ConnectionHelpers;
+import io.airbyte.server.scheduler.EventRunner;
 import io.airbyte.validation.json.JsonValidationException;
 import java.io.IOException;
 import java.util.Collections;