From 20f54740862454565d31d18654a7c45dfd8d4bd9 Mon Sep 17 00:00:00 2001 From: alovew Date: Tue, 29 Nov 2022 11:17:15 -0800 Subject: [PATCH 01/24] refresh before syncs when feature flag is on --- .../commons/temporal/TemporalClient.java | 21 ++++++++---- .../temporal/scheduling/SyncWorkflow.java | 7 +++- .../commons/temporal/TemporalClientTest.java | 5 ++- .../ConnectionManagerWorkflowImpl.java | 13 +++++-- .../temporal/sync/SyncWorkflowImpl.java | 34 ++++++++++++++++++- .../temporal/sync/SyncWorkflowTest.java | 5 ++- 6 files changed, 73 insertions(+), 12 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index 9abceab23c756..b233a41dea728 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -8,6 +8,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; +import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.temporal.config.WorkerMode; import io.airbyte.commons.temporal.exception.DeletedWorkflowException; import io.airbyte.commons.temporal.exception.UnreachableWorkflowException; @@ -26,10 +27,12 @@ import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; +import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.StreamDescriptor; +import io.airbyte.validation.json.JsonValidationException; import io.micronaut.context.annotation.Requires; import io.temporal.api.common.v1.WorkflowType; import io.temporal.api.enums.v1.WorkflowExecutionStatus; @@ -398,12 +401,18 @@ public TemporalResponse submitSync(final long jobId, final i .withDestinationResourceRequirements(config.getDestinationResourceRequirements()); return execute(jobRunConfig, - () -> getWorkflowStub(SyncWorkflow.class, TemporalJobType.SYNC).run( - jobRunConfig, - sourceLauncherConfig, - destinationLauncherConfig, - input, - connectionId)); + () -> { + try { + return getWorkflowStub(SyncWorkflow.class, TemporalJobType.SYNC).run( + jobRunConfig, + sourceLauncherConfig, + destinationLauncherConfig, + input, + connectionId); + } catch (JsonValidationException | ConfigNotFoundException | IOException | ApiException e) { + throw new RuntimeException(e); + } + }); } public void migrateSyncIfNeeded(final Set connectionIds) { diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java index 513974b955340..3e92e979aaa95 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java @@ -4,12 +4,16 @@ package io.airbyte.commons.temporal.scheduling; +import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; +import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; +import io.airbyte.validation.json.JsonValidationException; import io.temporal.workflow.WorkflowInterface; import io.temporal.workflow.WorkflowMethod; +import java.io.IOException; import java.util.UUID; @WorkflowInterface @@ -20,6 +24,7 @@ StandardSyncOutput run(JobRunConfig jobRunConfig, IntegrationLauncherConfig sourceLauncherConfig, IntegrationLauncherConfig destinationLauncherConfig, StandardSyncInput syncInput, - UUID connectionId); + UUID connectionId) + throws JsonValidationException, ConfigNotFoundException, IOException, ApiException; } diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index 36b3eda388063..c3e0cf987ac47 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.when; import com.google.common.collect.Sets; +import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalClient.ManualOperationResult; import io.airbyte.commons.temporal.scheduling.CheckConnectionWorkflow; @@ -41,11 +42,13 @@ import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.helpers.LogClientSingleton; +import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.StreamDescriptor; +import io.airbyte.validation.json.JsonValidationException; import io.temporal.api.enums.v1.WorkflowExecutionStatus; import io.temporal.api.workflow.v1.WorkflowExecutionInfo; import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionResponse; @@ -264,7 +267,7 @@ void testSubmitDiscoverSchema() { } @Test - void testSubmitSync() { + void testSubmitSync() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { final SyncWorkflow discoverCatalogWorkflow = mock(SyncWorkflow.class); when(workflowClient.newWorkflowStub(SyncWorkflow.class, TemporalWorkflowUtils.buildWorkflowOptions(TemporalJobType.SYNC))) .thenReturn(discoverCatalogWorkflow); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java index d05db6d4fa736..db5eb9f7dd914 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.JsonNode; import datadog.trace.api.Trace; +import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.temporal.TemporalJobType; import io.airbyte.commons.temporal.TemporalWorkflowUtils; import io.airbyte.commons.temporal.exception.RetryableException; @@ -31,10 +32,12 @@ import io.airbyte.config.StandardSyncOutput; import io.airbyte.config.StandardSyncSummary; import io.airbyte.config.StandardSyncSummary.ReplicationStatus; +import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; +import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.helper.FailureHelper; import io.airbyte.workers.temporal.annotations.TemporalActivityStub; @@ -82,6 +85,7 @@ import io.temporal.workflow.CancellationScope; import io.temporal.workflow.ChildWorkflowOptions; import io.temporal.workflow.Workflow; +import java.io.IOException; import java.time.Duration; import java.time.Instant; import java.util.HashSet; @@ -254,7 +258,11 @@ private CancellationScope generateSyncWorkflowRunnable(final ConnectionUpdaterIn workflowState.setFailed(getFailStatus(checkFailureOutput)); reportFailure(connectionUpdaterInput, checkFailureOutput, FailureCause.CONNECTION); } else { - standardSyncOutput = runChildWorkflow(jobInputs); + try { + standardSyncOutput = runChildWorkflow(jobInputs); + } catch (JsonValidationException | ConfigNotFoundException | IOException | ApiException e) { + throw new RuntimeException(e); + } workflowState.setFailed(getFailStatus(standardSyncOutput)); if (workflowState.isFailed()) { @@ -849,7 +857,8 @@ private void reportJobStarting(final UUID connectionId) { * since the latter is a long running workflow, in the future, using a different Node pool would * make sense. */ - private StandardSyncOutput runChildWorkflow(final GeneratedJobInput jobInputs) { + private StandardSyncOutput runChildWorkflow(final GeneratedJobInput jobInputs) + throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { final String taskQueue = getSyncTaskQueue(); final SyncWorkflow childSync = Workflow.newChildWorkflowStub(SyncWorkflow.class, diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index 945c0abe2bc78..378261f68cb13 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -12,22 +12,31 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.WORKFLOW_TRACE_OPERATION_NAME; import datadog.trace.api.Trace; +import io.airbyte.api.client.invoker.generated.ApiException; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.NormalizationInput; import io.airbyte.config.NormalizationSummary; import io.airbyte.config.OperatorDbtInput; import io.airbyte.config.OperatorWebhookInput; +import io.airbyte.config.StandardSync.Status; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOperation; import io.airbyte.config.StandardSyncOperation.OperatorType; import io.airbyte.config.StandardSyncOutput; +import io.airbyte.config.StandardSyncSummary; +import io.airbyte.config.StandardSyncSummary.ReplicationStatus; import io.airbyte.config.WebhookOperationSummary; +import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.temporal.annotations.TemporalActivityStub; +import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivity; import io.temporal.workflow.Workflow; +import java.io.IOException; import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -55,6 +64,10 @@ public class SyncWorkflowImpl implements SyncWorkflow { private NormalizationSummaryCheckActivity normalizationSummaryCheckActivity; @TemporalActivityStub(activityOptionsBeanName = "shortActivityOptions") private WebhookOperationActivity webhookOperationActivity; + @TemporalActivityStub(activityOptionsBeanName = "shortActivityOptions") + private RefreshSchemaActivity refreshSchemaActivity; + @TemporalActivityStub(activityOptionsBeanName = "shortActivityOptions") + private ConfigFetchActivity configFetchActivity; @Trace(operationName = WORKFLOW_TRACE_OPERATION_NAME) @Override @@ -62,7 +75,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId) { + final UUID connectionId) + throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { ApmTraceUtils .addTagsToTrace(Map.of(ATTEMPT_NUMBER_KEY, jobRunConfig.getAttemptId(), CONNECTION_ID_KEY, connectionId.toString(), JOB_ID_KEY, @@ -72,6 +86,24 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final int version = Workflow.getVersion(VERSION_LABEL, Workflow.DEFAULT_VERSION, CURRENT_VERSION); final String taskQueue = Workflow.getInfo().getTaskQueue(); + + final UUID sourceCatalogId = configFetchActivity.getStandardSync(connectionId).getSourceCatalogId(); + + final EnvVariableFeatureFlags envVariableFeatureFlags = new EnvVariableFeatureFlags(); + if (envVariableFeatureFlags.autoDetectSchema()) { + if (refreshSchemaActivity.shouldRefreshSchema(sourceCatalogId)) { + LOGGER.info("Refreshing source schema..."); + refreshSchemaActivity.refreshSchema(sourceCatalogId); + } + final Status status = configFetchActivity.getStandardSync(connectionId).getStatus(); + if (Status.INACTIVE == status) { + LOGGER.info("Connection is disabled. Cancelling run."); + final StandardSyncOutput output = + new StandardSyncOutput().withStandardSyncSummary(new StandardSyncSummary().withStatus(ReplicationStatus.CANCELLED)); + return output; + } + } + StandardSyncOutput syncOutput = replicationActivity.replicate(jobRunConfig, sourceLauncherConfig, destinationLauncherConfig, syncInput, taskQueue); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index 31933b9c734d6..491cf316430b3 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -16,6 +16,7 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; @@ -33,9 +34,11 @@ import io.airbyte.config.StandardSyncSummary; import io.airbyte.config.StandardSyncSummary.ReplicationStatus; import io.airbyte.config.SyncStats; +import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.temporal.support.TemporalProxyHelper; import io.airbyte.workers.test_utils.TestConfigHelpers; import io.micronaut.context.BeanRegistration; @@ -188,7 +191,7 @@ public void tearDown() { } // bundle up all the temporal worker setup / execution into one method. - private StandardSyncOutput execute() { + private StandardSyncOutput execute() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { syncWorker.registerActivitiesImplementations(replicationActivity, normalizationActivity, dbtTransformationActivity, persistStateActivity, normalizationSummaryCheckActivity, webhookOperationActivity); testEnv.start(); From 87b0b7211f22d961ff95ce6abc3a2cd441aa4957 Mon Sep 17 00:00:00 2001 From: alovew Date: Tue, 29 Nov 2022 11:59:20 -0800 Subject: [PATCH 02/24] tests --- .../workers/config/ActivityBeanFactory.java | 7 +++++-- .../workers/temporal/sync/SyncWorkflowTest.java | 15 ++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java index b8821c67ae0e9..7c333e8a0b75e 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java @@ -24,6 +24,7 @@ import io.airbyte.workers.temporal.sync.NormalizationActivity; import io.airbyte.workers.temporal.sync.NormalizationSummaryCheckActivity; import io.airbyte.workers.temporal.sync.PersistStateActivity; +import io.airbyte.workers.temporal.sync.RefreshSchemaActivity; import io.airbyte.workers.temporal.sync.ReplicationActivity; import io.airbyte.workers.temporal.sync.WebhookOperationActivity; import io.micronaut.context.annotation.Factory; @@ -110,9 +111,11 @@ public List syncActivities( final DbtTransformationActivity dbtTransformationActivity, final PersistStateActivity persistStateActivity, final NormalizationSummaryCheckActivity normalizationSummaryCheckActivity, - final WebhookOperationActivity webhookOperationActivity) { + final WebhookOperationActivity webhookOperationActivity, + final ConfigFetchActivity configFetchActivity, + final RefreshSchemaActivity refreshSchemaActivity) { return List.of(replicationActivity, normalizationActivity, dbtTransformationActivity, persistStateActivity, normalizationSummaryCheckActivity, - webhookOperationActivity); + webhookOperationActivity, configFetchActivity, refreshSchemaActivity); } @Singleton diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index 491cf316430b3..3c122a9c09119 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -39,6 +39,7 @@ import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.validation.json.JsonValidationException; +import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivityImpl; import io.airbyte.workers.temporal.support.TemporalProxyHelper; import io.airbyte.workers.test_utils.TestConfigHelpers; import io.micronaut.context.BeanRegistration; @@ -82,6 +83,8 @@ class SyncWorkflowTest { private PersistStateActivityImpl persistStateActivity; private NormalizationSummaryCheckActivityImpl normalizationSummaryCheckActivity; private WebhookOperationActivityImpl webhookOperationActivity; + private RefreshSchemaActivityImpl refreshSchemaActivity; + private ConfigFetchActivityImpl configFetchActivity; // AIRBYTE CONFIGURATION private static final long JOB_ID = 11L; @@ -149,6 +152,8 @@ void setUp() throws IOException { persistStateActivity = mock(PersistStateActivityImpl.class); normalizationSummaryCheckActivity = mock(NormalizationSummaryCheckActivityImpl.class); webhookOperationActivity = mock(WebhookOperationActivityImpl.class); + refreshSchemaActivity = mock(RefreshSchemaActivityImpl.class); + configFetchActivity = mock(ConfigFetchActivityImpl.class); when(normalizationActivity.generateNormalizationInput(any(), any())).thenReturn(normalizationInput); when(normalizationSummaryCheckActivity.shouldRunNormalization(any(), any(), any())).thenReturn(true); @@ -193,7 +198,7 @@ public void tearDown() { // bundle up all the temporal worker setup / execution into one method. private StandardSyncOutput execute() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { syncWorker.registerActivitiesImplementations(replicationActivity, normalizationActivity, dbtTransformationActivity, - persistStateActivity, normalizationSummaryCheckActivity, webhookOperationActivity); + persistStateActivity, normalizationSummaryCheckActivity, webhookOperationActivity, refreshSchemaActivity, configFetchActivity); testEnv.start(); final SyncWorkflow workflow = client.newWorkflowStub(SyncWorkflow.class, WorkflowOptions.newBuilder().setTaskQueue(SYNC_QUEUE).build()); @@ -202,7 +207,7 @@ private StandardSyncOutput execute() throws JsonValidationException, ConfigNotFo } @Test - void testSuccess() { + void testSuccess() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { doReturn(replicationSuccessOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -243,7 +248,7 @@ void testReplicationFailure() { } @Test - void testReplicationFailedGracefully() { + void testReplicationFailedGracefully() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { doReturn(replicationFailOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -333,7 +338,7 @@ void testCancelDuringNormalization() { @Test @Disabled("This behavior has been disabled temporarily (OC Issue #741)") - void testSkipNormalization() throws IOException { + void testSkipNormalization() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { final SyncStats syncStats = new SyncStats().withRecordsCommitted(0L); final StandardSyncSummary standardSyncSummary = new StandardSyncSummary().withTotalStats(syncStats); final StandardSyncOutput replicationSuccessOutputNoRecordsCommitted = @@ -356,7 +361,7 @@ void testSkipNormalization() throws IOException { } @Test - void testWebhookOperation() { + void testWebhookOperation() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { when(replicationActivity.replicate(any(), any(), any(), any(), any())).thenReturn(new StandardSyncOutput()); final StandardSyncOperation webhookOperation = new StandardSyncOperation() .withOperationId(UUID.randomUUID()) From 998bd81330d8fcaff5ffa7900d04c1afb3b9f454 Mon Sep 17 00:00:00 2001 From: alovew Date: Tue, 29 Nov 2022 18:55:39 -0800 Subject: [PATCH 03/24] fix activity --- .../sync/RefreshSchemaActivityImpl.java | 15 +++++---- .../temporal/sync/SyncWorkflowImpl.java | 31 ++++++++++--------- .../activities/RefreshSchemaActivityTest.java | 9 ++++-- 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java index bc12f37d0ff6e..5b0e933cf2255 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java @@ -7,25 +7,27 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.ACTIVITY_TRACE_OPERATION_NAME; import datadog.trace.api.Trace; -import io.airbyte.api.client.generated.SourceApi; +import io.airbyte.api.client.AirbyteApiClient; import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRequestBody; import io.airbyte.config.ActorCatalogFetchEvent; import io.airbyte.config.persistence.ConfigRepository; +import jakarta.inject.Singleton; import java.io.IOException; import java.time.OffsetDateTime; import java.util.Optional; import java.util.UUID; +@Singleton public class RefreshSchemaActivityImpl implements RefreshSchemaActivity { private final Optional configRepository; - private final SourceApi sourceApi; + private final AirbyteApiClient airbyteApiClient; - public RefreshSchemaActivityImpl(Optional configRepository, SourceApi sourceApi) { + public RefreshSchemaActivityImpl(Optional configRepository, AirbyteApiClient airbyteApiClient) { this.configRepository = configRepository; - this.sourceApi = sourceApi; + this.airbyteApiClient = airbyteApiClient; } @Override @@ -36,14 +38,15 @@ public boolean shouldRefreshSchema(UUID sourceCatalogId) throws IOException { return false; } - return !schemaRefreshRanRecently(sourceCatalogId); + return !schemaRefreshRanRecently(sourceCatalogId) || true; } @Override public void refreshSchema(UUID sourceCatalogId) throws ApiException { SourceDiscoverSchemaRequestBody requestBody = new SourceDiscoverSchemaRequestBody().sourceId(sourceCatalogId).disableCache(true); - sourceApi.discoverSchemaForSource(requestBody); + + airbyteApiClient.getSourceApi().discoverSchemaForSource(requestBody); } private boolean schemaRefreshRanRecently(UUID sourceCatalogId) throws IOException { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index 378261f68cb13..4bdcae6b1cb7d 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -87,20 +87,23 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final int version = Workflow.getVersion(VERSION_LABEL, Workflow.DEFAULT_VERSION, CURRENT_VERSION); final String taskQueue = Workflow.getInfo().getTaskQueue(); - final UUID sourceCatalogId = configFetchActivity.getStandardSync(connectionId).getSourceCatalogId(); - - final EnvVariableFeatureFlags envVariableFeatureFlags = new EnvVariableFeatureFlags(); - if (envVariableFeatureFlags.autoDetectSchema()) { - if (refreshSchemaActivity.shouldRefreshSchema(sourceCatalogId)) { - LOGGER.info("Refreshing source schema..."); - refreshSchemaActivity.refreshSchema(sourceCatalogId); - } - final Status status = configFetchActivity.getStandardSync(connectionId).getStatus(); - if (Status.INACTIVE == status) { - LOGGER.info("Connection is disabled. Cancelling run."); - final StandardSyncOutput output = - new StandardSyncOutput().withStandardSyncSummary(new StandardSyncSummary().withStatus(ReplicationStatus.CANCELLED)); - return output; + if (version > Workflow.DEFAULT_VERSION) { + final EnvVariableFeatureFlags envVariableFeatureFlags = new EnvVariableFeatureFlags(); + if (envVariableFeatureFlags.autoDetectSchema()) { + final UUID sourceId = configFetchActivity.getStandardSync(connectionId).getSourceId(); + if (refreshSchemaActivity.shouldRefreshSchema(sourceId)) { + LOGGER.info("Refreshing source schema..."); + refreshSchemaActivity.refreshSchema(sourceId); + } + LOGGER.info("refreshed schema"); + final Status status = configFetchActivity.getStandardSync(connectionId).getStatus(); + LOGGER.info("status is: " + status); + if (Status.INACTIVE == status) { + LOGGER.info("Connection is disabled. Cancelling run."); + final StandardSyncOutput output = + new StandardSyncOutput().withStandardSyncSummary(new StandardSyncSummary().withStatus(ReplicationStatus.CANCELLED)); + return output; + } } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java index 26086b1dd0d95..9678ccca02aab 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java @@ -9,6 +9,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import io.airbyte.api.client.AirbyteApiClient; import io.airbyte.api.client.generated.SourceApi; import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRequestBody; @@ -23,6 +24,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -30,6 +32,7 @@ class RefreshSchemaActivityTest { static private ConfigRepository mConfigRepository; + static private AirbyteApiClient mApiClient; static private SourceApi mSourceApi; static private RefreshSchemaActivityImpl refreshSchemaActivity; @@ -38,9 +41,11 @@ class RefreshSchemaActivityTest { @BeforeEach void setUp() { - mConfigRepository = mock(ConfigRepository.class); + mApiClient = mock(AirbyteApiClient.class); mSourceApi = mock(SourceApi.class); - refreshSchemaActivity = new RefreshSchemaActivityImpl(Optional.of(mConfigRepository), mSourceApi); + mConfigRepository = mock(ConfigRepository.class); + Mockito.lenient().when(mApiClient.getSourceApi()).thenReturn(mSourceApi); + refreshSchemaActivity = new RefreshSchemaActivityImpl(Optional.of(mConfigRepository), mApiClient); } @Test From 291db748b14f4d686559ad26e6843852f48d32d6 Mon Sep 17 00:00:00 2001 From: alovew Date: Wed, 30 Nov 2022 10:19:19 -0800 Subject: [PATCH 04/24] add connection id --- .../persistence/job/DefaultJobPersistence.java | 4 +++- .../workers/temporal/sync/RefreshSchemaActivity.java | 3 ++- .../temporal/sync/RefreshSchemaActivityImpl.java | 12 +++++++++--- .../workers/temporal/sync/SyncWorkflowImpl.java | 6 ++++-- .../activities/RefreshSchemaActivityTest.java | 3 ++- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobPersistence.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobPersistence.java index 369e7463cf982..cd39b41e59f19 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobPersistence.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobPersistence.java @@ -348,7 +348,9 @@ public void writeOutput(final long jobId, final int attemptNumber, final JobOutp .execute(); final Long attemptId = getAttemptId(jobId, attemptNumber, ctx); - writeSyncStats(now, syncStats, attemptId, ctx); + if (syncStats != null) { + writeSyncStats(now, syncStats, attemptId, ctx); + } if (normalizationSummary != null) { ctx.insertInto(NORMALIZATION_SUMMARIES) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivity.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivity.java index 3da403ea19ed6..b6079a689a980 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivity.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivity.java @@ -18,6 +18,7 @@ public interface RefreshSchemaActivity { @ActivityMethod boolean shouldRefreshSchema(UUID sourceCatalogId) throws IOException; - public void refreshSchema(UUID sourceCatalogId) throws JsonValidationException, ConfigNotFoundException, IOException, ApiException; + public void refreshSchema(UUID sourceCatalogId, UUID connectionId) + throws JsonValidationException, ConfigNotFoundException, IOException, ApiException; } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java index 5b0e933cf2255..63de67be4bb10 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java @@ -17,7 +17,9 @@ import java.time.OffsetDateTime; import java.util.Optional; import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +@Slf4j @Singleton public class RefreshSchemaActivityImpl implements RefreshSchemaActivity { @@ -42,11 +44,15 @@ public boolean shouldRefreshSchema(UUID sourceCatalogId) throws IOException { } @Override - public void refreshSchema(UUID sourceCatalogId) throws ApiException { + public void refreshSchema(UUID sourceCatalogId, UUID connectionId) throws ApiException { SourceDiscoverSchemaRequestBody requestBody = - new SourceDiscoverSchemaRequestBody().sourceId(sourceCatalogId).disableCache(true); + new SourceDiscoverSchemaRequestBody().sourceId(sourceCatalogId).disableCache(true).connectionId(connectionId); - airbyteApiClient.getSourceApi().discoverSchemaForSource(requestBody); + try { + airbyteApiClient.getSourceApi().discoverSchemaForSource(requestBody); + } catch (final Exception e) { + log.info("Attempted schema refresh, but failed."); + } } private boolean schemaRefreshRanRecently(UUID sourceCatalogId) throws IOException { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index 4bdcae6b1cb7d..fe990ae255ba4 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -26,6 +26,7 @@ import io.airbyte.config.StandardSyncOutput; import io.airbyte.config.StandardSyncSummary; import io.airbyte.config.StandardSyncSummary.ReplicationStatus; +import io.airbyte.config.SyncStats; import io.airbyte.config.WebhookOperationSummary; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.metrics.lib.ApmTraceUtils; @@ -93,7 +94,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final UUID sourceId = configFetchActivity.getStandardSync(connectionId).getSourceId(); if (refreshSchemaActivity.shouldRefreshSchema(sourceId)) { LOGGER.info("Refreshing source schema..."); - refreshSchemaActivity.refreshSchema(sourceId); + refreshSchemaActivity.refreshSchema(sourceId, connectionId); } LOGGER.info("refreshed schema"); final Status status = configFetchActivity.getStandardSync(connectionId).getStatus(); @@ -101,7 +102,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, if (Status.INACTIVE == status) { LOGGER.info("Connection is disabled. Cancelling run."); final StandardSyncOutput output = - new StandardSyncOutput().withStandardSyncSummary(new StandardSyncSummary().withStatus(ReplicationStatus.CANCELLED)); + new StandardSyncOutput() + .withStandardSyncSummary(new StandardSyncSummary().withStatus(ReplicationStatus.CANCELLED).withTotalStats(new SyncStats())); return output; } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java index 9678ccca02aab..24c9807ce4caf 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java @@ -73,7 +73,8 @@ void testShouldRefreshSchemaRecentRefreshLessThan24HoursAgo() throws IOException @Test void testRefreshSchema() throws ApiException { UUID sourceId = UUID.randomUUID(); - refreshSchemaActivity.refreshSchema(sourceId); + UUID connectionId = UUID.randomUUID(); + refreshSchemaActivity.refreshSchema(sourceId, connectionId); SourceDiscoverSchemaRequestBody requestBody = new SourceDiscoverSchemaRequestBody().sourceId(sourceId).disableCache(true); verify(mSourceApi, times(1)).discoverSchemaForSource(requestBody); From 00939242e753d40e68888db8be5ef355f5589089 Mon Sep 17 00:00:00 2001 From: alovew Date: Wed, 30 Nov 2022 10:20:33 -0800 Subject: [PATCH 05/24] logging --- .../java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index fe990ae255ba4..9147d5b99c332 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -96,9 +96,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, LOGGER.info("Refreshing source schema..."); refreshSchemaActivity.refreshSchema(sourceId, connectionId); } - LOGGER.info("refreshed schema"); final Status status = configFetchActivity.getStandardSync(connectionId).getStatus(); - LOGGER.info("status is: " + status); if (Status.INACTIVE == status) { LOGGER.info("Connection is disabled. Cancelling run."); final StandardSyncOutput output = From 5c350081f94940b594a633d5647bf252e75fa56b Mon Sep 17 00:00:00 2001 From: alovew Date: Wed, 30 Nov 2022 10:57:03 -0800 Subject: [PATCH 06/24] inject envvariablefeatureflags --- .../io/airbyte/commons/temporal/TemporalClient.java | 8 ++++++-- .../commons/temporal/scheduling/SyncWorkflow.java | 4 +++- .../airbyte/commons/temporal/TemporalClientTest.java | 9 ++++++--- .../src/main/java/io/airbyte/server/ServerApp.java | 3 ++- .../temporal/sync/RefreshSchemaActivityImpl.java | 10 +++++----- .../workers/temporal/sync/SyncWorkflowImpl.java | 4 ++-- .../workers/temporal/sync/SyncWorkflowTest.java | 6 +++++- 7 files changed, 29 insertions(+), 15 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index b233a41dea728..ffc0a00489e28 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -9,6 +9,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; import io.airbyte.api.client.invoker.generated.ApiException; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.config.WorkerMode; import io.airbyte.commons.temporal.exception.DeletedWorkflowException; import io.airbyte.commons.temporal.exception.UnreachableWorkflowException; @@ -81,19 +82,22 @@ public class TemporalClient { private final StreamResetPersistence streamResetPersistence; private final ConnectionManagerUtils connectionManagerUtils; private final StreamResetRecordsHelper streamResetRecordsHelper; + private final EnvVariableFeatureFlags envVariableFeatureFlags; public TemporalClient(@Named("workspaceRootTemporal") final Path workspaceRoot, final WorkflowClient client, final WorkflowServiceStubs service, final StreamResetPersistence streamResetPersistence, final ConnectionManagerUtils connectionManagerUtils, - final StreamResetRecordsHelper streamResetRecordsHelper) { + final StreamResetRecordsHelper streamResetRecordsHelper, + final EnvVariableFeatureFlags envVariableFeatureFlags) { this.workspaceRoot = workspaceRoot; this.client = client; this.service = service; this.streamResetPersistence = streamResetPersistence; this.connectionManagerUtils = connectionManagerUtils; this.streamResetRecordsHelper = streamResetRecordsHelper; + this.envVariableFeatureFlags = envVariableFeatureFlags; } private final Set workflowNames = new HashSet<>(); @@ -408,7 +412,7 @@ public TemporalResponse submitSync(final long jobId, final i sourceLauncherConfig, destinationLauncherConfig, input, - connectionId); + connectionId, envVariableFeatureFlags); } catch (JsonValidationException | ConfigNotFoundException | IOException | ApiException e) { throw new RuntimeException(e); } diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java index 3e92e979aaa95..6cf1897ea77a1 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java @@ -5,6 +5,7 @@ package io.airbyte.commons.temporal.scheduling; import io.airbyte.api.client.invoker.generated.ApiException; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; import io.airbyte.config.persistence.ConfigNotFoundException; @@ -24,7 +25,8 @@ StandardSyncOutput run(JobRunConfig jobRunConfig, IntegrationLauncherConfig sourceLauncherConfig, IntegrationLauncherConfig destinationLauncherConfig, StandardSyncInput syncInput, - UUID connectionId) + UUID connectionId, + EnvVariableFeatureFlags envVariableFeatureFlags) throws JsonValidationException, ConfigNotFoundException, IOException, ApiException; } diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index c3e0cf987ac47..7d42d5aaabf21 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.Sets; import io.airbyte.api.client.invoker.generated.ApiException; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalClient.ManualOperationResult; import io.airbyte.commons.temporal.scheduling.CheckConnectionWorkflow; @@ -108,6 +109,7 @@ public class TemporalClientTest { private ConnectionManagerUtils connectionManagerUtils; private StreamResetRecordsHelper streamResetRecordsHelper; private Path workspaceRoot; + private EnvVariableFeatureFlags envVariableFeatureFlags; @BeforeEach void setup() throws IOException { @@ -123,9 +125,10 @@ void setup() throws IOException { mockWorkflowStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_RUNNING); connectionManagerUtils = spy(new ConnectionManagerUtils()); streamResetRecordsHelper = mock(StreamResetRecordsHelper.class); + envVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); temporalClient = spy(new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, connectionManagerUtils, - streamResetRecordsHelper)); + streamResetRecordsHelper, envVariableFeatureFlags)); } @Nested @@ -139,7 +142,7 @@ void init() { temporalClient = spy( new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, mConnectionManagerUtils, - streamResetRecordsHelper)); + streamResetRecordsHelper, envVariableFeatureFlags)); } @Test @@ -294,7 +297,7 @@ void testSubmitSync() throws JsonValidationException, ConfigNotFoundException, I .withDockerImage(IMAGE_NAME2); temporalClient.submitSync(JOB_ID, ATTEMPT_ID, syncConfig, CONNECTION_ID); - discoverCatalogWorkflow.run(JOB_RUN_CONFIG, LAUNCHER_CONFIG, destinationLauncherConfig, input, CONNECTION_ID); + discoverCatalogWorkflow.run(JOB_RUN_CONFIG, LAUNCHER_CONFIG, destinationLauncherConfig, input, CONNECTION_ID, envVariableFeatureFlags); verify(workflowClient).newWorkflowStub(SyncWorkflow.class, TemporalWorkflowUtils.buildWorkflowOptions(TemporalJobType.SYNC)); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index f4f3fd9eaf31c..a5e72f6b31e35 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -248,7 +248,8 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, temporalService, streamResetPersistence, connectionManagerUtils, - streamResetRecordsHelper); + streamResetRecordsHelper, + envVariableFeatureFlags); final OAuthConfigSupplier oAuthConfigSupplier = new OAuthConfigSupplier(configRepository, trackingClient); final DefaultSynchronousSchedulerClient syncSchedulerClient = diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java index 63de67be4bb10..2341b8a51eb49 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java @@ -40,7 +40,7 @@ public boolean shouldRefreshSchema(UUID sourceCatalogId) throws IOException { return false; } - return !schemaRefreshRanRecently(sourceCatalogId) || true; + return !schemaRefreshRanRecently(sourceCatalogId); } @Override @@ -48,11 +48,11 @@ public void refreshSchema(UUID sourceCatalogId, UUID connectionId) throws ApiExc SourceDiscoverSchemaRequestBody requestBody = new SourceDiscoverSchemaRequestBody().sourceId(sourceCatalogId).disableCache(true).connectionId(connectionId); - try { + try { airbyteApiClient.getSourceApi().discoverSchemaForSource(requestBody); - } catch (final Exception e) { - log.info("Attempted schema refresh, but failed."); - } + } catch (final Exception e) { + log.info("Attempted schema refresh, but failed."); + } } private boolean schemaRefreshRanRecently(UUID sourceCatalogId) throws IOException { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index 9147d5b99c332..747dbd2b8a632 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -76,7 +76,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId) + final UUID connectionId, + final EnvVariableFeatureFlags envVariableFeatureFlags) throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { ApmTraceUtils @@ -89,7 +90,6 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final String taskQueue = Workflow.getInfo().getTaskQueue(); if (version > Workflow.DEFAULT_VERSION) { - final EnvVariableFeatureFlags envVariableFeatureFlags = new EnvVariableFeatureFlags(); if (envVariableFeatureFlags.autoDetectSchema()) { final UUID sourceId = configFetchActivity.getStandardSync(connectionId).getSourceId(); if (refreshSchemaActivity.shouldRefreshSchema(sourceId)) { diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index 3c122a9c09119..d6feae6449429 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.api.client.invoker.generated.ApiException; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; @@ -118,6 +119,7 @@ class SyncWorkflowTest { private ActivityOptions longActivityOptions; private ActivityOptions shortActivityOptions; private TemporalProxyHelper temporalProxyHelper; + private EnvVariableFeatureFlags envVariableFeatureFlags; @BeforeEach void setUp() throws IOException { @@ -154,6 +156,7 @@ void setUp() throws IOException { webhookOperationActivity = mock(WebhookOperationActivityImpl.class); refreshSchemaActivity = mock(RefreshSchemaActivityImpl.class); configFetchActivity = mock(ConfigFetchActivityImpl.class); + envVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); when(normalizationActivity.generateNormalizationInput(any(), any())).thenReturn(normalizationInput); when(normalizationSummaryCheckActivity.shouldRunNormalization(any(), any(), any())).thenReturn(true); @@ -203,7 +206,8 @@ private StandardSyncOutput execute() throws JsonValidationException, ConfigNotFo final SyncWorkflow workflow = client.newWorkflowStub(SyncWorkflow.class, WorkflowOptions.newBuilder().setTaskQueue(SYNC_QUEUE).build()); - return workflow.run(JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, DESTINATION_LAUNCHER_CONFIG, syncInput, sync.getConnectionId()); + return workflow.run(JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, DESTINATION_LAUNCHER_CONFIG, syncInput, sync.getConnectionId(), + envVariableFeatureFlags); } @Test From ca636b2044cdc850f86ce655cd2e9c398e499c31 Mon Sep 17 00:00:00 2001 From: alovew Date: Wed, 30 Nov 2022 12:07:53 -0800 Subject: [PATCH 07/24] inject envvariable feature flags --- .../temporal/ConnectionManagerUtils.java | 22 ++++++++++++------- .../commons/temporal/TemporalClient.java | 15 ++++++++----- .../scheduling/ConnectionManagerWorkflow.java | 3 ++- .../commons/temporal/TemporalClientTest.java | 2 +- .../server/handlers/SchedulerHandler.java | 1 + .../ConnectionManagerWorkflowImpl.java | 15 ++++++++----- .../ConnectionManagerWorkflowTest.java | 4 +++- .../DbtFailureSyncWorkflow.java | 4 +++- .../testsyncworkflow/EmptySyncWorkflow.java | 4 +++- .../NormalizationFailureSyncWorkflow.java | 4 +++- ...NormalizationTraceFailureSyncWorkflow.java | 4 +++- .../PersistFailureSyncWorkflow.java | 4 +++- .../ReplicateFailureSyncWorkflow.java | 4 +++- .../SleepingSyncWorkflow.java | 4 +++- ...urceAndDestinationFailureSyncWorkflow.java | 4 +++- .../SyncWorkflowFailingOutputWorkflow.java | 4 +++- ...cWorkflowWithActivityFailureException.java | 4 +++- 17 files changed, 69 insertions(+), 33 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java index 3a5b3628e8e38..31f6ccd740cc4 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java @@ -4,6 +4,7 @@ package io.airbyte.commons.temporal; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.exception.DeletedWorkflowException; import io.airbyte.commons.temporal.exception.UnreachableWorkflowException; import io.airbyte.commons.temporal.scheduling.ConnectionManagerWorkflow; @@ -62,9 +63,10 @@ public void deleteWorkflowIfItExist(final WorkflowClient client, */ public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, final UUID connectionId, - final Function signalMethod) + final Function signalMethod, + final EnvVariableFeatureFlags envVariableFeatureFlags) throws DeletedWorkflowException { - return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.empty()); + return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.empty(), envVariableFeatureFlags); } /** @@ -85,9 +87,10 @@ public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final Workfl public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, final UUID connectionId, final Function> signalMethod, - final T signalArgument) + final T signalArgument, + final EnvVariableFeatureFlags envVariableFeatureFlags) throws DeletedWorkflowException { - return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.of(signalArgument)); + return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.of(signalArgument), envVariableFeatureFlags); } // This method unifies the logic of the above two, by using the optional signalArgument parameter to @@ -98,7 +101,8 @@ public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final Wo private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, final UUID connectionId, final Function signalMethod, - final Optional signalArgument) + final Optional signalArgument, + final EnvVariableFeatureFlags envVariableFeatureFlags) throws DeletedWorkflowException { try { final ConnectionManagerWorkflow connectionManagerWorkflow = getConnectionManagerWorkflow(client, connectionId); @@ -127,7 +131,7 @@ private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final W final ConnectionUpdaterInput startWorkflowInput = TemporalWorkflowUtils.buildStartWorkflowInput(connectionId); final BatchRequest batchRequest = client.newSignalWithStartRequest(); - batchRequest.add(connectionManagerWorkflow::run, startWorkflowInput); + batchRequest.add(connectionManagerWorkflow::run, startWorkflowInput, envVariableFeatureFlags); // retrieve the signal from the lambda final TemporalFunctionalInterfaceMarker signal = signalMethod.apply(connectionManagerWorkflow); @@ -161,10 +165,12 @@ public void safeTerminateWorkflow(final WorkflowClient client, final UUID connec safeTerminateWorkflow(client, getConnectionManagerName(connectionId), reason); } - public ConnectionManagerWorkflow startConnectionManagerNoSignal(final WorkflowClient client, final UUID connectionId) { + public ConnectionManagerWorkflow startConnectionManagerNoSignal(final WorkflowClient client, + final UUID connectionId, + final EnvVariableFeatureFlags envVariableFeatureFlags) { final ConnectionManagerWorkflow connectionManagerWorkflow = newConnectionManagerWorkflowStub(client, connectionId); final ConnectionUpdaterInput input = TemporalWorkflowUtils.buildStartWorkflowInput(connectionId); - WorkflowClient.start(connectionManagerWorkflow::run, input); + WorkflowClient.start(connectionManagerWorkflow::run, input, envVariableFeatureFlags); return connectionManagerWorkflow; } diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index ffc0a00489e28..bfeacc3f979f2 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -110,7 +110,7 @@ public void restartClosedWorkflowByStatus(final WorkflowExecutionStatus executio nonRunningWorkflow.forEach(connectionId -> { connectionManagerUtils.safeTerminateWorkflow(client, connectionId, "Terminating workflow in " + "unreachable state before starting a new workflow for this connection"); - connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId); + connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId, envVariableFeatureFlags); }); } @@ -214,7 +214,8 @@ public ManualOperationResult startNewManualSync(final UUID connectionId) { } try { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::submitManualSync); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::submitManualSync, + envVariableFeatureFlags); } catch (final DeletedWorkflowException e) { log.error("Can't sync a deleted connection.", e); return new ManualOperationResult( @@ -247,7 +248,7 @@ public ManualOperationResult startNewCancellation(final UUID connectionId) { final long jobId = connectionManagerUtils.getCurrentJobId(client, connectionId); try { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::cancelJob); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::cancelJob, envVariableFeatureFlags); } catch (final DeletedWorkflowException e) { log.error("Can't cancel a deleted workflow", e); return new ManualOperationResult( @@ -293,9 +294,11 @@ public ManualOperationResult resetConnection(final UUID connectionId, try { if (syncImmediatelyAfter) { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnectionAndSkipNextScheduling); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnectionAndSkipNextScheduling, + envVariableFeatureFlags); } else { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnection); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnection, + envVariableFeatureFlags); } } catch (final DeletedWorkflowException e) { log.error("Can't reset a deleted workflow", e); @@ -475,7 +478,7 @@ private T getWorkflowStub(final Class workflowClass, final TemporalJobTyp public ConnectionManagerWorkflow submitConnectionUpdaterAsync(final UUID connectionId) { log.info("Starting the scheduler temporal wf"); final ConnectionManagerWorkflow connectionManagerWorkflow = - connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId); + connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId, envVariableFeatureFlags); try { CompletableFuture.supplyAsync(() -> { try { diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionManagerWorkflow.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionManagerWorkflow.java index 7dc3a33acda58..b816d2b20f17b 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionManagerWorkflow.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionManagerWorkflow.java @@ -4,6 +4,7 @@ package io.airbyte.commons.temporal.scheduling; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.state.WorkflowState; import io.temporal.workflow.QueryMethod; import io.temporal.workflow.SignalMethod; @@ -25,7 +26,7 @@ public interface ConnectionManagerWorkflow { * for scheduling syncs. This workflow will run and then continue running until deleted. */ @WorkflowMethod - void run(ConnectionUpdaterInput connectionUpdaterInput); + void run(ConnectionUpdaterInput connectionUpdaterInput, EnvVariableFeatureFlags envVariableFeatureFlags); /** * Send a signal that will bypass the waiting time and run a sync. Nothing will happen if a sync is diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index 7d42d5aaabf21..d63e5226f9639 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -161,7 +161,7 @@ void testRestartFailed() { temporalClient.restartClosedWorkflowByStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_FAILED); verify(mConnectionManagerUtils).safeTerminateWorkflow(eq(workflowClient), eq(connectionId), anyString()); - verify(mConnectionManagerUtils).startConnectionManagerNoSignal(eq(workflowClient), eq(connectionId)); + verify(mConnectionManagerUtils).startConnectionManagerNoSignal(eq(workflowClient), eq(connectionId), any(EnvVariableFeatureFlags.class)); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java index 7621cb4baf936..988c7e839ffb6 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java @@ -237,6 +237,7 @@ public CheckConnectionRead checkDestinationConnectionFromDestinationIdForUpdate( public SourceDiscoverSchemaRead discoverSchemaForSourceFromSourceId(final SourceDiscoverSchemaRequestBody discoverSchemaRequestBody) throws ConfigNotFoundException, IOException, JsonValidationException { + log.info("discover schema request: " + discoverSchemaRequestBody); final SourceConnection source = configRepository.getSourceConnection(discoverSchemaRequestBody.getSourceId()); final StandardSourceDefinition sourceDef = configRepository.getStandardSourceDefinition(source.getSourceDefinitionId()); final String imageName = DockerUtils.getTaggedImageName(sourceDef.getDockerRepository(), sourceDef.getDockerImageTag()); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java index db5eb9f7dd914..8ec5160675af7 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.JsonNode; import datadog.trace.api.Trace; import io.airbyte.api.client.invoker.generated.ApiException; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.TemporalJobType; import io.airbyte.commons.temporal.TemporalWorkflowUtils; import io.airbyte.commons.temporal.exception.RetryableException; @@ -159,7 +160,8 @@ public class ConnectionManagerWorkflowImpl implements ConnectionManagerWorkflow @Trace(operationName = WORKFLOW_TRACE_OPERATION_NAME) @Override - public void run(final ConnectionUpdaterInput connectionUpdaterInput) throws RetryableException { + public void run(final ConnectionUpdaterInput connectionUpdaterInput, final EnvVariableFeatureFlags envVariableFeatureFlags) + throws RetryableException { try { ApmTraceUtils.addTagsToTrace(Map.of(CONNECTION_ID_KEY, connectionUpdaterInput.getConnectionId())); @@ -170,7 +172,7 @@ public void run(final ConnectionUpdaterInput connectionUpdaterInput) throws Retr recordMetric(new RecordMetricInput(connectionUpdaterInput, Optional.empty(), OssMetricsRegistry.TEMPORAL_WORKFLOW_ATTEMPT, null)); try { - cancellableSyncWorkflow = generateSyncWorkflowRunnable(connectionUpdaterInput); + cancellableSyncWorkflow = generateSyncWorkflowRunnable(connectionUpdaterInput, envVariableFeatureFlags); cancellableSyncWorkflow.run(); } catch (final CanceledFailure cf) { // When a scope is cancelled temporal will throw a CanceledFailure as you can see here: @@ -208,7 +210,8 @@ public void run(final ConnectionUpdaterInput connectionUpdaterInput) throws Retr } @SuppressWarnings("PMD.EmptyIfStmt") - private CancellationScope generateSyncWorkflowRunnable(final ConnectionUpdaterInput connectionUpdaterInput) { + private CancellationScope generateSyncWorkflowRunnable(final ConnectionUpdaterInput connectionUpdaterInput, + EnvVariableFeatureFlags envVariableFeatureFlags) { return Workflow.newCancellationScope(() -> { connectionId = connectionUpdaterInput.getConnectionId(); @@ -259,7 +262,7 @@ private CancellationScope generateSyncWorkflowRunnable(final ConnectionUpdaterIn reportFailure(connectionUpdaterInput, checkFailureOutput, FailureCause.CONNECTION); } else { try { - standardSyncOutput = runChildWorkflow(jobInputs); + standardSyncOutput = runChildWorkflow(jobInputs, envVariableFeatureFlags); } catch (JsonValidationException | ConfigNotFoundException | IOException | ApiException e) { throw new RuntimeException(e); } @@ -857,7 +860,7 @@ private void reportJobStarting(final UUID connectionId) { * since the latter is a long running workflow, in the future, using a different Node pool would * make sense. */ - private StandardSyncOutput runChildWorkflow(final GeneratedJobInput jobInputs) + private StandardSyncOutput runChildWorkflow(final GeneratedJobInput jobInputs, EnvVariableFeatureFlags envVariableFeatureFlags) throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { final String taskQueue = getSyncTaskQueue(); @@ -874,7 +877,7 @@ private StandardSyncOutput runChildWorkflow(final GeneratedJobInput jobInputs) jobInputs.getSourceLauncherConfig(), jobInputs.getDestinationLauncherConfig(), jobInputs.getSyncInput(), - connectionId); + connectionId, envVariableFeatureFlags); } /** diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java index 6171a40674365..4aa3f7b6c9bb1 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java @@ -8,6 +8,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.TemporalJobType; import io.airbyte.commons.temporal.scheduling.ConnectionManagerWorkflow; import io.airbyte.commons.temporal.scheduling.ConnectionUpdaterInput; @@ -127,6 +128,7 @@ class ConnectionManagerWorkflowTest { mock(WorkflowConfigActivity.class, Mockito.withSettings().withoutAnnotations()); private static final RouteToSyncTaskQueueActivity mRouteToSyncTaskQueueActivity = mock(RouteToSyncTaskQueueActivity.class, Mockito.withSettings().withoutAnnotations()); + private static final EnvVariableFeatureFlags mEnvVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); private static final String EVENT = "event = "; private TestWorkflowEnvironment testEnv; @@ -1558,7 +1560,7 @@ public boolean matches(final JobCancelledInputWithAttemptNumber arg) { private static void startWorkflowAndWaitUntilReady(final ConnectionManagerWorkflow workflow, final ConnectionUpdaterInput input) throws InterruptedException { - WorkflowClient.start(workflow::run, input); + WorkflowClient.start(workflow::run, input, mEnvVariableFeatureFlags); boolean isReady = false; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java index 677740d2f796f..b8e49cabc662c 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java @@ -4,6 +4,7 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; @@ -25,7 +26,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId) { + final UUID connectionId, + final EnvVariableFeatureFlags envVariableFeatureFlags) { throw new ActivityFailure(1L, 1L, ACTIVITY_TYPE_DBT_RUN, "someId", RetryState.RETRY_STATE_UNSPECIFIED, "someIdentity", CAUSE); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/EmptySyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/EmptySyncWorkflow.java index 3f93c6ba87d24..002ffac48428f 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/EmptySyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/EmptySyncWorkflow.java @@ -4,6 +4,7 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; @@ -18,7 +19,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId) { + final UUID connectionId, + final EnvVariableFeatureFlags envVariableFeatureFlags) { return new StandardSyncOutput(); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java index 99e52c941c206..66ad03eaaac77 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java @@ -4,6 +4,7 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; @@ -25,7 +26,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId) { + final UUID connectionId, + final EnvVariableFeatureFlags envVariableFeatureFlags) { throw new ActivityFailure(1L, 1L, ACTIVITY_TYPE_NORMALIZE, "someId", RetryState.RETRY_STATE_UNSPECIFIED, "someIdentity", CAUSE); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationTraceFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationTraceFailureSyncWorkflow.java index 1845f0ac0e459..8b2c1aa4efa91 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationTraceFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationTraceFailureSyncWorkflow.java @@ -5,6 +5,7 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.FailureReason; import io.airbyte.config.FailureReason.FailureOrigin; @@ -30,7 +31,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId) { + final UUID connectionId, + final EnvVariableFeatureFlags envVariableFeatureFlags) { return new StandardSyncOutput() .withNormalizationSummary(new NormalizationSummary() diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java index e8b704a6512c0..b2d4c2dcc955a 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java @@ -4,6 +4,7 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; @@ -25,7 +26,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId) { + final UUID connectionId, + final EnvVariableFeatureFlags envVariableFeatureFlags) { throw new ActivityFailure(1L, 1L, ACTIVITY_TYPE_PERSIST, "someId", RetryState.RETRY_STATE_UNSPECIFIED, "someIdentity", CAUSE); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java index d4c64152e2cc4..9f0179b31477e 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java @@ -4,6 +4,7 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; @@ -25,7 +26,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId) { + final UUID connectionId, + final EnvVariableFeatureFlags envVariableFeatureFlags) { throw new ActivityFailure(1L, 1L, ACTIVITY_TYPE_REPLICATE, "someId", RetryState.RETRY_STATE_UNSPECIFIED, "someIdentity", CAUSE); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SleepingSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SleepingSyncWorkflow.java index c2f24529de127..f035c70a205ac 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SleepingSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SleepingSyncWorkflow.java @@ -4,6 +4,7 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; @@ -22,7 +23,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId) { + final UUID connectionId, + final EnvVariableFeatureFlags envVariableFeatureFlags) { Workflow.sleep(RUN_TIME); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java index 19a8bdabbe1c6..ad20d9adee92a 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java @@ -5,6 +5,7 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.FailureReason; import io.airbyte.config.FailureReason.FailureOrigin; @@ -31,7 +32,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId) { + final UUID connectionId, + final EnvVariableFeatureFlags envVariableFeatureFlags) { return new StandardSyncOutput() .withFailures(FAILURE_REASONS.stream().toList()) diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowFailingOutputWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowFailingOutputWorkflow.java index 07925086dcc7e..0d028cd66e848 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowFailingOutputWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowFailingOutputWorkflow.java @@ -4,6 +4,7 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; @@ -22,7 +23,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId) { + final UUID connectionId, + final EnvVariableFeatureFlags envVariableFeatureFlags) { return null; } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowWithActivityFailureException.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowWithActivityFailureException.java index 2a897d73dabd1..e77dfe469ff99 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowWithActivityFailureException.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowWithActivityFailureException.java @@ -4,6 +4,7 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; @@ -24,7 +25,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId) { + final UUID connectionId, + final EnvVariableFeatureFlags envVariableFeatureFlags) { Workflow.sleep(SleepingSyncWorkflow.RUN_TIME); throw new ActivityFailure(1L, 1L, "Replication", "id", RetryState.RETRY_STATE_RETRY_POLICY_NOT_SET, "identity", From c6fd2df3d4f3a9991caa259a96194faf4c0a85eb Mon Sep 17 00:00:00 2001 From: alovew Date: Wed, 30 Nov 2022 12:10:05 -0800 Subject: [PATCH 08/24] remove logging --- .../main/java/io/airbyte/server/handlers/SchedulerHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java index 988c7e839ffb6..7621cb4baf936 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java @@ -237,7 +237,6 @@ public CheckConnectionRead checkDestinationConnectionFromDestinationIdForUpdate( public SourceDiscoverSchemaRead discoverSchemaForSourceFromSourceId(final SourceDiscoverSchemaRequestBody discoverSchemaRequestBody) throws ConfigNotFoundException, IOException, JsonValidationException { - log.info("discover schema request: " + discoverSchemaRequestBody); final SourceConnection source = configRepository.getSourceConnection(discoverSchemaRequestBody.getSourceId()); final StandardSourceDefinition sourceDef = configRepository.getStandardSourceDefinition(source.getSourceDefinitionId()); final String imageName = DockerUtils.getTaggedImageName(sourceDef.getDockerRepository(), sourceDef.getDockerImageTag()); From 2370fdfda00d346a1d5cf1ad323cebe42d6f7ae4 Mon Sep 17 00:00:00 2001 From: alovew Date: Wed, 30 Nov 2022 13:53:35 -0800 Subject: [PATCH 09/24] inject into activities instead of workflow --- .../temporal/ConnectionManagerUtils.java | 21 +++++-------- .../commons/temporal/TemporalClient.java | 23 +++++--------- .../temporal/scheduling/SyncWorkflow.java | 3 +- .../commons/temporal/TemporalClientTest.java | 11 +++---- .../sync/RefreshSchemaActivityImpl.java | 13 ++++++-- .../temporal/sync/SyncWorkflowImpl.java | 30 ++++++++----------- .../activities/RefreshSchemaActivityTest.java | 5 +++- .../DbtFailureSyncWorkflow.java | 3 +- .../testsyncworkflow/EmptySyncWorkflow.java | 3 +- .../NormalizationFailureSyncWorkflow.java | 3 +- ...NormalizationTraceFailureSyncWorkflow.java | 3 +- .../PersistFailureSyncWorkflow.java | 3 +- .../ReplicateFailureSyncWorkflow.java | 3 +- .../SleepingSyncWorkflow.java | 3 +- ...urceAndDestinationFailureSyncWorkflow.java | 3 +- .../SyncWorkflowFailingOutputWorkflow.java | 3 +- 16 files changed, 57 insertions(+), 76 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java index 31f6ccd740cc4..a86f9aa80b984 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java @@ -4,7 +4,6 @@ package io.airbyte.commons.temporal; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.exception.DeletedWorkflowException; import io.airbyte.commons.temporal.exception.UnreachableWorkflowException; import io.airbyte.commons.temporal.scheduling.ConnectionManagerWorkflow; @@ -63,10 +62,9 @@ public void deleteWorkflowIfItExist(final WorkflowClient client, */ public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, final UUID connectionId, - final Function signalMethod, - final EnvVariableFeatureFlags envVariableFeatureFlags) + final Function signalMethod) throws DeletedWorkflowException { - return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.empty(), envVariableFeatureFlags); + return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.empty()); } /** @@ -87,10 +85,9 @@ public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final Workfl public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, final UUID connectionId, final Function> signalMethod, - final T signalArgument, - final EnvVariableFeatureFlags envVariableFeatureFlags) + final T signalArgument) throws DeletedWorkflowException { - return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.of(signalArgument), envVariableFeatureFlags); + return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.of(signalArgument)); } // This method unifies the logic of the above two, by using the optional signalArgument parameter to @@ -101,8 +98,7 @@ public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final Wo private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, final UUID connectionId, final Function signalMethod, - final Optional signalArgument, - final EnvVariableFeatureFlags envVariableFeatureFlags) + final Optional signalArgument) throws DeletedWorkflowException { try { final ConnectionManagerWorkflow connectionManagerWorkflow = getConnectionManagerWorkflow(client, connectionId); @@ -131,7 +127,7 @@ private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final W final ConnectionUpdaterInput startWorkflowInput = TemporalWorkflowUtils.buildStartWorkflowInput(connectionId); final BatchRequest batchRequest = client.newSignalWithStartRequest(); - batchRequest.add(connectionManagerWorkflow::run, startWorkflowInput, envVariableFeatureFlags); + batchRequest.add(connectionManagerWorkflow::run, startWorkflowInput); // retrieve the signal from the lambda final TemporalFunctionalInterfaceMarker signal = signalMethod.apply(connectionManagerWorkflow); @@ -166,11 +162,10 @@ public void safeTerminateWorkflow(final WorkflowClient client, final UUID connec } public ConnectionManagerWorkflow startConnectionManagerNoSignal(final WorkflowClient client, - final UUID connectionId, - final EnvVariableFeatureFlags envVariableFeatureFlags) { + final UUID connectionId) { final ConnectionManagerWorkflow connectionManagerWorkflow = newConnectionManagerWorkflowStub(client, connectionId); final ConnectionUpdaterInput input = TemporalWorkflowUtils.buildStartWorkflowInput(connectionId); - WorkflowClient.start(connectionManagerWorkflow::run, input, envVariableFeatureFlags); + WorkflowClient.start(connectionManagerWorkflow::run, input); return connectionManagerWorkflow; } diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index bfeacc3f979f2..b233a41dea728 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -9,7 +9,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; import io.airbyte.api.client.invoker.generated.ApiException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.config.WorkerMode; import io.airbyte.commons.temporal.exception.DeletedWorkflowException; import io.airbyte.commons.temporal.exception.UnreachableWorkflowException; @@ -82,22 +81,19 @@ public class TemporalClient { private final StreamResetPersistence streamResetPersistence; private final ConnectionManagerUtils connectionManagerUtils; private final StreamResetRecordsHelper streamResetRecordsHelper; - private final EnvVariableFeatureFlags envVariableFeatureFlags; public TemporalClient(@Named("workspaceRootTemporal") final Path workspaceRoot, final WorkflowClient client, final WorkflowServiceStubs service, final StreamResetPersistence streamResetPersistence, final ConnectionManagerUtils connectionManagerUtils, - final StreamResetRecordsHelper streamResetRecordsHelper, - final EnvVariableFeatureFlags envVariableFeatureFlags) { + final StreamResetRecordsHelper streamResetRecordsHelper) { this.workspaceRoot = workspaceRoot; this.client = client; this.service = service; this.streamResetPersistence = streamResetPersistence; this.connectionManagerUtils = connectionManagerUtils; this.streamResetRecordsHelper = streamResetRecordsHelper; - this.envVariableFeatureFlags = envVariableFeatureFlags; } private final Set workflowNames = new HashSet<>(); @@ -110,7 +106,7 @@ public void restartClosedWorkflowByStatus(final WorkflowExecutionStatus executio nonRunningWorkflow.forEach(connectionId -> { connectionManagerUtils.safeTerminateWorkflow(client, connectionId, "Terminating workflow in " + "unreachable state before starting a new workflow for this connection"); - connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId, envVariableFeatureFlags); + connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId); }); } @@ -214,8 +210,7 @@ public ManualOperationResult startNewManualSync(final UUID connectionId) { } try { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::submitManualSync, - envVariableFeatureFlags); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::submitManualSync); } catch (final DeletedWorkflowException e) { log.error("Can't sync a deleted connection.", e); return new ManualOperationResult( @@ -248,7 +243,7 @@ public ManualOperationResult startNewCancellation(final UUID connectionId) { final long jobId = connectionManagerUtils.getCurrentJobId(client, connectionId); try { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::cancelJob, envVariableFeatureFlags); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::cancelJob); } catch (final DeletedWorkflowException e) { log.error("Can't cancel a deleted workflow", e); return new ManualOperationResult( @@ -294,11 +289,9 @@ public ManualOperationResult resetConnection(final UUID connectionId, try { if (syncImmediatelyAfter) { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnectionAndSkipNextScheduling, - envVariableFeatureFlags); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnectionAndSkipNextScheduling); } else { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnection, - envVariableFeatureFlags); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnection); } } catch (final DeletedWorkflowException e) { log.error("Can't reset a deleted workflow", e); @@ -415,7 +408,7 @@ public TemporalResponse submitSync(final long jobId, final i sourceLauncherConfig, destinationLauncherConfig, input, - connectionId, envVariableFeatureFlags); + connectionId); } catch (JsonValidationException | ConfigNotFoundException | IOException | ApiException e) { throw new RuntimeException(e); } @@ -478,7 +471,7 @@ private T getWorkflowStub(final Class workflowClass, final TemporalJobTyp public ConnectionManagerWorkflow submitConnectionUpdaterAsync(final UUID connectionId) { log.info("Starting the scheduler temporal wf"); final ConnectionManagerWorkflow connectionManagerWorkflow = - connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId, envVariableFeatureFlags); + connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId); try { CompletableFuture.supplyAsync(() -> { try { diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java index 6cf1897ea77a1..31d549290e6e9 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java @@ -25,8 +25,7 @@ StandardSyncOutput run(JobRunConfig jobRunConfig, IntegrationLauncherConfig sourceLauncherConfig, IntegrationLauncherConfig destinationLauncherConfig, StandardSyncInput syncInput, - UUID connectionId, - EnvVariableFeatureFlags envVariableFeatureFlags) + UUID connectionId) throws JsonValidationException, ConfigNotFoundException, IOException, ApiException; } diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index d63e5226f9639..c3e0cf987ac47 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -23,7 +23,6 @@ import com.google.common.collect.Sets; import io.airbyte.api.client.invoker.generated.ApiException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalClient.ManualOperationResult; import io.airbyte.commons.temporal.scheduling.CheckConnectionWorkflow; @@ -109,7 +108,6 @@ public class TemporalClientTest { private ConnectionManagerUtils connectionManagerUtils; private StreamResetRecordsHelper streamResetRecordsHelper; private Path workspaceRoot; - private EnvVariableFeatureFlags envVariableFeatureFlags; @BeforeEach void setup() throws IOException { @@ -125,10 +123,9 @@ void setup() throws IOException { mockWorkflowStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_RUNNING); connectionManagerUtils = spy(new ConnectionManagerUtils()); streamResetRecordsHelper = mock(StreamResetRecordsHelper.class); - envVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); temporalClient = spy(new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, connectionManagerUtils, - streamResetRecordsHelper, envVariableFeatureFlags)); + streamResetRecordsHelper)); } @Nested @@ -142,7 +139,7 @@ void init() { temporalClient = spy( new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, mConnectionManagerUtils, - streamResetRecordsHelper, envVariableFeatureFlags)); + streamResetRecordsHelper)); } @Test @@ -161,7 +158,7 @@ void testRestartFailed() { temporalClient.restartClosedWorkflowByStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_FAILED); verify(mConnectionManagerUtils).safeTerminateWorkflow(eq(workflowClient), eq(connectionId), anyString()); - verify(mConnectionManagerUtils).startConnectionManagerNoSignal(eq(workflowClient), eq(connectionId), any(EnvVariableFeatureFlags.class)); + verify(mConnectionManagerUtils).startConnectionManagerNoSignal(eq(workflowClient), eq(connectionId)); } } @@ -297,7 +294,7 @@ void testSubmitSync() throws JsonValidationException, ConfigNotFoundException, I .withDockerImage(IMAGE_NAME2); temporalClient.submitSync(JOB_ID, ATTEMPT_ID, syncConfig, CONNECTION_ID); - discoverCatalogWorkflow.run(JOB_RUN_CONFIG, LAUNCHER_CONFIG, destinationLauncherConfig, input, CONNECTION_ID, envVariableFeatureFlags); + discoverCatalogWorkflow.run(JOB_RUN_CONFIG, LAUNCHER_CONFIG, destinationLauncherConfig, input, CONNECTION_ID); verify(workflowClient).newWorkflowStub(SyncWorkflow.class, TemporalWorkflowUtils.buildWorkflowOptions(TemporalJobType.SYNC)); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java index 2341b8a51eb49..b2d94b6f14e5f 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java @@ -10,6 +10,7 @@ import io.airbyte.api.client.AirbyteApiClient; import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRequestBody; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.config.ActorCatalogFetchEvent; import io.airbyte.config.persistence.ConfigRepository; import jakarta.inject.Singleton; @@ -26,17 +27,19 @@ public class RefreshSchemaActivityImpl implements RefreshSchemaActivity { private final Optional configRepository; private final AirbyteApiClient airbyteApiClient; + private final EnvVariableFeatureFlags envVariableFeatureFlags; - public RefreshSchemaActivityImpl(Optional configRepository, AirbyteApiClient airbyteApiClient) { + public RefreshSchemaActivityImpl(Optional configRepository, AirbyteApiClient airbyteApiClient, EnvVariableFeatureFlags envVariableFeatureFlags) { this.configRepository = configRepository; this.airbyteApiClient = airbyteApiClient; + this.envVariableFeatureFlags = envVariableFeatureFlags; } @Override @Trace(operationName = ACTIVITY_TRACE_OPERATION_NAME) public boolean shouldRefreshSchema(UUID sourceCatalogId) throws IOException { // if job persistence is unavailable, default to skipping the schema refresh - if (configRepository.isEmpty()) { + if (configRepository.isEmpty() || !envVariableFeatureFlags.autoDetectSchema()) { return false; } @@ -45,13 +48,17 @@ public boolean shouldRefreshSchema(UUID sourceCatalogId) throws IOException { @Override public void refreshSchema(UUID sourceCatalogId, UUID connectionId) throws ApiException { + if(!envVariableFeatureFlags.autoDetectSchema()){ + return; + } + SourceDiscoverSchemaRequestBody requestBody = new SourceDiscoverSchemaRequestBody().sourceId(sourceCatalogId).disableCache(true).connectionId(connectionId); try { airbyteApiClient.getSourceApi().discoverSchemaForSource(requestBody); } catch (final Exception e) { - log.info("Attempted schema refresh, but failed."); + log.error("Attempted schema refresh, but failed with error: ", e); } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index 747dbd2b8a632..e95088cf33ba5 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -13,7 +13,6 @@ import datadog.trace.api.Trace; import io.airbyte.api.client.invoker.generated.ApiException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.NormalizationInput; import io.airbyte.config.NormalizationSummary; @@ -76,8 +75,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId, - final EnvVariableFeatureFlags envVariableFeatureFlags) + final UUID connectionId) throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { ApmTraceUtils @@ -90,20 +88,18 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final String taskQueue = Workflow.getInfo().getTaskQueue(); if (version > Workflow.DEFAULT_VERSION) { - if (envVariableFeatureFlags.autoDetectSchema()) { - final UUID sourceId = configFetchActivity.getStandardSync(connectionId).getSourceId(); - if (refreshSchemaActivity.shouldRefreshSchema(sourceId)) { - LOGGER.info("Refreshing source schema..."); - refreshSchemaActivity.refreshSchema(sourceId, connectionId); - } - final Status status = configFetchActivity.getStandardSync(connectionId).getStatus(); - if (Status.INACTIVE == status) { - LOGGER.info("Connection is disabled. Cancelling run."); - final StandardSyncOutput output = - new StandardSyncOutput() - .withStandardSyncSummary(new StandardSyncSummary().withStatus(ReplicationStatus.CANCELLED).withTotalStats(new SyncStats())); - return output; - } + final UUID sourceId = configFetchActivity.getStandardSync(connectionId).getSourceId(); + if (refreshSchemaActivity.shouldRefreshSchema(sourceId)) { + LOGGER.info("Refreshing source schema..."); + refreshSchemaActivity.refreshSchema(sourceId, connectionId); + } + final Status status = configFetchActivity.getStandardSync(connectionId).getStatus(); + if (Status.INACTIVE == status) { + LOGGER.info("Connection is disabled. Cancelling run."); + final StandardSyncOutput output = + new StandardSyncOutput() + .withStandardSyncSummary(new StandardSyncSummary().withStatus(ReplicationStatus.CANCELLED).withTotalStats(new SyncStats())); + return output; } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java index 24c9807ce4caf..749be5b358965 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java @@ -13,6 +13,7 @@ import io.airbyte.api.client.generated.SourceApi; import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRequestBody; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.config.ActorCatalogFetchEvent; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.workers.temporal.sync.RefreshSchemaActivityImpl; @@ -34,6 +35,7 @@ class RefreshSchemaActivityTest { static private AirbyteApiClient mApiClient; static private SourceApi mSourceApi; + static private EnvVariableFeatureFlags mEnvVariableFeatureFlags; static private RefreshSchemaActivityImpl refreshSchemaActivity; @@ -44,8 +46,9 @@ void setUp() { mApiClient = mock(AirbyteApiClient.class); mSourceApi = mock(SourceApi.class); mConfigRepository = mock(ConfigRepository.class); + mEnvVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); Mockito.lenient().when(mApiClient.getSourceApi()).thenReturn(mSourceApi); - refreshSchemaActivity = new RefreshSchemaActivityImpl(Optional.of(mConfigRepository), mApiClient); + refreshSchemaActivity = new RefreshSchemaActivityImpl(Optional.of(mConfigRepository), mApiClient, mEnvVariableFeatureFlags); } @Test diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java index b8e49cabc662c..a89d08e45e4d5 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java @@ -26,8 +26,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId, - final EnvVariableFeatureFlags envVariableFeatureFlags) { + final UUID connectionId) { throw new ActivityFailure(1L, 1L, ACTIVITY_TYPE_DBT_RUN, "someId", RetryState.RETRY_STATE_UNSPECIFIED, "someIdentity", CAUSE); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/EmptySyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/EmptySyncWorkflow.java index 002ffac48428f..0c0abfd9d066f 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/EmptySyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/EmptySyncWorkflow.java @@ -19,8 +19,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId, - final EnvVariableFeatureFlags envVariableFeatureFlags) { + final UUID connectionId) { return new StandardSyncOutput(); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java index 66ad03eaaac77..f165effa0e042 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java @@ -26,8 +26,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId, - final EnvVariableFeatureFlags envVariableFeatureFlags) { + final UUID connectionId) { throw new ActivityFailure(1L, 1L, ACTIVITY_TYPE_NORMALIZE, "someId", RetryState.RETRY_STATE_UNSPECIFIED, "someIdentity", CAUSE); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationTraceFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationTraceFailureSyncWorkflow.java index 8b2c1aa4efa91..664debd1ec7da 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationTraceFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationTraceFailureSyncWorkflow.java @@ -31,8 +31,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId, - final EnvVariableFeatureFlags envVariableFeatureFlags) { + final UUID connectionId) { return new StandardSyncOutput() .withNormalizationSummary(new NormalizationSummary() diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java index b2d4c2dcc955a..0de104c7b1c61 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java @@ -26,8 +26,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId, - final EnvVariableFeatureFlags envVariableFeatureFlags) { + final UUID connectionId) { throw new ActivityFailure(1L, 1L, ACTIVITY_TYPE_PERSIST, "someId", RetryState.RETRY_STATE_UNSPECIFIED, "someIdentity", CAUSE); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java index 9f0179b31477e..f2da0d76dfbb5 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java @@ -26,8 +26,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId, - final EnvVariableFeatureFlags envVariableFeatureFlags) { + final UUID connectionId) { throw new ActivityFailure(1L, 1L, ACTIVITY_TYPE_REPLICATE, "someId", RetryState.RETRY_STATE_UNSPECIFIED, "someIdentity", CAUSE); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SleepingSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SleepingSyncWorkflow.java index f035c70a205ac..9d8dbe2ac5d8d 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SleepingSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SleepingSyncWorkflow.java @@ -23,8 +23,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId, - final EnvVariableFeatureFlags envVariableFeatureFlags) { + final UUID connectionId) { Workflow.sleep(RUN_TIME); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java index ad20d9adee92a..6c3f3b93ec435 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java @@ -32,8 +32,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId, - final EnvVariableFeatureFlags envVariableFeatureFlags) { + final UUID connectionId) { return new StandardSyncOutput() .withFailures(FAILURE_REASONS.stream().toList()) diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowFailingOutputWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowFailingOutputWorkflow.java index 0d028cd66e848..8e8f26a0a98d4 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowFailingOutputWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowFailingOutputWorkflow.java @@ -23,8 +23,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId, - final EnvVariableFeatureFlags envVariableFeatureFlags) { + final UUID connectionId) { return null; } From 8302e235bfbbe801284c5d33fde2c2b48f8a94cd Mon Sep 17 00:00:00 2001 From: alovew Date: Wed, 30 Nov 2022 14:00:00 -0800 Subject: [PATCH 10/24] remove from connection manager workflow --- .../commons/temporal/TemporalClient.java | 21 ++++++------------- .../scheduling/ConnectionManagerWorkflow.java | 3 +-- .../temporal/scheduling/SyncWorkflow.java | 1 - .../commons/temporal/TemporalClientTest.java | 5 +---- .../java/io/airbyte/server/ServerApp.java | 3 +-- .../ConnectionManagerWorkflowImpl.java | 14 ++++++------- .../sync/RefreshSchemaActivityImpl.java | 6 ++++-- .../ConnectionManagerWorkflowTest.java | 4 +--- .../DbtFailureSyncWorkflow.java | 1 - .../testsyncworkflow/EmptySyncWorkflow.java | 1 - .../NormalizationFailureSyncWorkflow.java | 1 - ...NormalizationTraceFailureSyncWorkflow.java | 1 - .../PersistFailureSyncWorkflow.java | 1 - .../ReplicateFailureSyncWorkflow.java | 1 - .../SleepingSyncWorkflow.java | 1 - ...urceAndDestinationFailureSyncWorkflow.java | 1 - .../SyncWorkflowFailingOutputWorkflow.java | 1 - 17 files changed, 20 insertions(+), 46 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index b233a41dea728..9abceab23c756 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -8,7 +8,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.temporal.config.WorkerMode; import io.airbyte.commons.temporal.exception.DeletedWorkflowException; import io.airbyte.commons.temporal.exception.UnreachableWorkflowException; @@ -27,12 +26,10 @@ import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; -import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.StreamDescriptor; -import io.airbyte.validation.json.JsonValidationException; import io.micronaut.context.annotation.Requires; import io.temporal.api.common.v1.WorkflowType; import io.temporal.api.enums.v1.WorkflowExecutionStatus; @@ -401,18 +398,12 @@ public TemporalResponse submitSync(final long jobId, final i .withDestinationResourceRequirements(config.getDestinationResourceRequirements()); return execute(jobRunConfig, - () -> { - try { - return getWorkflowStub(SyncWorkflow.class, TemporalJobType.SYNC).run( - jobRunConfig, - sourceLauncherConfig, - destinationLauncherConfig, - input, - connectionId); - } catch (JsonValidationException | ConfigNotFoundException | IOException | ApiException e) { - throw new RuntimeException(e); - } - }); + () -> getWorkflowStub(SyncWorkflow.class, TemporalJobType.SYNC).run( + jobRunConfig, + sourceLauncherConfig, + destinationLauncherConfig, + input, + connectionId)); } public void migrateSyncIfNeeded(final Set connectionIds) { diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionManagerWorkflow.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionManagerWorkflow.java index b816d2b20f17b..7dc3a33acda58 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionManagerWorkflow.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/ConnectionManagerWorkflow.java @@ -4,7 +4,6 @@ package io.airbyte.commons.temporal.scheduling; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.state.WorkflowState; import io.temporal.workflow.QueryMethod; import io.temporal.workflow.SignalMethod; @@ -26,7 +25,7 @@ public interface ConnectionManagerWorkflow { * for scheduling syncs. This workflow will run and then continue running until deleted. */ @WorkflowMethod - void run(ConnectionUpdaterInput connectionUpdaterInput, EnvVariableFeatureFlags envVariableFeatureFlags); + void run(ConnectionUpdaterInput connectionUpdaterInput); /** * Send a signal that will bypass the waiting time and run a sync. Nothing will happen if a sync is diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java index 31d549290e6e9..3e92e979aaa95 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java @@ -5,7 +5,6 @@ package io.airbyte.commons.temporal.scheduling; import io.airbyte.api.client.invoker.generated.ApiException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; import io.airbyte.config.persistence.ConfigNotFoundException; diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index c3e0cf987ac47..36b3eda388063 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -22,7 +22,6 @@ import static org.mockito.Mockito.when; import com.google.common.collect.Sets; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalClient.ManualOperationResult; import io.airbyte.commons.temporal.scheduling.CheckConnectionWorkflow; @@ -42,13 +41,11 @@ import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.helpers.LogClientSingleton; -import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.StreamDescriptor; -import io.airbyte.validation.json.JsonValidationException; import io.temporal.api.enums.v1.WorkflowExecutionStatus; import io.temporal.api.workflow.v1.WorkflowExecutionInfo; import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionResponse; @@ -267,7 +264,7 @@ void testSubmitDiscoverSchema() { } @Test - void testSubmitSync() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { + void testSubmitSync() { final SyncWorkflow discoverCatalogWorkflow = mock(SyncWorkflow.class); when(workflowClient.newWorkflowStub(SyncWorkflow.class, TemporalWorkflowUtils.buildWorkflowOptions(TemporalJobType.SYNC))) .thenReturn(discoverCatalogWorkflow); diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index a5e72f6b31e35..f4f3fd9eaf31c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -248,8 +248,7 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, temporalService, streamResetPersistence, connectionManagerUtils, - streamResetRecordsHelper, - envVariableFeatureFlags); + streamResetRecordsHelper); final OAuthConfigSupplier oAuthConfigSupplier = new OAuthConfigSupplier(configRepository, trackingClient); final DefaultSynchronousSchedulerClient syncSchedulerClient = diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java index 8ec5160675af7..4bd1f55836b91 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java @@ -12,7 +12,6 @@ import com.fasterxml.jackson.databind.JsonNode; import datadog.trace.api.Trace; import io.airbyte.api.client.invoker.generated.ApiException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.TemporalJobType; import io.airbyte.commons.temporal.TemporalWorkflowUtils; import io.airbyte.commons.temporal.exception.RetryableException; @@ -160,7 +159,7 @@ public class ConnectionManagerWorkflowImpl implements ConnectionManagerWorkflow @Trace(operationName = WORKFLOW_TRACE_OPERATION_NAME) @Override - public void run(final ConnectionUpdaterInput connectionUpdaterInput, final EnvVariableFeatureFlags envVariableFeatureFlags) + public void run(final ConnectionUpdaterInput connectionUpdaterInput) throws RetryableException { try { ApmTraceUtils.addTagsToTrace(Map.of(CONNECTION_ID_KEY, connectionUpdaterInput.getConnectionId())); @@ -172,7 +171,7 @@ public void run(final ConnectionUpdaterInput connectionUpdaterInput, final EnvVa recordMetric(new RecordMetricInput(connectionUpdaterInput, Optional.empty(), OssMetricsRegistry.TEMPORAL_WORKFLOW_ATTEMPT, null)); try { - cancellableSyncWorkflow = generateSyncWorkflowRunnable(connectionUpdaterInput, envVariableFeatureFlags); + cancellableSyncWorkflow = generateSyncWorkflowRunnable(connectionUpdaterInput); cancellableSyncWorkflow.run(); } catch (final CanceledFailure cf) { // When a scope is cancelled temporal will throw a CanceledFailure as you can see here: @@ -210,8 +209,7 @@ public void run(final ConnectionUpdaterInput connectionUpdaterInput, final EnvVa } @SuppressWarnings("PMD.EmptyIfStmt") - private CancellationScope generateSyncWorkflowRunnable(final ConnectionUpdaterInput connectionUpdaterInput, - EnvVariableFeatureFlags envVariableFeatureFlags) { + private CancellationScope generateSyncWorkflowRunnable(final ConnectionUpdaterInput connectionUpdaterInput) { return Workflow.newCancellationScope(() -> { connectionId = connectionUpdaterInput.getConnectionId(); @@ -262,7 +260,7 @@ private CancellationScope generateSyncWorkflowRunnable(final ConnectionUpdaterIn reportFailure(connectionUpdaterInput, checkFailureOutput, FailureCause.CONNECTION); } else { try { - standardSyncOutput = runChildWorkflow(jobInputs, envVariableFeatureFlags); + standardSyncOutput = runChildWorkflow(jobInputs); } catch (JsonValidationException | ConfigNotFoundException | IOException | ApiException e) { throw new RuntimeException(e); } @@ -860,7 +858,7 @@ private void reportJobStarting(final UUID connectionId) { * since the latter is a long running workflow, in the future, using a different Node pool would * make sense. */ - private StandardSyncOutput runChildWorkflow(final GeneratedJobInput jobInputs, EnvVariableFeatureFlags envVariableFeatureFlags) + private StandardSyncOutput runChildWorkflow(final GeneratedJobInput jobInputs) throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { final String taskQueue = getSyncTaskQueue(); @@ -877,7 +875,7 @@ private StandardSyncOutput runChildWorkflow(final GeneratedJobInput jobInputs, E jobInputs.getSourceLauncherConfig(), jobInputs.getDestinationLauncherConfig(), jobInputs.getSyncInput(), - connectionId, envVariableFeatureFlags); + connectionId); } /** diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java index b2d94b6f14e5f..c60aa8fbec0f4 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java @@ -29,7 +29,9 @@ public class RefreshSchemaActivityImpl implements RefreshSchemaActivity { private final AirbyteApiClient airbyteApiClient; private final EnvVariableFeatureFlags envVariableFeatureFlags; - public RefreshSchemaActivityImpl(Optional configRepository, AirbyteApiClient airbyteApiClient, EnvVariableFeatureFlags envVariableFeatureFlags) { + public RefreshSchemaActivityImpl(Optional configRepository, + AirbyteApiClient airbyteApiClient, + EnvVariableFeatureFlags envVariableFeatureFlags) { this.configRepository = configRepository; this.airbyteApiClient = airbyteApiClient; this.envVariableFeatureFlags = envVariableFeatureFlags; @@ -48,7 +50,7 @@ public boolean shouldRefreshSchema(UUID sourceCatalogId) throws IOException { @Override public void refreshSchema(UUID sourceCatalogId, UUID connectionId) throws ApiException { - if(!envVariableFeatureFlags.autoDetectSchema()){ + if (!envVariableFeatureFlags.autoDetectSchema()) { return; } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java index 4aa3f7b6c9bb1..6171a40674365 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java @@ -8,7 +8,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.TemporalJobType; import io.airbyte.commons.temporal.scheduling.ConnectionManagerWorkflow; import io.airbyte.commons.temporal.scheduling.ConnectionUpdaterInput; @@ -128,7 +127,6 @@ class ConnectionManagerWorkflowTest { mock(WorkflowConfigActivity.class, Mockito.withSettings().withoutAnnotations()); private static final RouteToSyncTaskQueueActivity mRouteToSyncTaskQueueActivity = mock(RouteToSyncTaskQueueActivity.class, Mockito.withSettings().withoutAnnotations()); - private static final EnvVariableFeatureFlags mEnvVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); private static final String EVENT = "event = "; private TestWorkflowEnvironment testEnv; @@ -1560,7 +1558,7 @@ public boolean matches(final JobCancelledInputWithAttemptNumber arg) { private static void startWorkflowAndWaitUntilReady(final ConnectionManagerWorkflow workflow, final ConnectionUpdaterInput input) throws InterruptedException { - WorkflowClient.start(workflow::run, input, mEnvVariableFeatureFlags); + WorkflowClient.start(workflow::run, input); boolean isReady = false; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java index a89d08e45e4d5..677740d2f796f 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/DbtFailureSyncWorkflow.java @@ -4,7 +4,6 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/EmptySyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/EmptySyncWorkflow.java index 0c0abfd9d066f..3f93c6ba87d24 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/EmptySyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/EmptySyncWorkflow.java @@ -4,7 +4,6 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java index f165effa0e042..99e52c941c206 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationFailureSyncWorkflow.java @@ -4,7 +4,6 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationTraceFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationTraceFailureSyncWorkflow.java index 664debd1ec7da..1845f0ac0e459 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationTraceFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/NormalizationTraceFailureSyncWorkflow.java @@ -5,7 +5,6 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.FailureReason; import io.airbyte.config.FailureReason.FailureOrigin; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java index 0de104c7b1c61..e8b704a6512c0 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/PersistFailureSyncWorkflow.java @@ -4,7 +4,6 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java index f2da0d76dfbb5..d4c64152e2cc4 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/ReplicateFailureSyncWorkflow.java @@ -4,7 +4,6 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SleepingSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SleepingSyncWorkflow.java index 9d8dbe2ac5d8d..c2f24529de127 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SleepingSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SleepingSyncWorkflow.java @@ -4,7 +4,6 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java index 6c3f3b93ec435..19a8bdabbe1c6 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SourceAndDestinationFailureSyncWorkflow.java @@ -5,7 +5,6 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.FailureReason; import io.airbyte.config.FailureReason.FailureOrigin; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowFailingOutputWorkflow.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowFailingOutputWorkflow.java index 8e8f26a0a98d4..07925086dcc7e 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowFailingOutputWorkflow.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowFailingOutputWorkflow.java @@ -4,7 +4,6 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; From c0f6bb14722b21134e6c6b160c1097287f5fab18 Mon Sep 17 00:00:00 2001 From: alovew Date: Wed, 30 Nov 2022 14:03:25 -0800 Subject: [PATCH 11/24] remove in tests --- .../io/airbyte/commons/temporal/ConnectionManagerUtils.java | 3 +-- .../temporal/scheduling/ConnectionManagerWorkflowImpl.java | 3 +-- .../SyncWorkflowWithActivityFailureException.java | 4 +--- .../io/airbyte/workers/temporal/sync/SyncWorkflowTest.java | 6 +----- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java index a86f9aa80b984..3a5b3628e8e38 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java @@ -161,8 +161,7 @@ public void safeTerminateWorkflow(final WorkflowClient client, final UUID connec safeTerminateWorkflow(client, getConnectionManagerName(connectionId), reason); } - public ConnectionManagerWorkflow startConnectionManagerNoSignal(final WorkflowClient client, - final UUID connectionId) { + public ConnectionManagerWorkflow startConnectionManagerNoSignal(final WorkflowClient client, final UUID connectionId) { final ConnectionManagerWorkflow connectionManagerWorkflow = newConnectionManagerWorkflowStub(client, connectionId); final ConnectionUpdaterInput input = TemporalWorkflowUtils.buildStartWorkflowInput(connectionId); WorkflowClient.start(connectionManagerWorkflow::run, input); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java index 4bd1f55836b91..db5eb9f7dd914 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java @@ -159,8 +159,7 @@ public class ConnectionManagerWorkflowImpl implements ConnectionManagerWorkflow @Trace(operationName = WORKFLOW_TRACE_OPERATION_NAME) @Override - public void run(final ConnectionUpdaterInput connectionUpdaterInput) - throws RetryableException { + public void run(final ConnectionUpdaterInput connectionUpdaterInput) throws RetryableException { try { ApmTraceUtils.addTagsToTrace(Map.of(CONNECTION_ID_KEY, connectionUpdaterInput.getConnectionId())); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowWithActivityFailureException.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowWithActivityFailureException.java index e77dfe469ff99..2a897d73dabd1 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowWithActivityFailureException.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/testsyncworkflow/SyncWorkflowWithActivityFailureException.java @@ -4,7 +4,6 @@ package io.airbyte.workers.temporal.scheduling.testsyncworkflow; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; @@ -25,8 +24,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId, - final EnvVariableFeatureFlags envVariableFeatureFlags) { + final UUID connectionId) { Workflow.sleep(SleepingSyncWorkflow.RUN_TIME); throw new ActivityFailure(1L, 1L, "Replication", "id", RetryState.RETRY_STATE_RETRY_POLICY_NOT_SET, "identity", diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index d6feae6449429..3c122a9c09119 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -17,7 +17,6 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.api.client.invoker.generated.ApiException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; @@ -119,7 +118,6 @@ class SyncWorkflowTest { private ActivityOptions longActivityOptions; private ActivityOptions shortActivityOptions; private TemporalProxyHelper temporalProxyHelper; - private EnvVariableFeatureFlags envVariableFeatureFlags; @BeforeEach void setUp() throws IOException { @@ -156,7 +154,6 @@ void setUp() throws IOException { webhookOperationActivity = mock(WebhookOperationActivityImpl.class); refreshSchemaActivity = mock(RefreshSchemaActivityImpl.class); configFetchActivity = mock(ConfigFetchActivityImpl.class); - envVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); when(normalizationActivity.generateNormalizationInput(any(), any())).thenReturn(normalizationInput); when(normalizationSummaryCheckActivity.shouldRunNormalization(any(), any(), any())).thenReturn(true); @@ -206,8 +203,7 @@ private StandardSyncOutput execute() throws JsonValidationException, ConfigNotFo final SyncWorkflow workflow = client.newWorkflowStub(SyncWorkflow.class, WorkflowOptions.newBuilder().setTaskQueue(SYNC_QUEUE).build()); - return workflow.run(JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, DESTINATION_LAUNCHER_CONFIG, syncInput, sync.getConnectionId(), - envVariableFeatureFlags); + return workflow.run(JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, DESTINATION_LAUNCHER_CONFIG, syncInput, sync.getConnectionId()); } @Test From 8acca503a07c2e5f3962010ef2c2590594941cb4 Mon Sep 17 00:00:00 2001 From: alovew Date: Wed, 30 Nov 2022 14:12:14 -0800 Subject: [PATCH 12/24] try catch --- .../commons/temporal/TemporalClient.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index 9abceab23c756..f967490001dd2 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -8,6 +8,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; +import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.temporal.config.WorkerMode; import io.airbyte.commons.temporal.exception.DeletedWorkflowException; import io.airbyte.commons.temporal.exception.UnreachableWorkflowException; @@ -26,10 +27,12 @@ import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; +import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.StreamDescriptor; +import io.airbyte.validation.json.JsonValidationException; import io.micronaut.context.annotation.Requires; import io.temporal.api.common.v1.WorkflowType; import io.temporal.api.enums.v1.WorkflowExecutionStatus; @@ -398,12 +401,18 @@ public TemporalResponse submitSync(final long jobId, final i .withDestinationResourceRequirements(config.getDestinationResourceRequirements()); return execute(jobRunConfig, - () -> getWorkflowStub(SyncWorkflow.class, TemporalJobType.SYNC).run( - jobRunConfig, - sourceLauncherConfig, - destinationLauncherConfig, - input, - connectionId)); + () -> { + try { + return getWorkflowStub(SyncWorkflow.class, TemporalJobType.SYNC).run( + jobRunConfig, + sourceLauncherConfig, + destinationLauncherConfig, + input, + connectionId); + } catch (JsonValidationException|ConfigNotFoundException|IOException|ApiException e) { + throw new RuntimeException(e); + } + }); } public void migrateSyncIfNeeded(final Set connectionIds) { From b9a869ba987ab532780fcb2e77e716fd1bd99643 Mon Sep 17 00:00:00 2001 From: alovew Date: Thu, 1 Dec 2022 12:59:02 -0800 Subject: [PATCH 13/24] auto detect schema version --- .../java/io/airbyte/commons/temporal/TemporalClient.java | 2 +- .../io/airbyte/commons/temporal/TemporalClientTest.java | 5 ++++- .../io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java | 6 +++++- .../scheduling/activities/RefreshSchemaActivityTest.java | 3 ++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index f967490001dd2..b233a41dea728 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -409,7 +409,7 @@ public TemporalResponse submitSync(final long jobId, final i destinationLauncherConfig, input, connectionId); - } catch (JsonValidationException|ConfigNotFoundException|IOException|ApiException e) { + } catch (JsonValidationException | ConfigNotFoundException | IOException | ApiException e) { throw new RuntimeException(e); } }); diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index 36b3eda388063..c3e0cf987ac47 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.when; import com.google.common.collect.Sets; +import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalClient.ManualOperationResult; import io.airbyte.commons.temporal.scheduling.CheckConnectionWorkflow; @@ -41,11 +42,13 @@ import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.helpers.LogClientSingleton; +import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.StreamDescriptor; +import io.airbyte.validation.json.JsonValidationException; import io.temporal.api.enums.v1.WorkflowExecutionStatus; import io.temporal.api.workflow.v1.WorkflowExecutionInfo; import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionResponse; @@ -264,7 +267,7 @@ void testSubmitDiscoverSchema() { } @Test - void testSubmitSync() { + void testSubmitSync() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { final SyncWorkflow discoverCatalogWorkflow = mock(SyncWorkflow.class); when(workflowClient.newWorkflowStub(SyncWorkflow.class, TemporalWorkflowUtils.buildWorkflowOptions(TemporalJobType.SYNC))) .thenReturn(discoverCatalogWorkflow); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index e95088cf33ba5..b0de6e336f0d2 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -51,6 +51,8 @@ public class SyncWorkflowImpl implements SyncWorkflow { private static final int CURRENT_VERSION = 2; private static final String NORMALIZATION_SUMMARY_CHECK_TAG = "normalization_summary_check"; private static final int NORMALIZATION_SUMMARY_CHECK_CURRENT_VERSION = 1; + private static final String AUTO_DETECT_SCHEMA_TAG = "auto_detect_schema"; + private static final int AUTO_DETECT_SCHEMA_VERSION = 1; @TemporalActivityStub(activityOptionsBeanName = "longRunActivityOptions") private ReplicationActivity replicationActivity; @@ -87,7 +89,9 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final int version = Workflow.getVersion(VERSION_LABEL, Workflow.DEFAULT_VERSION, CURRENT_VERSION); final String taskQueue = Workflow.getInfo().getTaskQueue(); - if (version > Workflow.DEFAULT_VERSION) { + final int autoDetectSchemaVersion = + Workflow.getVersion(AUTO_DETECT_SCHEMA_TAG, Workflow.DEFAULT_VERSION, AUTO_DETECT_SCHEMA_VERSION); + if (autoDetectSchemaVersion >= AUTO_DETECT_SCHEMA_VERSION) { final UUID sourceId = configFetchActivity.getStandardSync(connectionId).getSourceId(); if (refreshSchemaActivity.shouldRefreshSchema(sourceId)) { LOGGER.info("Refreshing source schema..."); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java index 749be5b358965..8de5b43df48c8 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java @@ -48,6 +48,7 @@ void setUp() { mConfigRepository = mock(ConfigRepository.class); mEnvVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); Mockito.lenient().when(mApiClient.getSourceApi()).thenReturn(mSourceApi); + when(mEnvVariableFeatureFlags.autoDetectSchema()).thenReturn(true); refreshSchemaActivity = new RefreshSchemaActivityImpl(Optional.of(mConfigRepository), mApiClient, mEnvVariableFeatureFlags); } @@ -79,7 +80,7 @@ void testRefreshSchema() throws ApiException { UUID connectionId = UUID.randomUUID(); refreshSchemaActivity.refreshSchema(sourceId, connectionId); SourceDiscoverSchemaRequestBody requestBody = - new SourceDiscoverSchemaRequestBody().sourceId(sourceId).disableCache(true); + new SourceDiscoverSchemaRequestBody().sourceId(sourceId).disableCache(true).connectionId(connectionId); verify(mSourceApi, times(1)).discoverSchemaForSource(requestBody); } From 6d99d6cbabc3b078a5d8a9126e590f5e13f06ef9 Mon Sep 17 00:00:00 2001 From: alovew Date: Thu, 1 Dec 2022 13:16:37 -0800 Subject: [PATCH 14/24] Catch errors --- .../commons/temporal/TemporalClient.java | 21 +++++---------- .../temporal/scheduling/SyncWorkflow.java | 7 +---- .../commons/temporal/TemporalClientTest.java | 5 +--- .../ConnectionManagerWorkflowImpl.java | 13 ++------- .../temporal/sync/SyncWorkflowImpl.java | 27 ++++++++++++++----- .../temporal/sync/SyncWorkflowTest.java | 13 ++++----- 6 files changed, 35 insertions(+), 51 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index b233a41dea728..9abceab23c756 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -8,7 +8,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.temporal.config.WorkerMode; import io.airbyte.commons.temporal.exception.DeletedWorkflowException; import io.airbyte.commons.temporal.exception.UnreachableWorkflowException; @@ -27,12 +26,10 @@ import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; -import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.StreamDescriptor; -import io.airbyte.validation.json.JsonValidationException; import io.micronaut.context.annotation.Requires; import io.temporal.api.common.v1.WorkflowType; import io.temporal.api.enums.v1.WorkflowExecutionStatus; @@ -401,18 +398,12 @@ public TemporalResponse submitSync(final long jobId, final i .withDestinationResourceRequirements(config.getDestinationResourceRequirements()); return execute(jobRunConfig, - () -> { - try { - return getWorkflowStub(SyncWorkflow.class, TemporalJobType.SYNC).run( - jobRunConfig, - sourceLauncherConfig, - destinationLauncherConfig, - input, - connectionId); - } catch (JsonValidationException | ConfigNotFoundException | IOException | ApiException e) { - throw new RuntimeException(e); - } - }); + () -> getWorkflowStub(SyncWorkflow.class, TemporalJobType.SYNC).run( + jobRunConfig, + sourceLauncherConfig, + destinationLauncherConfig, + input, + connectionId)); } public void migrateSyncIfNeeded(final Set connectionIds) { diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java index 3e92e979aaa95..513974b955340 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/SyncWorkflow.java @@ -4,16 +4,12 @@ package io.airbyte.commons.temporal.scheduling; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; -import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; -import io.airbyte.validation.json.JsonValidationException; import io.temporal.workflow.WorkflowInterface; import io.temporal.workflow.WorkflowMethod; -import java.io.IOException; import java.util.UUID; @WorkflowInterface @@ -24,7 +20,6 @@ StandardSyncOutput run(JobRunConfig jobRunConfig, IntegrationLauncherConfig sourceLauncherConfig, IntegrationLauncherConfig destinationLauncherConfig, StandardSyncInput syncInput, - UUID connectionId) - throws JsonValidationException, ConfigNotFoundException, IOException, ApiException; + UUID connectionId); } diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index c3e0cf987ac47..36b3eda388063 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -22,7 +22,6 @@ import static org.mockito.Mockito.when; import com.google.common.collect.Sets; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalClient.ManualOperationResult; import io.airbyte.commons.temporal.scheduling.CheckConnectionWorkflow; @@ -42,13 +41,11 @@ import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.helpers.LogClientSingleton; -import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.StreamDescriptor; -import io.airbyte.validation.json.JsonValidationException; import io.temporal.api.enums.v1.WorkflowExecutionStatus; import io.temporal.api.workflow.v1.WorkflowExecutionInfo; import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionResponse; @@ -267,7 +264,7 @@ void testSubmitDiscoverSchema() { } @Test - void testSubmitSync() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { + void testSubmitSync() { final SyncWorkflow discoverCatalogWorkflow = mock(SyncWorkflow.class); when(workflowClient.newWorkflowStub(SyncWorkflow.class, TemporalWorkflowUtils.buildWorkflowOptions(TemporalJobType.SYNC))) .thenReturn(discoverCatalogWorkflow); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java index db5eb9f7dd914..d05db6d4fa736 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java @@ -11,7 +11,6 @@ import com.fasterxml.jackson.databind.JsonNode; import datadog.trace.api.Trace; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.temporal.TemporalJobType; import io.airbyte.commons.temporal.TemporalWorkflowUtils; import io.airbyte.commons.temporal.exception.RetryableException; @@ -32,12 +31,10 @@ import io.airbyte.config.StandardSyncOutput; import io.airbyte.config.StandardSyncSummary; import io.airbyte.config.StandardSyncSummary.ReplicationStatus; -import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; -import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.helper.FailureHelper; import io.airbyte.workers.temporal.annotations.TemporalActivityStub; @@ -85,7 +82,6 @@ import io.temporal.workflow.CancellationScope; import io.temporal.workflow.ChildWorkflowOptions; import io.temporal.workflow.Workflow; -import java.io.IOException; import java.time.Duration; import java.time.Instant; import java.util.HashSet; @@ -258,11 +254,7 @@ private CancellationScope generateSyncWorkflowRunnable(final ConnectionUpdaterIn workflowState.setFailed(getFailStatus(checkFailureOutput)); reportFailure(connectionUpdaterInput, checkFailureOutput, FailureCause.CONNECTION); } else { - try { - standardSyncOutput = runChildWorkflow(jobInputs); - } catch (JsonValidationException | ConfigNotFoundException | IOException | ApiException e) { - throw new RuntimeException(e); - } + standardSyncOutput = runChildWorkflow(jobInputs); workflowState.setFailed(getFailStatus(standardSyncOutput)); if (workflowState.isFailed()) { @@ -857,8 +849,7 @@ private void reportJobStarting(final UUID connectionId) { * since the latter is a long running workflow, in the future, using a different Node pool would * make sense. */ - private StandardSyncOutput runChildWorkflow(final GeneratedJobInput jobInputs) - throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { + private StandardSyncOutput runChildWorkflow(final GeneratedJobInput jobInputs) { final String taskQueue = getSyncTaskQueue(); final SyncWorkflow childSync = Workflow.newChildWorkflowStub(SyncWorkflow.class, diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index b0de6e336f0d2..e7d0eaa902ccf 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -77,8 +77,7 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final IntegrationLauncherConfig sourceLauncherConfig, final IntegrationLauncherConfig destinationLauncherConfig, final StandardSyncInput syncInput, - final UUID connectionId) - throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { + final UUID connectionId) { ApmTraceUtils .addTagsToTrace(Map.of(ATTEMPT_NUMBER_KEY, jobRunConfig.getAttemptId(), CONNECTION_ID_KEY, connectionId.toString(), JOB_ID_KEY, @@ -92,12 +91,26 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final int autoDetectSchemaVersion = Workflow.getVersion(AUTO_DETECT_SCHEMA_TAG, Workflow.DEFAULT_VERSION, AUTO_DETECT_SCHEMA_VERSION); if (autoDetectSchemaVersion >= AUTO_DETECT_SCHEMA_VERSION) { - final UUID sourceId = configFetchActivity.getStandardSync(connectionId).getSourceId(); - if (refreshSchemaActivity.shouldRefreshSchema(sourceId)) { - LOGGER.info("Refreshing source schema..."); - refreshSchemaActivity.refreshSchema(sourceId, connectionId); + UUID sourceId = null; + try { + sourceId = configFetchActivity.getStandardSync(connectionId).getSourceId(); + } catch (JsonValidationException | ConfigNotFoundException | IOException e) { + LOGGER.error("An error occurred fetching the connection during schema refresh processing. Skipping schema refresh. ", e); + } + try { + if (sourceId != null && refreshSchemaActivity.shouldRefreshSchema(sourceId)) { + LOGGER.info("Refreshing source schema..."); + refreshSchemaActivity.refreshSchema(sourceId, connectionId); + } + } catch (IOException | JsonValidationException | ConfigNotFoundException | ApiException e) { + LOGGER.error("An error occurred while refreshing the source schema. Skipping schema refresh. ", e); + } + Status status = null; + try { + status = configFetchActivity.getStandardSync(connectionId).getStatus(); + } catch (JsonValidationException | ConfigNotFoundException | IOException e) { + LOGGER.error("An error occurred while fetching the connection status. ", e); } - final Status status = configFetchActivity.getStandardSync(connectionId).getStatus(); if (Status.INACTIVE == status) { LOGGER.info("Connection is disabled. Cancelling run."); final StandardSyncOutput output = diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index 3c122a9c09119..31622620314bd 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -16,7 +16,6 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; @@ -34,11 +33,9 @@ import io.airbyte.config.StandardSyncSummary; import io.airbyte.config.StandardSyncSummary.ReplicationStatus; import io.airbyte.config.SyncStats; -import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivityImpl; import io.airbyte.workers.temporal.support.TemporalProxyHelper; import io.airbyte.workers.test_utils.TestConfigHelpers; @@ -196,7 +193,7 @@ public void tearDown() { } // bundle up all the temporal worker setup / execution into one method. - private StandardSyncOutput execute() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { + private StandardSyncOutput execute() { syncWorker.registerActivitiesImplementations(replicationActivity, normalizationActivity, dbtTransformationActivity, persistStateActivity, normalizationSummaryCheckActivity, webhookOperationActivity, refreshSchemaActivity, configFetchActivity); testEnv.start(); @@ -207,7 +204,7 @@ private StandardSyncOutput execute() throws JsonValidationException, ConfigNotFo } @Test - void testSuccess() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { + void testSuccess() { doReturn(replicationSuccessOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -248,7 +245,7 @@ void testReplicationFailure() { } @Test - void testReplicationFailedGracefully() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { + void testReplicationFailedGracefully() { doReturn(replicationFailOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -338,7 +335,7 @@ void testCancelDuringNormalization() { @Test @Disabled("This behavior has been disabled temporarily (OC Issue #741)") - void testSkipNormalization() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { + void testSkipNormalization() { final SyncStats syncStats = new SyncStats().withRecordsCommitted(0L); final StandardSyncSummary standardSyncSummary = new StandardSyncSummary().withTotalStats(syncStats); final StandardSyncOutput replicationSuccessOutputNoRecordsCommitted = @@ -361,7 +358,7 @@ void testSkipNormalization() throws IOException, JsonValidationException, Config } @Test - void testWebhookOperation() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { + void testWebhookOperation() { when(replicationActivity.replicate(any(), any(), any(), any(), any())).thenReturn(new StandardSyncOutput()); final StandardSyncOperation webhookOperation = new StandardSyncOperation() .withOperationId(UUID.randomUUID()) From 9b013feb5f5715a18c7069250cfdfc34c9150962 Mon Sep 17 00:00:00 2001 From: alovew Date: Thu, 1 Dec 2022 14:15:09 -0800 Subject: [PATCH 15/24] tests --- .../temporal/sync/SyncWorkflowTest.java | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index 31622620314bd..e27c2492d1e79 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -16,6 +16,7 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; @@ -26,6 +27,7 @@ import io.airbyte.config.OperatorWebhookInput; import io.airbyte.config.ResourceRequirements; import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.Status; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOperation; import io.airbyte.config.StandardSyncOperation.OperatorType; @@ -33,9 +35,11 @@ import io.airbyte.config.StandardSyncSummary; import io.airbyte.config.StandardSyncSummary.ReplicationStatus; import io.airbyte.config.SyncStats; +import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivityImpl; import io.airbyte.workers.temporal.support.TemporalProxyHelper; import io.airbyte.workers.test_utils.TestConfigHelpers; @@ -86,6 +90,7 @@ class SyncWorkflowTest { // AIRBYTE CONFIGURATION private static final long JOB_ID = 11L; private static final int ATTEMPT_ID = 21; + private static final UUID SOURCE_ID = UUID.randomUUID(); private static final JobRunConfig JOB_RUN_CONFIG = new JobRunConfig() .withJobId(String.valueOf(JOB_ID)) .withAttemptId((long) ATTEMPT_ID); @@ -111,13 +116,14 @@ class SyncWorkflowTest { private StandardSyncSummary standardSyncSummary; private StandardSyncSummary failedSyncSummary; private SyncStats syncStats; + private StandardSync standardSync; private NormalizationSummary normalizationSummary; private ActivityOptions longActivityOptions; private ActivityOptions shortActivityOptions; private TemporalProxyHelper temporalProxyHelper; @BeforeEach - void setUp() throws IOException { + void setUp() throws IOException, JsonValidationException, ConfigNotFoundException { testEnv = TestWorkflowEnvironment.newInstance(); syncWorker = testEnv.newWorker(SYNC_QUEUE); client = testEnv.getWorkflowClient(); @@ -134,6 +140,8 @@ void setUp() throws IOException { normalizationSummary = new NormalizationSummary(); + standardSync = new StandardSync().withSourceId(SOURCE_ID).withStatus(Status.ACTIVE); + normalizationInput = new NormalizationInput() .withDestinationConfiguration(syncInput.getDestinationConfiguration()) .withCatalog(syncInput.getCatalog()) @@ -155,6 +163,9 @@ void setUp() throws IOException { when(normalizationActivity.generateNormalizationInput(any(), any())).thenReturn(normalizationInput); when(normalizationSummaryCheckActivity.shouldRunNormalization(any(), any(), any())).thenReturn(true); + when(configFetchActivity.getStandardSync(sync.getConnectionId())).thenReturn(standardSync); + when(refreshSchemaActivity.shouldRefreshSchema(SOURCE_ID)).thenReturn(true); + longActivityOptions = ActivityOptions.newBuilder() .setScheduleToCloseTimeout(Duration.ofDays(3)) .setStartToCloseTimeout(Duration.ofDays(3)) @@ -204,7 +215,7 @@ private StandardSyncOutput execute() { } @Test - void testSuccess() { + void testSuccess() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { doReturn(replicationSuccessOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -223,6 +234,8 @@ void testSuccess() { verifyNormalize(normalizationActivity, normalizationInput); verifyDbtTransform(dbtTransformationActivity, syncInput.getResourceRequirements(), operatorDbtInput); + verifyShouldRefreshSchema(refreshSchemaActivity); + verifyRefreshSchema(refreshSchemaActivity, sync); assertEquals( replicationSuccessOutput.withNormalizationSummary(normalizationSummary).getStandardSyncSummary(), actualOutput.getStandardSyncSummary()); @@ -376,6 +389,15 @@ void testWebhookOperation() { assertEquals(actualOutput.getWebhookOperationSummary().getSuccesses().get(0), WEBHOOK_CONFIG_ID); } + @Test + void testSkipReplicationAfterRefreshSchema() throws JsonValidationException, ConfigNotFoundException, IOException { + when(configFetchActivity.getStandardSync(any())).thenReturn(new StandardSync().withSourceId(UUID.randomUUID()).withStatus(Status.INACTIVE)); + StandardSyncOutput output = execute(); + verifyNoInteractions(replicationActivity); + verifyNoInteractions(normalizationActivity); + assertEquals(output.getStandardSyncSummary().getStatus(), ReplicationStatus.CANCELLED); + } + @SuppressWarnings("ResultOfMethodCallIgnored") private void cancelWorkflow() { final WorkflowServiceBlockingStub temporalService = testEnv.getWorkflowService().blockingStub(); @@ -428,4 +450,12 @@ private static void verifyDbtTransform(final DbtTransformationActivity dbtTransf operatorDbtInput); } + private static void verifyShouldRefreshSchema(final RefreshSchemaActivity refreshSchemaActivity) throws IOException { + verify(refreshSchemaActivity).shouldRefreshSchema(SOURCE_ID); + } + + private static void verifyRefreshSchema(final RefreshSchemaActivity refreshSchemaActivity, final StandardSync sync) + throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { + verify(refreshSchemaActivity).refreshSchema(SOURCE_ID, sync.getConnectionId()); + } } From f09f87b442a09dbaaf5be93fd503e71b33c63405 Mon Sep 17 00:00:00 2001 From: alovew Date: Thu, 1 Dec 2022 14:41:54 -0800 Subject: [PATCH 16/24] fix test --- .../io/airbyte/workers/temporal/sync/SyncWorkflowTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index e27c2492d1e79..8c41089a771ab 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -389,14 +389,14 @@ void testWebhookOperation() { assertEquals(actualOutput.getWebhookOperationSummary().getSuccesses().get(0), WEBHOOK_CONFIG_ID); } - @Test - void testSkipReplicationAfterRefreshSchema() throws JsonValidationException, ConfigNotFoundException, IOException { + @Test + void testSkipReplicationAfterRefreshSchema() throws JsonValidationException, ConfigNotFoundException, IOException { when(configFetchActivity.getStandardSync(any())).thenReturn(new StandardSync().withSourceId(UUID.randomUUID()).withStatus(Status.INACTIVE)); StandardSyncOutput output = execute(); verifyNoInteractions(replicationActivity); verifyNoInteractions(normalizationActivity); assertEquals(output.getStandardSyncSummary().getStatus(), ReplicationStatus.CANCELLED); - } + } @SuppressWarnings("ResultOfMethodCallIgnored") private void cancelWorkflow() { @@ -458,4 +458,5 @@ private static void verifyRefreshSchema(final RefreshSchemaActivity refreshSchem throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { verify(refreshSchemaActivity).refreshSchema(SOURCE_ID, sync.getConnectionId()); } + } From 3fc37513efcc51c2d57c91916b5d4def85902b81 Mon Sep 17 00:00:00 2001 From: alovew Date: Thu, 1 Dec 2022 16:52:29 -0800 Subject: [PATCH 17/24] update tests --- .../temporal/sync/SyncWorkflowImpl.java | 34 +++++++------------ .../temporal/sync/SyncWorkflowTest.java | 28 +++++++++++---- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index e7d0eaa902ccf..52967cfc4c5fe 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -91,32 +91,22 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final int autoDetectSchemaVersion = Workflow.getVersion(AUTO_DETECT_SCHEMA_TAG, Workflow.DEFAULT_VERSION, AUTO_DETECT_SCHEMA_VERSION); if (autoDetectSchemaVersion >= AUTO_DETECT_SCHEMA_VERSION) { - UUID sourceId = null; try { - sourceId = configFetchActivity.getStandardSync(connectionId).getSourceId(); - } catch (JsonValidationException | ConfigNotFoundException | IOException e) { - LOGGER.error("An error occurred fetching the connection during schema refresh processing. Skipping schema refresh. ", e); - } - try { - if (sourceId != null && refreshSchemaActivity.shouldRefreshSchema(sourceId)) { + final UUID sourceId = configFetchActivity.getStandardSync(connectionId).getSourceId(); + if (refreshSchemaActivity.shouldRefreshSchema(sourceId)) { LOGGER.info("Refreshing source schema..."); refreshSchemaActivity.refreshSchema(sourceId, connectionId); } - } catch (IOException | JsonValidationException | ConfigNotFoundException | ApiException e) { - LOGGER.error("An error occurred while refreshing the source schema. Skipping schema refresh. ", e); - } - Status status = null; - try { - status = configFetchActivity.getStandardSync(connectionId).getStatus(); - } catch (JsonValidationException | ConfigNotFoundException | IOException e) { - LOGGER.error("An error occurred while fetching the connection status. ", e); - } - if (Status.INACTIVE == status) { - LOGGER.info("Connection is disabled. Cancelling run."); - final StandardSyncOutput output = - new StandardSyncOutput() - .withStandardSyncSummary(new StandardSyncSummary().withStatus(ReplicationStatus.CANCELLED).withTotalStats(new SyncStats())); - return output; + final Status status = configFetchActivity.getStandardSync(connectionId).getStatus(); + if (Status.INACTIVE == status) { + LOGGER.info("Connection is disabled. Cancelling run."); + final StandardSyncOutput output = + new StandardSyncOutput() + .withStandardSyncSummary(new StandardSyncSummary().withStatus(ReplicationStatus.CANCELLED).withTotalStats(new SyncStats())); + return output; + } + } catch (JsonValidationException | ConfigNotFoundException | IOException | ApiException e) { + LOGGER.error("An error occurred during schema refresh processing. Skipping schema refresh. ", e); } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index 8c41089a771ab..837d56f8c1240 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -242,7 +242,7 @@ void testSuccess() throws IOException, JsonValidationException, ConfigNotFoundEx } @Test - void testReplicationFailure() { + void testReplicationFailure() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { doThrow(new IllegalArgumentException("induced exception")).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -251,6 +251,8 @@ void testReplicationFailure() { assertThrows(WorkflowFailedException.class, this::execute); + verifyShouldRefreshSchema(refreshSchemaActivity); + verifyRefreshSchema(refreshSchemaActivity, sync); verifyReplication(replicationActivity, syncInput); verifyNoInteractions(persistStateActivity); verifyNoInteractions(normalizationActivity); @@ -258,7 +260,7 @@ void testReplicationFailure() { } @Test - void testReplicationFailedGracefully() { + void testReplicationFailedGracefully() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { doReturn(replicationFailOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -272,6 +274,8 @@ void testReplicationFailedGracefully() { final StandardSyncOutput actualOutput = execute(); + verifyShouldRefreshSchema(refreshSchemaActivity); + verifyRefreshSchema(refreshSchemaActivity, sync); verifyReplication(replicationActivity, syncInput); verifyPersistState(persistStateActivity, sync, replicationFailOutput, syncInput.getCatalog()); verifyNormalize(normalizationActivity, normalizationInput); @@ -283,7 +287,7 @@ void testReplicationFailedGracefully() { } @Test - void testNormalizationFailure() { + void testNormalizationFailure() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { doReturn(replicationSuccessOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -297,6 +301,8 @@ void testNormalizationFailure() { assertThrows(WorkflowFailedException.class, this::execute); + verifyShouldRefreshSchema(refreshSchemaActivity); + verifyRefreshSchema(refreshSchemaActivity, sync); verifyReplication(replicationActivity, syncInput); verifyPersistState(persistStateActivity, sync, replicationSuccessOutput, syncInput.getCatalog()); verifyNormalize(normalizationActivity, normalizationInput); @@ -304,7 +310,7 @@ void testNormalizationFailure() { } @Test - void testCancelDuringReplication() { + void testCancelDuringReplication() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { doAnswer(ignored -> { cancelWorkflow(); return replicationSuccessOutput; @@ -316,6 +322,8 @@ void testCancelDuringReplication() { assertThrows(WorkflowFailedException.class, this::execute); + verifyShouldRefreshSchema(refreshSchemaActivity); + verifyRefreshSchema(refreshSchemaActivity, sync); verifyReplication(replicationActivity, syncInput); verifyNoInteractions(persistStateActivity); verifyNoInteractions(normalizationActivity); @@ -323,7 +331,7 @@ void testCancelDuringReplication() { } @Test - void testCancelDuringNormalization() { + void testCancelDuringNormalization() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { doReturn(replicationSuccessOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -340,6 +348,8 @@ void testCancelDuringNormalization() { assertThrows(WorkflowFailedException.class, this::execute); + verifyShouldRefreshSchema(refreshSchemaActivity); + verifyRefreshSchema(refreshSchemaActivity, sync); verifyReplication(replicationActivity, syncInput); verifyPersistState(persistStateActivity, sync, replicationSuccessOutput, syncInput.getCatalog()); verifyNormalize(normalizationActivity, normalizationInput); @@ -348,7 +358,7 @@ void testCancelDuringNormalization() { @Test @Disabled("This behavior has been disabled temporarily (OC Issue #741)") - void testSkipNormalization() { + void testSkipNormalization() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { final SyncStats syncStats = new SyncStats().withRecordsCommitted(0L); final StandardSyncSummary standardSyncSummary = new StandardSyncSummary().withTotalStats(syncStats); final StandardSyncOutput replicationSuccessOutputNoRecordsCommitted = @@ -363,6 +373,8 @@ void testSkipNormalization() { execute(); + verifyShouldRefreshSchema(refreshSchemaActivity); + verifyRefreshSchema(refreshSchemaActivity, sync); verifyReplication(replicationActivity, syncInput); verifyPersistState(persistStateActivity, sync, replicationSuccessOutputNoRecordsCommitted, syncInput.getCatalog()); verifyNoInteractions(normalizationActivity); @@ -390,9 +402,11 @@ void testWebhookOperation() { } @Test - void testSkipReplicationAfterRefreshSchema() throws JsonValidationException, ConfigNotFoundException, IOException { + void testSkipReplicationAfterRefreshSchema() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { when(configFetchActivity.getStandardSync(any())).thenReturn(new StandardSync().withSourceId(UUID.randomUUID()).withStatus(Status.INACTIVE)); StandardSyncOutput output = execute(); + verifyShouldRefreshSchema(refreshSchemaActivity); + verifyRefreshSchema(refreshSchemaActivity, sync); verifyNoInteractions(replicationActivity); verifyNoInteractions(normalizationActivity); assertEquals(output.getStandardSyncSummary().getStatus(), ReplicationStatus.CANCELLED); From 825e42a71acce850d79d4bb1a1988447dae02bbd Mon Sep 17 00:00:00 2001 From: alovew Date: Fri, 2 Dec 2022 11:11:47 -0800 Subject: [PATCH 18/24] fix test --- .../java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index 837d56f8c1240..ced994bdde45c 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -403,7 +403,7 @@ void testWebhookOperation() { @Test void testSkipReplicationAfterRefreshSchema() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { - when(configFetchActivity.getStandardSync(any())).thenReturn(new StandardSync().withSourceId(UUID.randomUUID()).withStatus(Status.INACTIVE)); + when(configFetchActivity.getStandardSync(any())).thenReturn(new StandardSync().withSourceId(SOURCE_ID).withStatus(Status.INACTIVE)); StandardSyncOutput output = execute(); verifyShouldRefreshSchema(refreshSchemaActivity); verifyRefreshSchema(refreshSchemaActivity, sync); From a35fc502e5e3dd668715319fcb94052fb038ae8f Mon Sep 17 00:00:00 2001 From: alovew Date: Fri, 2 Dec 2022 16:29:11 -0800 Subject: [PATCH 19/24] use sourceApi directly --- .../workers/config/ApiClientBeanFactory.java | 24 +++++++++++++++++++ .../sync/RefreshSchemaActivityImpl.java | 10 ++++---- .../activities/RefreshSchemaActivityTest.java | 7 ++---- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java index 19168b4678da0..afddd2640cc2a 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java @@ -9,6 +9,7 @@ import com.auth0.jwt.algorithms.Algorithm; import com.google.auth.oauth2.ServiceAccountCredentials; import io.airbyte.api.client.AirbyteApiClient; +import io.airbyte.api.client.generated.SourceApi; import io.airbyte.commons.temporal.config.WorkerMode; import io.micronaut.context.BeanProvider; import io.micronaut.context.annotation.Factory; @@ -57,6 +58,29 @@ public AirbyteApiClient airbyteApiClient( })); } + @Singleton + public SourceApi sourceApi( + @Value("${airbyte.internal.api.auth-header.name}") final String airbyteApiAuthHeaderName, + @Value("${airbyte.internal.api.host}") final String airbyteApiHost, + @Named("internalApiAuthToken") final BeanProvider internalApiAuthToken, + @Named("internalApiScheme") final String internalApiScheme) { + return new SourceApi( + new io.airbyte.api.client.invoker.generated.ApiClient() + .setScheme(internalApiScheme) + .setHost(parseHostName(airbyteApiHost)) + .setPort(parsePort(airbyteApiHost)) + .setBasePath("/api") + .setHttpClientBuilder(HttpClient.newBuilder().version(Version.HTTP_1_1)) + .setRequestInterceptor(builder -> { + builder.setHeader("User-Agent", "WorkerApp"); + // internalApiAuthToken is in BeanProvider because we want to create a new token each + // time we send a request. + if (!airbyteApiAuthHeaderName.isBlank()) { + builder.setHeader(airbyteApiAuthHeaderName, internalApiAuthToken.get()); + } + })); + } + @Singleton public HttpClient httpClient() { return HttpClient.newHttpClient(); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java index c60aa8fbec0f4..79a3858d6b7e4 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java @@ -7,7 +7,7 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.ACTIVITY_TRACE_OPERATION_NAME; import datadog.trace.api.Trace; -import io.airbyte.api.client.AirbyteApiClient; +import io.airbyte.api.client.generated.SourceApi; import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRequestBody; import io.airbyte.commons.features.EnvVariableFeatureFlags; @@ -26,14 +26,14 @@ public class RefreshSchemaActivityImpl implements RefreshSchemaActivity { private final Optional configRepository; - private final AirbyteApiClient airbyteApiClient; + private final SourceApi sourceApi; private final EnvVariableFeatureFlags envVariableFeatureFlags; public RefreshSchemaActivityImpl(Optional configRepository, - AirbyteApiClient airbyteApiClient, + SourceApi sourceApi, EnvVariableFeatureFlags envVariableFeatureFlags) { this.configRepository = configRepository; - this.airbyteApiClient = airbyteApiClient; + this.sourceApi = sourceApi; this.envVariableFeatureFlags = envVariableFeatureFlags; } @@ -58,7 +58,7 @@ public void refreshSchema(UUID sourceCatalogId, UUID connectionId) throws ApiExc new SourceDiscoverSchemaRequestBody().sourceId(sourceCatalogId).disableCache(true).connectionId(connectionId); try { - airbyteApiClient.getSourceApi().discoverSchemaForSource(requestBody); + sourceApi.discoverSchemaForSource(requestBody); } catch (final Exception e) { log.error("Attempted schema refresh, but failed with error: ", e); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java index 8de5b43df48c8..61113727539c4 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java @@ -25,15 +25,12 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) class RefreshSchemaActivityTest { static private ConfigRepository mConfigRepository; - - static private AirbyteApiClient mApiClient; static private SourceApi mSourceApi; static private EnvVariableFeatureFlags mEnvVariableFeatureFlags; @@ -47,9 +44,9 @@ void setUp() { mSourceApi = mock(SourceApi.class); mConfigRepository = mock(ConfigRepository.class); mEnvVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); - Mockito.lenient().when(mApiClient.getSourceApi()).thenReturn(mSourceApi); + mSourceApi = mock(SourceApi.class); when(mEnvVariableFeatureFlags.autoDetectSchema()).thenReturn(true); - refreshSchemaActivity = new RefreshSchemaActivityImpl(Optional.of(mConfigRepository), mApiClient, mEnvVariableFeatureFlags); + refreshSchemaActivity = new RefreshSchemaActivityImpl(Optional.of(mConfigRepository), mSourceApi, mEnvVariableFeatureFlags); } @Test From 6362a97ebefbaf0e3e9df7f8a44a5e321693bba3 Mon Sep 17 00:00:00 2001 From: alovew Date: Fri, 2 Dec 2022 16:54:03 -0800 Subject: [PATCH 20/24] fix test --- .../scheduling/activities/RefreshSchemaActivityTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java index 61113727539c4..014a00fe86a15 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java @@ -40,7 +40,6 @@ class RefreshSchemaActivityTest { @BeforeEach void setUp() { - mApiClient = mock(AirbyteApiClient.class); mSourceApi = mock(SourceApi.class); mConfigRepository = mock(ConfigRepository.class); mEnvVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); From ed7c615cb92348b9a6fa64d19a109b1b61894d64 Mon Sep 17 00:00:00 2001 From: alovew Date: Fri, 2 Dec 2022 17:16:15 -0800 Subject: [PATCH 21/24] formatting --- .../scheduling/activities/RefreshSchemaActivityTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java index 014a00fe86a15..d120bf8c7f453 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/RefreshSchemaActivityTest.java @@ -9,7 +9,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import io.airbyte.api.client.AirbyteApiClient; import io.airbyte.api.client.generated.SourceApi; import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRequestBody; From c3faa67edf3dfd9116a09c9b005fda48d1fa2190 Mon Sep 17 00:00:00 2001 From: alovew Date: Mon, 5 Dec 2022 15:05:58 -0800 Subject: [PATCH 22/24] Move error handling into activities --- .../activities/ConfigFetchActivity.java | 8 +++++ .../activities/ConfigFetchActivityImpl.java | 22 ++++++++++++ .../temporal/sync/RefreshSchemaActivity.java | 9 ++--- .../sync/RefreshSchemaActivityImpl.java | 22 ++++++------ .../temporal/sync/SyncWorkflowImpl.java | 35 ++++++++----------- .../temporal/sync/SyncWorkflowTest.java | 33 ++++++++--------- 6 files changed, 74 insertions(+), 55 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivity.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivity.java index f5eba653530cf..f936f9d93788b 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivity.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivity.java @@ -5,12 +5,14 @@ package io.airbyte.workers.temporal.scheduling.activities; import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.Status; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.validation.json.JsonValidationException; import io.temporal.activity.ActivityInterface; import io.temporal.activity.ActivityMethod; import java.io.IOException; import java.time.Duration; +import java.util.Optional; import java.util.UUID; import lombok.AllArgsConstructor; import lombok.Data; @@ -19,6 +21,12 @@ @ActivityInterface public interface ConfigFetchActivity { + @ActivityMethod + Optional getSourceId(UUID connectionId); + + @ActivityMethod + Optional getStatus(UUID connectionId); + @Data @NoArgsConstructor @AllArgsConstructor diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java index 7653502fc2571..d8776e6ae55f1 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/ConfigFetchActivityImpl.java @@ -176,4 +176,26 @@ public GetMaxAttemptOutput getMaxAttempt() { return new GetMaxAttemptOutput(syncJobMaxAttempts); } + @Override + public Optional getSourceId(UUID connectionId) { + try { + final StandardSync standardSync = getStandardSync(connectionId); + return Optional.ofNullable(standardSync.getSourceId()); + } catch (JsonValidationException | ConfigNotFoundException | IOException e) { + log.info("Encountered an error fetching the connection's Source ID: ", e); + return Optional.empty(); + } + } + + @Override + public Optional getStatus(UUID connectionId) { + try { + final StandardSync standardSync = getStandardSync(connectionId); + return Optional.ofNullable(standardSync.getStatus()); + } catch (JsonValidationException | ConfigNotFoundException | IOException e) { + log.info("Encountered an error fetching the connection's status: ", e); + return Optional.empty(); + } + } + } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivity.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivity.java index b6079a689a980..d69eea15c9d5f 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivity.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivity.java @@ -4,21 +4,16 @@ package io.airbyte.workers.temporal.sync; -import io.airbyte.api.client.invoker.generated.ApiException; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.validation.json.JsonValidationException; import io.temporal.activity.ActivityInterface; import io.temporal.activity.ActivityMethod; -import java.io.IOException; import java.util.UUID; @ActivityInterface public interface RefreshSchemaActivity { @ActivityMethod - boolean shouldRefreshSchema(UUID sourceCatalogId) throws IOException; + boolean shouldRefreshSchema(UUID sourceCatalogId); - public void refreshSchema(UUID sourceCatalogId, UUID connectionId) - throws JsonValidationException, ConfigNotFoundException, IOException, ApiException; + public void refreshSchema(UUID sourceCatalogId, UUID connectionId); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java index 79a3858d6b7e4..9edb9abda26c8 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java @@ -8,7 +8,6 @@ import datadog.trace.api.Trace; import io.airbyte.api.client.generated.SourceApi; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRequestBody; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.config.ActorCatalogFetchEvent; @@ -39,7 +38,7 @@ public RefreshSchemaActivityImpl(Optional configRepository, @Override @Trace(operationName = ACTIVITY_TRACE_OPERATION_NAME) - public boolean shouldRefreshSchema(UUID sourceCatalogId) throws IOException { + public boolean shouldRefreshSchema(UUID sourceCatalogId) { // if job persistence is unavailable, default to skipping the schema refresh if (configRepository.isEmpty() || !envVariableFeatureFlags.autoDetectSchema()) { return false; @@ -49,7 +48,7 @@ public boolean shouldRefreshSchema(UUID sourceCatalogId) throws IOException { } @Override - public void refreshSchema(UUID sourceCatalogId, UUID connectionId) throws ApiException { + public void refreshSchema(UUID sourceCatalogId, UUID connectionId) { if (!envVariableFeatureFlags.autoDetectSchema()) { return; } @@ -64,14 +63,17 @@ public void refreshSchema(UUID sourceCatalogId, UUID connectionId) throws ApiExc } } - private boolean schemaRefreshRanRecently(UUID sourceCatalogId) throws IOException { - Optional mostRecentFetchEvent = configRepository.get().getMostRecentActorCatalogFetchEventForSource(sourceCatalogId); - - if (mostRecentFetchEvent.isEmpty()) { - return false; + private boolean schemaRefreshRanRecently(UUID sourceCatalogId) { + try { + Optional mostRecentFetchEvent = configRepository.get().getMostRecentActorCatalogFetchEventForSource(sourceCatalogId); + if (mostRecentFetchEvent.isEmpty()) { + return false; + } + return mostRecentFetchEvent.get().getCreatedAt() > OffsetDateTime.now().minusHours(24l).toEpochSecond(); + } catch (IOException e) { + log.info("Encountered an error fetching most recent actor catalog fetch event: ", e); + return true; } - - return mostRecentFetchEvent.get().getCreatedAt() > OffsetDateTime.now().minusHours(24l).toEpochSecond(); } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index 52967cfc4c5fe..e0c4b0f55af8e 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -12,7 +12,6 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.WORKFLOW_TRACE_OPERATION_NAME; import datadog.trace.api.Trace; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; import io.airbyte.config.NormalizationInput; import io.airbyte.config.NormalizationSummary; @@ -27,16 +26,13 @@ import io.airbyte.config.StandardSyncSummary.ReplicationStatus; import io.airbyte.config.SyncStats; import io.airbyte.config.WebhookOperationSummary; -import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.temporal.annotations.TemporalActivityStub; import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivity; import io.temporal.workflow.Workflow; -import java.io.IOException; import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -90,23 +86,22 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, final int autoDetectSchemaVersion = Workflow.getVersion(AUTO_DETECT_SCHEMA_TAG, Workflow.DEFAULT_VERSION, AUTO_DETECT_SCHEMA_VERSION); + if (autoDetectSchemaVersion >= AUTO_DETECT_SCHEMA_VERSION) { - try { - final UUID sourceId = configFetchActivity.getStandardSync(connectionId).getSourceId(); - if (refreshSchemaActivity.shouldRefreshSchema(sourceId)) { - LOGGER.info("Refreshing source schema..."); - refreshSchemaActivity.refreshSchema(sourceId, connectionId); - } - final Status status = configFetchActivity.getStandardSync(connectionId).getStatus(); - if (Status.INACTIVE == status) { - LOGGER.info("Connection is disabled. Cancelling run."); - final StandardSyncOutput output = - new StandardSyncOutput() - .withStandardSyncSummary(new StandardSyncSummary().withStatus(ReplicationStatus.CANCELLED).withTotalStats(new SyncStats())); - return output; - } - } catch (JsonValidationException | ConfigNotFoundException | IOException | ApiException e) { - LOGGER.error("An error occurred during schema refresh processing. Skipping schema refresh. ", e); + final Optional sourceId = configFetchActivity.getSourceId(connectionId); + + if (!sourceId.isEmpty() && refreshSchemaActivity.shouldRefreshSchema(sourceId.get())) { + LOGGER.info("Refreshing source schema..."); + refreshSchemaActivity.refreshSchema(sourceId.get(), connectionId); + } + + final Optional status = configFetchActivity.getStatus(connectionId); + if (!status.isEmpty() && Status.INACTIVE == status.get()) { + LOGGER.info("Connection is disabled. Cancelling run."); + final StandardSyncOutput output = + new StandardSyncOutput() + .withStandardSyncSummary(new StandardSyncSummary().withStatus(ReplicationStatus.CANCELLED).withTotalStats(new SyncStats())); + return output; } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java index ced994bdde45c..75b2963c28224 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/sync/SyncWorkflowTest.java @@ -16,7 +16,6 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.temporal.scheduling.SyncWorkflow; @@ -59,6 +58,7 @@ import java.io.IOException; import java.time.Duration; import java.util.List; +import java.util.Optional; import java.util.UUID; import org.apache.commons.lang3.tuple.ImmutablePair; import org.junit.jupiter.api.AfterEach; @@ -116,14 +116,13 @@ class SyncWorkflowTest { private StandardSyncSummary standardSyncSummary; private StandardSyncSummary failedSyncSummary; private SyncStats syncStats; - private StandardSync standardSync; private NormalizationSummary normalizationSummary; private ActivityOptions longActivityOptions; private ActivityOptions shortActivityOptions; private TemporalProxyHelper temporalProxyHelper; @BeforeEach - void setUp() throws IOException, JsonValidationException, ConfigNotFoundException { + void setUp() { testEnv = TestWorkflowEnvironment.newInstance(); syncWorker = testEnv.newWorker(SYNC_QUEUE); client = testEnv.getWorkflowClient(); @@ -140,8 +139,6 @@ void setUp() throws IOException, JsonValidationException, ConfigNotFoundExceptio normalizationSummary = new NormalizationSummary(); - standardSync = new StandardSync().withSourceId(SOURCE_ID).withStatus(Status.ACTIVE); - normalizationInput = new NormalizationInput() .withDestinationConfiguration(syncInput.getDestinationConfiguration()) .withCatalog(syncInput.getCatalog()) @@ -163,8 +160,9 @@ void setUp() throws IOException, JsonValidationException, ConfigNotFoundExceptio when(normalizationActivity.generateNormalizationInput(any(), any())).thenReturn(normalizationInput); when(normalizationSummaryCheckActivity.shouldRunNormalization(any(), any(), any())).thenReturn(true); - when(configFetchActivity.getStandardSync(sync.getConnectionId())).thenReturn(standardSync); + when(configFetchActivity.getSourceId(sync.getConnectionId())).thenReturn(Optional.of(SOURCE_ID)); when(refreshSchemaActivity.shouldRefreshSchema(SOURCE_ID)).thenReturn(true); + when(configFetchActivity.getStatus(sync.getConnectionId())).thenReturn(Optional.of(Status.ACTIVE)); longActivityOptions = ActivityOptions.newBuilder() .setScheduleToCloseTimeout(Duration.ofDays(3)) @@ -215,7 +213,7 @@ private StandardSyncOutput execute() { } @Test - void testSuccess() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { + void testSuccess() { doReturn(replicationSuccessOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -242,7 +240,7 @@ void testSuccess() throws IOException, JsonValidationException, ConfigNotFoundEx } @Test - void testReplicationFailure() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { + void testReplicationFailure() { doThrow(new IllegalArgumentException("induced exception")).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -260,7 +258,7 @@ void testReplicationFailure() throws IOException, JsonValidationException, Confi } @Test - void testReplicationFailedGracefully() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { + void testReplicationFailedGracefully() { doReturn(replicationFailOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -287,7 +285,7 @@ void testReplicationFailedGracefully() throws IOException, JsonValidationExcepti } @Test - void testNormalizationFailure() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { + void testNormalizationFailure() { doReturn(replicationSuccessOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -310,7 +308,7 @@ void testNormalizationFailure() throws IOException, JsonValidationException, Con } @Test - void testCancelDuringReplication() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { + void testCancelDuringReplication() { doAnswer(ignored -> { cancelWorkflow(); return replicationSuccessOutput; @@ -331,7 +329,7 @@ void testCancelDuringReplication() throws IOException, JsonValidationException, } @Test - void testCancelDuringNormalization() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { + void testCancelDuringNormalization() { doReturn(replicationSuccessOutput).when(replicationActivity).replicate( JOB_RUN_CONFIG, SOURCE_LAUNCHER_CONFIG, @@ -358,7 +356,7 @@ void testCancelDuringNormalization() throws IOException, JsonValidationException @Test @Disabled("This behavior has been disabled temporarily (OC Issue #741)") - void testSkipNormalization() throws IOException, JsonValidationException, ConfigNotFoundException, ApiException { + void testSkipNormalization() { final SyncStats syncStats = new SyncStats().withRecordsCommitted(0L); final StandardSyncSummary standardSyncSummary = new StandardSyncSummary().withTotalStats(syncStats); final StandardSyncOutput replicationSuccessOutputNoRecordsCommitted = @@ -402,8 +400,8 @@ void testWebhookOperation() { } @Test - void testSkipReplicationAfterRefreshSchema() throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { - when(configFetchActivity.getStandardSync(any())).thenReturn(new StandardSync().withSourceId(SOURCE_ID).withStatus(Status.INACTIVE)); + void testSkipReplicationAfterRefreshSchema() throws JsonValidationException, ConfigNotFoundException, IOException { + when(configFetchActivity.getStatus(any())).thenReturn(Optional.of(Status.INACTIVE)); StandardSyncOutput output = execute(); verifyShouldRefreshSchema(refreshSchemaActivity); verifyRefreshSchema(refreshSchemaActivity, sync); @@ -464,12 +462,11 @@ private static void verifyDbtTransform(final DbtTransformationActivity dbtTransf operatorDbtInput); } - private static void verifyShouldRefreshSchema(final RefreshSchemaActivity refreshSchemaActivity) throws IOException { + private static void verifyShouldRefreshSchema(final RefreshSchemaActivity refreshSchemaActivity) { verify(refreshSchemaActivity).shouldRefreshSchema(SOURCE_ID); } - private static void verifyRefreshSchema(final RefreshSchemaActivity refreshSchemaActivity, final StandardSync sync) - throws JsonValidationException, ConfigNotFoundException, IOException, ApiException { + private static void verifyRefreshSchema(final RefreshSchemaActivity refreshSchemaActivity, final StandardSync sync) { verify(refreshSchemaActivity).refreshSchema(SOURCE_ID, sync.getConnectionId()); } From 6b706302117ef9ee72db1a323406697b0854f46b Mon Sep 17 00:00:00 2001 From: alovew Date: Mon, 5 Dec 2022 15:49:19 -0800 Subject: [PATCH 23/24] separate singleton for apiclient --- .../workers/config/ApiClientBeanFactory.java | 66 ++++++++----------- 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java index afddd2640cc2a..c35c88dc1e10a 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/ApiClientBeanFactory.java @@ -10,6 +10,7 @@ import com.google.auth.oauth2.ServiceAccountCredentials; import io.airbyte.api.client.AirbyteApiClient; import io.airbyte.api.client.generated.SourceApi; +import io.airbyte.api.client.invoker.generated.ApiClient; import io.airbyte.commons.temporal.config.WorkerMode; import io.micronaut.context.BeanProvider; import io.micronaut.context.annotation.Factory; @@ -36,49 +37,34 @@ public class ApiClientBeanFactory { private static final int JWT_TTL_MINUTES = 5; @Singleton - public AirbyteApiClient airbyteApiClient( - @Value("${airbyte.internal.api.auth-header.name}") final String airbyteApiAuthHeaderName, - @Value("${airbyte.internal.api.host}") final String airbyteApiHost, - @Named("internalApiAuthToken") final BeanProvider internalApiAuthToken, - @Named("internalApiScheme") final String internalApiScheme) { - return new AirbyteApiClient( - new io.airbyte.api.client.invoker.generated.ApiClient() - .setScheme(internalApiScheme) - .setHost(parseHostName(airbyteApiHost)) - .setPort(parsePort(airbyteApiHost)) - .setBasePath("/api") - .setHttpClientBuilder(HttpClient.newBuilder().version(Version.HTTP_1_1)) - .setRequestInterceptor(builder -> { - builder.setHeader("User-Agent", "WorkerApp"); - // internalApiAuthToken is in BeanProvider because we want to create a new token each - // time we send a request. - if (!airbyteApiAuthHeaderName.isBlank()) { - builder.setHeader(airbyteApiAuthHeaderName, internalApiAuthToken.get()); - } - })); - } - - @Singleton - public SourceApi sourceApi( - @Value("${airbyte.internal.api.auth-header.name}") final String airbyteApiAuthHeaderName, + public ApiClient apiClient(@Value("${airbyte.internal.api.auth-header.name}") final String airbyteApiAuthHeaderName, @Value("${airbyte.internal.api.host}") final String airbyteApiHost, @Named("internalApiAuthToken") final BeanProvider internalApiAuthToken, @Named("internalApiScheme") final String internalApiScheme) { - return new SourceApi( - new io.airbyte.api.client.invoker.generated.ApiClient() - .setScheme(internalApiScheme) - .setHost(parseHostName(airbyteApiHost)) - .setPort(parsePort(airbyteApiHost)) - .setBasePath("/api") - .setHttpClientBuilder(HttpClient.newBuilder().version(Version.HTTP_1_1)) - .setRequestInterceptor(builder -> { - builder.setHeader("User-Agent", "WorkerApp"); - // internalApiAuthToken is in BeanProvider because we want to create a new token each - // time we send a request. - if (!airbyteApiAuthHeaderName.isBlank()) { - builder.setHeader(airbyteApiAuthHeaderName, internalApiAuthToken.get()); - } - })); + return new io.airbyte.api.client.invoker.generated.ApiClient() + .setScheme(internalApiScheme) + .setHost(parseHostName(airbyteApiHost)) + .setPort(parsePort(airbyteApiHost)) + .setBasePath("/api") + .setHttpClientBuilder(HttpClient.newBuilder().version(Version.HTTP_1_1)) + .setRequestInterceptor(builder -> { + builder.setHeader("User-Agent", "WorkerApp"); + // internalApiAuthToken is in BeanProvider because we want to create a new token each + // time we send a request. + if (!airbyteApiAuthHeaderName.isBlank()) { + builder.setHeader(airbyteApiAuthHeaderName, internalApiAuthToken.get()); + } + }); + } + + @Singleton + public AirbyteApiClient airbyteApiClient(ApiClient apiClient) { + return new AirbyteApiClient(apiClient); + } + + @Singleton + public SourceApi sourceApi(final ApiClient apiClient) { + return new SourceApi(apiClient); } @Singleton From c98cd7b7750719a0a935bf8d620b87d8b6c264ee Mon Sep 17 00:00:00 2001 From: alovew Date: Mon, 5 Dec 2022 16:59:36 -0800 Subject: [PATCH 24/24] add comments --- .../workers/temporal/sync/RefreshSchemaActivityImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java index 9edb9abda26c8..c30631dcf6985 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/RefreshSchemaActivityImpl.java @@ -59,6 +59,7 @@ public void refreshSchema(UUID sourceCatalogId, UUID connectionId) { try { sourceApi.discoverSchemaForSource(requestBody); } catch (final Exception e) { + // catching this exception because we don't want to block replication due to a failed schema refresh log.error("Attempted schema refresh, but failed with error: ", e); } } @@ -71,6 +72,7 @@ private boolean schemaRefreshRanRecently(UUID sourceCatalogId) { } return mostRecentFetchEvent.get().getCreatedAt() > OffsetDateTime.now().minusHours(24l).toEpochSecond(); } catch (IOException e) { + // catching this exception because we don't want to block replication due to a failed schema refresh log.info("Encountered an error fetching most recent actor catalog fetch event: ", e); return true; }