diff --git a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java index cda5b52f58644..20db344f2fd63 100644 --- a/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java +++ b/airbyte-scheduler/persistence/src/main/java/io/airbyte/scheduler/persistence/JobPersistence.java @@ -75,12 +75,12 @@ public interface JobPersistence { // /** - * Create a new attempt for a job. Throws {@link IllegalStateException} if the job is already in a - * terminal state. + * Create a new attempt for a job and return its attempt number. Throws + * {@link IllegalStateException} if the job is already in a terminal state. * * @param jobId job for which an attempt will be created * @param logPath path where logs should be written for the attempt - * @return id of the attempt + * @return The attempt number of the created attempt (see {@link DefaultJobPersistence}) * @throws IOException exception due to interaction with persistence */ int createAttempt(long jobId, Path logPath) throws IOException; diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index e37a5b9a44824..ea4d418ed7c05 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -33,7 +33,7 @@ dependencies { implementation project(':airbyte-scheduler:models') testImplementation 'io.temporal:temporal-testing:1.8.1' - testImplementation 'io.temporal:temporal-testing-junit5:1.5.0' // versioned separately from rest of temporal + testImplementation 'com.jayway.jsonpath:json-path:2.7.0' testImplementation "org.flywaydb:flyway-core:7.14.0" testImplementation 'org.mockito:mockito-inline:4.0.0' testImplementation 'org.postgresql:postgresql:42.2.18' diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java index c654f034880be..16ecec71a2fbc 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java @@ -21,15 +21,20 @@ import io.airbyte.workers.temporal.scheduling.activities.GenerateInputActivity; import io.airbyte.workers.temporal.scheduling.activities.GenerateInputActivity.GeneratedJobInput; import io.airbyte.workers.temporal.scheduling.activities.GenerateInputActivity.SyncInput; +import io.airbyte.workers.temporal.scheduling.activities.GenerateInputActivity.SyncInputWithAttemptNumber; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptCreationInput; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptCreationOutput; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptFailureInput; +import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptNumberCreationOutput; +import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptNumberFailureInput; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobCancelledInput; +import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobCancelledInputWithAttemptNumber; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobCreationInput; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobCreationOutput; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobFailureInput; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobSuccessInput; +import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobSuccessInputWithAttemptNumber; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.ReportJobStartInput; import io.airbyte.workers.temporal.scheduling.shared.ActivityConfiguration; import io.airbyte.workers.temporal.scheduling.state.WorkflowInternalState; @@ -60,6 +65,9 @@ public class ConnectionManagerWorkflowImpl implements ConnectionManagerWorkflow private static final int TASK_QUEUE_CHANGE_CURRENT_VERSION = 1; private static final int AUTO_DISABLE_FAILING_CONNECTION_CHANGE_CURRENT_VERSION = 1; + private static final String RENAME_ATTEMPT_ID_TO_NUMBER_TAG = "rename_attempt_id_to_number"; + private static final int RENAME_ATTEMPT_ID_TO_NUMBER_CURRENT_VERSION = 1; + private WorkflowState workflowState = new WorkflowState(UUID.randomUUID(), new NoopStateListener()); private final WorkflowInternalState workflowInternalState = new WorkflowInternalState(); @@ -147,7 +155,7 @@ private CancellationScope generateSyncWorkflowRunnable(final ConnectionUpdaterIn workflowInternalState.setJobId(getOrCreateJobId(connectionUpdaterInput)); - workflowInternalState.setAttemptId(createAttemptId(workflowInternalState.getJobId())); + workflowInternalState.setAttemptNumber(createAttempt(workflowInternalState.getJobId())); final GeneratedJobInput jobInputs = getJobInput(); @@ -183,13 +191,13 @@ private CancellationScope generateSyncWorkflowRunnable(final ConnectionUpdaterIn af.getActivityType(), af.getCause(), workflowInternalState.getJobId(), - workflowInternalState.getAttemptId())); + workflowInternalState.getAttemptNumber())); reportFailure(connectionUpdaterInput, standardSyncOutput); prepareForNextRunAndContinueAsNew(connectionUpdaterInput); } else { workflowInternalState.getFailures().add( FailureHelper.unknownOriginFailure(childWorkflowFailure.getCause(), workflowInternalState.getJobId(), - workflowInternalState.getAttemptId())); + workflowInternalState.getAttemptNumber())); reportFailure(connectionUpdaterInput, standardSyncOutput); prepareForNextRunAndContinueAsNew(connectionUpdaterInput); } @@ -199,20 +207,41 @@ private CancellationScope generateSyncWorkflowRunnable(final ConnectionUpdaterIn private void reportSuccess(final ConnectionUpdaterInput connectionUpdaterInput, final StandardSyncOutput standardSyncOutput) { workflowState.setSuccess(true); - runMandatoryActivity(jobCreationAndStatusUpdateActivity::jobSuccess, new JobSuccessInput( - workflowInternalState.getJobId(), - workflowInternalState.getAttemptId(), - standardSyncOutput)); + final int attemptCreationVersion = + Workflow.getVersion(RENAME_ATTEMPT_ID_TO_NUMBER_TAG, Workflow.DEFAULT_VERSION, RENAME_ATTEMPT_ID_TO_NUMBER_CURRENT_VERSION); + + if (attemptCreationVersion < RENAME_ATTEMPT_ID_TO_NUMBER_CURRENT_VERSION) { + runMandatoryActivity(jobCreationAndStatusUpdateActivity::jobSuccess, new JobSuccessInput( + workflowInternalState.getJobId(), + workflowInternalState.getAttemptNumber(), + standardSyncOutput)); + } else { + runMandatoryActivity(jobCreationAndStatusUpdateActivity::jobSuccessWithAttemptNumber, new JobSuccessInputWithAttemptNumber( + workflowInternalState.getJobId(), + workflowInternalState.getAttemptNumber(), + standardSyncOutput)); + } resetNewConnectionInput(connectionUpdaterInput); } private void reportFailure(final ConnectionUpdaterInput connectionUpdaterInput, final StandardSyncOutput standardSyncOutput) { - runMandatoryActivity(jobCreationAndStatusUpdateActivity::attemptFailure, new AttemptFailureInput( - workflowInternalState.getJobId(), - workflowInternalState.getAttemptId(), - standardSyncOutput, - FailureHelper.failureSummary(workflowInternalState.getFailures(), workflowInternalState.getPartialSuccess()))); + final int attemptCreationVersion = + Workflow.getVersion(RENAME_ATTEMPT_ID_TO_NUMBER_TAG, Workflow.DEFAULT_VERSION, RENAME_ATTEMPT_ID_TO_NUMBER_CURRENT_VERSION); + + if (attemptCreationVersion < RENAME_ATTEMPT_ID_TO_NUMBER_CURRENT_VERSION) { + runMandatoryActivity(jobCreationAndStatusUpdateActivity::attemptFailure, new AttemptFailureInput( + workflowInternalState.getJobId(), + workflowInternalState.getAttemptNumber(), + standardSyncOutput, + FailureHelper.failureSummary(workflowInternalState.getFailures(), workflowInternalState.getPartialSuccess()))); + } else { + runMandatoryActivity(jobCreationAndStatusUpdateActivity::attemptFailureWithAttemptNumber, new AttemptNumberFailureInput( + workflowInternalState.getJobId(), + workflowInternalState.getAttemptNumber(), + standardSyncOutput, + FailureHelper.failureSummary(workflowInternalState.getFailures(), workflowInternalState.getPartialSuccess()))); + } final int maxAttempt = configFetchActivity.getMaxAttempt().getMaxAttempt(); final int attemptNumber = connectionUpdaterInput.getAttemptNumber(); @@ -306,20 +335,20 @@ public WorkflowState getState() { @Override public JobInformation getJobInformation() { final Long jobId = workflowInternalState.getJobId(); - final Integer attemptId = workflowInternalState.getAttemptId(); + final Integer attemptNumber = workflowInternalState.getAttemptNumber(); return new JobInformation( jobId == null ? NON_RUNNING_JOB_ID : jobId, - attemptId == null ? NON_RUNNING_ATTEMPT_ID : attemptId); + attemptNumber == null ? NON_RUNNING_ATTEMPT_ID : attemptNumber); } @Override public QuarantinedInformation getQuarantinedInformation() { final Long jobId = workflowInternalState.getJobId(); - final Integer attemptId = workflowInternalState.getAttemptId(); + final Integer attemptNumber = workflowInternalState.getAttemptNumber(); return new QuarantinedInformation( connectionId, jobId == null ? NON_RUNNING_JOB_ID : jobId, - attemptId == null ? NON_RUNNING_ATTEMPT_ID : attemptId, + attemptNumber == null ? NON_RUNNING_ATTEMPT_ID : attemptNumber, workflowState.isQuarantined()); } @@ -419,15 +448,31 @@ private Long getOrCreateJobId(final ConnectionUpdaterInput connectionUpdaterInpu /** * Create a new attempt for a given jobId + * + * @param jobId - the jobId associated with the new attempt + * + * @return The attempt number */ - private Integer createAttemptId(final long jobId) { - final AttemptCreationOutput attemptCreationOutput = + private Integer createAttempt(final long jobId) { + final int attemptCreationVersion = + Workflow.getVersion(RENAME_ATTEMPT_ID_TO_NUMBER_TAG, Workflow.DEFAULT_VERSION, RENAME_ATTEMPT_ID_TO_NUMBER_CURRENT_VERSION); + + // Retrieve the attempt number but name it attempt id + if (attemptCreationVersion < RENAME_ATTEMPT_ID_TO_NUMBER_CURRENT_VERSION) { + final AttemptCreationOutput attemptCreationOutput = + runMandatoryActivityWithOutput( + jobCreationAndStatusUpdateActivity::createNewAttempt, + new AttemptCreationInput( + jobId)); + return attemptCreationOutput.getAttemptId(); + } + + final AttemptNumberCreationOutput attemptNumberCreationOutput = runMandatoryActivityWithOutput( - jobCreationAndStatusUpdateActivity::createNewAttempt, + jobCreationAndStatusUpdateActivity::createNewAttemptNumber, new AttemptCreationInput( jobId)); - - return attemptCreationOutput.getAttemptId(); + return attemptNumberCreationOutput.getAttemptNumber(); } /** @@ -436,14 +481,30 @@ private Integer createAttemptId(final long jobId) { */ private GeneratedJobInput getJobInput() { final Long jobId = workflowInternalState.getJobId(); - final Integer attemptId = workflowInternalState.getAttemptId(); - final SyncInput getSyncInputActivitySyncInput = new SyncInput( - attemptId, + final Integer attemptNumber = workflowInternalState.getAttemptNumber(); + final int attemptCreationVersion = + Workflow.getVersion(RENAME_ATTEMPT_ID_TO_NUMBER_TAG, Workflow.DEFAULT_VERSION, RENAME_ATTEMPT_ID_TO_NUMBER_CURRENT_VERSION); + + if (attemptCreationVersion < RENAME_ATTEMPT_ID_TO_NUMBER_CURRENT_VERSION) { + final SyncInput getSyncInputActivitySyncInput = new SyncInput( + attemptNumber, + jobId, + workflowState.isResetConnection()); + + final GeneratedJobInput syncWorkflowInputs = runMandatoryActivityWithOutput( + getSyncInputActivity::getSyncWorkflowInput, + getSyncInputActivitySyncInput); + + return syncWorkflowInputs; + } + + final SyncInputWithAttemptNumber getSyncInputActivitySyncInput = new SyncInputWithAttemptNumber( + attemptNumber, jobId, workflowState.isResetConnection()); final GeneratedJobInput syncWorkflowInputs = runMandatoryActivityWithOutput( - getSyncInputActivity::getSyncWorkflowInput, + getSyncInputActivity::getSyncWorkflowInputWithAttemptNumber, getSyncInputActivitySyncInput); return syncWorkflowInputs; @@ -529,14 +590,25 @@ private void deleteConnectionBeforeTerminatingTheWorkflow() { private void reportCancelledAndContinueWith(final boolean isReset, final ConnectionUpdaterInput connectionUpdaterInput) { workflowState.setContinueAsReset(isReset); final Long jobId = workflowInternalState.getJobId(); - final Integer attemptId = workflowInternalState.getAttemptId(); + final Integer attemptNumber = workflowInternalState.getAttemptNumber(); final Set failures = workflowInternalState.getFailures(); final Boolean partialSuccess = workflowInternalState.getPartialSuccess(); - runMandatoryActivity(jobCreationAndStatusUpdateActivity::jobCancelled, - new JobCancelledInput( - jobId, - attemptId, - FailureHelper.failureSummaryForCancellation(jobId, attemptId, failures, partialSuccess))); + final int attemptCreationVersion = + Workflow.getVersion(RENAME_ATTEMPT_ID_TO_NUMBER_TAG, Workflow.DEFAULT_VERSION, RENAME_ATTEMPT_ID_TO_NUMBER_CURRENT_VERSION); + + if (attemptCreationVersion < RENAME_ATTEMPT_ID_TO_NUMBER_CURRENT_VERSION) { + runMandatoryActivity(jobCreationAndStatusUpdateActivity::jobCancelled, + new JobCancelledInput( + jobId, + attemptNumber, + FailureHelper.failureSummaryForCancellation(jobId, attemptNumber, failures, partialSuccess))); + } else { + runMandatoryActivity(jobCreationAndStatusUpdateActivity::jobCancelledWithAttemptNumber, + new JobCancelledInputWithAttemptNumber( + jobId, + attemptNumber, + FailureHelper.failureSummaryForCancellation(jobId, attemptNumber, failures, partialSuccess))); + } resetNewConnectionInput(connectionUpdaterInput); prepareForNextRunAndContinueAsNew(connectionUpdaterInput); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivity.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivity.java index 0a1ed70c30083..60191311ef344 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivity.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivity.java @@ -27,6 +27,17 @@ class SyncInput { } + @Data + @NoArgsConstructor + @AllArgsConstructor + class SyncInputWithAttemptNumber { + + private int attemptNumber; + private long jobId; + private boolean reset; + + } + @Data @NoArgsConstructor @AllArgsConstructor @@ -45,4 +56,10 @@ class GeneratedJobInput { @ActivityMethod GeneratedJobInput getSyncWorkflowInput(SyncInput input); + /** + * This generate the input needed by the child sync workflow + */ + @ActivityMethod + GeneratedJobInput getSyncWorkflowInputWithAttemptNumber(SyncInputWithAttemptNumber input); + } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java index 30edaf108adcc..bcb939d022c88 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/GenerateInputActivityImpl.java @@ -76,4 +76,12 @@ public GeneratedJobInput getSyncWorkflowInput(final SyncInput input) { } } + @Override + public GeneratedJobInput getSyncWorkflowInputWithAttemptNumber(final SyncInputWithAttemptNumber input) { + return getSyncWorkflowInput(new SyncInput( + input.getAttemptNumber(), + input.getJobId(), + input.isReset())); + } + } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivity.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivity.java index aa45b53b0e8c0..2de587c2a15bd 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivity.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivity.java @@ -72,6 +72,24 @@ class AttemptCreationOutput { @ActivityMethod AttemptCreationOutput createNewAttempt(AttemptCreationInput input) throws RetryableException; + @Data + @NoArgsConstructor + @AllArgsConstructor + class AttemptNumberCreationOutput { + + private int attemptNumber; + + } + + /** + * Create a new attempt for a given job ID + * + * @param input POJO containing the jobId + * @return A POJO containing the attemptNumber + */ + @ActivityMethod + AttemptNumberCreationOutput createNewAttemptNumber(AttemptCreationInput input) throws RetryableException; + @Data @NoArgsConstructor @AllArgsConstructor @@ -89,6 +107,23 @@ class JobSuccessInput { @ActivityMethod void jobSuccess(JobSuccessInput input); + @Data + @NoArgsConstructor + @AllArgsConstructor + class JobSuccessInputWithAttemptNumber { + + private long jobId; + private int attemptNumber; + private StandardSyncOutput standardSyncOutput; + + } + + /** + * Set a job status as successful + */ + @ActivityMethod + void jobSuccessWithAttemptNumber(JobSuccessInputWithAttemptNumber input); + @Data @NoArgsConstructor @AllArgsConstructor @@ -123,6 +158,24 @@ class AttemptFailureInput { @ActivityMethod void attemptFailure(AttemptFailureInput input); + @Data + @NoArgsConstructor + @AllArgsConstructor + class AttemptNumberFailureInput { + + private long jobId; + private int attemptNumber; + private StandardSyncOutput standardSyncOutput; + private AttemptFailureSummary attemptFailureSummary; + + } + + /** + * Set an attempt status as failed + */ + @ActivityMethod + void attemptFailureWithAttemptNumber(AttemptNumberFailureInput input); + @Data @NoArgsConstructor @AllArgsConstructor @@ -140,6 +193,23 @@ class JobCancelledInput { @ActivityMethod void jobCancelled(JobCancelledInput input); + @Data + @NoArgsConstructor + @AllArgsConstructor + class JobCancelledInputWithAttemptNumber { + + private long jobId; + private int attemptNumber; + private AttemptFailureSummary attemptFailureSummary; + + } + + /** + * Set a job status as cancelled + */ + @ActivityMethod + void jobCancelledWithAttemptNumber(JobCancelledInputWithAttemptNumber input); + @Data @NoArgsConstructor @AllArgsConstructor diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java index e027966751744..b9a76065e7761 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java @@ -128,6 +128,24 @@ public AttemptCreationOutput createNewAttempt(final AttemptCreationInput input) } } + @Override + public AttemptNumberCreationOutput createNewAttemptNumber(final AttemptCreationInput input) throws RetryableException { + try { + final long jobId = input.getJobId(); + final Job createdJob = jobPersistence.getJob(jobId); + + final WorkerRun workerRun = temporalWorkerRunFactory.create(createdJob); + final Path logFilePath = workerRun.getJobRoot().resolve(LogClientSingleton.LOG_FILENAME); + final int persistedAttemptNumber = jobPersistence.createAttempt(jobId, logFilePath); + emitJobIdToReleaseStagesMetric(MetricsRegistry.ATTEMPT_CREATED_BY_RELEASE_STAGE, jobId); + + LogClientSingleton.getInstance().setJobMdc(workerEnvironment, logConfigs, workerRun.getJobRoot()); + return new AttemptNumberCreationOutput(persistedAttemptNumber); + } catch (final IOException e) { + throw new RetryableException(e); + } + } + @Override public void jobSuccess(final JobSuccessInput input) { try { @@ -152,6 +170,14 @@ public void jobSuccess(final JobSuccessInput input) { } } + @Override + public void jobSuccessWithAttemptNumber(final JobSuccessInputWithAttemptNumber input) { + jobSuccess(new JobSuccessInput( + input.getJobId(), + input.getAttemptNumber(), + input.getStandardSyncOutput())); + } + @Override public void jobFailure(final JobFailureInput input) { try { @@ -191,6 +217,15 @@ public void attemptFailure(final AttemptFailureInput input) { } } + @Override + public void attemptFailureWithAttemptNumber(final AttemptNumberFailureInput input) { + attemptFailure(new AttemptFailureInput( + input.getJobId(), + input.getAttemptNumber(), + input.getStandardSyncOutput(), + input.getAttemptFailureSummary())); + } + @Override public void jobCancelled(final JobCancelledInput input) { try { @@ -209,6 +244,14 @@ public void jobCancelled(final JobCancelledInput input) { } } + @Override + public void jobCancelledWithAttemptNumber(final JobCancelledInputWithAttemptNumber input) { + jobCancelled(new JobCancelledInput( + input.getJobId(), + input.getAttemptNumber(), + input.getAttemptFailureSummary())); + } + @Override public void reportJobStart(final ReportJobStartInput input) { try { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/state/WorkflowInternalState.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/state/WorkflowInternalState.java index 822bbfe0ba480..d9bdcb2d58000 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/state/WorkflowInternalState.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/state/WorkflowInternalState.java @@ -17,7 +17,7 @@ public class WorkflowInternalState { private Long jobId = null; - private Integer attemptId = null; + private Integer attemptNumber = null; // StandardSyncOutput standardSyncOutput = null; private final Set failures = new HashSet<>(); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java index ca897b5afe808..8353b91169401 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java @@ -16,13 +16,14 @@ import io.airbyte.workers.temporal.scheduling.activities.ConfigFetchActivity.ScheduleRetrieverOutput; import io.airbyte.workers.temporal.scheduling.activities.ConnectionDeletionActivity; import io.airbyte.workers.temporal.scheduling.activities.GenerateInputActivity.GeneratedJobInput; -import io.airbyte.workers.temporal.scheduling.activities.GenerateInputActivity.SyncInput; +import io.airbyte.workers.temporal.scheduling.activities.GenerateInputActivity.SyncInputWithAttemptNumber; import io.airbyte.workers.temporal.scheduling.activities.GenerateInputActivityImpl; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity; -import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptCreationOutput; -import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptFailureInput; -import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobCancelledInput; +import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptNumberCreationOutput; +import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptNumberFailureInput; +import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobCancelledInputWithAttemptNumber; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobCreationOutput; +import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobSuccessInputWithAttemptNumber; import io.airbyte.workers.temporal.scheduling.state.WorkflowState; import io.airbyte.workers.temporal.scheduling.state.listener.TestStateListener; import io.airbyte.workers.temporal.scheduling.state.listener.WorkflowStateChangedListener.ChangedStateEvent; @@ -121,11 +122,11 @@ public void setUp() { .thenReturn(new JobCreationOutput( 1L)); - Mockito.when(mJobCreationAndStatusUpdateActivity.createNewAttempt(Mockito.any())) - .thenReturn(new AttemptCreationOutput( + Mockito.when(mJobCreationAndStatusUpdateActivity.createNewAttemptNumber(Mockito.any())) + .thenReturn(new AttemptNumberCreationOutput( 1)); - Mockito.when(mGenerateInputActivityImpl.getSyncWorkflowInput(Mockito.any(SyncInput.class))) + Mockito.when(mGenerateInputActivityImpl.getSyncWorkflowInputWithAttemptNumber(Mockito.any(SyncInputWithAttemptNumber.class))) .thenReturn( new GeneratedJobInput( new JobRunConfig(), @@ -510,7 +511,8 @@ public void cancelRunning() throws InterruptedException { .filteredOn(changedStateEvent -> changedStateEvent.getField() == StateField.CANCELLED && changedStateEvent.isValue()) .hasSizeGreaterThanOrEqualTo(1); - Mockito.verify(mJobCreationAndStatusUpdateActivity).jobCancelled(Mockito.argThat(new HasCancellationFailure(JOB_ID, ATTEMPT_ID))); + Mockito.verify(mJobCreationAndStatusUpdateActivity) + .jobCancelledWithAttemptNumber(Mockito.argThat(new HasCancellationFailure(JOB_ID, ATTEMPT_ID))); } @RepeatedTest(10) @@ -594,7 +596,7 @@ public void resetCancelRunningWorkflow() throws InterruptedException { .filteredOn(changedStateEvent -> changedStateEvent.getField() == StateField.RESET && changedStateEvent.isValue()) .hasSizeGreaterThanOrEqualTo(1); - Mockito.verify(mJobCreationAndStatusUpdateActivity).jobCancelled(Mockito.any()); + Mockito.verify(mJobCreationAndStatusUpdateActivity).jobCancelledWithAttemptNumber(Mockito.any(JobCancelledInputWithAttemptNumber.class)); } @@ -687,7 +689,7 @@ public void updatedSignalReceivedWhileRunning() throws InterruptedException { .filteredOn(changedStateEvent -> changedStateEvent.getField() == StateField.UPDATED && changedStateEvent.isValue()) .hasSizeGreaterThanOrEqualTo(1); - Mockito.verify(mJobCreationAndStatusUpdateActivity).jobSuccess(Mockito.any()); + Mockito.verify(mJobCreationAndStatusUpdateActivity).jobSuccessWithAttemptNumber(Mockito.any(JobSuccessInputWithAttemptNumber.class)); } } @@ -747,8 +749,10 @@ public void testSourceAndDestinationFailuresRecorded() throws InterruptedExcepti workflow.submitManualSync(); testEnv.sleep(Duration.ofMinutes(1L)); // any time after no-waiting manual run - Mockito.verify(mJobCreationAndStatusUpdateActivity).attemptFailure(Mockito.argThat(new HasFailureFromOrigin(FailureOrigin.SOURCE))); - Mockito.verify(mJobCreationAndStatusUpdateActivity).attemptFailure(Mockito.argThat(new HasFailureFromOrigin(FailureOrigin.DESTINATION))); + Mockito.verify(mJobCreationAndStatusUpdateActivity) + .attemptFailureWithAttemptNumber(Mockito.argThat(new HasFailureFromOrigin(FailureOrigin.SOURCE))); + Mockito.verify(mJobCreationAndStatusUpdateActivity) + .attemptFailureWithAttemptNumber(Mockito.argThat(new HasFailureFromOrigin(FailureOrigin.DESTINATION))); } @RepeatedTest(10) @@ -783,7 +787,8 @@ public void testNormalizationFailure() throws InterruptedException { workflow.submitManualSync(); testEnv.sleep(Duration.ofMinutes(1L)); // any time after no-waiting manual run - Mockito.verify(mJobCreationAndStatusUpdateActivity).attemptFailure(Mockito.argThat(new HasFailureFromOrigin(FailureOrigin.NORMALIZATION))); + Mockito.verify(mJobCreationAndStatusUpdateActivity) + .attemptFailureWithAttemptNumber(Mockito.argThat(new HasFailureFromOrigin(FailureOrigin.NORMALIZATION))); } @RepeatedTest(10) @@ -818,7 +823,8 @@ public void testDbtFailureRecorded() throws InterruptedException { workflow.submitManualSync(); testEnv.sleep(Duration.ofMinutes(1L)); // any time after no-waiting manual run - Mockito.verify(mJobCreationAndStatusUpdateActivity).attemptFailure(Mockito.argThat(new HasFailureFromOrigin(FailureOrigin.DBT))); + Mockito.verify(mJobCreationAndStatusUpdateActivity) + .attemptFailureWithAttemptNumber(Mockito.argThat(new HasFailureFromOrigin(FailureOrigin.DBT))); } @RepeatedTest(10) @@ -853,7 +859,8 @@ public void testPersistenceFailureRecorded() throws InterruptedException { workflow.submitManualSync(); testEnv.sleep(Duration.ofMinutes(1L)); // any time after no-waiting manual run - Mockito.verify(mJobCreationAndStatusUpdateActivity).attemptFailure(Mockito.argThat(new HasFailureFromOrigin(FailureOrigin.PERSISTENCE))); + Mockito.verify(mJobCreationAndStatusUpdateActivity) + .attemptFailureWithAttemptNumber(Mockito.argThat(new HasFailureFromOrigin(FailureOrigin.PERSISTENCE))); } @RepeatedTest(10) @@ -888,7 +895,8 @@ public void testReplicationFailureRecorded() throws InterruptedException { workflow.submitManualSync(); testEnv.sleep(Duration.ofMinutes(1L)); // any time after no-waiting manual run - Mockito.verify(mJobCreationAndStatusUpdateActivity).attemptFailure(Mockito.argThat(new HasFailureFromOrigin(FailureOrigin.REPLICATION))); + Mockito.verify(mJobCreationAndStatusUpdateActivity) + .attemptFailureWithAttemptNumber(Mockito.argThat(new HasFailureFromOrigin(FailureOrigin.REPLICATION))); } } @@ -906,12 +914,13 @@ public static Stream getSetupFailingFailingActivityBeforeRun() { return Stream.of( Arguments.of(new Thread(() -> Mockito.when(mJobCreationAndStatusUpdateActivity.createNewJob(Mockito.any())) .thenThrow(ApplicationFailure.newNonRetryableFailure("", "")))), - Arguments.of(new Thread(() -> Mockito.when(mJobCreationAndStatusUpdateActivity.createNewAttempt(Mockito.any())) + Arguments.of(new Thread(() -> Mockito.when(mJobCreationAndStatusUpdateActivity.createNewAttemptNumber(Mockito.any())) .thenThrow(ApplicationFailure.newNonRetryableFailure("", "")))), Arguments.of(new Thread(() -> Mockito.doThrow(ApplicationFailure.newNonRetryableFailure("", "")) .when(mJobCreationAndStatusUpdateActivity).reportJobStart(Mockito.any()))), - Arguments.of(new Thread(() -> Mockito.when(mGenerateInputActivityImpl.getSyncWorkflowInput(Mockito.any())) - .thenThrow(ApplicationFailure.newNonRetryableFailure("", ""))))); + Arguments.of(new Thread( + () -> Mockito.when(mGenerateInputActivityImpl.getSyncWorkflowInputWithAttemptNumber(Mockito.any(SyncInputWithAttemptNumber.class))) + .thenThrow(ApplicationFailure.newNonRetryableFailure("", ""))))); } @ParameterizedTest @@ -996,10 +1005,10 @@ public static Stream getSetupFailingFailingActivityAfterRun() { return Stream.of( Arguments.of((Consumer) ((ConnectionManagerWorkflow workflow) -> System.out.println("do Nothing")), new Thread(() -> Mockito.doThrow(ApplicationFailure.newNonRetryableFailure("", "")) - .when(mJobCreationAndStatusUpdateActivity).jobSuccess(Mockito.any()))), + .when(mJobCreationAndStatusUpdateActivity).jobSuccessWithAttemptNumber(Mockito.any(JobSuccessInputWithAttemptNumber.class)))), Arguments.of((Consumer) ((ConnectionManagerWorkflow workflow) -> workflow.cancelJob()), new Thread(() -> Mockito.doThrow(ApplicationFailure.newNonRetryableFailure("", "")) - .when(mJobCreationAndStatusUpdateActivity).jobCancelled(Mockito.any()))), + .when(mJobCreationAndStatusUpdateActivity).jobCancelledWithAttemptNumber(Mockito.any(JobCancelledInputWithAttemptNumber.class)))), Arguments.of((Consumer) ((ConnectionManagerWorkflow workflow) -> workflow.deleteConnection()), new Thread(() -> Mockito.doThrow(ApplicationFailure.newNonRetryableFailure("", "")) .when(mConnectionDeletionActivity).deleteConnection(Mockito.any())))); @@ -1044,7 +1053,7 @@ void testGetStuckAfterRun(final Consumer signalSender Assertions.assertThat(events) .filteredOn(changedStateEvent -> changedStateEvent.getField() == StateField.QUARANTINED && changedStateEvent.isValue()) - .hasSize(1); + .hasSizeGreaterThanOrEqualTo(1); } } @@ -1127,7 +1136,10 @@ public void failedResetContinueAsReset() throws InterruptedException { @DisplayName("Test that we are getting stuck if the report of a failure happen") void testGetStuckAfterRun() throws InterruptedException { Mockito.doThrow(ApplicationFailure.newNonRetryableFailure("", "")) - .when(mJobCreationAndStatusUpdateActivity).attemptFailure(Mockito.any()); + .when(mJobCreationAndStatusUpdateActivity).attemptFailureWithAttemptNumber(Mockito.any()); + + Mockito.when(mConfigFetchActivity.getMaxAttempt()) + .thenReturn(new GetMaxAttemptOutput(3)); final UUID testId = UUID.randomUUID(); final TestStateListener testStateListener = new TestStateListener(); @@ -1206,7 +1218,7 @@ public void failedResetJobWaitsOnRestart() throws InterruptedException { } - private class HasFailureFromOrigin implements ArgumentMatcher { + private class HasFailureFromOrigin implements ArgumentMatcher { private final FailureOrigin expectedFailureOrigin; @@ -1215,26 +1227,26 @@ public HasFailureFromOrigin(final FailureOrigin failureOrigin) { } @Override - public boolean matches(final AttemptFailureInput arg) { + public boolean matches(final AttemptNumberFailureInput arg) { return arg.getAttemptFailureSummary().getFailures().stream().anyMatch(f -> f.getFailureOrigin().equals(expectedFailureOrigin)); } } - private class HasCancellationFailure implements ArgumentMatcher { + private class HasCancellationFailure implements ArgumentMatcher { private final long expectedJobId; - private final int expectedAttemptId; + private final int expectedAttemptNumber; - public HasCancellationFailure(final long jobId, final int attemptId) { + public HasCancellationFailure(final long jobId, final int attemptNumber) { this.expectedJobId = jobId; - this.expectedAttemptId = attemptId; + this.expectedAttemptNumber = attemptNumber; } @Override - public boolean matches(final JobCancelledInput arg) { + public boolean matches(final JobCancelledInputWithAttemptNumber arg) { return arg.getAttemptFailureSummary().getFailures().stream().anyMatch(f -> f.getFailureType().equals(FailureType.MANUAL_CANCELLATION)) - && arg.getJobId() == expectedJobId && arg.getAttemptId() == expectedAttemptId; + && arg.getJobId() == expectedJobId && arg.getAttemptNumber() == expectedAttemptNumber; } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/WorkflowReplayingTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/WorkflowReplayingTest.java new file mode 100644 index 0000000000000..bba04f6cea42a --- /dev/null +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/WorkflowReplayingTest.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.temporal.scheduling; + +import io.temporal.testing.WorkflowReplayer; +import java.io.File; +import java.net.URL; +import org.junit.jupiter.api.Test; + +// TODO: Auto generation of the input and more scenario coverage +public class WorkflowReplayingTest { + + @Test + public void replaySimpleSuccessfulWorkflow() throws Exception { + final URL historyPath = getClass().getClassLoader().getResource("workflowHistory.json"); + + final File historyFile = new File(historyPath.toURI()); + + WorkflowReplayer.replayWorkflowExecution(historyFile, ConnectionManagerWorkflowImpl.class); + } + +} diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java index 5f480cf4fb80e..f643bc4bd07ee 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java @@ -29,6 +29,7 @@ import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptCreationInput; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptCreationOutput; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptFailureInput; +import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.AttemptNumberCreationOutput; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobCancelledInput; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobCreationInput; import io.airbyte.workers.temporal.scheduling.activities.JobCreationAndStatusUpdateActivity.JobCreationOutput; @@ -84,6 +85,7 @@ public class JobCreationAndStatusUpdateActivityTest { private static final UUID CONNECTION_ID = UUID.randomUUID(); private static final long JOB_ID = 123L; private static final int ATTEMPT_ID = 0; + private static final int ATTEMPT_NUMBER = 1; private static final StandardSyncOutput standardSyncOutput = new StandardSyncOutput() .withStandardSyncSummary( new StandardSyncSummary() @@ -164,6 +166,56 @@ public void createAttemptThrowException() throws IOException { .hasCauseInstanceOf(IOException.class); } + @Test + @DisplayName("Test attempt creation") + public void createAttemptNumber() throws IOException { + Mockito.when(mConfigRepository.getDatabase()).thenReturn(Mockito.mock(ExceptionWrappingDatabase.class)); + + final Job mJob = Mockito.mock(Job.class); + + Mockito.when(mJobPersistence.getJob(JOB_ID)) + .thenReturn(mJob); + + final WorkerRun mWorkerRun = Mockito.mock(WorkerRun.class); + + Mockito.when(mTemporalWorkerRunFactory.create(mJob)) + .thenReturn(mWorkerRun); + + final Path mPath = Mockito.mock(Path.class); + final Path path = Path.of("test"); + Mockito.when(mPath.resolve(Mockito.anyString())) + .thenReturn(path); + Mockito.when(mWorkerRun.getJobRoot()) + .thenReturn(mPath); + + Mockito.when(mJobPersistence.createAttempt(JOB_ID, path)) + .thenReturn(ATTEMPT_NUMBER); + + final LogClientSingleton mLogClientSingleton = Mockito.mock(LogClientSingleton.class); + try (final MockedStatic utilities = Mockito.mockStatic(LogClientSingleton.class)) { + utilities.when(() -> LogClientSingleton.getInstance()) + .thenReturn(mLogClientSingleton); + + final AttemptNumberCreationOutput output = jobCreationAndStatusUpdateActivity.createNewAttemptNumber(new AttemptCreationInput( + JOB_ID)); + + Mockito.verify(mLogClientSingleton).setJobMdc(mWorkerEnvironment, mLogConfigs, mPath); + Assertions.assertThat(output.getAttemptNumber()).isEqualTo(ATTEMPT_NUMBER); + } + } + + @Test + @DisplayName("Test exception errors are properly wrapped") + public void createAttemptNumberThrowException() throws IOException { + Mockito.when(mJobPersistence.getJob(JOB_ID)) + .thenThrow(new IOException()); + + Assertions.assertThatThrownBy(() -> jobCreationAndStatusUpdateActivity.createNewAttemptNumber(new AttemptCreationInput( + JOB_ID))) + .isInstanceOf(RetryableException.class) + .hasCauseInstanceOf(IOException.class); + } + } @Nested diff --git a/airbyte-workers/src/test/resources/workflowHistory.json b/airbyte-workers/src/test/resources/workflowHistory.json new file mode 100644 index 0000000000000..a83619c8ea298 --- /dev/null +++ b/airbyte-workers/src/test/resources/workflowHistory.json @@ -0,0 +1,939 @@ +{ + "events": [ + { + "eventId": "1", + "eventTime": "2022-03-08T22:47:57.534705300Z", + "eventType": "WorkflowExecutionStarted", + "taskId": "1048664", + "workflowExecutionStartedEventAttributes": { + "workflowType": { + "name": "ConnectionManagerWorkflow" + }, + "taskQueue": { + "name": "CONNECTION_UPDATER", + "kind": "Normal" + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJjb25uZWN0aW9uSWQiOiJiOTE2ZmQ4Ni1hNWE2LTQ4YWMtODg4MC00OTc4NzRjZjUzY2YiLCJqb2JJZCI6bnVsbCwiYXR0ZW1wdElkIjpudWxsLCJmcm9tRmFpbHVyZSI6ZmFsc2UsImF0dGVtcHROdW1iZXIiOjEsIndvcmtmbG93U3RhdGUiOm51bGwsInJlc2V0Q29ubmVjdGlvbiI6ZmFsc2V9" + } + ] + }, + "workflowExecutionTimeout": "0s", + "workflowRunTimeout": "0s", + "workflowTaskTimeout": "10s", + "originalExecutionRunId": "4e6584e1-b030-4e42-a7bc-15dee5d195c0", + "identity": "1@3de809efb2ed", + "firstExecutionRunId": "4e6584e1-b030-4e42-a7bc-15dee5d195c0", + "retryPolicy": { + "initialInterval": "1s", + "backoffCoefficient": 2, + "maximumInterval": "100s", + "maximumAttempts": 1 + }, + "attempt": 1, + "firstWorkflowTaskBackoff": "0s", + "header": {} + } + }, + { + "eventId": "2", + "eventTime": "2022-03-08T22:47:57.534975800Z", + "eventType": "WorkflowTaskScheduled", + "taskId": "1048665", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "CONNECTION_UPDATER", + "kind": "Normal" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "3", + "eventTime": "2022-03-08T22:47:57.563121800Z", + "eventType": "WorkflowTaskStarted", + "taskId": "1048669", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "2", + "identity": "1@2741f9c3f558", + "requestId": "e9cf205d-de02-4139-b00d-fab56c4b9fd7" + } + }, + { + "eventId": "4", + "eventTime": "2022-03-08T22:47:57.646973200Z", + "eventType": "WorkflowTaskCompleted", + "taskId": "1048672", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "2", + "startedEventId": "3", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "5", + "eventTime": "2022-03-08T22:47:57.647029Z", + "eventType": "ActivityTaskScheduled", + "taskId": "1048673", + "activityTaskScheduledEventAttributes": { + "activityId": "c45be44b-784b-3a0c-9473-e80129b65969", + "activityType": { + "name": "GetTimeToWait" + }, + "taskQueue": { + "name": "CONNECTION_UPDATER", + "kind": "Normal" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJjb25uZWN0aW9uSWQiOiJiOTE2ZmQ4Ni1hNWE2LTQ4YWMtODg4MC00OTc4NzRjZjUzY2YifQ==" + } + ] + }, + "scheduleToCloseTimeout": "0s", + "scheduleToStartTimeout": "0s", + "startToCloseTimeout": "120s", + "heartbeatTimeout": "30s", + "workflowTaskCompletedEventId": "4", + "retryPolicy": { + "initialInterval": "30s", + "backoffCoefficient": 2, + "maximumInterval": "3000s", + "maximumAttempts": 10 + } + } + }, + { + "eventId": "6", + "eventTime": "2022-03-08T22:47:57.668813100Z", + "eventType": "ActivityTaskStarted", + "taskId": "1048677", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "5", + "identity": "1@2741f9c3f558", + "requestId": "df9001bd-0c42-4415-a631-0a37ee3f7698", + "attempt": 1 + } + }, + { + "eventId": "7", + "eventTime": "2022-03-08T22:47:57.856240400Z", + "eventType": "ActivityTaskCompleted", + "taskId": "1048678", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0aW1lVG9XYWl0IjowLjB9" + } + ] + }, + "scheduledEventId": "5", + "startedEventId": "6", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "8", + "eventTime": "2022-03-08T22:47:57.856293200Z", + "eventType": "WorkflowTaskScheduled", + "taskId": "1048679", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "1@2741f9c3f558:e7111d28-2117-4614-9bb1-8a73524b789f", + "kind": "Sticky" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "9", + "eventTime": "2022-03-08T22:47:57.876328300Z", + "eventType": "WorkflowTaskStarted", + "taskId": "1048683", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "8", + "identity": "e7111d28-2117-4614-9bb1-8a73524b789f", + "requestId": "80932560-7d5d-4f5f-9982-561857b07f50" + } + }, + { + "eventId": "10", + "eventTime": "2022-03-08T22:47:57.921753Z", + "eventType": "WorkflowTaskCompleted", + "taskId": "1048686", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "8", + "startedEventId": "9", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "11", + "eventTime": "2022-03-08T22:47:57.921814200Z", + "eventType": "ActivityTaskScheduled", + "taskId": "1048687", + "activityTaskScheduledEventAttributes": { + "activityId": "c7b20a16-db46-3dd1-b8ac-e2a93d3a8e0d", + "activityType": { + "name": "CreateNewJob" + }, + "taskQueue": { + "name": "CONNECTION_UPDATER", + "kind": "Normal" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJjb25uZWN0aW9uSWQiOiJiOTE2ZmQ4Ni1hNWE2LTQ4YWMtODg4MC00OTc4NzRjZjUzY2YiLCJyZXNldCI6ZmFsc2V9" + } + ] + }, + "scheduleToCloseTimeout": "0s", + "scheduleToStartTimeout": "0s", + "startToCloseTimeout": "120s", + "heartbeatTimeout": "30s", + "workflowTaskCompletedEventId": "10", + "retryPolicy": { + "initialInterval": "30s", + "backoffCoefficient": 2, + "maximumInterval": "3000s", + "maximumAttempts": 10 + } + } + }, + { + "eventId": "12", + "eventTime": "2022-03-08T22:47:57.942341Z", + "eventType": "ActivityTaskStarted", + "taskId": "1048691", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "11", + "identity": "1@2741f9c3f558", + "requestId": "29e77ce9-f715-4f19-9fe2-b5b94201d0b3", + "attempt": 1 + } + }, + { + "eventId": "13", + "eventTime": "2022-03-08T22:47:58.268669700Z", + "eventType": "ActivityTaskCompleted", + "taskId": "1048692", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJqb2JJZCI6MX0=" + } + ] + }, + "scheduledEventId": "11", + "startedEventId": "12", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "14", + "eventTime": "2022-03-08T22:47:58.268723100Z", + "eventType": "WorkflowTaskScheduled", + "taskId": "1048693", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "1@2741f9c3f558:e7111d28-2117-4614-9bb1-8a73524b789f", + "kind": "Sticky" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "15", + "eventTime": "2022-03-08T22:47:58.283301600Z", + "eventType": "WorkflowTaskStarted", + "taskId": "1048697", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "14", + "identity": "e7111d28-2117-4614-9bb1-8a73524b789f", + "requestId": "18d2f139-0794-4dfb-b36d-1448df3eb350" + } + }, + { + "eventId": "16", + "eventTime": "2022-03-08T22:47:58.302388600Z", + "eventType": "WorkflowTaskCompleted", + "taskId": "1048700", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "14", + "startedEventId": "15", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "17", + "eventTime": "2022-03-08T22:47:58.302431Z", + "eventType": "ActivityTaskScheduled", + "taskId": "1048701", + "activityTaskScheduledEventAttributes": { + "activityId": "3c1b8fa0-437b-3bc2-a365-352e9a5d765d", + "activityType": { + "name": "CreateNewAttempt" + }, + "taskQueue": { + "name": "CONNECTION_UPDATER", + "kind": "Normal" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJqb2JJZCI6MX0=" + } + ] + }, + "scheduleToCloseTimeout": "0s", + "scheduleToStartTimeout": "0s", + "startToCloseTimeout": "120s", + "heartbeatTimeout": "30s", + "workflowTaskCompletedEventId": "16", + "retryPolicy": { + "initialInterval": "30s", + "backoffCoefficient": 2, + "maximumInterval": "3000s", + "maximumAttempts": 10 + } + } + }, + { + "eventId": "18", + "eventTime": "2022-03-08T22:47:58.314956300Z", + "eventType": "ActivityTaskStarted", + "taskId": "1048705", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "17", + "identity": "1@2741f9c3f558", + "requestId": "0221b660-4f40-4bcf-9e6a-2a9d5898bb91", + "attempt": 1 + } + }, + { + "eventId": "19", + "eventTime": "2022-03-08T22:47:58.400030800Z", + "eventType": "ActivityTaskCompleted", + "taskId": "1048706", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJhdHRlbXB0SWQiOjB9" + } + ] + }, + "scheduledEventId": "17", + "startedEventId": "18", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "20", + "eventTime": "2022-03-08T22:47:58.400072800Z", + "eventType": "WorkflowTaskScheduled", + "taskId": "1048707", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "1@2741f9c3f558:e7111d28-2117-4614-9bb1-8a73524b789f", + "kind": "Sticky" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "21", + "eventTime": "2022-03-08T22:47:58.414415400Z", + "eventType": "WorkflowTaskStarted", + "taskId": "1048711", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "20", + "identity": "e7111d28-2117-4614-9bb1-8a73524b789f", + "requestId": "e4110bad-579c-4ac2-a3eb-3836d7d6f841" + } + }, + { + "eventId": "22", + "eventTime": "2022-03-08T22:47:58.431563800Z", + "eventType": "WorkflowTaskCompleted", + "taskId": "1048714", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "20", + "startedEventId": "21", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "23", + "eventTime": "2022-03-08T22:47:58.431607100Z", + "eventType": "ActivityTaskScheduled", + "taskId": "1048715", + "activityTaskScheduledEventAttributes": { + "activityId": "5aa065bf-5ef2-3e24-b560-c6b3c1f4e2bc", + "activityType": { + "name": "GetSyncWorkflowInput" + }, + "taskQueue": { + "name": "CONNECTION_UPDATER", + "kind": "Normal" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJhdHRlbXB0SWQiOjAsImpvYklkIjoxLCJyZXNldCI6ZmFsc2V9" + } + ] + }, + "scheduleToCloseTimeout": "0s", + "scheduleToStartTimeout": "0s", + "startToCloseTimeout": "120s", + "heartbeatTimeout": "30s", + "workflowTaskCompletedEventId": "22", + "retryPolicy": { + "initialInterval": "30s", + "backoffCoefficient": 2, + "maximumInterval": "3000s", + "maximumAttempts": 10 + } + } + }, + { + "eventId": "24", + "eventTime": "2022-03-08T22:47:58.445218800Z", + "eventType": "ActivityTaskStarted", + "taskId": "1048719", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "23", + "identity": "1@2741f9c3f558", + "requestId": "7165cc86-d137-4b0f-906b-a7e52a1074e4", + "attempt": 1 + } + }, + { + "eventId": "25", + "eventTime": "2022-03-08T22:47:58.471038600Z", + "eventType": "ActivityTaskCompleted", + "taskId": "1048720", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJqb2JSdW5Db25maWciOnsiam9iSWQiOiIxIiwiYXR0ZW1wdElkIjowfSwic291cmNlTGF1bmNoZXJDb25maWciOnsiam9iSWQiOiIxIiwiYXR0ZW1wdElkIjowLCJkb2NrZXJJbWFnZSI6ImFpcmJ5dGUvc291cmNlLXBva2VhcGk6MC4xLjQifSwiZGVzdGluYXRpb25MYXVuY2hlckNvbmZpZyI6eyJqb2JJZCI6IjEiLCJhdHRlbXB0SWQiOjAsImRvY2tlckltYWdlIjoiYWlyYnl0ZS9kZXN0aW5hdGlvbi1sb2NhbC1qc29uOjAuMi4xMCJ9LCJzeW5jSW5wdXQiOnsibmFtZXNwYWNlRGVmaW5pdGlvbiI6ImRlc3RpbmF0aW9uIiwibmFtZXNwYWNlRm9ybWF0IjoiJHtTT1VSQ0VfTkFNRVNQQUNFfSIsInByZWZpeCI6IiIsInNvdXJjZUNvbmZpZ3VyYXRpb24iOnsicG9rZW1vbl9uYW1lIjoiZGl0dG8ifSwiZGVzdGluYXRpb25Db25maWd1cmF0aW9uIjp7ImRlc3RpbmF0aW9uX3BhdGgiOiIvdG1wIn0sIm9wZXJhdGlvblNlcXVlbmNlIjpbXSwiY2F0YWxvZyI6eyJzdHJlYW1zIjpbeyJzdHJlYW0iOnsibmFtZSI6InBva2Vtb24iLCJqc29uX3NjaGVtYSI6eyJ0eXBlIjoib2JqZWN0IiwiJHNjaGVtYSI6Imh0dHA6Ly9qc29uLXNjaGVtYS5vcmcvZHJhZnQtMDcvc2NoZW1hIyIsInByb3BlcnRpZXMiOnsiaWQiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sImZvcm1zIjp7InR5cGUiOlsibnVsbCIsImFycmF5Il0sIml0ZW1zIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX19LCJtb3ZlcyI6eyJ0eXBlIjpbIm51bGwiLCJhcnJheSJdLCJpdGVtcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJtb3ZlIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX0sInZlcnNpb25fZ3JvdXBfZGV0YWlscyI6eyJ0eXBlIjpbIm51bGwiLCJhcnJheSJdLCJpdGVtcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ2ZXJzaW9uX2dyb3VwIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX0sImxldmVsX2xlYXJuZWRfYXQiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfSwibW92ZV9sZWFybl9tZXRob2QiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fX19fX19fSwib3JkZXIiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfSwic3RhdHMiOnsidHlwZSI6WyJudWxsIiwiYXJyYXkiXSwiaXRlbXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsic3RhdCI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ1cmwiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19LCJlZmZvcnQiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfSwiYmFzZV9zdGF0Ijp7InR5cGUiOlsibnVsbCIsImludGVnZXIiXX19fX0sInR5cGVzIjp7InR5cGUiOlsibnVsbCIsImFycmF5Il0sIml0ZW1zIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InNsb3QiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfSwidHlwZSI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ1cmwiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19fX19LCJoZWlnaHQiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfSwid2VpZ2h0Ijp7InR5cGUiOlsibnVsbCIsImludGVnZXIiXX0sInNwZWNpZXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fSwic3ByaXRlcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJiYWNrX3NoaW55Ijp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwiYmFja19mZW1hbGUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJmcm9udF9zaGlueSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sImJhY2tfZGVmYXVsdCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sImZyb250X2ZlbWFsZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sImZyb250X2RlZmF1bHQiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJiYWNrX3NoaW55X2ZlbWFsZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sImZyb250X3NoaW55X2ZlbWFsZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fSwiYWJpbGl0aWVzIjp7InR5cGUiOlsibnVsbCIsImFycmF5Il0sIml0ZW1zIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InNsb3QiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfSwiYWJpbGl0eSI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ1cmwiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19LCJpc19oaWRkZW4iOnsidHlwZSI6WyJudWxsIiwiYm9vbGVhbiJdfX19fSwiaGVsZF9pdGVtcyI6eyJ0eXBlIjpbIm51bGwiLCJhcnJheSJdLCJpdGVtcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJpdGVtIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX0sInZlcnNpb25fZGV0YWlscyI6eyJ0eXBlIjpbIm51bGwiLCJhcnJheSJdLCJpdGVtcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJyYXJpdHkiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfSwidmVyc2lvbiI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ1cmwiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19fX19fX19LCJpc19kZWZhdWx0ICI6eyJ0eXBlIjpbIm51bGwiLCJib29sZWFuIl19LCJnYW1lX2luZGljZXMiOnsidHlwZSI6WyJudWxsIiwiYXJyYXkiXSwiaXRlbXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidmVyc2lvbiI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ1cmwiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19LCJnYW1lX2luZGV4Ijp7InR5cGUiOlsibnVsbCIsImludGVnZXIiXX19fX0sImJhc2VfZXhwZXJpZW5jZSI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJsb2NhdGlvbl9hcmVhX2VuY291bnRlcnMiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX0sInN1cHBvcnRlZF9zeW5jX21vZGVzIjpbImZ1bGxfcmVmcmVzaCJdLCJkZWZhdWx0X2N1cnNvcl9maWVsZCI6W10sInNvdXJjZV9kZWZpbmVkX3ByaW1hcnlfa2V5IjpbXX0sInN5bmNfbW9kZSI6ImZ1bGxfcmVmcmVzaCIsImN1cnNvcl9maWVsZCI6W10sImRlc3RpbmF0aW9uX3N5bmNfbW9kZSI6ImFwcGVuZCIsInByaW1hcnlfa2V5IjpbXX1dfSwicmVzb3VyY2VSZXF1aXJlbWVudHMiOnt9LCJzb3VyY2VSZXNvdXJjZVJlcXVpcmVtZW50cyI6e30sImRlc3RpbmF0aW9uUmVzb3VyY2VSZXF1aXJlbWVudHMiOnt9fX0=" + } + ] + }, + "scheduledEventId": "23", + "startedEventId": "24", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "26", + "eventTime": "2022-03-08T22:47:58.471218800Z", + "eventType": "WorkflowTaskScheduled", + "taskId": "1048721", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "1@2741f9c3f558:e7111d28-2117-4614-9bb1-8a73524b789f", + "kind": "Sticky" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "27", + "eventTime": "2022-03-08T22:47:58.485851600Z", + "eventType": "WorkflowTaskStarted", + "taskId": "1048725", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "26", + "identity": "e7111d28-2117-4614-9bb1-8a73524b789f", + "requestId": "289a7723-efac-4cfa-bad2-f0c022b27421" + } + }, + { + "eventId": "28", + "eventTime": "2022-03-08T22:47:58.513022200Z", + "eventType": "WorkflowTaskCompleted", + "taskId": "1048728", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "26", + "startedEventId": "27", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "29", + "eventTime": "2022-03-08T22:47:58.513073500Z", + "eventType": "ActivityTaskScheduled", + "taskId": "1048729", + "activityTaskScheduledEventAttributes": { + "activityId": "95686aea-a2ac-3e1e-a512-0790d3a4e95f", + "activityType": { + "name": "ReportJobStart" + }, + "taskQueue": { + "name": "CONNECTION_UPDATER", + "kind": "Normal" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJqb2JJZCI6MX0=" + } + ] + }, + "scheduleToCloseTimeout": "0s", + "scheduleToStartTimeout": "0s", + "startToCloseTimeout": "120s", + "heartbeatTimeout": "30s", + "workflowTaskCompletedEventId": "28", + "retryPolicy": { + "initialInterval": "30s", + "backoffCoefficient": 2, + "maximumInterval": "3000s", + "maximumAttempts": 10 + } + } + }, + { + "eventId": "30", + "eventTime": "2022-03-08T22:47:58.528653400Z", + "eventType": "ActivityTaskStarted", + "taskId": "1048733", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "29", + "identity": "1@2741f9c3f558", + "requestId": "0e4b03aa-2493-4f7d-b832-4e98e13551da", + "attempt": 1 + } + }, + { + "eventId": "31", + "eventTime": "2022-03-08T22:47:58.668827900Z", + "eventType": "ActivityTaskCompleted", + "taskId": "1048734", + "activityTaskCompletedEventAttributes": { + "scheduledEventId": "29", + "startedEventId": "30", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "32", + "eventTime": "2022-03-08T22:47:58.668874600Z", + "eventType": "WorkflowTaskScheduled", + "taskId": "1048735", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "1@2741f9c3f558:e7111d28-2117-4614-9bb1-8a73524b789f", + "kind": "Sticky" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "33", + "eventTime": "2022-03-08T22:47:58.682929200Z", + "eventType": "WorkflowTaskStarted", + "taskId": "1048739", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "32", + "identity": "e7111d28-2117-4614-9bb1-8a73524b789f", + "requestId": "caa61b0a-5d34-48a8-ab16-997d3ba9eab5" + } + }, + { + "eventId": "34", + "eventTime": "2022-03-08T22:47:58.735519100Z", + "eventType": "WorkflowTaskCompleted", + "taskId": "1048742", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "32", + "startedEventId": "33", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "35", + "eventTime": "2022-03-08T22:47:58.735579100Z", + "eventType": "MarkerRecorded", + "taskId": "1048743", + "markerRecordedEventAttributes": { + "markerName": "Version", + "details": { + "changeId": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "InRhc2tfcXVldWVfY2hhbmdlX2Zyb21fY29ubmVjdGlvbl91cGRhdGVyX3RvX3N5bmMi" + } + ] + }, + "version": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "MQ==" + } + ] + } + }, + "workflowTaskCompletedEventId": "34" + } + }, + { + "eventId": "36", + "eventTime": "2022-03-08T22:47:58.735800Z", + "eventType": "StartChildWorkflowExecutionInitiated", + "taskId": "1048744", + "startChildWorkflowExecutionInitiatedEventAttributes": { + "namespace": "default", + "workflowId": "sync_1", + "workflowType": { + "name": "SyncWorkflow" + }, + "taskQueue": { + "name": "SYNC" + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJqb2JJZCI6IjEiLCJhdHRlbXB0SWQiOjB9" + }, + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJqb2JJZCI6IjEiLCJhdHRlbXB0SWQiOjAsImRvY2tlckltYWdlIjoiYWlyYnl0ZS9zb3VyY2UtcG9rZWFwaTowLjEuNCJ9" + }, + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJqb2JJZCI6IjEiLCJhdHRlbXB0SWQiOjAsImRvY2tlckltYWdlIjoiYWlyYnl0ZS9kZXN0aW5hdGlvbi1sb2NhbC1qc29uOjAuMi4xMCJ9" + }, + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJuYW1lc3BhY2VEZWZpbml0aW9uIjoiZGVzdGluYXRpb24iLCJuYW1lc3BhY2VGb3JtYXQiOiIke1NPVVJDRV9OQU1FU1BBQ0V9IiwicHJlZml4IjoiIiwic291cmNlQ29uZmlndXJhdGlvbiI6eyJwb2tlbW9uX25hbWUiOiJkaXR0byJ9LCJkZXN0aW5hdGlvbkNvbmZpZ3VyYXRpb24iOnsiZGVzdGluYXRpb25fcGF0aCI6Ii90bXAifSwib3BlcmF0aW9uU2VxdWVuY2UiOltdLCJjYXRhbG9nIjp7InN0cmVhbXMiOlt7InN0cmVhbSI6eyJuYW1lIjoicG9rZW1vbiIsImpzb25fc2NoZW1hIjp7InR5cGUiOiJvYmplY3QiLCIkc2NoZW1hIjoiaHR0cDovL2pzb24tc2NoZW1hLm9yZy9kcmFmdC0wNy9zY2hlbWEjIiwicHJvcGVydGllcyI6eyJpZCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwiZm9ybXMiOnsidHlwZSI6WyJudWxsIiwiYXJyYXkiXSwiaXRlbXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fX0sIm1vdmVzIjp7InR5cGUiOlsibnVsbCIsImFycmF5Il0sIml0ZW1zIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7Im1vdmUiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fSwidmVyc2lvbl9ncm91cF9kZXRhaWxzIjp7InR5cGUiOlsibnVsbCIsImFycmF5Il0sIml0ZW1zIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InZlcnNpb25fZ3JvdXAiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fSwibGV2ZWxfbGVhcm5lZF9hdCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJtb3ZlX2xlYXJuX21ldGhvZCI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ1cmwiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19fX19fX19LCJvcmRlciI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJzdGF0cyI6eyJ0eXBlIjpbIm51bGwiLCJhcnJheSJdLCJpdGVtcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJzdGF0Ijp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX0sImVmZm9ydCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJiYXNlX3N0YXQiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfX19fSwidHlwZXMiOnsidHlwZSI6WyJudWxsIiwiYXJyYXkiXSwiaXRlbXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsic2xvdCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJ0eXBlIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX19fX0sImhlaWdodCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJ3ZWlnaHQiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfSwic3BlY2llcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ1cmwiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19LCJzcHJpdGVzIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7ImJhY2tfc2hpbnkiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJiYWNrX2ZlbWFsZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sImZyb250X3NoaW55Ijp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwiYmFja19kZWZhdWx0Ijp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwiZnJvbnRfZmVtYWxlIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwiZnJvbnRfZGVmYXVsdCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sImJhY2tfc2hpbnlfZmVtYWxlIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwiZnJvbnRfc2hpbnlfZmVtYWxlIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19LCJhYmlsaXRpZXMiOnsidHlwZSI6WyJudWxsIiwiYXJyYXkiXSwiaXRlbXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsic2xvdCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJhYmlsaXR5Ijp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX0sImlzX2hpZGRlbiI6eyJ0eXBlIjpbIm51bGwiLCJib29sZWFuIl19fX19LCJoZWxkX2l0ZW1zIjp7InR5cGUiOlsibnVsbCIsImFycmF5Il0sIml0ZW1zIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7Iml0ZW0iOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fSwidmVyc2lvbl9kZXRhaWxzIjp7InR5cGUiOlsibnVsbCIsImFycmF5Il0sIml0ZW1zIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InJhcml0eSI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJ2ZXJzaW9uIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX19fX19fX0sImlzX2RlZmF1bHQgIjp7InR5cGUiOlsibnVsbCIsImJvb2xlYW4iXX0sImdhbWVfaW5kaWNlcyI6eyJ0eXBlIjpbIm51bGwiLCJhcnJheSJdLCJpdGVtcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ2ZXJzaW9uIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX0sImdhbWVfaW5kZXgiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfX19fSwiYmFzZV9leHBlcmllbmNlIjp7InR5cGUiOlsibnVsbCIsImludGVnZXIiXX0sImxvY2F0aW9uX2FyZWFfZW5jb3VudGVycyI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fSwic3VwcG9ydGVkX3N5bmNfbW9kZXMiOlsiZnVsbF9yZWZyZXNoIl0sImRlZmF1bHRfY3Vyc29yX2ZpZWxkIjpbXSwic291cmNlX2RlZmluZWRfcHJpbWFyeV9rZXkiOltdfSwic3luY19tb2RlIjoiZnVsbF9yZWZyZXNoIiwiY3Vyc29yX2ZpZWxkIjpbXSwiZGVzdGluYXRpb25fc3luY19tb2RlIjoiYXBwZW5kIiwicHJpbWFyeV9rZXkiOltdfV19LCJyZXNvdXJjZVJlcXVpcmVtZW50cyI6e30sInNvdXJjZVJlc291cmNlUmVxdWlyZW1lbnRzIjp7fSwiZGVzdGluYXRpb25SZXNvdXJjZVJlcXVpcmVtZW50cyI6e319" + }, + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "ImI5MTZmZDg2LWE1YTYtNDhhYy04ODgwLTQ5Nzg3NGNmNTNjZiI=" + } + ] + }, + "workflowExecutionTimeout": "0s", + "workflowRunTimeout": "0s", + "workflowTaskTimeout": "10s", + "parentClosePolicy": "RequestCancel", + "workflowTaskCompletedEventId": "34", + "workflowIdReusePolicy": "AllowDuplicate", + "header": {} + } + }, + { + "eventId": "37", + "eventTime": "2022-03-08T22:47:58.762930500Z", + "eventType": "ChildWorkflowExecutionStarted", + "taskId": "1048747", + "childWorkflowExecutionStartedEventAttributes": { + "namespace": "default", + "initiatedEventId": "36", + "workflowExecution": { + "workflowId": "sync_1", + "runId": "140640f0-c577-4d9c-8777-4e7e40b64241" + }, + "workflowType": { + "name": "SyncWorkflow" + }, + "header": {} + } + }, + { + "eventId": "38", + "eventTime": "2022-03-08T22:47:58.762971900Z", + "eventType": "WorkflowTaskScheduled", + "taskId": "1048748", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "1@2741f9c3f558:e7111d28-2117-4614-9bb1-8a73524b789f", + "kind": "Sticky" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "39", + "eventTime": "2022-03-08T22:47:58.774051900Z", + "eventType": "WorkflowTaskStarted", + "taskId": "1048752", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "38", + "identity": "e7111d28-2117-4614-9bb1-8a73524b789f", + "requestId": "ca98144b-47f8-486d-b260-2e1dc42bd68e" + } + }, + { + "eventId": "40", + "eventTime": "2022-03-08T22:47:58.794905100Z", + "eventType": "WorkflowTaskCompleted", + "taskId": "1048755", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "38", + "startedEventId": "39", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "41", + "eventTime": "2022-03-08T22:48:02.408058300Z", + "eventType": "ChildWorkflowExecutionCompleted", + "taskId": "1048757", + "childWorkflowExecutionCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJzdGFuZGFyZFN5bmNTdW1tYXJ5Ijp7InN0YXR1cyI6ImNvbXBsZXRlZCIsInJlY29yZHNTeW5jZWQiOjEsImJ5dGVzU3luY2VkIjoyMjcxNSwic3RhcnRUaW1lIjoxNjQ2Nzc5Njc4OTM5LCJlbmRUaW1lIjoxNjQ2Nzc5NjgyMjM4LCJ0b3RhbFN0YXRzIjp7InJlY29yZHNFbWl0dGVkIjoxLCJieXRlc0VtaXR0ZWQiOjIyNzE1LCJzdGF0ZU1lc3NhZ2VzRW1pdHRlZCI6MCwicmVjb3Jkc0NvbW1pdHRlZCI6MX0sInN0cmVhbVN0YXRzIjpbeyJzdHJlYW1OYW1lIjoicG9rZW1vbiIsInN0YXRzIjp7InJlY29yZHNFbWl0dGVkIjoxLCJieXRlc0VtaXR0ZWQiOjIyNzE1LCJyZWNvcmRzQ29tbWl0dGVkIjoxfX1dfSwib3V0cHV0X2NhdGFsb2ciOnsic3RyZWFtcyI6W3sic3RyZWFtIjp7Im5hbWUiOiJwb2tlbW9uIiwianNvbl9zY2hlbWEiOnsidHlwZSI6Im9iamVjdCIsIiRzY2hlbWEiOiJodHRwOi8vanNvbi1zY2hlbWEub3JnL2RyYWZ0LTA3L3NjaGVtYSMiLCJwcm9wZXJ0aWVzIjp7ImlkIjp7InR5cGUiOlsibnVsbCIsImludGVnZXIiXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJmb3JtcyI6eyJ0eXBlIjpbIm51bGwiLCJhcnJheSJdLCJpdGVtcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ1cmwiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19fSwibW92ZXMiOnsidHlwZSI6WyJudWxsIiwiYXJyYXkiXSwiaXRlbXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsibW92ZSI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ1cmwiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19LCJ2ZXJzaW9uX2dyb3VwX2RldGFpbHMiOnsidHlwZSI6WyJudWxsIiwiYXJyYXkiXSwiaXRlbXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidmVyc2lvbl9ncm91cCI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ1cmwiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19LCJsZXZlbF9sZWFybmVkX2F0Ijp7InR5cGUiOlsibnVsbCIsImludGVnZXIiXX0sIm1vdmVfbGVhcm5fbWV0aG9kIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX19fX19fX0sIm9yZGVyIjp7InR5cGUiOlsibnVsbCIsImludGVnZXIiXX0sInN0YXRzIjp7InR5cGUiOlsibnVsbCIsImFycmF5Il0sIml0ZW1zIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InN0YXQiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fSwiZWZmb3J0Ijp7InR5cGUiOlsibnVsbCIsImludGVnZXIiXX0sImJhc2Vfc3RhdCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19fX19LCJ0eXBlcyI6eyJ0eXBlIjpbIm51bGwiLCJhcnJheSJdLCJpdGVtcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJzbG90Ijp7InR5cGUiOlsibnVsbCIsImludGVnZXIiXX0sInR5cGUiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fX19fSwiaGVpZ2h0Ijp7InR5cGUiOlsibnVsbCIsImludGVnZXIiXX0sIndlaWdodCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJzcGVjaWVzIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX0sInNwcml0ZXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsiYmFja19zaGlueSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sImJhY2tfZmVtYWxlIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwiZnJvbnRfc2hpbnkiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJiYWNrX2RlZmF1bHQiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJmcm9udF9mZW1hbGUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJmcm9udF9kZWZhdWx0Ijp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwiYmFja19zaGlueV9mZW1hbGUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJmcm9udF9zaGlueV9mZW1hbGUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX0sImFiaWxpdGllcyI6eyJ0eXBlIjpbIm51bGwiLCJhcnJheSJdLCJpdGVtcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJzbG90Ijp7InR5cGUiOlsibnVsbCIsImludGVnZXIiXX0sImFiaWxpdHkiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fSwiaXNfaGlkZGVuIjp7InR5cGUiOlsibnVsbCIsImJvb2xlYW4iXX19fX0sImhlbGRfaXRlbXMiOnsidHlwZSI6WyJudWxsIiwiYXJyYXkiXSwiaXRlbXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsiaXRlbSI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ1cmwiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19LCJ2ZXJzaW9uX2RldGFpbHMiOnsidHlwZSI6WyJudWxsIiwiYXJyYXkiXSwiaXRlbXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsicmFyaXR5Ijp7InR5cGUiOlsibnVsbCIsImludGVnZXIiXX0sInZlcnNpb24iOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fX19fX19fSwiaXNfZGVmYXVsdCAiOnsidHlwZSI6WyJudWxsIiwiYm9vbGVhbiJdfSwiZ2FtZV9pbmRpY2VzIjp7InR5cGUiOlsibnVsbCIsImFycmF5Il0sIml0ZW1zIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InZlcnNpb24iOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fSwiZ2FtZV9pbmRleCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19fX19LCJiYXNlX2V4cGVyaWVuY2UiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfSwibG9jYXRpb25fYXJlYV9lbmNvdW50ZXJzIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19LCJzdXBwb3J0ZWRfc3luY19tb2RlcyI6WyJmdWxsX3JlZnJlc2giXSwiZGVmYXVsdF9jdXJzb3JfZmllbGQiOltdLCJzb3VyY2VfZGVmaW5lZF9wcmltYXJ5X2tleSI6W119LCJzeW5jX21vZGUiOiJmdWxsX3JlZnJlc2giLCJjdXJzb3JfZmllbGQiOltdLCJkZXN0aW5hdGlvbl9zeW5jX21vZGUiOiJhcHBlbmQiLCJwcmltYXJ5X2tleSI6W119XX0sImZhaWx1cmVzIjpbXX0=" + } + ] + }, + "namespace": "default", + "workflowExecution": { + "workflowId": "sync_1", + "runId": "140640f0-c577-4d9c-8777-4e7e40b64241" + }, + "workflowType": { + "name": "SyncWorkflow" + }, + "initiatedEventId": "36", + "startedEventId": "37" + } + }, + { + "eventId": "42", + "eventTime": "2022-03-08T22:48:02.408127200Z", + "eventType": "WorkflowTaskScheduled", + "taskId": "1048758", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "1@2741f9c3f558:e7111d28-2117-4614-9bb1-8a73524b789f", + "kind": "Sticky" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "43", + "eventTime": "2022-03-08T22:48:02.422112800Z", + "eventType": "WorkflowTaskStarted", + "taskId": "1048762", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "42", + "identity": "e7111d28-2117-4614-9bb1-8a73524b789f", + "requestId": "c0916d42-c83c-4e9d-805a-29ca5f979624" + } + }, + { + "eventId": "44", + "eventTime": "2022-03-08T22:48:02.454203Z", + "eventType": "WorkflowTaskCompleted", + "taskId": "1048765", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "42", + "startedEventId": "43", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "45", + "eventTime": "2022-03-08T22:48:02.454256Z", + "eventType": "ActivityTaskScheduled", + "taskId": "1048766", + "activityTaskScheduledEventAttributes": { + "activityId": "b169a729-47bc-38f7-a315-c1a4b6d96466", + "activityType": { + "name": "JobSuccess" + }, + "taskQueue": { + "name": "CONNECTION_UPDATER", + "kind": "Normal" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJqb2JJZCI6MSwiYXR0ZW1wdElkIjowLCJzdGFuZGFyZFN5bmNPdXRwdXQiOnsic3RhbmRhcmRTeW5jU3VtbWFyeSI6eyJzdGF0dXMiOiJjb21wbGV0ZWQiLCJyZWNvcmRzU3luY2VkIjoxLCJieXRlc1N5bmNlZCI6MjI3MTUsInN0YXJ0VGltZSI6MTY0Njc3OTY3ODkzOSwiZW5kVGltZSI6MTY0Njc3OTY4MjIzOCwidG90YWxTdGF0cyI6eyJyZWNvcmRzRW1pdHRlZCI6MSwiYnl0ZXNFbWl0dGVkIjoyMjcxNSwic3RhdGVNZXNzYWdlc0VtaXR0ZWQiOjAsInJlY29yZHNDb21taXR0ZWQiOjF9LCJzdHJlYW1TdGF0cyI6W3sic3RyZWFtTmFtZSI6InBva2Vtb24iLCJzdGF0cyI6eyJyZWNvcmRzRW1pdHRlZCI6MSwiYnl0ZXNFbWl0dGVkIjoyMjcxNSwicmVjb3Jkc0NvbW1pdHRlZCI6MX19XX0sIm91dHB1dF9jYXRhbG9nIjp7InN0cmVhbXMiOlt7InN0cmVhbSI6eyJuYW1lIjoicG9rZW1vbiIsImpzb25fc2NoZW1hIjp7InR5cGUiOiJvYmplY3QiLCIkc2NoZW1hIjoiaHR0cDovL2pzb24tc2NoZW1hLm9yZy9kcmFmdC0wNy9zY2hlbWEjIiwicHJvcGVydGllcyI6eyJpZCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwiZm9ybXMiOnsidHlwZSI6WyJudWxsIiwiYXJyYXkiXSwiaXRlbXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fX0sIm1vdmVzIjp7InR5cGUiOlsibnVsbCIsImFycmF5Il0sIml0ZW1zIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7Im1vdmUiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fSwidmVyc2lvbl9ncm91cF9kZXRhaWxzIjp7InR5cGUiOlsibnVsbCIsImFycmF5Il0sIml0ZW1zIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InZlcnNpb25fZ3JvdXAiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fSwibGV2ZWxfbGVhcm5lZF9hdCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJtb3ZlX2xlYXJuX21ldGhvZCI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ1cmwiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19fX19fX19LCJvcmRlciI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJzdGF0cyI6eyJ0eXBlIjpbIm51bGwiLCJhcnJheSJdLCJpdGVtcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJzdGF0Ijp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX0sImVmZm9ydCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJiYXNlX3N0YXQiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfX19fSwidHlwZXMiOnsidHlwZSI6WyJudWxsIiwiYXJyYXkiXSwiaXRlbXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsic2xvdCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJ0eXBlIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX19fX0sImhlaWdodCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJ3ZWlnaHQiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfSwic3BlY2llcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ1cmwiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJuYW1lIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19LCJzcHJpdGVzIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7ImJhY2tfc2hpbnkiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19LCJiYWNrX2ZlbWFsZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sImZyb250X3NoaW55Ijp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwiYmFja19kZWZhdWx0Ijp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwiZnJvbnRfZmVtYWxlIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwiZnJvbnRfZGVmYXVsdCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sImJhY2tfc2hpbnlfZmVtYWxlIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwiZnJvbnRfc2hpbnlfZmVtYWxlIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfX19LCJhYmlsaXRpZXMiOnsidHlwZSI6WyJudWxsIiwiYXJyYXkiXSwiaXRlbXMiOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsic2xvdCI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJhYmlsaXR5Ijp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX0sImlzX2hpZGRlbiI6eyJ0eXBlIjpbIm51bGwiLCJib29sZWFuIl19fX19LCJoZWxkX2l0ZW1zIjp7InR5cGUiOlsibnVsbCIsImFycmF5Il0sIml0ZW1zIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7Iml0ZW0iOnsidHlwZSI6WyJudWxsIiwib2JqZWN0Il0sInByb3BlcnRpZXMiOnsidXJsIjp7InR5cGUiOlsibnVsbCIsInN0cmluZyJdfSwibmFtZSI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fSwidmVyc2lvbl9kZXRhaWxzIjp7InR5cGUiOlsibnVsbCIsImFycmF5Il0sIml0ZW1zIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InJhcml0eSI6eyJ0eXBlIjpbIm51bGwiLCJpbnRlZ2VyIl19LCJ2ZXJzaW9uIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX19fX19fX0sImlzX2RlZmF1bHQgIjp7InR5cGUiOlsibnVsbCIsImJvb2xlYW4iXX0sImdhbWVfaW5kaWNlcyI6eyJ0eXBlIjpbIm51bGwiLCJhcnJheSJdLCJpdGVtcyI6eyJ0eXBlIjpbIm51bGwiLCJvYmplY3QiXSwicHJvcGVydGllcyI6eyJ2ZXJzaW9uIjp7InR5cGUiOlsibnVsbCIsIm9iamVjdCJdLCJwcm9wZXJ0aWVzIjp7InVybCI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX0sIm5hbWUiOnsidHlwZSI6WyJudWxsIiwic3RyaW5nIl19fX0sImdhbWVfaW5kZXgiOnsidHlwZSI6WyJudWxsIiwiaW50ZWdlciJdfX19fSwiYmFzZV9leHBlcmllbmNlIjp7InR5cGUiOlsibnVsbCIsImludGVnZXIiXX0sImxvY2F0aW9uX2FyZWFfZW5jb3VudGVycyI6eyJ0eXBlIjpbIm51bGwiLCJzdHJpbmciXX19fSwic3VwcG9ydGVkX3N5bmNfbW9kZXMiOlsiZnVsbF9yZWZyZXNoIl0sImRlZmF1bHRfY3Vyc29yX2ZpZWxkIjpbXSwic291cmNlX2RlZmluZWRfcHJpbWFyeV9rZXkiOltdfSwic3luY19tb2RlIjoiZnVsbF9yZWZyZXNoIiwiY3Vyc29yX2ZpZWxkIjpbXSwiZGVzdGluYXRpb25fc3luY19tb2RlIjoiYXBwZW5kIiwicHJpbWFyeV9rZXkiOltdfV19LCJmYWlsdXJlcyI6W119fQ==" + } + ] + }, + "scheduleToCloseTimeout": "0s", + "scheduleToStartTimeout": "0s", + "startToCloseTimeout": "120s", + "heartbeatTimeout": "30s", + "workflowTaskCompletedEventId": "44", + "retryPolicy": { + "initialInterval": "30s", + "backoffCoefficient": 2, + "maximumInterval": "3000s", + "maximumAttempts": 10 + } + } + }, + { + "eventId": "46", + "eventTime": "2022-03-08T22:48:02.437049800Z", + "eventType": "ActivityTaskStarted", + "taskId": "1048770", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "45", + "identity": "1@2741f9c3f558", + "requestId": "9d2fc180-ec33-42a2-a259-d29afb281992", + "attempt": 1 + } + }, + { + "eventId": "47", + "eventTime": "2022-03-08T22:48:02.664164100Z", + "eventType": "ActivityTaskCompleted", + "taskId": "1048771", + "activityTaskCompletedEventAttributes": { + "scheduledEventId": "45", + "startedEventId": "46", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "48", + "eventTime": "2022-03-08T22:48:02.664217700Z", + "eventType": "WorkflowTaskScheduled", + "taskId": "1048772", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "1@2741f9c3f558:e7111d28-2117-4614-9bb1-8a73524b789f", + "kind": "Sticky" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "49", + "eventTime": "2022-03-08T22:48:02.676895300Z", + "eventType": "WorkflowTaskStarted", + "taskId": "1048776", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "48", + "identity": "e7111d28-2117-4614-9bb1-8a73524b789f", + "requestId": "b43c8f30-2500-47d6-a8f6-aa2cd0d99218" + } + }, + { + "eventId": "50", + "eventTime": "2022-03-08T22:48:02.709745Z", + "eventType": "WorkflowTaskCompleted", + "taskId": "1048779", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "48", + "startedEventId": "49", + "identity": "1@2741f9c3f558" + } + }, + { + "eventId": "51", + "eventTime": "2022-03-08T22:48:02.709811400Z", + "eventType": "WorkflowExecutionContinuedAsNew", + "taskId": "1048780", + "workflowExecutionContinuedAsNewEventAttributes": { + "newExecutionRunId": "e81cf38b-7f11-4eeb-8c85-301778bf2671", + "workflowType": { + "name": "ConnectionManagerWorkflow" + }, + "taskQueue": { + "name": "CONNECTION_UPDATER", + "kind": "Normal" + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJjb25uZWN0aW9uSWQiOiJiOTE2ZmQ4Ni1hNWE2LTQ4YWMtODg4MC00OTc4NzRjZjUzY2YiLCJqb2JJZCI6bnVsbCwiYXR0ZW1wdElkIjpudWxsLCJmcm9tRmFpbHVyZSI6ZmFsc2UsImF0dGVtcHROdW1iZXIiOjEsIndvcmtmbG93U3RhdGUiOm51bGwsInJlc2V0Q29ubmVjdGlvbiI6ZmFsc2V9" + } + ] + }, + "workflowRunTimeout": "0s", + "workflowTaskTimeout": "10s", + "workflowTaskCompletedEventId": "50", + "header": {} + } + } + ] +}