diff --git a/.env b/.env index 0ee0e5b5aa5156..3d688ebd7154d1 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -VERSION=0.7.2-alpha +VERSION=0.8.0-alpha DATABASE_USER=docker DATABASE_PASSWORD=docker DATABASE_DB=airbyte diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json index 6cc4634b055dda..6c7fffc24cadca 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json @@ -2,6 +2,6 @@ "destinationDefinitionId": "22f6c74f-5699-40ff-833c-4a879ea40133", "name": "BigQuery", "dockerRepository": "airbyte/destination-bigquery", - "dockerImageTag": "0.1.10", + "dockerImageTag": "0.1.11", "documentationUrl": "https://hub.docker.com/r/airbyte/integration-singer-bigquery-destination" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index d6240818a27b0f..d14e11ba10550d 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -11,7 +11,7 @@ - destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 name: BigQuery dockerRepository: airbyte/destination-bigquery - dockerImageTag: 0.1.10 + dockerImageTag: 0.1.11 documentationUrl: https://hub.docker.com/r/airbyte/integration-singer-bigquery-destination - destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba name: Snowflake diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index 1f5451a4f23ee5..ec1b66680fae22 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobRetrier.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobRetrier.java index e8a9b3c08067b2..325ac97e6b95b5 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobRetrier.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobRetrier.java @@ -28,9 +28,10 @@ import io.airbyte.scheduler.persistence.JobPersistence; import java.io.IOException; import java.time.Instant; +import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; -import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,24 +53,35 @@ public JobRetrier(JobPersistence jobPersistence, Supplier timeSupplier) public void run() { LOGGER.info("Running job-retrier..."); - listFailedJobs() + final AtomicInteger failedJobs = new AtomicInteger(); + final AtomicInteger retriedJobs = new AtomicInteger(); + final List incompleteJobs = incompleteJobs(); + + incompleteJobs .forEach(job -> { + LOGGER.info("weeee"); if (hasReachedMaxAttempt(job)) { failJob(job); + failedJobs.incrementAndGet(); return; } if (shouldRetry(job)) { + retriedJobs.incrementAndGet(); resetJob(job); } }); - LOGGER.info("Completed job-retrier..."); + LOGGER.info("Completed Job-Retrier..."); + LOGGER.info("Job-Retrier Summary. Incomplete jobs: {}, Job set to retry: {}, Jobs set to failed: {}", + incompleteJobs.size(), + failedJobs.get(), + retriedJobs.get()); } - private Stream listFailedJobs() { + private List incompleteJobs() { try { - return persistence.listJobsWithStatus(JobConfig.ConfigType.SYNC, JobStatus.INCOMPLETE).stream(); + return persistence.listJobsWithStatus(JobConfig.ConfigType.SYNC, JobStatus.INCOMPLETE); } catch (IOException e) { throw new RuntimeException("failed to fetch failed jobs", e); } diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobScheduler.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobScheduler.java index b7fa42778f19c9..22ff1dfab08b0a 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobScheduler.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobScheduler.java @@ -39,6 +39,7 @@ import java.time.Instant; import java.util.List; import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiPredicate; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -80,14 +81,17 @@ public void run() { scheduleSyncJobs(); - LOGGER.info("Completed job-scheduler..."); + LOGGER.info("Completed Job-Scheduler..."); } catch (Throwable e) { LOGGER.error("Job Scheduler Error", e); } } private void scheduleSyncJobs() throws IOException { - for (StandardSync connection : getAllActiveConnections()) { + final AtomicInteger jobsScheduled = new AtomicInteger(); + final List activeConnections = getAllActiveConnections(); + + for (StandardSync connection : activeConnections) { final Optional previousJobOptional = jobPersistence.getLastSyncJob(connection.getConnectionId()); final StandardSyncSchedule standardSyncSchedule = getStandardSyncSchedule(connection); @@ -95,6 +99,7 @@ private void scheduleSyncJobs() throws IOException { jobFactory.create(connection.getConnectionId()); } } + LOGGER.info("Job-Scheduler Summary. Active connections: {}, Jobs scheduler: {}", activeConnections.size(), jobsScheduled.get()); } private StandardSyncSchedule getStandardSyncSchedule(StandardSync connection) { diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobSubmitter.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobSubmitter.java index 2a31cbeae848a1..e31e1399f730eb 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobSubmitter.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/JobSubmitter.java @@ -80,9 +80,10 @@ public void run() { oldestPendingJob.ifPresent(job -> { trackSubmission(job); submitJob(job); + LOGGER.info("Job-Submitter Summary. Submitted job with scope {}", job.getScope()); }); - LOGGER.info("Completed job-submitter..."); + LOGGER.info("Completed Job-Submitter..."); } catch (Throwable e) { LOGGER.error("Job Submitter Error", e); } diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index d0e6bec0e2f5ef..0afe49791a496f 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -25,6 +25,7 @@ Each stream will be output into its own table in BigQuery. Each table will conta | Feature | Supported?\(Yes/No\) | Notes | | :--- | :--- | :--- | | Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | ## Getting started diff --git a/docs/integrations/destinations/local-csv.md b/docs/integrations/destinations/local-csv.md index 0eae4175c46c60..54efc307e64a32 100644 --- a/docs/integrations/destinations/local-csv.md +++ b/docs/integrations/destinations/local-csv.md @@ -19,6 +19,7 @@ Each stream will be output into its own file. Each file will contain 3 columns: | Feature | Supported | | :--- | :--- | | Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | | #### Performance considerations diff --git a/docs/integrations/destinations/postgres.md b/docs/integrations/destinations/postgres.md index e28a452e297ec7..398dacfa0566cd 100644 --- a/docs/integrations/destinations/postgres.md +++ b/docs/integrations/destinations/postgres.md @@ -21,6 +21,7 @@ Each stream will be output into its own table in Postgres. Each table will conta | Feature | Supported?\(Yes/No\) | Notes | | :--- | :--- | :--- | | Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | ## Getting started diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index d0af1d7cbf761d..79ca47ff768619 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -21,6 +21,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Feature | Supported?\(Yes/No\) | Notes | | :--- | :--- | :--- | | Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | #### Target Database diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index cfa1cf8c38bf88..2405e397a6dd6b 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -19,6 +19,8 @@ Each stream will be output into its own table in Snowflake. Each table will cont | Feature | Supported?\(Yes/No\) | Notes | | :--- | :--- | :--- | | Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | + ## Getting started diff --git a/docs/integrations/sources/intercom.md b/docs/integrations/sources/intercom.md index d44937358c8d85..0c562a79bab304 100644 --- a/docs/integrations/sources/intercom.md +++ b/docs/integrations/sources/intercom.md @@ -31,7 +31,7 @@ If there are more endpoints you'd like Airbyte to support, please [create an iss | Feature | Supported? | | :--- | :--- | | Full Refresh Sync | Yes | -| Incremental Sync | No | +| Incremental - Append Sync | Yes | | Replicate Incremental Deletes | No | | SSL connection | Yes | diff --git a/docs/integrations/sources/mixpanel.md b/docs/integrations/sources/mixpanel.md index 247ffae6dd5d32..fa8062982955f5 100644 --- a/docs/integrations/sources/mixpanel.md +++ b/docs/integrations/sources/mixpanel.md @@ -25,7 +25,7 @@ If there are more endpoints you'd like Airbyte to support, please [create an iss | Feature | Supported? | | :--- | :--- | | Full Refresh Sync | Yes | -| Incremental Sync | No | +| Incremental - Append Sync | Yes | | Replicate Incremental Deletes | No | | SSL connection | Yes | diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index c72f232a1acdfc..09c0f338b74f44 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -33,7 +33,7 @@ If you do not see a type in this list, assume that it is coerced into a string. | Feature | Supported | | :--- | :--- | | Full Refresh Sync | Yes | -| Incremental Sync | No | +| Incremental - Append Sync | Yes | | Replicate Incremental Deletes | No | | Logical Replication \(WAL\) | No | | SSL Support | Yes | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 8959fa2579cc3a..f09815162861ab 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -52,7 +52,7 @@ Postgres data types are mapped to the following data types when synchronizing da | Feature | Supported | | :--- | :--- | | Full Refresh Sync | Yes | -| Incremental Sync | No | +| Incremental - Append Sync | Yes | | Replicate Incremental Deletes | No | | Logical Replication \(WAL\) | No | | SSL Support | Yes | diff --git a/docs/integrations/sources/twilio.md b/docs/integrations/sources/twilio.md index 758211c2bd0ac8..6bd1fe8ac0f9ad 100644 --- a/docs/integrations/sources/twilio.md +++ b/docs/integrations/sources/twilio.md @@ -46,7 +46,7 @@ This Source is capable of syncing the following core Streams: | Feature | Supported?\(Yes/No\) | Notes | | :--- | :--- | :--- | | Full Refresh Sync | yes | | -| Incremental Sync | no | | +| Incremental - Append Sync | Yes | ### Performance considerations