🚧 Synced with airbytehq/master

.bumpversion.cfg

@@ -1,10 +1,10 @@
 [bumpversion]
-current_version = 0.39.42-alpha
+current_version = 0.40.28
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
 serialize = 
-	{major}.{minor}.{patch}-alpha
+	{major}.{minor}.{patch}
 
 [bumpversion:file:.bumpversion.cfg]
 
@@ -14,48 +14,56 @@ serialize =
 
 [bumpversion:file:airbyte-container-orchestrator/Dockerfile]
 
+[bumpversion:file:airbyte-cron/Dockerfile]
+
 [bumpversion:file:airbyte-metrics/reporter/Dockerfile]
 
-[bumpversion:file:airbyte-server/Dockerfile]
+[bumpversion:file:airbyte-proxy/Dockerfile]
 
-[bumpversion:file:airbyte-webapp/package.json]
+[bumpversion:file:airbyte-server/Dockerfile]
 
 [bumpversion:file:airbyte-webapp/package-lock.json]
 
+[bumpversion:file:airbyte-webapp/package.json]
+
 [bumpversion:file:airbyte-workers/Dockerfile]
 
-[bumpversion:file:charts/airbyte/Chart.yaml]
+[bumpversion:file:charts/airbyte-bootloader/Chart.yaml]
 
-[bumpversion:file:charts/airbyte-worker/Chart.yaml]
+[bumpversion:file:charts/airbyte-cron/Chart.yaml]
+
+[bumpversion:file:charts/airbyte-server/Chart.yaml]
 
 [bumpversion:file:charts/airbyte-temporal/Chart.yaml]
 
 [bumpversion:file:charts/airbyte-webapp/Chart.yaml]
 
-[bumpversion:file:charts/airbyte-server/Chart.yaml]
+[bumpversion:file:charts/airbyte-worker/Chart.yaml]
 
-[bumpversion:file:charts/airbyte-bootloader/Chart.yaml]
+[bumpversion:file:charts/airbyte/Chart.yaml]
 
-[bumpversion:file:charts/airbyte/values.yaml]
+[bumpversion:file:charts/airbyte-connector-builder-server/Chart.yaml]
 
 [bumpversion:file:charts/airbyte/README.md]
 
 [bumpversion:file:docs/operator-guides/upgrading-airbyte.md]
 
+[bumpversion:file:kube/overlays/stable-with-resource-limits/.env]
+
+[bumpversion:file:kube/overlays/stable-with-resource-limits/kustomization.yaml]
+
 [bumpversion:file:kube/overlays/stable/.env]
 
 [bumpversion:file:kube/overlays/stable/kustomization.yaml]
 
-[bumpversion:file:kube/overlays/stable-with-resource-limits/.env]
+[bumpversion:file:octavia-cli/Dockerfile]
 
-[bumpversion:file:kube/overlays/stable-with-resource-limits/kustomization.yaml]
+[bumpversion:file:octavia-cli/README.md]
 
 [bumpversion:file:octavia-cli/install.sh]
 
-[bumpversion:file:octavia-cli/README.md]
+[bumpversion:file:octavia-cli/setup.py]
 
-[bumpversion:file:octavia-cli/Dockerfile]
+[bumpversion:file:airbyte-connector-builder-server/Dockerfile]
 
-[bumpversion:file:octavia-cli/setup.py]
-serialize = 
-	{major}.{minor}.{patch}
+[bumpversion:file:airbyte-connector-builder-server/setup.py]

.env

@@ -3,14 +3,14 @@
 # Variables with defaults have been omitted to avoid duplication of defaults.
 # The only exception to the non-default rule are env vars related to scaling.
 #
-# See https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java
+# See https://github.com/airbytehq/airbyte/blob/master/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java
 # for the latest environment variables.
 #
 # # Contributors - please organise this env file according to the above linked file.
 
 
 ### SHARED ###
-VERSION=0.39.42-alpha
+VERSION=0.40.28
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data
@@ -32,6 +32,11 @@ LOCAL_DOCKER_MOUNT=/tmp/airbyte_local
 # Issue: https://github.com/airbytehq/airbyte/issues/577
 HACK_LOCAL_ROOT_PARENT=/tmp
 
+# Proxy Configuration
+# Set to empty values, e.g. "" to disable basic auth
+BASIC_AUTH_USERNAME=airbyte
+BASIC_AUTH_PASSWORD=password
+BASIC_AUTH_PROXY_TIMEOUT=600
 
 ### DATABASE ###
 # Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
@@ -42,21 +47,22 @@ DATABASE_PORT=5432
 DATABASE_DB=airbyte
 # translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} (do not include the username or password here)
 DATABASE_URL=jdbc:postgresql://db:5432/airbyte
-JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.29.15.001
+JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.40.26.001
 
 # Airbyte Internal Config Database, defaults to Job Database if empty. Explicitly left empty to mute docker compose warnings.
 CONFIG_DATABASE_USER=
 CONFIG_DATABASE_PASSWORD=
 CONFIG_DATABASE_URL=
-CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.35.15.001
+CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.40.23.002
 
 ### AIRBYTE SERVICES ###
 TEMPORAL_HOST=airbyte-temporal:7233
 INTERNAL_API_HOST=airbyte-server:8001
+CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server:80
 WEBAPP_URL=http://localhost:8000/
 # Although not present as an env var, required for webapp configuration.
 API_URL=/api/v1/
-
+CONNECTOR_BUILDER_API_URL=/connector-builder-api
 
 ### JOBS ###
 # Relevant to scaling.
@@ -67,6 +73,10 @@ JOB_MAIN_CONTAINER_CPU_LIMIT=
 JOB_MAIN_CONTAINER_MEMORY_REQUEST=
 JOB_MAIN_CONTAINER_MEMORY_LIMIT=
 
+NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT=
+NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST=
+NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT=
+NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST=
 
 ### LOGGING/MONITORING/TRACKING ###
 TRACKING_STRATEGY=segment
@@ -77,21 +87,24 @@ LOG_LEVEL=INFO
 
 ### APPLICATIONS ###
 # Worker #
+WORKERS_MICRONAUT_ENVIRONMENTS=control-plane
+# Cron #
+CRON_MICRONAUT_ENVIRONMENTS=control-plane
 # Relevant to scaling.
 MAX_SYNC_WORKERS=5
 MAX_SPEC_WORKERS=5
 MAX_CHECK_WORKERS=5
 MAX_DISCOVER_WORKERS=5
+MAX_NOTIFY_WORKERS=5
+SHOULD_RUN_NOTIFY_WORKFLOWS=false
 # Temporal Activity configuration
 ACTIVITY_MAX_ATTEMPT=
 ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS=
 ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS=
 WORKFLOW_FAILURE_RESTART_DELAY_SECONDS=
 
-
 ### FEATURE FLAGS ###
 AUTO_DISABLE_FAILING_CONNECTIONS=false
-EXPOSE_SECRETS_IN_EXPORT=false
 FORCE_MIGRATE_SECRET_STORE=false
 
 ### MONITORING FLAGS ###
@@ -100,4 +113,5 @@ METRIC_CLIENT=
 # Useful only when metric client is set to be otel. Must start with http:// or https://.
 OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317"
 
-USE_STREAM_CAPABLE_STATE=false
+USE_STREAM_CAPABLE_STATE=true
+AUTO_DETECT_SCHEMA=false

.env.dev

@@ -4,6 +4,7 @@ VERSION=dev
 DATABASE_USER=docker
 DATABASE_PASSWORD=docker
 DATABASE_DB=airbyte
+DATABASE_URL=jdbc:postgresql://db:5432/airbyte
 CONFIG_ROOT=/data
 WORKSPACE_ROOT=/tmp/workspace
 DATA_DOCKER_MOUNT=airbyte_data_dev
@@ -22,8 +23,16 @@ HACK_LOCAL_ROOT_PARENT=/tmp
 WEBAPP_URL=http://localhost:8000/
 API_URL=/api/v1/
 INTERNAL_API_HOST=airbyte-server:8001
+CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server:80
 SYNC_JOB_MAX_ATTEMPTS=3
 SYNC_JOB_MAX_TIMEOUT_DAYS=3
+WORKERS_MICRONAUT_ENVIRONMENTS=control-plane
+CRON_MICRONAUT_ENVIRONMENTS=control-plane
+AUTO_DETECT_SCHEMA=false
 
 # Sentry
 SENTRY_DSN=""
+
+# Migration Configuration
+CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.35.15.001
+JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.29.15.001

.github/.gitignore

@@ -0,0 +1 @@
+connector_org_review_requirements.yaml

.github/CODEOWNERS

@@ -1,25 +1,38 @@
-# Frontend code
-/airbyte-webapp-e2e-tests/  @airbytehq/frontend
-/airbyte-webapp/  @airbytehq/frontend
-## Exclude the package(-lock).json from code ownership to prevent version bump PRs from triggering codeowners review
-/airbyte-webapp/package.json
-/airbyte-webapp/package-lock.json
-
 # CDK and SAT
-/airbyte-cdk/ @airbytehq/api-connectors-dx
-/airbyte-integrations/bases/source-acceptance-tests/ @airbytehq/api-connectors-dx
-/airbyte-integrations/connector-templates/ @airbytehq/api-connectors-dx
+/airbyte-cdk/ @airbytehq/connector-extensibility
+/airbyte-integrations/bases/source-acceptance-tests/ @airbytehq/connector-extensibility
+/airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility
+
+# Oauth
+/airbyte-oauth/ @airbytehq/connector-operations
+/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java @airbytehq/connector-operations
+/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java @airbytehq/connector-operations
 
 # Protocol related items
 /airbyte-protocol/ @airbytehq/protocol-reviewers
 /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
 
+# Airbyte Maintainers
+airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @airbytehq/airbyte-maintainers
+airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseConfigDatabaseTest.java @airbytehq/airbyte-maintainers
+airbyte-commons/src/main/ @airbytehq/airbyte-maintainers
+airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java @airbytehq/airbyte-maintainers
+airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java @airbytehq/airbyte-maintainers
+airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @airbytehq/airbyte-maintainers
+airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/ @airbytehq/airbyte-maintainers
+airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java @airbytehq/airbyte-maintainers
+airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java @airbytehq/airbyte-maintainers
+airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @airbytehq/airbyte-maintainers
+*build.gradle @airbytehq/airbyte-maintainers
+airbyte-integrations/connectors/**/build.gradle # ignore gradle files for individual connectors.
+
 # Normalization
 /airbyte-integrations/bases/base-normalization/ @airbytehq/normalization
 
 # JDBC-based connectors
 /airbyte-integrations/bases/base-java/ @airbytehq/jdbc-connectors
 /airbyte-integrations/connectors/source-jdbc/ @airbytehq/jdbc-connectors
+/airbyte-integrations/connectors/source-alloydb/ @airbytehq/jdbc-connectors
 /airbyte-integrations/connectors/source-bigquery/ @airbytehq/jdbc-connectors
 /airbyte-integrations/connectors/source-clickhouse/ @airbytehq/jdbc-connectors
 /airbyte-integrations/connectors/source-cockroachdb/ @airbytehq/jdbc-connectors
@@ -43,4 +56,4 @@
 /airbyte-integrations/connectors/destination-postgres/ @airbytehq/jdbc-connectors
 /airbyte-integrations/connectors/destination-redshift/ @airbytehq/jdbc-connectors
 /airbyte-integrations/connectors/destination-rockset/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-snowflake/ @airbytehq/jdbc-connectors
+/airbyte-integrations/connectors/source-snowflake/ @airbytehq/jdbc-connectors

.github/ISSUE_TEMPLATE/bug-report.md

@@ -9,16 +9,12 @@ assignees: ''
 <!-- 
 Welcome to Airbyte!
 
-We're really appreciate your report and know that this will help us build an 
+We appreciate your report and know that this will help us build an 
 amazing tool. If you want to contribute yourself, you can find a good place 
 to start by searching for the good-first-issues label or maybe... by trying 
 to solve this one? (we can help debug this with you!)
 
-Right now we are in alpha, so we're releasing versions a lot more frequently than 
-normal. You can help us get to the root of the problem faster by filling out the
-questionnaire below!
-
-It's really important having all information and context. 
+It's really important to have all the information and context. 
 You can remove the examples bellow and fill out with your information.
 
 --> 

.github/actions/build-and-push-branch/action.yml

@@ -13,14 +13,24 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - name: Delete default old docker and replace it with a new one
+      shell: bash
+      run: |
+        sudo rm /var/lib/dpkg/lock
+        sudo rm /var/lib/dpkg/lock-frontend
+        sudo apt-get remove docker.io || sudo apt-get remove docker
+        curl -fsSL https://get.docker.com | bash -
+        sudo rm -f /var/lib/dpkg/lock
+        sudo rm -f /var/lib/dpkg/lock-frontend
+
     - name: Build
       id: build
       uses: ./.github/actions/build-branch
       with:
         branch_version_tag: ${{ inputs.branch_version_tag }}
 
     - name: Login to Docker (on Master)
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         username: ${{ inputs.dockerhub_username }}
         password: ${{ inputs.dockerhub_token }}
@@ -29,5 +39,5 @@ runs:
       run: |
         GIT_REVISION=$(git rev-parse HEAD)
         [ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1
-        VERSION=${{ steps.build.outputs.branch_version_tag }} GIT_REVISION=$GIT_REVISION docker-compose -f docker-compose-cloud.build.yaml push
+        VERSION=${{ steps.build.outputs.branch_version_tag }} GIT_REVISION=$GIT_REVISION docker compose -f docker-compose-cloud.buildx.yaml push
       shell: bash