Merge branch 'master' into pnilan/source-chargebee-schema-update

airbytehq · Jan 26, 2024 · f5051e8 · f5051e8
2 parents 69116f5 + 7fe9b09
commit f5051e8
Show file tree

Hide file tree

Showing 268 changed files with 6,296 additions and 17,023 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.50.44
+current_version = 0.50.45
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

diff --git a/.github/actions/airbyte-ci-requirements/action.yml b/.github/actions/airbyte-ci-requirements/action.yml
@@ -84,7 +84,7 @@ runs:
       id: get-dagger-version
       shell: bash
       run: |
-        dagger_version=$(airbyte-ci ${{ inputs.airbyte_ci_command }} --ci-requirements | tail -n 1 | jq -r '.dagger_version')
+        dagger_version=$(airbyte-ci --disable-update-check ${{ inputs.airbyte_ci_command }} --ci-requirements | tail -n 1 | jq -r '.dagger_version')
         echo "dagger_version=${dagger_version}" >> "$GITHUB_OUTPUT"
 
     - name: Get runner name

diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml
@@ -83,6 +83,9 @@ inputs:
     description: "URL to airbyte-ci binary"
     required: false
     default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
+  python_registry_token:
+    description: "Python registry API token to publish python package"
+    required: false
 
 runs:
   using: "composite"
@@ -154,7 +157,7 @@ runs:
       shell: bash
       run: |
         export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
-        airbyte-ci --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
+        airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
       env:
         CI_CONTEXT: "${{ inputs.context }}"
         CI_GIT_BRANCH: ${{ inputs.git_branch || github.head_ref }}
@@ -182,3 +185,4 @@ runs:
         CI: "True"
         TAILSCALE_AUTH_KEY: ${{ inputs.tailscale_auth_key }}
         DOCKER_REGISTRY_MIRROR_URL: ${{ inputs.docker_registry_mirror_url }}
+        PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }}
diff --git a/.github/workflows/publish-airbyte-lib-command-manually.yml b/.github/workflows/publish-airbyte-lib-command-manually.yml
@@ -0,0 +1,57 @@
+name: Publish AirbyteLib Manually
+on: workflow_dispatch
+
+concurrency:
+  group: publish-airbyte-lib
+  cancel-in-progress: false
+
+jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "publish"
+          runner_size: "large"
+          # Getting ci requirements for connectors publish command as there is no special one for poetry publish
+          airbyte_ci_command: "connectors publish"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
+  publish_connectors:
+    name: Publish airbyte-lib
+    needs: get_ci_runner
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+      - name: Publish
+        id: publish-airbyte-lib
+        uses: ./.github/actions/run-dagger-pipeline
+        with:
+          context: "manual"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
+          spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
+          s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
+          subcommand: 'poetry --package-path=airbyte-lib publish --registry-url="https://test.pypi.org/legacy/"'
+          python_registry_token: ${{ secrets.PYPI_TOKEN }}
diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml
@@ -63,6 +63,7 @@ jobs:
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
           subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release"
+          python_registry_token: ${{ secrets.PYPI_TOKEN }}
 
       - name: Publish connectors [manual]
         id: publish-connectors
@@ -84,6 +85,7 @@ jobs:
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
           subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}"
+          python_registry_token: ${{ secrets.PYPI_TOKEN }}
 
   set-instatus-incident-on-failure:
     name: Create Instatus Incident on Failure

diff --git a/.gitignore b/.gitignore
@@ -69,13 +69,6 @@ resources/examples/airflow/logs/*
 # Summary.md keeps getting added and we just don't like it
 docs/SUMMARY.md
 
-# Files generated by unit tests
-**/specs_secrets_mask.yaml
-
-# Files generated when downloading connector registry
-**/init-oss/src/main/resources/seed/oss_registry.json
-**/init-oss/src/main/resources/seed/oss_catalog.json
-
 # Output Files generated by scripts
 lowcode_connector_names.txt
 num_lowcode_connectors.csv

diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md
@@ -166,6 +166,13 @@ MavenLocal debugging steps:
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                        |
 |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.15.2  | 2024-01-25 | [\#34441](https://github.com/airbytehq/airbyte/pull/34441) | Improve airbyte-api build performance.                                                                                                                         |
+| 0.15.1  | 2024-01-25 | [\#34451](https://github.com/airbytehq/airbyte/pull/34451) | Async destinations: Better logging when we fail to parse an AirbyteMessage                                                                                     |
+| 0.15.0  | 2024-01-23 | [\#34441](https://github.com/airbytehq/airbyte/pull/34441) | Removed connector registry and micronaut dependencies.                                                                                                         |
+| 0.14.2  | 2024-01-24 | [\#34458](https://github.com/airbytehq/airbyte/pull/34458) | Handle case-sensitivity in sentry error grouping                                                                                                               |
+| 0.14.1  | 2024-01-24 | [\#34468](https://github.com/airbytehq/airbyte/pull/34468) | Add wait for process to be done before ending sync in destination BaseTDTest                                                                                   |
+| 0.14.0  | 2024-01-23 | [\#34461](https://github.com/airbytehq/airbyte/pull/34461) | Revert non backward compatible signature changes from 0.13.1                                                                                                   |
+| 0.13.3  | 2024-01-23 | [\#34077](https://github.com/airbytehq/airbyte/pull/34077) | Denote if destinations fully support Destinations V2                                                                                                           |
 | 0.13.2  | 2024-01-18 | [\#34364](https://github.com/airbytehq/airbyte/pull/34364) | Better logging in mongo db source connector                                                                                                                    |
 | 0.13.1  | 2024-01-18 | [\#34236](https://github.com/airbytehq/airbyte/pull/34236) | Add postCreateTable hook in destination JdbcSqlGenerator                                                                                                       |
 | 0.13.0  | 2024-01-16 | [\#34177](https://github.com/airbytehq/airbyte/pull/34177) | Add `useExpensiveSafeCasting` param in JdbcSqlGenerator methods; add JdbcTypingDedupingTest fixture; other DV2-related changes                                 |

diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle
@@ -9,12 +9,6 @@ java {
 }
 
 dependencies {
-    annotationProcessor platform(libs.micronaut.bom)
-    annotationProcessor libs.bundles.micronaut.annotation.processor
-
-    implementation platform(libs.micronaut.bom)
-    implementation libs.bundles.micronaut
-
     implementation group: 'joda-time', name: 'joda-time', version: '2.12.5'
     implementation 'io.fabric8:kubernetes-client:5.12.2'
     implementation 'com.auth0:java-jwt:3.19.2'
@@ -33,11 +27,8 @@ dependencies {
     implementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss')
     implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation')
 
-    testAnnotationProcessor platform(libs.micronaut.bom)
-    testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor
     testAnnotationProcessor libs.jmh.annotations
 
-    testImplementation libs.bundles.micronaut.test
     testImplementation 'com.jayway.jsonpath:json-path:2.7.0'
     testImplementation 'org.mockito:mockito-inline:4.7.0'
     testImplementation libs.postgresql

diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle
@@ -11,58 +11,24 @@ java {
     }
 }
 
-def specFile = "$projectDir/src/main/openapi/config.yaml"
+String specFile = "$projectDir/src/main/openapi/config.yaml"
+String serverOutputDir = "$buildDir/generated/api/server"
+String clientOutputDir = "$buildDir/generated/api/client"
+String docsOutputDir = "$buildDir/generated/api/docs"
+Map<String,String> schemaMappingsValue = [
+        'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
+        'SourceDefinitionSpecification'     : 'com.fasterxml.jackson.databind.JsonNode',
+        'SourceConfiguration'               : 'com.fasterxml.jackson.databind.JsonNode',
+        'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode',
+        'DestinationConfiguration'          : 'com.fasterxml.jackson.databind.JsonNode',
+        'StreamJsonSchema'                  : 'com.fasterxml.jackson.databind.JsonNode',
+        'StateBlob'                         : 'com.fasterxml.jackson.databind.JsonNode',
+        'FieldSchema'                       : 'com.fasterxml.jackson.databind.JsonNode',
+]
 
 def generate = tasks.register('generate')
 
-// Deprecated -- can be removed once airbyte-server is converted to use the per-domain endpoints generated by generateApiServer
-def generateApiServerLegacy = tasks.register('generateApiServerLegacy', GenerateTask) {
-    def serverOutputDir = "$buildDir/generated/api/server"
-
-    inputs.file specFile
-    outputs.dir serverOutputDir
-
-    generatorName = "jaxrs-spec"
-    inputSpec = specFile
-    outputDir = serverOutputDir
-
-    apiPackage = "io.airbyte.api.generated"
-    invokerPackage = "io.airbyte.api.invoker.generated"
-    modelPackage = "io.airbyte.api.model.generated"
-
-    schemaMappings.set([
-            'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
-            'SourceDefinitionSpecification'     : 'com.fasterxml.jackson.databind.JsonNode',
-            'SourceConfiguration'               : 'com.fasterxml.jackson.databind.JsonNode',
-            'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode',
-            'DestinationConfiguration'          : 'com.fasterxml.jackson.databind.JsonNode',
-            'StreamJsonSchema'                  : 'com.fasterxml.jackson.databind.JsonNode',
-            'StateBlob'                         : 'com.fasterxml.jackson.databind.JsonNode',
-            'FieldSchema'                       : 'com.fasterxml.jackson.databind.JsonNode',
-    ])
-
-    generateApiDocumentation = false
-
-    configOptions.set([
-            dateLibrary  : "java8",
-            generatePom  : "false",
-            interfaceOnly: "true",
-            /*
-            JAX-RS generator does not respect nullable properties defined in the OpenApi Spec.
-            It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json.
-            The below Jackson annotation is made to only keep non null values in serialized json.
-            We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above.
-            Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details.
-            */
-            additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)",
-    ])
-}
-generate.configure {
-    dependsOn generateApiServerLegacy
-}
-
 def generateApiServer = tasks.register('generateApiServer', GenerateTask) {
-    def serverOutputDir = "$buildDir/generated/api/server"
 
     inputs.file specFile
     outputs.dir serverOutputDir
@@ -75,16 +41,7 @@ def generateApiServer = tasks.register('generateApiServer', GenerateTask) {
     invokerPackage = "io.airbyte.api.invoker.generated"
     modelPackage = "io.airbyte.api.model.generated"
 
-    schemaMappings.set([
-            'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
-            'SourceDefinitionSpecification'     : 'com.fasterxml.jackson.databind.JsonNode',
-            'SourceConfiguration'               : 'com.fasterxml.jackson.databind.JsonNode',
-            'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode',
-            'DestinationConfiguration'          : 'com.fasterxml.jackson.databind.JsonNode',
-            'StreamJsonSchema'                  : 'com.fasterxml.jackson.databind.JsonNode',
-            'StateBlob'                         : 'com.fasterxml.jackson.databind.JsonNode',
-            'FieldSchema'                       : 'com.fasterxml.jackson.databind.JsonNode',
-    ])
+    schemaMappings.set(schemaMappingsValue)
 
     generateApiDocumentation = false
 
@@ -100,17 +57,13 @@ def generateApiServer = tasks.register('generateApiServer', GenerateTask) {
             Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details.
             */
             additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)",
-
-            // Generate separate classes for each endpoint "domain"
-            useTags: "true"
     ])
 }
 generate.configure {
     dependsOn generateApiServer
 }
 
 def generateApiClient = tasks.register('generateApiClient', GenerateTask) {
-    def clientOutputDir = "$buildDir/generated/api/client"
 
     inputs.file specFile
     outputs.dir clientOutputDir
@@ -123,16 +76,7 @@ def generateApiClient = tasks.register('generateApiClient', GenerateTask) {
     invokerPackage = "io.airbyte.api.client.invoker.generated"
     modelPackage = "io.airbyte.api.client.model.generated"
 
-    schemaMappings.set([
-            'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
-            'SourceDefinitionSpecification'     : 'com.fasterxml.jackson.databind.JsonNode',
-            'SourceConfiguration'               : 'com.fasterxml.jackson.databind.JsonNode',
-            'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode',
-            'DestinationConfiguration'          : 'com.fasterxml.jackson.databind.JsonNode',
-            'StreamJsonSchema'                  : 'com.fasterxml.jackson.databind.JsonNode',
-            'StateBlob'                         : 'com.fasterxml.jackson.databind.JsonNode',
-            'FieldSchema'                       : 'com.fasterxml.jackson.databind.JsonNode',
-    ])
+    schemaMappings.set(schemaMappingsValue)
 
     library = "native"
 
@@ -149,7 +93,6 @@ generate.configure {
 }
 
 def generateApiDocs = tasks.register('generateApiDocs', GenerateTask) {
-    def docsOutputDir = "$buildDir/generated/api/docs"
 
     generatorName = "html"
     inputSpec = specFile
@@ -159,16 +102,7 @@ def generateApiDocs = tasks.register('generateApiDocs', GenerateTask) {
     invokerPackage = "io.airbyte.api.client.invoker.generated"
     modelPackage = "io.airbyte.api.client.model.generated"
 
-    schemaMappings.set([
-            'OAuthConfiguration'                : 'com.fasterxml.jackson.databind.JsonNode',
-            'SourceDefinitionSpecification'     : 'com.fasterxml.jackson.databind.JsonNode',
-            'SourceConfiguration'               : 'com.fasterxml.jackson.databind.JsonNode',
-            'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode',
-            'DestinationConfiguration'          : 'com.fasterxml.jackson.databind.JsonNode',
-            'StreamJsonSchema'                  : 'com.fasterxml.jackson.databind.JsonNode',
-            'StateBlob'                         : 'com.fasterxml.jackson.databind.JsonNode',
-            'FieldSchema'                       : 'com.fasterxml.jackson.databind.JsonNode',
-    ])
+    schemaMappings.set(schemaMappingsValue)
 
     generateApiDocumentation = false
 
@@ -190,9 +124,25 @@ def generateApiDocs = tasks.register('generateApiDocs', GenerateTask) {
         }
     }
 }
-generate.configure {
+def deleteExistingDocs = tasks.register('deleteOldApiDocs', Delete) {
+    delete rootProject.file("docs/reference/api/generated-api-html")
+}
+deleteExistingDocs.configure {
     dependsOn generateApiDocs
 }
+def copyApiDocs = tasks.register('copyApiDocs', Copy) {
+    from(docsOutputDir) {
+        include "**/*.html"
+    }
+    into rootProject.file("docs/reference/api/generated-api-html")
+    includeEmptyDirs = false
+}
+copyApiDocs.configure {
+    dependsOn deleteExistingDocs
+}
+generate.configure {
+    dependsOn copyApiDocs
+}
 
 dependencies {
     implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310'
@@ -209,7 +159,11 @@ dependencies {
 sourceSets {
     main {
         java {
-            srcDirs "$buildDir/generated/api/server/src/gen/java", "$buildDir/generated/api/client/src/main/java", "$projectDir/src/main/java"
+            srcDirs([
+                "$projectDir/src/main/java",
+                "${serverOutputDir}/src/gen/java",
+                "${clientOutputDir}/src/main/java",
+            ])
         }
         resources {
             srcDir "$projectDir/src/main/openapi/"

diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle
@@ -5,12 +5,6 @@ java {
 }
 
 dependencies {
-    annotationProcessor libs.bundles.micronaut.annotation.processor
-    testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor
-
-    implementation libs.bundles.micronaut.annotation
-    testImplementation libs.bundles.micronaut.test
-
     implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons')
     implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation')
 }
diff --git a/...te-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java b/...te-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java
@@ -9,8 +9,6 @@
 import io.airbyte.commons.protocol.migrations.MigrationContainer;
 import io.airbyte.commons.version.Version;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
-import jakarta.annotation.PostConstruct;
-import jakarta.inject.Singleton;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -21,7 +19,6 @@
  * This class is intended to apply the transformations required to go from one version of the
  * AirbyteProtocol to another.
  */
-@Singleton
 public class AirbyteMessageMigrator {
 
   private final MigrationContainer<AirbyteMessageMigration<?, ?>> migrationContainer;
@@ -30,7 +27,6 @@ public AirbyteMessageMigrator(final List<AirbyteMessageMigration<?, ?>> migratio
     migrationContainer = new MigrationContainer<>(migrations);
   }
 
-  @PostConstruct
   public void initialize() {
     migrationContainer.initialize();
   }