Merge branch 'master' into feature/migrate-opsgenie-cdk

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.50.30
+current_version = 0.50.31
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.github/CODEOWNERS

@@ -1,3 +1,11 @@
+# Vector db connectors
+/airbyte-integrations/connectors/destination-pinecone @airbytehq/ai-language-models
+/airbyte-integrations/connectors/destination-weaviate @airbytehq/ai-language-models
+/airbyte-integrations/connectors/destination-milvus @airbytehq/ai-language-models
+/airbyte-integrations/connectors/destination-qdrant @airbytehq/ai-language-models
+/airbyte-integrations/connectors/destination-chroma @airbytehq/ai-language-models
+/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based @airbytehq/ai-language-models
+
 # CDK and Connector Acceptance Tests
 /airbyte-cdk/python @airbytehq/connector-extensibility
 /airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility

.github/ISSUE_TEMPLATE/2-issue-platform.yaml

@@ -0,0 +1,62 @@
+name: 🐛 Report a platform, infra or deployment bug
+description: Use this template when you have a problem operating Airbyte platform
+labels: [type/bug, area/platform, needs-triage]
+body:
+  - type: markdown
+    attributes:
+      value: >
+        <p align="center">
+          <a target="_blank" href="https://airbyte.com">
+            <image>
+              <source srcset="https://raw.githubusercontent.com/airbytehq/airbyte/master/.github/octavia-issue-template.svg">
+              <img alt="octavia-welcome" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/.github/octavia-issue-template.svg" width="auto" height="120">
+            </image>
+          </a>
+        </p>
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report...
+        Make sure to update this issue with a concise title and provide all information you have to 
+        help us debug the problem together. Issues not following the template will be closed.
+  - type: dropdown
+    id: deploy
+    validations:
+      required: true
+    attributes:
+      label: What method are you using to run Airbyte?
+      multiple: false
+      options:
+        - Docker
+        - Kubernetes
+  - type: input
+    id: platform-version
+    attributes:
+      label: Platform Version or Helm Chart Version
+      description: "Some examples are: (eg. 0.44.1, 0.30.0), you can find the version in the left bottom in Airbyte UI or in the .env / value.yaml file"
+    validations:
+      required: true
+  - type: dropdown
+    id: step
+    attributes:
+      label: What step the error happened?
+      multiple: false
+      options:
+        - On deploy
+        - During the Sync
+        - Upgrading the Platform or Helm Chart
+        - Other
+  - type: textarea
+    id: description
+    attributes:
+      label: Revelant information
+      description: Please give any additional information you have and steps to reproduce the problem.
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: |
+        Please copy and paste any relevant log output. 
+        This will be automatically formatted into code, so no need for backticks.
+        We strongly recommend to upload the log file for further debugging.
+      render: shell

.github/pull_request_template.md

@@ -91,19 +91,19 @@ If this is a community PR, the Airbyte engineer reviewing this PR is responsible
 ### Airbyter
 
 Before merging:
-- [ ] Pull Request description explains what problem it is solving
-- [ ] Code change is unit tested
-- [ ] Build and my-py check pass
-- [ ] Smoke test the change on at least one affected connector
+- Pull Request description explains what problem it is solving
+- Code change is unit tested
+- Build and my-py check pass
+- Smoke test the change on at least one affected connector
    - On Github: Run [this workflow](https://github.com/airbytehq/airbyte/actions/workflows/connectors_tests.yml), passing `--use-local-cdk --name=source-<connector>` as options
    - Locally: `airbyte-ci connectors --use-local-cdk --name=source-<connector> test`
-- [ ] PR is reviewed and approved
+- PR is reviewed and approved
 
 After merging:
-- [ ] [Publish the CDK](https://github.com/airbytehq/airbyte/actions/workflows/publish-cdk-command-manually.yml)
+- [Publish the CDK](https://github.com/airbytehq/airbyte/actions/workflows/publish-cdk-command-manually.yml)
    - The CDK does not follow proper semantic versioning. Choose minor if this the change has significant user impact or is a breaking change. Choose patch otherwise.
    - Write a thoughtful changelog message so we know what was updated.
-- [ ] Merge the platform PR that was auto-created for updating the Connector Builder's CDK version
+- Merge the platform PR that was auto-created for updating the Connector Builder's CDK version
    - This step is optional if the change does not affect the connector builder or declarative connectors.
 
 </details>

.github/workflows/airbyte-ci-tests.yml

@@ -11,36 +11,94 @@ on:
       - opened
       - reopened
       - synchronize
-    paths:
-      - airbyte-ci/**
+    paths-ignore:
+      - "**/*.md"
 jobs:
   run-airbyte-ci-tests:
     name: Run Airbyte CI tests
     runs-on: "conn-prod-xlarge-runner"
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref }}
+      - name: Get changed files
+        uses: dorny/paths-filter@v2
+        id: changes
+        with:
+          filters: |
+            ops:
+              - 'airbyte-ci/connectors/connector_ops/**'
+            base_images:
+              - 'airbyte-ci/connectors/connector_ops/**'
+              - 'airbyte-ci/connectors/base_images/**'
+            pipelines:
+              - 'airbyte-ci/connectors/connector_ops/**'
+              - 'airbyte-ci/connectors/base_images/**'
+              - 'airbyte-ci/connectors/pipelines/**'
+            metadata-lib:
+              - 'airbyte-ci/connectors/metadata/lib/**'
+            metadata-orchestrator:
+              - 'airbyte-ci/connectors/metadata/lib/**'
+              - 'airbyte-ci/connectors/metadata/orchestrator/**'
+
+      - name: Run airbyte-ci/connectors/connector_ops tests
+        if: steps.changes.outputs.ops == 'true'
+        id: run-airbyte-ci-connectors-connector-ops-tests
+        uses: ./.github/actions/run-dagger-pipeline
+        with:
+          context: "pull_request"
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          subcommand: "test airbyte-ci/connectors/connector_ops"
+
       - name: Run airbyte-ci/connectors/pipelines tests
         id: run-airbyte-ci-connectors-pipelines-tests
+        if: steps.changes.outputs.pipelines == 'true'
         uses: ./.github/actions/run-dagger-pipeline
         with:
           context: "pull_request"
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           subcommand: "test airbyte-ci/connectors/pipelines"
+
       - name: Run airbyte-ci/connectors/base_images tests
         id: run-airbyte-ci-connectors-base-images-tests
+        if: steps.changes.outputs.base_images == 'true'
         uses: ./.github/actions/run-dagger-pipeline
         with:
           context: "pull_request"
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           subcommand: "test airbyte-ci/connectors/base_images"
+
+      - name: Run test pipeline for the metadata lib
+        id: metadata-lib-test-pipeline
+        if: steps.changes.outputs.metadata-lib == 'true'
+        uses: ./.github/actions/run-dagger-pipeline
+        with:
+          subcommand: "test airbyte-ci/connectors/metadata_service/lib/"
+          context: "pull_request"
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+      - name: Run test for the metadata orchestrator
+        id: metadata-orchestrator-test-pipeline
+        if: steps.changes.outputs.metadata-orchestrator == 'true'
+        uses: ./.github/actions/run-dagger-pipeline
+        with:
+          subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/"
+          context: "pull_request"
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}

.github/workflows/connector-performance-command.yml

@@ -1,10 +1,24 @@
 name: Connector Performance Harness
 on:
+  workflow_call:
+    inputs:
+      connector:
+        type: string
+        required: true
+      dataset:
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       connector:
         description: "Airbyte Connector"
+        type: choice
         required: true
+        options:
+          - connectors/source-postgres
+          - connectors/source-mysql
+          - connectors/destination-snowflake
+        default: "connectors/source-postgres"
       repo:
         description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos."
         required: false
@@ -102,6 +116,7 @@ jobs:
         with:
           repository: ${{ github.event.inputs.repo }}
           ref: ${{ github.event.inputs.gitref }}
+          fetch-depth: 0 # This is to fetch the main branch in case we are running on a different branch.
       - name: Install Java
         uses: actions/setup-java@v3
         with:
@@ -124,6 +139,7 @@ jobs:
           echo "harness_type=$the_harness" >> "$GITHUB_OUTPUT"
       - name: Write harness credentials
         run: |
+          export PATH="$PATH:/root/.local/bin"
           ci_credentials connectors-performance/$HARNESS_TYPE write-to-storage
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -144,6 +160,8 @@ jobs:
           connector_name=$(echo ${{ github.event.inputs.connector }} | cut -d / -f 2)
           echo "Running ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check"
           ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check
+        env:
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
       - name: KIND Kubernetes Cluster Setup
         uses: helm/kind-action@v1.4.0
         with:
@@ -159,16 +177,15 @@ jobs:
           PREFIX: '{"type":"LOG","log":{"level":"INFO","message":"INFO i.a.i.p.PerformanceTest(runTest):165'
           SUFFIX: '"}}'
           HARNESS_TYPE: ${{ steps.which-harness.outputs.harness_type }}
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
         run: |
           kubectl apply -f ./tools/bin/admin-service-account.yaml
           connector_name=$(echo $CONN | cut -d / -f 2)
           kind load docker-image airbyte/$connector_name:dev --name chart-testing
           kind load docker-image airbyte/$HARNESS_TYPE:dev --name chart-testing
-          # envsubst requires variables to be exported
+          # envsubst requires variables to be exported or setup in the env field in this step.
           export CONNECTOR_IMAGE_NAME=${CONN/connectors/airbyte}:dev
           export DATASET=$DS
-          export STREAM_NUMBER=$STREAM_NUMBER
-          export SYNC_MODE=$SYNC_MODE
           export HARNESS=$HARNESS_TYPE
           envsubst < ./tools/bin/run-harness-process.yaml | kubectl create -f -
           echo "harness is ${{ steps.which-harness.outputs.harness_type }}"
@@ -180,6 +197,7 @@ jobs:
           kubectl logs --tail=1 $POD | while read line ; do line=${line#"$PREFIX"}; line=${line%"$SUFFIX"}; echo $line >> $GITHUB_OUTPUT ; done
           echo "$EOF" >> $GITHUB_OUTPUT
       - name: Link comment to workflow run
+        if: github.event.inputs.comment-id
         uses: peter-evans/create-or-update-comment@v2
         with:
           reactions: "+1"

.github/workflows/connector-performance-cron.yml

@@ -0,0 +1,34 @@
+name: Connector Performance Harness Cron
+on: workflow_dispatch
+
+jobs:
+  postgres-1m-run:
+    uses: ./.github/workflows/connector-performance-command.yml
+    with:
+      connector: connectors/source-postgres
+      dataset: 1m
+  postgres-10m-run:
+    uses: ./.github/workflows/connector-performance-command.yml
+    with:
+      connector: connectors/source-postgres
+      dataset: 10m
+  postgres-20m-run:
+    uses: ./.github/workflows/connector-performance-command.yml
+    with:
+      connector: connectors/source-postgres
+      dataset: 20m
+  mysql-1m-run:
+    uses: ./.github/workflows/connector-performance-command.yml
+    with:
+      connector: connectors/source-mysql
+      dataset: 1m
+  mysql-10m-run:
+    uses: ./.github/workflows/connector-performance-command.yml
+    with:
+      connector: connectors/source-mysql
+      dataset: 10m
+  mysql-20m-run:
+    uses: ./.github/workflows/connector-performance-command.yml
+    with:
+      connector: connectors/source-mysql
+      dataset: 20m

.github/workflows/connector_teams_review_requirements.yml

@@ -2,6 +2,11 @@ name: Connector Ops CI - Check review requirements
 
 on:
   pull_request:
+    types:
+      - opened
+      - reopened
+      - ready_for_review
+      - synchronize
     paths:
       - "airbyte-integrations/connectors/source-**"
   pull_request_review:
@@ -12,7 +17,7 @@ jobs:
     name: "Check if a review is required from Connector teams"
     runs-on: ubuntu-latest
 
-    if: ${{ github.repository == 'airbytehq/airbyte' }}
+    if: ${{ github.repository == 'airbytehq/airbyte' && github.event.pull_request.draft == false }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -39,4 +44,5 @@ jobs:
         with:
           status: ${{ steps.get-mandatory-reviewers.outputs.MANDATORY_REVIEWERS  }}
           token: ${{ secrets.OCTAVIA_4_ROOT_ACCESS }}
+          request-reviews: true
           requirements-file: .github/connector_org_review_requirements.yaml

.github/workflows/format.yml

@@ -13,7 +13,7 @@ jobs:
   format-and-commit:
     runs-on: ubuntu-latest
     name: "Apply All Formatting Rules"
-    timeout-minutes: 20
+    timeout-minutes: 40
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3