Merge branch 'master' into ddavydov/#4842-source-sendgrid-change-star…

…time-param-type-to-datetime

.github/workflows/gradle.yml

@@ -298,11 +298,15 @@ jobs:
           github-token: ${{ needs.find_valid_pat.outputs.pat }}
   frontend-build:
     name: "Frontend: Build"
-    needs: start-frontend-runner
+    needs:
+      - start-frontend-runner
+      - find_valid_pat
     runs-on: ${{ needs.start-frontend-runner.outputs.label }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Cache Build Artifacts
         uses: ./.github/actions/cache-build-artifacts
@@ -333,6 +337,17 @@ jobs:
 
       - name: Build :airbyte-webapp
         run: SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-webapp:build --scan
+
+      - name: Publish Storybook to Chromatic
+        uses: chromaui/action@v1
+        with:
+          projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
+          token: ${{ needs.find_valid_pat.outputs.pat }}
+          workingDir: ./airbyte-webapp/
+          storybookBuildDir: build/storybook/
+          exitOnceUploaded: true
+
+
   frontend-test:
     name: "Frontend: Run End-to-End Tests"
     needs:
@@ -685,6 +700,166 @@ jobs:
           label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }}
           ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }}
 
+
+  ## Kube Acceptance Tests
+  # Docker acceptance tests run as part of the build job.
+  # In case of self-hosted EC2 errors, remove this block.
+  start-helm-acceptance-test-runner:
+    name: "Platform: Start Helm Acceptance Test Runner"
+    needs:
+      - changes
+      - find_valid_pat
+    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
+    if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - name: Start AWS Runner
+        id: start-ec2-runner
+        uses: ./.github/actions/start-aws-runner
+        with:
+          # github-self-hosted-runner-ubuntu-20-with-150gdisk-docker-20.10.7-and-socat
+          ec2-image-id: ami-0c1a9bc22624339d8
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
+
+
+  # Todo: Kyrylo turn this on.
+#  helm-acceptance-test:
+#    name: "Platform: Acceptance Tests (Helm)"
+#    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
+#    needs: start-helm-acceptance-test-runner # required to start the main job when the runner is ready
+#    runs-on: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
+#    environment: more-secrets
+#    timeout-minutes: 90
+#    steps:
+#      - name: Checkout Airbyte
+#        uses: actions/checkout@v2
+#
+#      - name: Cache Build Artifacts
+#        uses: ./.github/actions/cache-build-artifacts
+#        with:
+#          cache-key: ${{ secrets.CACHE_VERSION }}
+#          cache-python: "false"
+#
+#      - uses: actions/setup-java@v1
+#        with:
+#          java-version: "17"
+#
+#      - uses: actions/setup-node@v2
+#        with:
+#          node-version: "lts/gallium"
+#
+#      - name: Install unzip
+#        shell: bash
+#        run: sudo apt-get update && sudo apt-get install -y unzip
+#
+#      - uses: azure/setup-helm@v3
+#        with:
+#          version: 'latest'
+#          token: ${{ secrets.GITHUB_TOKEN }}
+#        id: install
+#
+#      - name: Fix EC-2 Runner
+#        run: |
+#          mkdir -p /home/runner
+#
+#      - name: Set up CI Gradle Properties
+#        run: |
+#          mkdir -p ~/.gradle/
+#          cat > ~/.gradle/gradle.properties <<EOF
+#          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+#            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+#            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+#            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+#            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+#          org.gradle.workers.max=8
+#          org.gradle.vfs.watch=false
+#          EOF
+#
+#      - name: Create cluster config file
+#        run: |
+#          cat > /tmp/kind-config.yaml <<EOF
+#          kind: Cluster
+#          apiVersion: kind.x-k8s.io/v1alpha4
+#          nodes:
+#          - role: control-plane
+#          - role: worker
+#          EOF
+#
+#      - name: KIND Kubernetes Cluster Setup
+#        uses: helm/kind-action@v1.2.0
+#        with:
+#          node_image: kindest/node:v1.21.2
+#          config: /tmp/kind-config.yaml
+#        # In case of self-hosted EC2 errors, remove this env block.
+#        env:
+#          USER: root
+#          HOME: /home/runner
+#          CHANGE_MINIKUBE_NONE_USER: true
+#
+#      - name: Build Platform Docker Images
+#        run: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
+#
+#      - name: Run Helm End-to-End Acceptance Tests
+#        env:
+#          USER: root
+#          HOME: /home/runner
+#          # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS
+#          AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
+#          SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
+#          SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
+#        run: |
+#          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube_helm.sh
+#      - name: Generate Test Report
+#        uses: dorny/test-reporter@v1
+#        if: always()    # run this step even if previous step failed
+#        with:
+#          name: Platform Helm E2E Test Report
+#          path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml'
+#          reporter: java-junit
+#
+#      - uses: actions/upload-artifact@v2
+#        if: failure()
+#        with:
+#          name: Kubernetes Logs
+#          path: /tmp/kubernetes_logs/*
+
+  # In case of self-hosted EC2 errors, remove this block.
+  stop-helm-acceptance-test-runner:
+    name: "Platform: Stop Helm Acceptance Test EC2 Runner"
+    timeout-minutes: 10
+    needs:
+      - start-helm-acceptance-test-runner # required to get output from the start-runner job
+      - helm-acceptance-test # required to wait when the main job is done
+      - find_valid_pat
+    runs-on: ubuntu-latest
+    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
+    # Thus, we check for skipped here.
+    if: ${{ always() && needs.start-helm-acceptance-test-runner.result != 'skipped'}}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Stop EC2 runner
+        uses: supertopher/ec2-github-runner@base64v1.0.10
+        with:
+          mode: stop
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
+          label: ${{ needs.start-helm-acceptance-test-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-helm-acceptance-test-runner.outputs.ec2-instance-id }}
+
+
+
   notify-failure-slack-channel:
     name: "Notify Slack Channel on Build Failures"
     runs-on: ubuntu-latest
@@ -694,6 +869,7 @@ jobs:
       - octavia-cli-build
       - platform-build
       - kube-acceptance-test
+      - helm-acceptance-test
     if: ${{ failure() && github.ref == 'refs/heads/master' }}
     steps:
       - name: Publish to OSS Build Failure Slack Channel
@@ -718,6 +894,7 @@ jobs:
       - octavia-cli-build
       - platform-build
       - kube-acceptance-test
+      - helm-acceptance-test
     if: success()
     steps:
       - name: Get Previous Workflow Status

.github/workflows/publish-helm-charts.yml

@@ -56,7 +56,7 @@ jobs:
       - name: "Helm package"
         shell: bash
         run: |
-          declare -a StringArray=("airbyte-bootloader" "airbyte-server" "airbyte-temporal" "airbyte-webapp" "airbyte-pod-sweeper" "airbyte-worker", "airbyte-metrics")
+          declare -a StringArray=("airbyte-bootloader" "airbyte-server" "airbyte-temporal" "airbyte-webapp" "airbyte-pod-sweeper" "airbyte-worker" "airbyte-metrics")
           for val in ${StringArray[@]}; do
             cd ./airbyte/charts/${val} && helm dep update && cd $GITHUB_WORKSPACE
             helm package ./airbyte/charts/${val} -d airbyte-oss  --version ${{ needs.generate-semantic-version.outputs.next-version }}

.vscode/frontend.code-workspace

@@ -18,9 +18,9 @@
   "extensions": {
     "recommendations": [
       "dbaeumer.vscode-eslint",
+      "stylelint.vscode-stylelint",
       "esbenp.prettier-vscode",
-      "ms-vsliveshare.vsliveshare",
-      "eamodio.gitlens"
+      "eamodio.gitlens",
     ]
   },
   "settings": {
@@ -33,6 +33,8 @@
     "editor.detectIndentation": true,
     "eslint.format.enable": true,
     "eslint.run": "onType",
+    "stylelint.enable": true,
+    "stylelint.validate": ["css", "scss"],
     "[javascript]": {
       "editor.formatOnSave": true,
       "editor.defaultFormatter": "dbaeumer.vscode-eslint",
@@ -57,6 +59,13 @@
     "[json]": {
       "editor.formatOnSave": true,
       "editor.defaultFormatter": "esbenp.prettier-vscode"
+    },
+    "[scss]": {
+      "editor.formatOnSave": true,
+      "editor.defaultFormatter": "esbenp.prettier-vscode",
+      "editor.codeActionsOnSave": {
+        "source.fixAll.stylelint": true
+      }
     }
   }
 }

.vscode/settings.json

@@ -8,6 +8,7 @@
   "editor.detectIndentation": true,
   "eslint.format.enable": true,
   "eslint.run": "onType",
+  "stylelint.enable": true,
   "stylelint.validate": ["css", "scss"],
   "[javascript]": {
     "editor.formatOnSave": true,
@@ -33,5 +34,12 @@
   "[json]": {
     "editor.formatOnSave": true,
     "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[scss]": {
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "editor.codeActionsOnSave": {
+      "source.fixAll.stylelint": true
+    }
   }
 }

airbyte-analytics/src/main/java/io/airbyte/analytics/TrackingIdentity.java

@@ -9,6 +9,7 @@
 import java.util.Optional;
 import java.util.UUID;
 
+@SuppressWarnings("PMD.CompareObjectsWithEquals")
 public class TrackingIdentity {
 
   private final AirbyteVersion airbyteVersion;

airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java

@@ -136,10 +136,11 @@ void testTrackWithMetadata() {
   }
 
   private static ImmutableMap<String, Object> filterTrackedAtProperty(final Map<String, ?> properties) {
-    assertTrue(properties.containsKey("tracked_at"));
+    final String trackedAtKey = "tracked_at";
+    assertTrue(properties.containsKey(trackedAtKey));
     final Builder<String, Object> builder = ImmutableMap.builder();
     properties.forEach((key, value) -> {
-      if (!"tracked_at".equals(key)) {
+      if (!trackedAtKey.equals(key)) {
         builder.put(key, value);
       }
     });

airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java

@@ -75,7 +75,7 @@ public ApiResponse<File> getLogsWithHttpInfo(final LogsRequestBody logsRequestBo
       if (memberVarResponseInterceptor != null) {
         memberVarResponseInterceptor.accept(localVarResponse);
       }
-      if (localVarResponse.statusCode() / 100 != 2) {
+      if (isErrorResponse(localVarResponse)) {
         throw new ApiException(localVarResponse.statusCode(),
             "getLogs call received non-success response",
             localVarResponse.headers(),
@@ -100,6 +100,10 @@ public ApiResponse<File> getLogsWithHttpInfo(final LogsRequestBody logsRequestBo
     }
   }
 
+  private Boolean isErrorResponse(final HttpResponse<InputStream> httpResponse) {
+    return httpResponse.statusCode() / 100 != 2;
+  }
+
   private HttpRequest.Builder getLogsRequestBuilder(final LogsRequestBody logsRequestBody) throws ApiException {
     // verify the required parameter 'logsRequestBody' is set
     if (logsRequestBody == null) {

airbyte-api/src/main/openapi/config.yaml

@@ -2509,6 +2509,9 @@ components:
           format: uri
         icon:
           type: string
+        protocolVersion:
+          description: The Airbyte Protocol version supported by the connector
+          type: string
         releaseStage:
           $ref: "#/components/schemas/ReleaseStage"
         releaseDate:
@@ -2869,6 +2872,9 @@ components:
           format: uri
         icon:
           type: string
+        protocolVersion:
+          description: The Airbyte Protocol version supported by the connector
+          type: string
         releaseStage:
           $ref: "#/components/schemas/ReleaseStage"
         releaseDate:

airbyte-bootloader/build.gradle

@@ -24,25 +24,15 @@ application {
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-bootloader-*.tar'
-    }
-    into 'build/docker/bin'
-}
-
-Task dockerBuildTask = getDockerBuildTask("bootloader", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)
-
 // produce reproducible archives
 // (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives)
 tasks.withType(AbstractArchiveTask) {
     preserveFileTimestamps = false
     reproducibleFileOrder = true
 }
 
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
+}
+
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-bootloader/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=bootloader

airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java

@@ -128,7 +128,7 @@ void testBootloaderAppBlankDb() throws Exception {
       bootloader.load();
 
       val jobsMigrator = new JobsDatabaseMigrator(jobDatabase, jobsFlyway);
-      assertEquals("0.35.62.001", jobsMigrator.getLatestMigration().getVersion().getVersion());
+      assertEquals("0.40.4.001", jobsMigrator.getLatestMigration().getVersion().getVersion());
 
       val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway);
       // this line should change with every new migration

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.1.83
+- Fix per-stream to send legacy format for connectors that override read
+
+## 0.1.82
+- Freeze dataclasses-jsonschema to 2.15.1
+
 ## 0.1.81
 - Fix regression in `_checkpoint_state` arg