🎉 Source Salesforce: add optional parameters to filter properties within streams

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.35.9-alpha
+current_version = 0.35.13-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.35.9-alpha
+VERSION=0.35.13-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.env.dev

@@ -27,4 +27,3 @@ SYNC_JOB_MAX_TIMEOUT_DAYS=3
 
 # Sentry
 SENTRY_DSN=""
-

.github/workflows/platform-project-automation.yml

@@ -0,0 +1,34 @@
+# See https://github.com/marketplace/actions/project-beta-automations for guidance
+
+name: Platform Project Automation
+on:
+  issues:
+    types: [labeled]
+
+env:
+  GH_PROJECT_TOKEN: ${{ secrets.PARKER_PAT_FOR_PLATFORM_PROJECT_AUTOMATION }}
+  ORG: airbytehq
+  PROJECT_ID: 6 # https://github.com/orgs/airbytehq/projects/6/views/8
+  FIELD_STATUS: Status
+  STATUS_TODO: Todo
+  FIELD_DATE_ADDED: Date Added
+
+jobs:
+  add-area-platform-issues-to-platform-project:
+    runs-on: ubuntu-latest
+    name: Add area/platform issue to Platform Project
+    steps:
+      - name: Set current date env var
+        id: set_date
+        run: echo ::set-output name=CURRENT_DATE::$(date +'%Y-%m-%dT%H:%M:%S%z')
+
+      - name: Add issue to project if labelled with area/platform
+        uses: leonsteinhaeuser/project-beta-automations@v1.1.0
+        if: contains(github.event.issue.labels.*.name, 'area/platform')
+        with:
+          gh_token: ${{ env.GH_PROJECT_TOKEN }}
+          organization: ${{ env.ORG }}
+          project_id: ${{ env.PROJECT_ID }}
+          resource_node_id: ${{ github.event.issue.node_id }}
+          operation_mode: custom_field
+          custom_field_values: '[{\"name\": \"Status\",\"type\": \"single_select\",\"value\": \"${{ env.STATUS_TODO }}\"},{\"name\": \"${{ env.FIELD_DATE_ADDED }}\",\"type\": \"date\",\"value\": \"${{ steps.set_date.outputs.CURRENT_DATE }}\"}]'

.github/workflows/publish-external-command.yml

@@ -0,0 +1,113 @@
+name: Publish External Connector Image
+on:
+  workflow_dispatch:
+    inputs:
+      connector:
+        description: "Airbyte Connector image"
+        required: true
+      version:
+        description: "Airbyte Connector version"
+        required: true
+      comment-id:
+        description: "The comment-id of the slash command. Used to update the comment with the status."
+        required: false
+
+jobs:
+  ## Gradle Build
+  # In case of self-hosted EC2 errors, remove this block.
+  start-publish-image-runner:
+    name: Start Build EC2 Runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+        with:
+          repository: ${{github.event.pull_request.head.repo.full_name}} # always use the branch's repository
+      - name: Start AWS Runner
+        id: start-ec2-runner
+        uses: ./.github/actions/start-aws-runner
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          # 80 gb disk
+          ec2-image-id: ami-0d648081937c75a73
+  publish-image:
+    needs: start-publish-image-runner
+    runs-on: ${{ needs.start-publish-image-runner.outputs.label }}
+    environment: more-secrets
+    steps:
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
+        with:
+          service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }}
+          export_default_credentials: true
+      - name: Link comment to workflow run
+        if: github.event.inputs.comment-id
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+        with:
+          repository: ${{github.event.pull_request.head.repo.full_name}} # always use the branch's repository
+      - run: |
+          echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD}
+          ./tools/integrations/manage.sh publish_external ${{ github.event.inputs.connector }} ${{ github.event.inputs.version }}
+        name: publish ${{ github.event.inputs.connector }}
+        id: publish
+        env:
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
+          TZ: UTC
+      - name: Add Success Comment
+        if: github.event.inputs.comment-id && success()
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+      - name: Add Failure Comment
+        if: github.event.inputs.comment-id && !success()
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          body: |
+            > :x: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+      - name: Slack Notification - Failure
+        if: failure()
+        uses: rtCamp/action-slack-notify@master
+        env:
+          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
+          SLACK_USERNAME: Buildozer
+          SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
+          SLACK_COLOR: DC143C
+          SLACK_TITLE: "Failed to publish connector ${{ github.event.inputs.connector }} from branch ${{ github.ref	 }}"
+          SLACK_FOOTER: ""
+  # In case of self-hosted EC2 errors, remove this block.
+  stop-publish-image-runner:
+    name: Stop Build EC2 Runner
+    needs:
+      - start-publish-image-runner # required to get output from the start-runner job
+      - publish-image # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          label: ${{ needs.start-publish-image-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-publish-image-runner.outputs.ec2-instance-id }}

.github/workflows/slash-commands.yml

@@ -19,6 +19,7 @@ jobs:
             test
             test-performance
             publish
+            publish-external
             publish-cdk
             gke-kube-test
             run-specific-test

.pre-commit-config.yaml

@@ -11,11 +11,17 @@ repos:
     rev: 21.11b1
     hooks:
       - id: black
+        args: ["--line-length=140"]
   - repo: https://github.com/timothycrosley/isort
     rev: 5.10.1
     hooks:
       - id: isort
-        args: ["--dont-follow-links", "--jobs=-1"]
+        args:
+          [
+            "--settings-path=tools/python/.isort.cfg",
+            "--dont-follow-links",
+            "--jobs=-1",
+          ]
         additional_dependencies: ["colorama"]
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v2.5.0
@@ -34,12 +40,14 @@ repos:
     rev: v0.0.1a2.post1
     hooks:
       - id: pyproject-flake8
+        args: ["--config=tools/python/.flake8"]
         additional_dependencies: ["mccabe"]
         alias: flake8
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.910-1
     hooks:
       - id: mypy
+        args: ["--config-file=tools/python/.mypy.ini"]
         exclude: |
           (?x)^.*(
               octavia-cli/unit_tests/|

airbyte-api/src/main/openapi/config.yaml

@@ -18,6 +18,7 @@ info:
     * All backwards incompatible changes will happen in major version bumps. We will not make backwards incompatible changes in minor version bumps. Examples of non-breaking changes (includes but not limited to...):
       * Adding fields to request or response bodies.
       * Adding new HTTP endpoints.
+    * All `web_backend` APIs are not considered public APIs and are not guaranteeing backwards compatibility.
 
   version: "1.0.0"
   title: Airbyte Configuration API
@@ -53,7 +54,9 @@ tags:
   - name: db_migration
     description: Database migration related resources.
   - name: web_backend
-    description: Connection between sources and destinations.
+    description: |
+      Endpoints for the Airbyte web application. Those APIs should not be called outside the web application implementation and are not
+      guaranteeing any backwards compatibility.
   - name: health
     description: Healthchecks
   - name: deployment
@@ -182,6 +185,29 @@ paths:
           $ref: "#/components/responses/NotFoundResponse"
         "422":
           $ref: "#/components/responses/InvalidInputResponse"
+  /v1/workspaces/update_name:
+    post:
+      tags:
+        - workspace
+      summary: Update workspace name
+      operationId: updateWorkspaceName
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/WorkspaceUpdateName"
+        required: true
+      responses:
+        "200":
+          description: Successful operation
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/WorkspaceRead"
+        "404":
+          $ref: "#/components/responses/NotFoundResponse"
+        "422":
+          $ref: "#/components/responses/InvalidInputResponse"
   /v1/workspaces/tag_feedback_status_as_done:
     post:
       tags:
@@ -1969,6 +1995,16 @@ components:
           type: boolean
         feedbackDone:
           type: boolean
+    WorkspaceUpdateName:
+      type: object
+      required:
+        - workspaceId
+        - name
+      properties:
+        workspaceId:
+          $ref: "#/components/schemas/WorkspaceId"
+        name:
+          type: string
     WorkspaceUpdate:
       type: object
       required:

airbyte-bootloader/Dockerfile

@@ -5,6 +5,6 @@ ENV APPLICATION airbyte-bootloader
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.9-alpha.tar /app
+ADD bin/${APPLICATION}-0.35.13-alpha.tar /app
 
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.9-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.13-alpha/bin/${APPLICATION}"]

airbyte-bootloader/build.gradle

@@ -11,7 +11,9 @@ dependencies {
     implementation project(':airbyte-db:lib')
     implementation project(":airbyte-json-validation")
     implementation project(':airbyte-scheduler:persistence')
+    implementation project(':airbyte-scheduler:models')
 
+    implementation 'io.temporal:temporal-sdk:1.6.0'
     implementation "org.flywaydb:flyway-core:7.14.0"
 
     testImplementation "org.testcontainers:postgresql:1.15.3"

airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java

@@ -5,6 +5,8 @@
 package io.airbyte.bootloader;
 
 import com.google.common.annotations.VisibleForTesting;
+import io.airbyte.commons.features.EnvVariableFeatureFlags;
+import io.airbyte.commons.features.FeatureFlags;
 import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.commons.version.AirbyteVersion;
 import io.airbyte.config.Configs;
@@ -19,9 +21,14 @@
 import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator;
 import io.airbyte.db.instance.jobs.JobsDatabaseInstance;
 import io.airbyte.db.instance.jobs.JobsDatabaseMigrator;
+import io.airbyte.scheduler.models.Job;
+import io.airbyte.scheduler.models.JobStatus;
 import io.airbyte.scheduler.persistence.DefaultJobPersistence;
 import io.airbyte.scheduler.persistence.JobPersistence;
 import io.airbyte.validation.json.JsonValidationException;
+import io.temporal.client.WorkflowClient;
+import io.temporal.serviceclient.WorkflowServiceStubs;
+import io.temporal.serviceclient.WorkflowServiceStubsOptions;
 import java.io.IOException;
 import java.util.Optional;
 import java.util.UUID;
@@ -47,10 +54,12 @@ public class BootloaderApp {
 
   private final Configs configs;
   private Runnable postLoadExecution;
+  private FeatureFlags featureFlags;
 
   @VisibleForTesting
-  public BootloaderApp(Configs configs) {
+  public BootloaderApp(Configs configs, FeatureFlags featureFlags) {
     this.configs = configs;
+    this.featureFlags = featureFlags;
   }
 
   /**
@@ -61,9 +70,10 @@ public BootloaderApp(Configs configs) {
    * @param configs
    * @param postLoadExecution
    */
-  public BootloaderApp(Configs configs, Runnable postLoadExecution) {
+  public BootloaderApp(Configs configs, Runnable postLoadExecution, FeatureFlags featureFlags) {
     this.configs = configs;
     this.postLoadExecution = postLoadExecution;
+    this.featureFlags = featureFlags;
   }
 
   public BootloaderApp() {
@@ -80,6 +90,7 @@ public BootloaderApp() {
         e.printStackTrace();
       }
     };
+    featureFlags = new EnvVariableFeatureFlags();
   }
 
   public void load() throws Exception {
@@ -207,4 +218,16 @@ private static void runFlywayMigration(final Configs configs, final Database con
     }
   }
 
+  private static void cleanupZombies(final JobPersistence jobPersistence) throws IOException {
+    final Configs configs = new EnvConfigs();
+    WorkflowClient wfClient =
+        WorkflowClient.newInstance(WorkflowServiceStubs.newInstance(
+            WorkflowServiceStubsOptions.newBuilder().setTarget(configs.getTemporalHost()).build()));
+    for (final Job zombieJob : jobPersistence.listJobsWithStatus(JobStatus.RUNNING)) {
+      LOGGER.info("Kill zombie job {} for connection {}", zombieJob.getId(), zombieJob.getScope());
+      wfClient.newUntypedWorkflowStub("sync_" + zombieJob.getId())
+          .terminate("Zombie");
+    }
+  }
+
 }