Airbyte v40 (#295)

Switch to Airbyte v0.40.9 (same as our SaaS product)

---------

Co-authored-by: Chris Wu <chris@faros.ai>

.env

@@ -25,7 +25,7 @@ FAROS_AIRBYTE_FORCE_SETUP=false
 FAROS_INIT_IMAGE=farosai/faros-ce-init:latest
 
 ############################## Airbyte ########################################
-VERSION=0.39.37-alpha
+VERSION=0.40.9
 # Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
 DATABASE_USER=docker
 DATABASE_PASSWORD=docker
@@ -74,7 +74,7 @@ N8N_DOCKER_MOUNT=n8n_data
 #
 # # Contributors - please organise this env file according to the above linked file.
 
-# Source: https://github.com/airbytehq/airbyte/blob/v0.39.37-alpha/.env
+# Source: https://github.com/airbytehq/airbyte/blob/v0.40.9/.env
 
 ### SHARED ###
 
@@ -125,6 +125,10 @@ JOB_MAIN_CONTAINER_CPU_LIMIT=
 JOB_MAIN_CONTAINER_MEMORY_REQUEST=
 JOB_MAIN_CONTAINER_MEMORY_LIMIT=
 
+NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT=
+NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST=
+NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT=
+NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST=
 
 ### LOGGING/MONITORING/TRACKING ###
 TRACKING_STRATEGY=logging
@@ -135,6 +139,7 @@ LOG_LEVEL=INFO
 
 ### APPLICATIONS ###
 # Worker #
+WORKERS_MICRONAUT_ENVIRONMENTS=control
 # Relevant to scaling.
 MAX_SYNC_WORKERS=5
 MAX_SPEC_WORKERS=5
@@ -158,13 +163,15 @@ METRIC_CLIENT=
 # Useful only when metric client is set to be otel. Must start with http:// or https://.
 OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317"
 
-USE_STREAM_CAPABLE_STATE=false
+USE_STREAM_CAPABLE_STATE=true
 
-# extra settings to limit warnings during CE startup
-PAPERCUPS_STORYTIME=disabled
-RUN_DATABASE_MIGRATION_ON_STARTUP=true
-SECRET_PERSISTENCE=NONE
-WORKER_ENVIRONMENT=docker
-NEW_SCHEDULER=
+# extra settings to limit warnings during CE setup
+RUN_DATABASE_MIGRATION_ON_STARTUP=true	
+SECRET_PERSISTENCE=NONE	
+WORKER_ENVIRONMENT=docker	
+NEW_SCHEDULER=	
 DEPLOYMENT_MODE=
+JOB_ERROR_REPORTING_STRATEGY=	
 JOB_ERROR_REPORTING_SENTRY_DSN=
+LOG_CONNECTOR_MESSAGES=
+TEMPORAL_HISTORY_RETENTION_IN_DAYS=

.github/workflows/pr.yml

@@ -53,9 +53,34 @@ jobs:
         run: npm run test:unit -- --coverage
         working-directory: init
 
-      # In order to optimize for speed, we use the latest init image.
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            farosai/faros-ce-init
+          flavor: |
+            latest=auto
+          tags: |
+            type=sha,format=long,prefix=
+            type=raw,value=latest,enable=true
+
+      - name: Set up Docker QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          load: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
       - name: Start services
-        run: FAROS_EMAIL=integration.tests@faros.ai FAROS_INIT_IMAGE=farosai/faros-ce-init:latest docker compose up --quiet-pull -d
+        run: FAROS_EMAIL=integration.tests@faros.ai FAROS_INIT_IMAGE=farosai/faros-ce-init:${{ github.sha }} docker compose up --quiet-pull -d
 
       - name: Show logs
         run: docker compose logs --tail all
@@ -69,9 +94,8 @@ jobs:
           echo "Faros init container exit code was ${CONTAINER_EXIT_CODE}"
           [[ $CONTAINER_EXIT_CODE -eq 0 ]]
 
-      - name: Get destination id & Hasura Admin Secret
+      - name: Get Hasura Admin Secret
         run: |
-          echo "DESTINATION_ID=$(cat ${{ github.workspace }}/init/resources/airbyte/workspace/airbyte_config/DESTINATION_CONNECTION.yaml | yq '.[0].destinationId')" >> "$GITHUB_ENV"
           echo $(cat .env | grep "^HASURA_GRAPHQL_ADMIN_SECRET") >> "$GITHUB_ENV"
 
       - name: Run integration tests (init)

.github/workflows/release.yml

@@ -89,9 +89,8 @@ jobs:
           echo "Faros init container exit code was ${CONTAINER_EXIT_CODE}"
           [[ $CONTAINER_EXIT_CODE -eq 0 ]]
 
-      - name: Get destination id & Hasura Admin Secret
+      - name: Get Hasura Admin Secret
         run: |
-          echo "DESTINATION_ID=$(cat ${{ github.workspace }}/init/resources/airbyte/workspace/airbyte_config/DESTINATION_CONNECTION.yaml | yq '.[0].destinationId')" >> "$GITHUB_ENV"
           echo $(cat .env | grep "^HASURA_GRAPHQL_ADMIN_SECRET") >> "$GITHUB_ENV"
 
       - name: Run integration tests (init)

cli/src/airbyte/airbyte-client.ts

@@ -42,6 +42,35 @@ export class Airbyte {
     );
   }
 
+  async getFirstWorkspace(): Promise<string> {
+    const response = await this.api.post('/workspaces/list', {});
+    return response.data.workspaces[0].workspaceId as string;
+  }
+
+  async findFarosSource(name: string): Promise<string> {
+    const workspaceId = await this.getFirstWorkspace();
+    const response = await this.api
+      .post('/sources/list', {workspaceId})
+      .catch((err) => {
+        throw wrapApiError(err, 'Failed to call /sources/list');
+      });
+    return response.data.sources.filter(
+      (source: any) => source.name === name
+    )[0].sourceId;
+  }
+
+  async findFarosConnection(name: string): Promise<string> {
+    const workspaceId = await this.getFirstWorkspace();
+    const response = await this.api
+      .post('/connections/list', {workspaceId})
+      .catch((err) => {
+        throw wrapApiError(err, 'Failed to call /connections/list');
+      });
+    return response.data.connections.filter(
+      (connection: any) => connection.name === name
+    )[0].connectionId;
+  }
+
   async setupSource(config: any): Promise<void> {
     display('Setting up source %s', Emoji.SETUP);
     await this.api
@@ -158,19 +187,22 @@ export class Airbyte {
     return `${this.airbyteUrl}/connections/${connectionId}/status`;
   }
 
-  async isActiveConnection(connectionId: string): Promise<boolean> {
+  async isActiveConnection(connectionName: string): Promise<boolean> {
+    const connection = await this.findFarosConnection(connectionName);
+
     const response = await this.api
       .post('/jobs/list', {
         configTypes: ['sync'],
-        configId: connectionId,
+        configId: connection,
       })
       .catch((err) => {
         throw wrapApiError(err, ' Failed to call /jobs/list');
       });
     return response.data.jobs[0]?.job.status === 'succeeded';
   }
 
-  async refresh(connectionId: string, sourceName: string): Promise<void> {
+  async refresh(connectionName: string, sourceName: string): Promise<void> {
+    const connectionId = await this.findFarosConnection(connectionName);
     const job = await this.triggerSync(connectionId);
     await this.track(job, connectionId, sourceName);
   }

cli/src/bitbucket/run.ts

@@ -20,8 +20,6 @@ import {
   runSelect,
 } from '../utils/prompts';
 
-const BITBUCKET_SOURCE_ID = '5a19e927-51a2-4d5f-9b26-b35aba0910e0';
-export const BITBUCKET_CONNECTION_ID = '2093cc9f-81d5-47df-8c14-d898c89f4c81';
 const DEFAULT_CUTOFF_DAYS = 30;
 const DEFAULT_API_URL = 'https://api.bitbucket.org/2.0';
 
@@ -167,6 +165,7 @@ export async function runBitbucket(cfg: BitbucketConfig): Promise<void> {
       }
     }
 
+    const bitbucketSourceId = await cfg.airbyte.findFarosSource('Bitbucket');
     await cfg.airbyte.setupSource({
       connectionConfiguration: {
         serverUrl,
@@ -178,15 +177,18 @@ export async function runBitbucket(cfg: BitbucketConfig): Promise<void> {
         pagelen: 10,
       },
       name: 'Bitbucket',
-      sourceId: BITBUCKET_SOURCE_ID,
+      sourceId: bitbucketSourceId,
     });
   } catch (error) {
     errorLog('Setup failed %s', Emoji.FAILURE, error);
     return;
   }
 
+  const bitbucketConnectionId = await cfg.airbyte.findFarosConnection(
+    'Bitbucket - Faros'
+  );
   await cfg.airbyte.triggerAndTrackSync(
-    BITBUCKET_CONNECTION_ID,
+    bitbucketConnectionId,
     'Bitbucket',
     cfg.cutoffDays || DEFAULT_CUTOFF_DAYS,
     repos?.length || 0

cli/src/github/run.ts

@@ -20,8 +20,6 @@ import {
   runSelect,
 } from '../utils/prompts';
 
-const GITHUB_SOURCE_ID = '5d9079ca-8173-406f-bfdb-41f19c62daff';
-export const GITHUB_CONNECTION_ID = '6421df4e-0c5a-4666-a530-9c01de683518';
 const DEFAULT_CUTOFF_DAYS = 30;
 
 interface GithubConfig {
@@ -125,6 +123,7 @@ export async function runGithub(cfg: GithubConfig): Promise<void> {
       }
     }
 
+    const githubSourceId = await cfg.airbyte.findFarosSource('GitHub');
     await cfg.airbyte.setupSource({
       connectionConfiguration: {
         repository: repos?.join(' '),
@@ -136,15 +135,18 @@ export async function runGithub(cfg: GithubConfig): Promise<void> {
         page_size_for_large_streams: 10,
       },
       name: 'GitHub',
-      sourceId: GITHUB_SOURCE_ID,
+      sourceId: githubSourceId,
     });
   } catch (error) {
     errorLog('Setup failed %s', Emoji.FAILURE, error);
     return;
   }
 
+  const githubConnectionId = await cfg.airbyte.findFarosConnection(
+    'GitHub - Faros'
+  );
   await cfg.airbyte.triggerAndTrackSync(
-    GITHUB_CONNECTION_ID,
+    githubConnectionId,
     'GitHub',
     cfg.cutoffDays || DEFAULT_CUTOFF_DAYS,
     repos?.length || 0

cli/src/gitlab/run.ts

@@ -21,8 +21,6 @@ import {
   runSelect,
 } from '../utils/prompts';
 
-const GITLAB_SOURCE_ID = '59c74ca4-8cbb-4c65-8cb7-66bf771190fb';
-export const GITLAB_CONNECTION_ID = 'cef1b90d-ab16-4645-a0e3-b81818b8ffc7';
 const DEFAULT_CUTOFF_DAYS = 30;
 const DEFAULT_API_URL = 'gitlab.com';
 
@@ -134,6 +132,7 @@ export async function runGitlab(cfg: GitLabConfig): Promise<void> {
       }
     }
 
+    const gitlabSourceId = await cfg.airbyte.findFarosSource('GitLab');
     await cfg.airbyte.setupSource({
       connectionConfiguration: {
         api_url,
@@ -142,15 +141,18 @@ export async function runGitlab(cfg: GitLabConfig): Promise<void> {
         private_token: token,
       },
       name: 'GitLab',
-      sourceId: GITLAB_SOURCE_ID,
+      sourceId: gitlabSourceId,
     });
   } catch (error) {
     errorLog('Setup failed %s', Emoji.FAILURE, error);
     return;
   }
 
+  const gitlabConnectionId = await cfg.airbyte.findFarosConnection(
+    'GitLab - Faros'
+  );
   await cfg.airbyte.triggerAndTrackSync(
-    GITLAB_CONNECTION_ID,
+    gitlabConnectionId,
     'GitLab',
     cfg.cutoffDays || DEFAULT_CUTOFF_DAYS,
     projects?.length || 0

cli/src/jira/run.ts

@@ -21,8 +21,6 @@ import {
   runSelect,
 } from '../utils/prompts';
 
-const JIRA_SOURCE_ID = '22852029-670c-4296-958e-c581fa76ae98';
-export const JIRA_CONNECTION_ID = '577ceecf-a92a-4785-b385-c41112d7f537';
 const DEFAULT_CUTOFF_DAYS = 30;
 
 interface JiraConfig {
@@ -154,6 +152,7 @@ export async function runJira(cfg: JiraConfig): Promise<void> {
       }
     }
 
+    const jiraSourceId = await cfg.airbyte.findFarosSource('Jira');
     await cfg.airbyte.setupSource({
       connectionConfiguration: {
         email,
@@ -166,15 +165,18 @@ export async function runJira(cfg: JiraConfig): Promise<void> {
         expand_issue_changelog: true,
       },
       name: 'Jira',
-      sourceId: JIRA_SOURCE_ID,
+      sourceId: jiraSourceId,
     });
   } catch (error) {
     errorLog('Setup failed %s', Emoji.FAILURE, error);
     return;
   }
 
+  const jiraConnectionId = await cfg.airbyte.findFarosConnection(
+    'Jira - Faros'
+  );
   await cfg.airbyte.triggerAndTrackSync(
-    JIRA_CONNECTION_ID,
+    jiraConnectionId,
     'Jira',
     cfg.cutoffDays || DEFAULT_CUTOFF_DAYS,
     projects?.length || 0

cli/src/refresh/run.ts

@@ -1,10 +1,6 @@
 import {Command} from 'commander';
 
 import {Airbyte} from '../airbyte/airbyte-client';
-import {BITBUCKET_CONNECTION_ID} from '../bitbucket/run';
-import {GITHUB_CONNECTION_ID} from '../github/run';
-import {GITLAB_CONNECTION_ID} from '../gitlab/run';
-import {JIRA_CONNECTION_ID} from '../jira/run';
 import {Metabase} from '../metabase/metabase-client';
 import {display, Emoji} from '../utils';
 
@@ -34,24 +30,24 @@ export async function runRefresh(cfg: RefreshConfig): Promise<void> {
   await cfg.airbyte.waitUntilHealthy();
   const work = [];
 
-  if (await cfg.airbyte.isActiveConnection(GITHUB_CONNECTION_ID)) {
+  if (await cfg.airbyte.isActiveConnection('GitHub - Faros')) {
     display('refreshing GitHub %s', Emoji.SYNC);
-    work.push(cfg.airbyte.refresh(GITHUB_CONNECTION_ID, 'GitHub'));
+    work.push(cfg.airbyte.refresh('GitHub - Faros', 'GitHub'));
   }
 
-  if (await cfg.airbyte.isActiveConnection(GITLAB_CONNECTION_ID)) {
+  if (await cfg.airbyte.isActiveConnection('GitLab - Faros')) {
     display('refreshing GitLab %s', Emoji.SYNC);
-    work.push(cfg.airbyte.refresh(GITLAB_CONNECTION_ID, 'GitLab'));
+    work.push(cfg.airbyte.refresh('GitLab - Faros', 'GitLab'));
   }
 
-  if (await cfg.airbyte.isActiveConnection(BITBUCKET_CONNECTION_ID)) {
+  if (await cfg.airbyte.isActiveConnection('Bitbucket - Faros')) {
     display('refreshing Bitbucket %s', Emoji.SYNC);
-    work.push(cfg.airbyte.refresh(BITBUCKET_CONNECTION_ID, 'Bitbucket'));
+    work.push(cfg.airbyte.refresh('Bitbucket - Faros', 'Bitbucket'));
   }
 
-  if (await cfg.airbyte.isActiveConnection(JIRA_CONNECTION_ID)) {
+  if (await cfg.airbyte.isActiveConnection('Jira - Faros')) {
     display('refreshing Jira %s', Emoji.SYNC);
-    work.push(cfg.airbyte.refresh(JIRA_CONNECTION_ID, 'Jira'));
+    work.push(cfg.airbyte.refresh('Jira - Faros', 'Jira'));
   }
 
   if (work.length === 0) {