diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 57ae97abdd2d1..8f7bd0d80423e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.40.9 +current_version = 0.40.10 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? diff --git a/.env b/.env index 5d7aa17eda4de..db1a7075928e4 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ ### SHARED ### -VERSION=0.40.9 +VERSION=0.40.10 # When using the airbyte-db via default docker image CONFIG_ROOT=/data @@ -57,7 +57,6 @@ WEBAPP_URL=http://localhost:8000/ # Although not present as an env var, required for webapp configuration. API_URL=/api/v1/ - ### JOBS ### # Relevant to scaling. SYNC_JOB_MAX_ATTEMPTS=3 @@ -81,7 +80,7 @@ LOG_LEVEL=INFO ### APPLICATIONS ### # Worker # -WORKERS_MICRONAUT_ENVIRONMENTS=control +WORKERS_MICRONAUT_ENVIRONMENTS=control-plane # Relevant to scaling. MAX_SYNC_WORKERS=5 MAX_SPEC_WORKERS=5 @@ -95,7 +94,6 @@ WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= ### FEATURE FLAGS ### AUTO_DISABLE_FAILING_CONNECTIONS=false -EXPOSE_SECRETS_IN_EXPORT=false FORCE_MIGRATE_SECRET_STORE=false ### MONITORING FLAGS ### diff --git a/.env.dev b/.env.dev index b1780d7ee8c12..687fbe07b5dc3 100644 --- a/.env.dev +++ b/.env.dev @@ -25,7 +25,7 @@ API_URL=/api/v1/ INTERNAL_API_HOST=airbyte-server:8001 SYNC_JOB_MAX_ATTEMPTS=3 SYNC_JOB_MAX_TIMEOUT_DAYS=3 -WORKERS_MICRONAUT_ENVIRONMENTS=control +WORKERS_MICRONAUT_ENVIRONMENTS=control-plane # Sentry SENTRY_DSN="" diff --git a/.github/workflows/assign-issue-to-project.yaml b/.github/workflows/assign-issue-to-project.yaml index a34d0cb6f2dc9..736ae88d15346 100644 --- a/.github/workflows/assign-issue-to-project.yaml +++ b/.github/workflows/assign-issue-to-project.yaml @@ -32,9 +32,3 @@ jobs: with: project: "https://github.com/airbytehq/airbyte/projects/15" column_name: "To do" - - name: Assign Onboarding Improvements PR to Project - uses: srggrs/assign-one-project-github-action@1.2.0 - if: contains(github.event.pull_request.labels.*.name, 'project/onboarding-improvements') - with: - project: "https://github.com/airbytehq/airbyte/projects/15" - column_name: "To do" diff --git a/.github/workflows/label-github-issues-by-context.yml b/.github/workflows/label-github-issues-by-context.yml new file mode 100644 index 0000000000000..48cc8f26d20d4 --- /dev/null +++ b/.github/workflows/label-github-issues-by-context.yml @@ -0,0 +1,27 @@ +name: "Add labels to github issues based on context" +on: + issues: + types: [opened, labeled, unlabeled] + +jobs: + shared-issues: + name: "Add Labels to Issues. Safe to Merge on fail" + runs-on: ubuntu-latest + steps: + - name: Checkout Airbyte Repo for PAT command + uses: actions/checkout@v2 + - name: Check PAT rate limits + # Cannot share PAT outside of JOB context + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.OCTAVIA_4_ROOT_ACCESS }} \ + ${{ secrets.OCTAVIA_PAT }} + - name: Run Issue Command from workflow-actions + uses: nick-fields/private-action-loader@v3 + with: + pal-repo-token: "${{ env.PAT }}" + pal-repo-name: airbytehq/workflow-actions@production + # the following input gets passed to the private + token: "${{ env.PAT }}" + # ref: https://github.com/airbytehq/workflow-actions/blob/main/src/bin_issue.ts + command: "issue" diff --git a/.github/workflows/labeler.yml b/.github/workflows/label-github-issues-by-path.yml similarity index 83% rename from .github/workflows/labeler.yml rename to .github/workflows/label-github-issues-by-path.yml index e1761c4c1771d..d59067a093ea7 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/label-github-issues-by-path.yml @@ -1,5 +1,5 @@ # the mapping from filepath to label -# is defined in .github/labeler.yml +# is defined in .github/label-github-issues-by-path.yml name: "Label PR based on filepath" on: diff --git a/.github/workflows/label-github-issues.yml b/.github/workflows/label-github-issues.yml deleted file mode 100644 index f23361864fe65..0000000000000 --- a/.github/workflows/label-github-issues.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: "Add labels to github issues based on context" -on: - issues: - types: [opened, labeled, unlabeled] - -jobs: - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: UUID ${{ github.event.inputs.uuid }} - run: true - - name: Checkout Airbyte - uses: actions/checkout@v2 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.OCTAVIA_PAT }} \ - ${{ secrets.OCTAVIA_4_ROOT_ACCESS }} - shared-issues: - name: "Add Labels to Issues. Safe to Merge on fail" - runs-on: ubuntu-latest - needs: - - find_valid_pat - steps: - - uses: nick-fields/private-action-loader@v3 - with: - pal-repo-token: "${{ needs.find_valid_pat.outputs.pat }}" - pal-repo-name: airbytehq/workflow-actions@production - # the following input gets passed to the private action - token: "${{ needs.find_valid_pat.outputs.pat }}" - # ref: https://github.com/airbytehq/workflow-actions/blob/main/src/bin_issue.ts - command: "issue" diff --git a/.github/workflows/label-prs-by-context.yml b/.github/workflows/label-prs-by-context.yml new file mode 100644 index 0000000000000..b733096e87c4e --- /dev/null +++ b/.github/workflows/label-prs-by-context.yml @@ -0,0 +1,28 @@ +# Runs internal automation for pull requests + +name: "Add labels to github PRs based on context" +on: + pull_request_target: + types: [opened, labeled, unlabeled, ready_for_review, synchronize, reopened] + +jobs: + shared-pr-labeller: + name: "Add Labels to PRs. Safe to Merge on fail" + runs-on: ubuntu-latest + steps: + - name: Checkout Airbyte Repo for PAT command + uses: actions/checkout@v2 + - name: Check PAT rate limits + # Cannot share PAT outside of JOB context + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.OCTAVIA_4_ROOT_ACCESS }} \ + ${{ secrets.OCTAVIA_PAT }} + - name: Run Issue Command from workflow-actions + uses: nick-fields/private-action-loader@v3 + with: + pal-repo-token: "${{ env.PAT }}" + pal-repo-name: airbytehq/workflow-actions@production + # the following input gets passed to the private action + token: "${{ env.PAT }}" + command: "pull" diff --git a/.github/workflows/notify-on-label.yml b/.github/workflows/notify-on-label.yml index 6d47413f7e55c..dbbb7a4b54ef8 100644 --- a/.github/workflows/notify-on-label.yml +++ b/.github/workflows/notify-on-label.yml @@ -1,6 +1,6 @@ # Notify users/teams when labels are added to an issue. -name: Notify when adding label to issue +name: Notify FE team for FE label on issues on: issues: @@ -8,12 +8,23 @@ on: jobs: notify: + name: "Notify FE team on label creation" runs-on: ubuntu-latest steps: - - uses: jenschelkopf/issue-label-notification-action@1.3 - with: - token: ${{ secrets.OCTAVIA_PAT }} - message: 'cc {recipients}' - # Specify a map of label -> team/user to notify - recipients: | - team/frontend=@airbytehq/frontend + - name: Checkout Airbyte Repo for PAT command + uses: actions/checkout@v2 + - name: Check PAT rate limits + # Cannot share PAT outside of JOB context + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.OCTAVIA_4_ROOT_ACCESS }} \ + ${{ secrets.OCTAVIA_PAT }} + # Updated name to reflect reality. Update name if you change recipients + - name: Notify FE team when on FE label creation + uses: jenschelkopf/issue-label-notification-action@1.3 + with: + token: "${{ env.PAT }}" + message: 'cc {recipients}' + # Specify a map of label -> team/user to notify + recipients: | + team/frontend=@airbytehq/frontend diff --git a/.github/workflows/notify-on-push-to-master.yml b/.github/workflows/notify-on-push-to-master.yml index 48f3a8f352711..d5891fdf310ec 100644 --- a/.github/workflows/notify-on-push-to-master.yml +++ b/.github/workflows/notify-on-push-to-master.yml @@ -1,4 +1,4 @@ -name: Notify Cloud of OSS Push to Master +name: Trigger action in cloud based on push on: push: branches: @@ -7,12 +7,21 @@ on: jobs: repo-sync: + name: "Fire a Repo Dispatch event to airbyte-cloud" runs-on: ubuntu-latest steps: + - name: Checkout Airbyte Repo for PAT command + uses: actions/checkout@v2 + - name: Check PAT rate limits + # Cannot share PAT outside of JOB context + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.OCTAVIA_4_ROOT_ACCESS }} \ + ${{ secrets.OCTAVIA_PAT }} - name: Repository Dispatch uses: peter-evans/repository-dispatch@v2 with: - token: ${{ secrets.OCTAVIA_PAT }} + token: ${{ env.PAT }} repository: airbytehq/airbyte-cloud event-type: oss-push-to-master client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}' diff --git a/.github/workflows/shared-pulls.yml b/.github/workflows/shared-pulls.yml deleted file mode 100644 index 23c065ed7b3bd..0000000000000 --- a/.github/workflows/shared-pulls.yml +++ /dev/null @@ -1,18 +0,0 @@ -# Runs internal automation for pull requests - -name: "Add metadata and labels to PRs based on context" -on: - pull_request_target: - types: [opened, labeled, unlabeled, ready_for_review, synchronize, reopened] - -jobs: - shared-pulls: - runs-on: ubuntu-latest - steps: - - uses: nick-fields/private-action-loader@v3 - with: - pal-repo-token: "${{ secrets.OCTAVIA_PAT }}" - pal-repo-name: airbytehq/workflow-actions@production - # the following input gets passed to the private action - token: "${{ secrets.OCTAVIA_PAT }}" - command: "pull" diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 02a91076c5f0b..6d13ebe89e34a 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -2756,6 +2756,8 @@ components: type: string sourceName: type: string + icon: + type: string SourceReadList: type: object required: @@ -3053,6 +3055,8 @@ components: type: string destinationName: type: string + icon: + type: string DestinationReadList: type: object required: @@ -3813,6 +3817,10 @@ components: type: array items: $ref: "#/components/schemas/AttemptRead" + JobCreatedAt: + description: epoch time of the latest sync job. null if no sync job has taken place. + type: integer + format: int64 JobStatus: type: string enum: @@ -4508,6 +4516,43 @@ components: type: object additionalProperties: true # Web Backend + WebBackendConnectionListItem: + type: object + description: Information about a connection that shows up in the connection list view. + required: + - connectionId + - name + - sourceId + - destinationId + - source + - destination + - status + - isSyncing + properties: + connectionId: + $ref: "#/components/schemas/ConnectionId" + name: + type: string + sourceId: + $ref: "#/components/schemas/SourceId" + destinationId: + $ref: "#/components/schemas/DestinationId" + scheduleType: + $ref: "#/components/schemas/ConnectionScheduleType" + scheduleData: + $ref: "#/components/schemas/ConnectionScheduleData" + status: + $ref: "#/components/schemas/ConnectionStatus" + source: + $ref: "#/components/schemas/SourceRead" + destination: + $ref: "#/components/schemas/DestinationRead" + latestSyncJobCreatedAt: + $ref: "#/components/schemas/JobCreatedAt" + latestSyncJobStatus: + $ref: "#/components/schemas/JobStatus" + isSyncing: + type: boolean WebBackendConnectionRead: type: object required: @@ -4562,9 +4607,7 @@ components: items: $ref: "#/components/schemas/OperationRead" latestSyncJobCreatedAt: - description: epoch time of the latest sync job. null if no sync job has taken place. - type: integer - format: int64 + $ref: "#/components/schemas/JobCreatedAt" latestSyncJobStatus: $ref: "#/components/schemas/JobStatus" isSyncing: @@ -4584,7 +4627,7 @@ components: connections: type: array items: - $ref: "#/components/schemas/WebBackendConnectionRead" + $ref: "#/components/schemas/WebBackendConnectionListItem" SyncMode: type: string enum: diff --git a/airbyte-bootloader/Dockerfile b/airbyte-bootloader/Dockerfile index c79f6786b2e68..d43cc448cac7a 100644 --- a/airbyte-bootloader/Dockerfile +++ b/airbyte-bootloader/Dockerfile @@ -2,7 +2,7 @@ ARG JDK_VERSION=17.0.4 ARG JDK_IMAGE=amazoncorretto:${JDK_VERSION} FROM ${JDK_IMAGE} -ARG VERSION=0.40.9 +ARG VERSION=0.40.10 ENV APPLICATION airbyte-bootloader ENV VERSION ${VERSION} diff --git a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java index f53e55a540ddd..5f71b0f725771 100644 --- a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java +++ b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java @@ -9,6 +9,7 @@ import io.airbyte.commons.lang.CloseableShutdownHook; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.commons.version.Version; import io.airbyte.config.Configs; import io.airbyte.config.EnvConfigs; import io.airbyte.config.StandardWorkspace; @@ -153,6 +154,11 @@ public void load() throws Exception { final AirbyteVersion currAirbyteVersion = configs.getAirbyteVersion(); assertNonBreakingMigration(jobPersistence, currAirbyteVersion); + final Version airbyteProtocolVersionMax = configs.getAirbyteProtocolVersionMax(); + final Version airbyteProtocolVersionMin = configs.getAirbyteProtocolVersionMin(); + // TODO ProtocolVersion validation should happen here + trackProtocolVersion(airbyteProtocolVersionMin, airbyteProtocolVersionMax); + // TODO Will be converted to an injected singleton during DI migration final DatabaseMigrator configDbMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway); final DatabaseMigrator jobDbMigrator = new JobsDatabaseMigrator(jobDatabase, jobsFlyway); @@ -185,7 +191,6 @@ private static Database getConfigDatabase(final DSLContext dslContext) throws IO private static ConfigPersistence getConfigPersistence(final Database configDatabase) throws IOException { final JsonSecretsProcessor jsonSecretsProcessor = JsonSecretsProcessor.builder() - .maskSecrets(true) .copySecrets(true) .build(); @@ -299,6 +304,12 @@ private static void assertNonBreakingMigration(final JobPersistence jobPersisten } } + private void trackProtocolVersion(final Version airbyteProtocolVersionMin, final Version airbyteProtocolVersionMax) throws IOException { + jobPersistence.setAirbyteProtocolVersionMin(airbyteProtocolVersionMin); + jobPersistence.setAirbyteProtocolVersionMax(airbyteProtocolVersionMax); + LOGGER.info("AirbyteProtocol version support range [{}:{}]", airbyteProtocolVersionMin.serialize(), airbyteProtocolVersionMax.serialize()); + } + static boolean isLegalUpgrade(final AirbyteVersion airbyteDatabaseVersion, final AirbyteVersion airbyteVersion) { // means there was no previous version so upgrade even needs to happen. always legal. if (airbyteDatabaseVersion == null) { diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java index 0c1ff9e14aed6..c5f89eef85505 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.commons.version.Version; import io.airbyte.config.Configs; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardWorkspace; @@ -60,6 +61,8 @@ class BootloaderAppTest { private DataSource configsDataSource; private DataSource jobsDataSource; private static final String DOCKER = "docker"; + private static final String PROTOCOL_VERSION_123 = "1.2.3"; + private static final String PROTOCOL_VERSION_124 = "1.2.4"; private static final String VERSION_0330_ALPHA = "0.33.0-alpha"; private static final String VERSION_0320_ALPHA = "0.32.0-alpha"; private static final String VERSION_0321_ALPHA = "0.32.1-alpha"; @@ -99,6 +102,8 @@ void testBootloaderAppBlankDb() throws Exception { when(mockedConfigs.getDatabaseUser()).thenReturn(container.getUsername()); when(mockedConfigs.getDatabasePassword()).thenReturn(container.getPassword()); when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(VERSION_0330_ALPHA)); + when(mockedConfigs.getAirbyteProtocolVersionMin()).thenReturn(new Version(PROTOCOL_VERSION_123)); + when(mockedConfigs.getAirbyteProtocolVersionMax()).thenReturn(new Version(PROTOCOL_VERSION_124)); when(mockedConfigs.runDatabaseMigrationOnStartup()).thenReturn(true); when(mockedConfigs.getConfigsDatabaseInitializationTimeoutMs()).thenReturn(60000L); when(mockedConfigs.getJobsDatabaseInitializationTimeoutMs()).thenReturn(60000L); @@ -137,6 +142,8 @@ void testBootloaderAppBlankDb() throws Exception { val jobsPersistence = new DefaultJobPersistence(jobDatabase); assertEquals(VERSION_0330_ALPHA, jobsPersistence.getVersion().get()); + assertEquals(new Version(PROTOCOL_VERSION_123), jobsPersistence.getAirbyteProtocolVersionMin().get()); + assertEquals(new Version(PROTOCOL_VERSION_124), jobsPersistence.getAirbyteProtocolVersionMax().get()); assertNotEquals(Optional.empty(), jobsPersistence.getDeployment().get()); } @@ -152,6 +159,8 @@ void testBootloaderAppRunSecretMigration() throws Exception { when(mockedConfigs.getDatabaseUser()).thenReturn(container.getUsername()); when(mockedConfigs.getDatabasePassword()).thenReturn(container.getPassword()); when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(VERSION_0330_ALPHA)); + when(mockedConfigs.getAirbyteProtocolVersionMin()).thenReturn(new Version(PROTOCOL_VERSION_123)); + when(mockedConfigs.getAirbyteProtocolVersionMax()).thenReturn(new Version(PROTOCOL_VERSION_123)); when(mockedConfigs.runDatabaseMigrationOnStartup()).thenReturn(true); when(mockedConfigs.getSecretPersistenceType()).thenReturn(TESTING_CONFIG_DB_TABLE); when(mockedConfigs.getConfigsDatabaseInitializationTimeoutMs()).thenReturn(60000L); @@ -161,7 +170,6 @@ void testBootloaderAppRunSecretMigration() throws Exception { final JsonSecretsProcessor jsonSecretsProcessor = JsonSecretsProcessor.builder() .copySecrets(true) - .maskSecrets(true) .build(); try (val configsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES); @@ -295,6 +303,8 @@ void testPostLoadExecutionExecutes() throws Exception { when(mockedConfigs.getDatabaseUser()).thenReturn(container.getUsername()); when(mockedConfigs.getDatabasePassword()).thenReturn(container.getPassword()); when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(VERSION_0330_ALPHA)); + when(mockedConfigs.getAirbyteProtocolVersionMin()).thenReturn(new Version(PROTOCOL_VERSION_123)); + when(mockedConfigs.getAirbyteProtocolVersionMax()).thenReturn(new Version(PROTOCOL_VERSION_123)); when(mockedConfigs.runDatabaseMigrationOnStartup()).thenReturn(true); when(mockedConfigs.getConfigsDatabaseInitializationTimeoutMs()).thenReturn(60000L); when(mockedConfigs.getJobsDatabaseInitializationTimeoutMs()).thenReturn(60000L); diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index b3878ed0e324f..ef00c23bd5c9b 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.1.91 +- Low-code: Rename LimitPaginator to DefaultPaginator and move page_size field to PaginationStrategy + +## 0.1.90 +- Fix error when TypeTransformer tries to warn about invalid transformations in arrays + ## 0.1.89 - Fix: properly emit state when a stream has empty slices, provided by an iterator diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json index 29ba32b36458d..f8ff83b6e52d9 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json @@ -46,7 +46,7 @@ }, "name": { "type": "string", - "default": "" + "default": "" }, "_name": { "type": "string", @@ -73,7 +73,7 @@ "type": "string" } ], - "default": "" + "default": "" }, "_primary_key": { "type": "string", @@ -81,16 +81,27 @@ }, "stream_cursor_field": { "anyOf": [ - { "type": "array", "items": { "type": "string" } }, - { "type": "string" } + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } ] }, "transformations": { "type": "array", "items": { "anyOf": [ - { "$ref": "#/definitions/AddFields" }, - { "$ref": "#/definitions/RemoveFields" } + { + "$ref": "#/definitions/AddFields" + }, + { + "$ref": "#/definitions/RemoveFields" + } ] } }, @@ -111,8 +122,12 @@ "properties": { "file_path": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "config": { @@ -158,7 +173,7 @@ }, "name": { "type": "string", - "default": "" + "default": "" }, "_name": { "type": "string", @@ -185,7 +200,7 @@ "type": "string" } ], - "default": "" + "default": "" }, "_primary_key": { "type": "string", @@ -193,8 +208,12 @@ }, "paginator": { "anyOf": [ - { "$ref": "#/definitions/LimitPaginator" }, - { "$ref": "#/definitions/NoPagination" } + { + "$ref": "#/definitions/DefaultPaginator" + }, + { + "$ref": "#/definitions/NoPagination" + } ] }, "stream_slicer": { @@ -236,14 +255,22 @@ }, "url_base": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "path": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "config": { @@ -285,8 +312,12 @@ }, "error_handler": { "anyOf": [ - { "$ref": "#/definitions/CompositeErrorHandler" }, - { "$ref": "#/definitions/DefaultErrorHandler" } + { + "$ref": "#/definitions/CompositeErrorHandler" + }, + { + "$ref": "#/definitions/DefaultErrorHandler" + } ] } } @@ -310,36 +341,52 @@ "anyOf": [ { "type": "object", - "additionalProperties": { "type": "string" } + "additionalProperties": { + "type": "string" + } }, - { "type": "string" } + { + "type": "string" + } ] }, "request_headers": { "anyOf": [ { "type": "object", - "additionalProperties": { "type": "string" } + "additionalProperties": { + "type": "string" + } }, - { "type": "string" } + { + "type": "string" + } ] }, "request_body_data": { "anyOf": [ { "type": "object", - "additionalProperties": { "type": "string" } + "additionalProperties": { + "type": "string" + } }, - { "type": "string" } + { + "type": "string" + } ] }, "request_body_json": { "anyOf": [ { "type": "object", - "additionalProperties": { "type": "string" } + "additionalProperties": { + "type": "string" + } }, - { "type": "string" } + { + "type": "string" + } ] } } @@ -354,10 +401,15 @@ }, "NoAuth": { "allOf": [ - { "$ref": "#/definitions/DeclarativeAuthenticator" }, - { "type": "object", "properties": {} } + { + "$ref": "#/definitions/DeclarativeAuthenticator" + }, + { + "type": "object", + "properties": {} + } ], - "description": "NoAuth()" + "description": "NoAuth(options: dataclasses.InitVar[typing.Mapping[str, typing.Any]])" }, "DeclarativeAuthenticator": { "type": "object", @@ -381,26 +433,42 @@ "properties": { "token_refresh_endpoint": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "client_id": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "client_secret": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "refresh_token": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "config": { @@ -414,22 +482,34 @@ }, "token_expiry_date": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "_token_expiry_date": {}, "access_token_name": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ], "default": "access_token" }, "expires_in_name": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ], "default": "expires_in" }, @@ -452,14 +532,22 @@ "properties": { "header": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "api_token": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "config": { @@ -481,8 +569,12 @@ "properties": { "api_token": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "config": { @@ -504,8 +596,12 @@ "properties": { "username": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "config": { @@ -513,8 +609,12 @@ }, "password": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ], "default": "" } @@ -536,15 +636,19 @@ "type": "array", "items": { "anyOf": [ - { "$ref": "#/definitions/CompositeErrorHandler" }, - { "$ref": "#/definitions/DefaultErrorHandler" } + { + "$ref": "#/definitions/CompositeErrorHandler" + }, + { + "$ref": "#/definitions/DefaultErrorHandler" + } ] } } } } ], - "description": "\n Error handler that sequentially iterates over a list of `ErrorHandler`s\n\n Sample config chaining 2 different retriers:\n error_handler:\n type: \"CompositeErrorHandler\"\n error_handlers:\n - response_filters:\n - predicate: \"{{ 'codase' in response }}\"\n action: RETRY\n backoff_strategies:\n - type: \"ConstantBackoffStrategy\"\n backoff_time_in_seconds: 5\n - response_filters:\n - http_codes: [403]\n action: RETRY\n backoff_strategies:\n - type: \"ConstantBackoffStrategy\"\n backoff_time_in_seconds: 10\n Attributes:\n error_handlers (List[ErrorHandler]): list of error handlers\n " + "description": "\n Error handler that sequentially iterates over a list of `ErrorHandler`s\n\n Sample config chaining 2 different retriers:\n error_handler:\n type: \"CompositeErrorHandler\"\n error_handlers:\n - response_filters:\n - predicate: \"{{ 'codase' in response }}\"\n action: RETRY\n backoff_strategies:\n - type: \"ConstantBackoffStrategy\"\n backoff_time_in_seconds: 5\n - response_filters:\n - http_codes: [ 403 ]\n action: RETRY\n backoff_strategies:\n - type: \"ConstantBackoffStrategy\"\n backoff_time_in_seconds: 10\n Attributes:\n error_handlers (List[ErrorHandler]): list of error handlers\n " }, "DefaultErrorHandler": { "allOf": [ @@ -562,7 +666,7 @@ }, "max_retries": { "type": "integer", - "default": "" + "default": "" }, "_max_retries": { "type": "integer", @@ -590,7 +694,7 @@ } } ], - "description": "\n Default error handler.\n\n By default, the handler will only retry server errors (HTTP 5XX) and too many requests (HTTP 429) with exponential backoff.\n\n If the response is successful, then return SUCCESS\n Otherwise, iterate over the response_filters.\n If any of the filter match the response, then return the appropriate status.\n If the match is RETRY, then iterate sequentially over the backoff_strategies and return the first non-None backoff time.\n\n Sample configs:\n\n 1. retry 10 times\n `\n error_handler:\n max_retries: 10\n `\n 2. backoff for 5 seconds\n `\n error_handler:\n backoff_strategies:\n - type: \"ConstantBackoffStrategy\"\n backoff_time_in_seconds: 5\n `\n 3. retry on HTTP 404\n `\n error_handler:\n response_filters:\n - http_codes: [404]\n action: RETRY\n `\n 4. ignore HTTP 404\n `\n error_handler:\n - http_codes: [404]\n action: IGNORE\n `\n 5. retry if error message contains `retrythisrequest!` substring\n `\n error_handler:\n response_filters:\n - error_message_contain: \"retrythisrequest!\"\n action: IGNORE\n `\n 6. retry if 'code' is a field present in the response body\n `\n error_handler:\n response_filters:\n - predicate: \"{{ 'code' in response }}\"\n action: IGNORE\n `\n\n 7. ignore 429 and retry on 404\n `\n error_handler:\n - http_codes: [429]\n action: IGNORE\n - http_codes: [404]\n action: RETRY\n `\n\n Attributes:\n response_filters (Optional[List[HttpResponseFilter]]): response filters to iterate on\n max_retries (Optional[int]): maximum retry attempts\n backoff_strategies (Optional[List[BackoffStrategy]]): list of backoff strategies to use to determine how long\n to wait before retrying\n " + "description": "\n Default error handler.\n\n By default, the handler will only retry server errors (HTTP 5XX) and too many requests (HTTP 429) with exponential backoff.\n\n If the response is successful, then return SUCCESS\n Otherwise, iterate over the response_filters.\n If any of the filter match the response, then return the appropriate status.\n If the match is RETRY, then iterate sequentially over the backoff_strategies and return the first non-None backoff time.\n\n Sample configs:\n\n 1. retry 10 times\n `\n error_handler:\n max_retries: 10\n `\n 2. backoff for 5 seconds\n `\n error_handler:\n backoff_strategies:\n - type: \"ConstantBackoffStrategy\"\n backoff_time_in_seconds: 5\n `\n 3. retry on HTTP 404\n `\n error_handler:\n response_filters:\n - http_codes: [ 404 ]\n action: RETRY\n `\n 4. ignore HTTP 404\n `\n error_handler:\n - http_codes: [ 404 ]\n action: IGNORE\n `\n 5. retry if error message contains `retrythisrequest!` substring\n `\n error_handler:\n response_filters:\n - error_message_contain: \"retrythisrequest!\"\n action: IGNORE\n `\n 6. retry if 'code' is a field present in the response body\n `\n error_handler:\n response_filters:\n - predicate: \"{{ 'code' in response }}\"\n action: IGNORE\n `\n\n 7. ignore 429 and retry on 404\n `\n error_handler:\n - http_codes: [ 429 ]\n action: IGNORE\n - http_codes: [ 404 ]\n action: RETRY\n `\n\n Attributes:\n response_filters (Optional[List[HttpResponseFilter]]): response filters to iterate on\n max_retries (Optional[int]): maximum retry attempts\n backoff_strategies (Optional[List[BackoffStrategy]]): list of backoff strategies to use to determine how long\n to wait before retrying\n " }, "HttpResponseFilter": { "type": "object", @@ -619,8 +723,12 @@ }, "predicate": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedBoolean" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedBoolean" + }, + { + "type": "string" + } ], "default": "" } @@ -661,10 +769,17 @@ }, "ExponentialBackoffStrategy": { "allOf": [ - { "$ref": "#/definitions/BackoffStrategy" }, + { + "$ref": "#/definitions/BackoffStrategy" + }, { "type": "object", - "properties": { "factor": { "type": "number", "default": 5 } } + "properties": { + "factor": { + "type": "number", + "default": 5 + } + } } ], "description": "\n Backoff strategy with an exponential backoff interval\n\n Attributes:\n factor (float): multiplicative factor\n " @@ -719,8 +834,13 @@ }, "Requester": { "allOf": [ - { "$ref": "#/definitions/RequestOptionsProvider" }, - { "type": "object", "properties": {} } + { + "$ref": "#/definitions/RequestOptionsProvider" + }, + { + "type": "object", + "properties": {} + } ] }, "RecordSelector": { @@ -756,8 +876,12 @@ "type": "array", "items": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] } }, @@ -775,8 +899,13 @@ }, "JsonDecoder": { "allOf": [ - { "$ref": "#/definitions/Decoder" }, - { "type": "object", "properties": {} } + { + "$ref": "#/definitions/Decoder" + }, + { + "type": "object", + "properties": {} + } ], "description": "\n Decoder strategy that returns the json-encoded content of a response, if any.\n " }, @@ -809,7 +938,7 @@ "properties": {}, "description": "\n Responsible for translating an HTTP response into a list of records by extracting records from the response and optionally filtering\n records based on a heuristic.\n " }, - "LimitPaginator": { + "DefaultPaginator": { "allOf": [ { "$ref": "#/definitions/Paginator" @@ -817,18 +946,13 @@ { "type": "object", "required": [ - "page_size", - "limit_option", "page_token_option", "pagination_strategy", "config", "url_base" ], "properties": { - "page_size": { - "type": "integer" - }, - "limit_option": { + "page_size_option": { "$ref": "#/definitions/RequestOption" }, "page_token_option": { @@ -852,8 +976,12 @@ }, "url_base": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "decoder": { @@ -864,7 +992,7 @@ } } ], - "description": "\n Limit paginator to request pages of results with a fixed size until the pagination strategy no longer returns a next_page_token\n\n Examples:\n 1.\n * fetches up to 10 records at a time by setting the \"limit\" request param to 10\n * updates the request path with \"{{ response._metadata.next }}\"\n paginator:\n type: \"LimitPaginator\"\n page_size: 10\n limit_option:\n inject_into: request_parameter\n field_name: page_size\n page_token_option:\n option_type: path\n pagination_strategy:\n type: \"CursorPagination\"\n cursor_value: \"{{ response._metadata.next }}\"\n `\n\n 2.\n * fetches up to 5 records at a time by setting the \"page_size\" header to 5\n * increments a record counter and set the request parameter \"offset\" to the value of the counter\n `\n paginator:\n type: \"LimitPaginator\"\n page_size: 5\n limit_option:\n inject_into: header\n field_name: page_size\n pagination_strategy:\n type: \"OffsetIncrement\"\n page_token:\n option_type: \"request_parameter\"\n field_name: \"offset\"\n `\n\n 3.\n * fetches up to 5 records at a time by setting the \"page_size\" request param to 5\n * increments a page counter and set the request parameter \"page\" to the value of the counter\n `\n paginator:\n type: \"LimitPaginator\"\n page_size: 5\n limit_option:\n inject_into: request_parameter\n field_name: page_size\n pagination_strategy:\n type: \"PageIncrement\"\n page_token:\n option_type: \"request_parameter\"\n field_name: \"page\"\n\n Attributes:\n page_size (int): the number of records to request\n limit_option (RequestOption): the request option to set the limit. Cannot be injected in the path.\n page_token_option (RequestOption): the request option to set the page token\n pagination_strategy (PaginationStrategy): Strategy defining how to get the next page token\n config (Config): connection config\n url_base (Union[InterpolatedString, str]): endpoint's base url\n decoder (Decoder): decoder to decode the response\n " + "description": "\n Default paginator to request pages of results with a fixed size until the pagination strategy no longer returns a next_page_token\n\n Examples:\n 1.\n * fetches up to 10 records at a time by setting the \"limit\" request param to 10\n * updates the request path with \"{{ response._metadata.next }}\"\n ```\n paginator:\n type: \"DefaultPaginator\"\n page_size_option:\n inject_into: request_parameter\n field_name: limit\n page_token_option:\n option_type: path\n pagination_strategy:\n type: \"CursorPagination\"\n cursor_value: \"{{ response._metadata.next }}\"\n page_size: 10\n ```\n\n 2.\n * fetches up to 5 records at a time by setting the \"page_size\" header to 5\n * increments a record counter and set the request parameter \"offset\" to the value of the counter\n ```\n paginator:\n type: \"DefaultPaginator\"\n page_size_option:\n inject_into: header\n field_name: page_size\n pagination_strategy:\n type: \"OffsetIncrement\"\n page_size: 5\n page_token:\n option_type: \"request_parameter\"\n field_name: \"offset\"\n ```\n\n 3.\n * fetches up to 5 records at a time by setting the \"page_size\" request param to 5\n * increments a page counter and set the request parameter \"page\" to the value of the counter\n ```\n paginator:\n type: \"DefaultPaginator\"\n page_size_option:\n inject_into: request_parameter\n field_name: page_size\n pagination_strategy:\n type: \"PageIncrement\"\n page_size: 5\n page_token_option:\n option_type: \"request_parameter\"\n field_name: \"page\"\n ```\n Attributes:\n page_size_option (Optional[RequestOption]): the request option to set the page size. Cannot be injected in the path.\n page_token_option (RequestOption): the request option to set the page token\n pagination_strategy (PaginationStrategy): Strategy defining how to get the next page token\n config (Config): connection config\n url_base (Union[InterpolatedString, str]): endpoint's base url\n decoder (Decoder): decoder to decode the response\n " }, "RequestOption": { "type": "object", @@ -897,17 +1025,28 @@ "properties": { "cursor_value": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "config": { "type": "object" }, + "page_size": { + "type": "integer" + }, "stop_condition": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedBoolean" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedBoolean" + }, + { + "type": "string" + } ] }, "decoder": { @@ -917,7 +1056,7 @@ } } ], - "description": "\n Pagination strategy that evaluates an interpolated string to define the next page token\n\n Attributes:\n cursor_value (Union[InterpolatedString, str]): template string evaluating to the cursor value\n config (Config): connection config\n stop_condition (Optional[InterpolatedBoolean]): template string evaluating when to stop paginating\n decoder (Decoder): decoder to decode the response\n " + "description": "\n Pagination strategy that evaluates an interpolated string to define the next page token\n\n Attributes:\n page_size (Optional[int]): the number of records to request\n cursor_value (Union[InterpolatedString, str]): template string evaluating to the cursor value\n config (Config): connection config\n stop_condition (Optional[InterpolatedBoolean]): template string evaluating when to stop paginating\n decoder (Decoder): decoder to decode the response\n " }, "PaginationStrategy": { "type": "object", @@ -960,15 +1099,25 @@ }, "Paginator": { "allOf": [ - { "$ref": "#/definitions/RequestOptionsProvider" }, - { "type": "object", "properties": {} } + { + "$ref": "#/definitions/RequestOptionsProvider" + }, + { + "type": "object", + "properties": {} + } ], "description": "\n Defines the token to use to fetch the next page of records from the API.\n\n If needed, the Paginator will set request options to be set on the HTTP request to fetch the next page of records.\n If the next_page_token is the path to the next page of records, then it should be accessed through the `path` method\n " }, "NoPagination": { "allOf": [ - { "$ref": "#/definitions/Paginator" }, - { "type": "object", "properties": {} } + { + "$ref": "#/definitions/Paginator" + }, + { + "type": "object", + "properties": {} + } ], "description": "\n Pagination implementation that never returns a next page.\n " }, @@ -1026,14 +1175,22 @@ "properties": { "start_datetime": { "anyOf": [ - { "$ref": "#/definitions/MinMaxDatetime" }, - { "type": "string" } + { + "$ref": "#/definitions/MinMaxDatetime" + }, + { + "type": "string" + } ] }, "end_datetime": { "anyOf": [ - { "$ref": "#/definitions/MinMaxDatetime" }, - { "type": "string" } + { + "$ref": "#/definitions/MinMaxDatetime" + }, + { + "type": "string" + } ] }, "step": { @@ -1041,8 +1198,12 @@ }, "cursor_field": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "datetime_format": { @@ -1071,8 +1232,12 @@ }, "lookback_window": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] } } @@ -1086,13 +1251,17 @@ "properties": { "datetime": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "datetime_format": { "type": "string", - "default": "" + "default": "" }, "_datetime_format": { "type": "string", @@ -1100,15 +1269,23 @@ }, "min_datetime": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ], "default": "" }, "max_datetime": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ], "default": "" } @@ -1117,8 +1294,13 @@ }, "StreamSlicer": { "allOf": [ - { "$ref": "#/definitions/RequestOptionsProvider" }, - { "type": "object", "properties": {} } + { + "$ref": "#/definitions/RequestOptionsProvider" + }, + { + "type": "object", + "properties": {} + } ], "description": "\n Slices the stream into a subset of records.\n Slices enable state checkpointing and data retrieval parallelization.\n\n The stream slicer keeps track of the cursor state as a dict of cursor_field -> cursor_value\n\n See the stream slicing section of the docs for more information.\n " }, @@ -1133,14 +1315,25 @@ "properties": { "slice_values": { "anyOf": [ - { "type": "array", "items": { "type": "string" } }, - { "type": "string" } + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } ] }, "cursor_field": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] }, "config": { @@ -1156,8 +1349,13 @@ }, "SingleSlice": { "allOf": [ - { "$ref": "#/definitions/StreamSlicer" }, - { "type": "object", "properties": {} } + { + "$ref": "#/definitions/StreamSlicer" + }, + { + "type": "object", + "properties": {} + } ], "description": "Stream slicer returning only a single stream slice" }, @@ -1242,8 +1440,12 @@ }, "value": { "anyOf": [ - { "$ref": "#/definitions/InterpolatedString" }, - { "type": "string" } + { + "$ref": "#/definitions/InterpolatedString" + }, + { + "type": "string" + } ] } }, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py index 484feb7d565f7..ff071affe2543 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py @@ -20,7 +20,7 @@ from airbyte_cdk.sources.declarative.requesters.error_handlers.composite_error_handler import CompositeErrorHandler from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import DefaultErrorHandler from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester -from airbyte_cdk.sources.declarative.requesters.paginators.limit_paginator import LimitPaginator +from airbyte_cdk.sources.declarative.requesters.paginators.default_paginator import DefaultPaginator from airbyte_cdk.sources.declarative.requesters.paginators.no_pagination import NoPagination from airbyte_cdk.sources.declarative.requesters.paginators.strategies.cursor_pagination_strategy import CursorPaginationStrategy from airbyte_cdk.sources.declarative.requesters.paginators.strategies.offset_increment import OffsetIncrement @@ -54,7 +54,7 @@ "InterpolatedBoolean": InterpolatedBoolean, "InterpolatedString": InterpolatedString, "JsonSchema": JsonSchema, - "LimitPaginator": LimitPaginator, + "DefaultPaginator": DefaultPaginator, "ListStreamSlicer": ListStreamSlicer, "MinMaxDatetime": MinMaxDatetime, "NoAuth": NoAuth, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py index 234504b2de0b4..e86a3f6462fb7 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py @@ -20,7 +20,7 @@ from airbyte_cdk.sources.declarative.requesters.error_handlers.error_handler import ErrorHandler from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import HttpResponseFilter from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester -from airbyte_cdk.sources.declarative.requesters.paginators.limit_paginator import RequestOption +from airbyte_cdk.sources.declarative.requesters.paginators.default_paginator import RequestOption from airbyte_cdk.sources.declarative.requesters.paginators.no_pagination import NoPagination from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider import ( diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py index d0310b21c199c..c8bd6fd13ffdb 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/__init__.py @@ -2,9 +2,9 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.sources.declarative.requesters.paginators.limit_paginator import LimitPaginator +from airbyte_cdk.sources.declarative.requesters.paginators.default_paginator import DefaultPaginator from airbyte_cdk.sources.declarative.requesters.paginators.no_pagination import NoPagination from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import PaginationStrategy -__all__ = ["LimitPaginator", "NoPagination", "PaginationStrategy", "Paginator"] +__all__ = ["DefaultPaginator", "NoPagination", "PaginationStrategy", "Paginator"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/limit_paginator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py similarity index 77% rename from airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/limit_paginator.py rename to airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py index bf9adcbd515b8..b88908aef8950 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/limit_paginator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py @@ -17,63 +17,63 @@ @dataclass -class LimitPaginator(Paginator, JsonSchemaMixin): +class DefaultPaginator(Paginator, JsonSchemaMixin): """ - Limit paginator to request pages of results with a fixed size until the pagination strategy no longer returns a next_page_token + Default paginator to request pages of results with a fixed size until the pagination strategy no longer returns a next_page_token Examples: 1. * fetches up to 10 records at a time by setting the "limit" request param to 10 * updates the request path with "{{ response._metadata.next }}" + ``` paginator: - type: "LimitPaginator" - page_size: 10 - limit_option: + type: "DefaultPaginator" + page_size_option: inject_into: request_parameter - field_name: page_size + field_name: limit page_token_option: option_type: path pagination_strategy: type: "CursorPagination" cursor_value: "{{ response._metadata.next }}" - ` + page_size: 10 + ``` 2. * fetches up to 5 records at a time by setting the "page_size" header to 5 * increments a record counter and set the request parameter "offset" to the value of the counter - ` + ``` paginator: - type: "LimitPaginator" - page_size: 5 - limit_option: + type: "DefaultPaginator" + page_size_option: inject_into: header field_name: page_size pagination_strategy: type: "OffsetIncrement" - page_token: + page_size: 5 + page_token_option: option_type: "request_parameter" field_name: "offset" - ` + ``` 3. * fetches up to 5 records at a time by setting the "page_size" request param to 5 * increments a page counter and set the request parameter "page" to the value of the counter - ` + ``` paginator: - type: "LimitPaginator" - page_size: 5 - limit_option: + type: "DefaultPaginator" + page_size_option: inject_into: request_parameter field_name: page_size pagination_strategy: type: "PageIncrement" - page_token: + page_size: 5 + page_token_option: option_type: "request_parameter" field_name: "page" - + ``` Attributes: - page_size (int): the number of records to request - limit_option (RequestOption): the request option to set the limit. Cannot be injected in the path. + page_size_option (Optional[RequestOption]): the request option to set the page size. Cannot be injected in the path. page_token_option (RequestOption): the request option to set the page token pagination_strategy (PaginationStrategy): Strategy defining how to get the next page token config (Config): connection config @@ -81,8 +81,7 @@ class LimitPaginator(Paginator, JsonSchemaMixin): decoder (Decoder): decoder to decode the response """ - page_size: int - limit_option: RequestOption + page_size_option: Optional[RequestOption] page_token_option: RequestOption pagination_strategy: PaginationStrategy config: Config @@ -92,8 +91,12 @@ class LimitPaginator(Paginator, JsonSchemaMixin): _token: Optional[Any] = field(init=False, repr=False, default=None) def __post_init__(self, options: Mapping[str, Any]): - if self.limit_option.inject_into == RequestOptionType.path: - raise ValueError("Limit parameter cannot be a path") + if self.page_size_option and self.page_size_option.inject_into == RequestOptionType.path: + raise ValueError("page_size_option cannot be set in as path") + if self.page_size_option and not self.pagination_strategy.get_page_size(): + raise ValueError("page_size_option cannot be set if the pagination strategy does not have a page_size") + if self.pagination_strategy.get_page_size() and not self.page_size_option: + raise ValueError("page_size_option must be set if the pagination strategy has a page_size") if isinstance(self.url_base, str): self.url_base = InterpolatedString(string=self.url_base, options=options) @@ -155,7 +158,7 @@ def _get_request_options(self, option_type: RequestOptionType) -> Mapping[str, A if self.page_token_option.inject_into == option_type: if option_type != RequestOptionType.path and self._token: options[self.page_token_option.field_name] = self._token - if self.limit_option.inject_into == option_type: + if self.page_size_option and self.pagination_strategy.get_page_size() and self.page_size_option.inject_into == option_type: if option_type != RequestOptionType.path: - options[self.limit_option.field_name] = self.page_size + options[self.page_size_option.field_name] = self.pagination_strategy.get_page_size() return options diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py index 0f56074af0161..5f324ac0673bd 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py @@ -21,6 +21,7 @@ class CursorPaginationStrategy(PaginationStrategy, JsonSchemaMixin): Pagination strategy that evaluates an interpolated string to define the next page token Attributes: + page_size (Optional[int]): the number of records to request cursor_value (Union[InterpolatedString, str]): template string evaluating to the cursor value config (Config): connection config stop_condition (Optional[InterpolatedBoolean]): template string evaluating when to stop paginating @@ -30,6 +31,7 @@ class CursorPaginationStrategy(PaginationStrategy, JsonSchemaMixin): cursor_value: Union[InterpolatedString, str] config: Config options: InitVar[Mapping[str, Any]] + page_size: Optional[int] = None stop_condition: Optional[Union[InterpolatedBoolean, str]] = None decoder: Decoder = JsonDecoder(options={}) @@ -57,3 +59,6 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin def reset(self): # No state to reset pass + + def get_page_size(self) -> Optional[int]: + return self.page_size diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py index e6ab8a03fb589..429de3a7a7e38 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py @@ -34,3 +34,6 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin def reset(self): self._offset = 0 + + def get_page_size(self) -> Optional[int]: + return self.page_size diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py index 46e112a0397f5..530c8fbcfcc74 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py @@ -34,3 +34,6 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin def reset(self): self._page = 0 + + def get_page_size(self) -> Optional[int]: + return self.page_size diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py index a2d9407a833dc..a92d0f489877d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py @@ -30,3 +30,9 @@ def reset(self): """ Reset the pagination's inner state """ + + @abstractmethod + def get_page_size(self) -> Optional[int]: + """ + :return: page size: The number of records to fetch in a page. Returns None if unspecified + """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py index f8478451adf45..c9e31fd47ba92 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py @@ -63,7 +63,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: "parsed YAML into declarative source", extra={"path_to_yaml_file": self._path_to_yaml, "source_name": self.name, "parsed_config": json.dumps(self._source_config)}, ) - return [self._factory.create_component(stream_config, config, True)() for stream_config in self._stream_configs()] + source_streams = [self._factory.create_component(stream_config, config, True)() for stream_config in self._stream_configs()] + for stream in source_streams: + # make sure the log level is always appied to the stream's logger + self._apply_log_level_to_stream_logger(self.logger, stream) + return source_streams def _read_and_parse_yaml_file(self, path_to_yaml_file): package = self.__class__.__module__.split(".")[0] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py index 81b52ec84b3ca..f0cb40d4c8bf2 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py @@ -190,7 +190,7 @@ def transform(self, record: Dict[str, Any], schema: Mapping[str, Any]): def get_error_message(self, e: ValidationError) -> str: instance_json_type = python_to_json[type(e.instance)] - key_path = "." + ".".join(e.path) + key_path = "." + ".".join(map(str, e.path)) return ( f"Failed to transform value {repr(e.instance)} of type '{instance_json_type}' to '{e.validator_value}', key path: '{key_path}'" ) diff --git a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.paginators.rst b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.paginators.rst index 8fc8f41b57b73..fd91f3f08af4c 100644 --- a/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.paginators.rst +++ b/airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.declarative.requesters.paginators.rst @@ -13,7 +13,7 @@ Submodules airbyte\_cdk.sources.declarative.requesters.paginators.limit\_paginator module ------------------------------------------------------------------------------ -.. automodule:: airbyte_cdk.sources.declarative.requesters.paginators.limit_paginator +.. automodule:: airbyte_cdk.sources.declarative.requesters.paginators.default_paginator :members: :undoc-members: :show-inheritance: diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index c0a9eaee61197..2b833673c5a57 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.1.89", + version="0.1.91", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py index 2f3600e03cd2d..24110cbff5700 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py @@ -12,31 +12,39 @@ @pytest.mark.parametrize( - "test_name, template_string, stop_condition, expected_token", + "test_name, template_string, stop_condition, expected_token, page_size", [ - ("test_static_token", "token", None, "token"), - ("test_token_from_config", "{{ config.config_key }}", None, "config_value"), - ("test_token_from_last_record", "{{ last_records[-1].id }}", None, 1), - ("test_token_from_response", "{{ response._metadata.content }}", None, "content_value"), - ("test_token_from_options", "{{ options.key }}", None, "value"), - ("test_token_not_found", "{{ response.invalid_key }}", None, None), - ("test_static_token_with_stop_condition_false", "token", InterpolatedBoolean("{{False}}", options={}), "token"), - ("test_static_token_with_stop_condition_true", "token", InterpolatedBoolean("{{True}}", options={}), None), - ("test_token_from_header", "{{ headers.next }}", InterpolatedBoolean("{{ not headers.has_more }}", options={}), "ready_to_go"), + ("test_static_token", "token", None, "token", None), + ("test_static_token_with_page_size", "token", None, "token", 5), + ("test_token_from_config", "{{ config.config_key }}", None, "config_value", None), + ("test_token_from_last_record", "{{ last_records[-1].id }}", None, 1, None), + ("test_token_from_response", "{{ response._metadata.content }}", None, "content_value", None), + ("test_token_from_options", "{{ options.key }}", None, "value", None), + ("test_token_not_found", "{{ response.invalid_key }}", None, None, None), + ("test_static_token_with_stop_condition_false", "token", InterpolatedBoolean("{{False}}", options={}), "token", None), + ("test_static_token_with_stop_condition_true", "token", InterpolatedBoolean("{{True}}", options={}), None, None), + ( + "test_token_from_header", + "{{ headers.next }}", + InterpolatedBoolean("{{ not headers.has_more }}", options={}), + "ready_to_go", + None, + ), ( "test_token_from_response_header_links", "{{ headers.link.next.url }}", InterpolatedBoolean("{{ not headers.link.next.url }}", options={}), "https://adventure.io/api/v1/records?page=2&per_page=100", + None, ), ], ) -def test_cursor_pagination_strategy(test_name, template_string, stop_condition, expected_token): +def test_cursor_pagination_strategy(test_name, template_string, stop_condition, expected_token, page_size): decoder = JsonDecoder(options={}) config = {"config_key": "config_value"} options = {"key": "value"} strategy = CursorPaginationStrategy( - cursor_value=template_string, config=config, stop_condition=stop_condition, decoder=decoder, options=options + page_size=page_size, cursor_value=template_string, config=config, stop_condition=stop_condition, decoder=decoder, options=options ) response = requests.Response() @@ -48,3 +56,4 @@ def test_cursor_pagination_strategy(test_name, template_string, stop_condition, token = strategy.next_page_token(response, last_records) assert expected_token == token + assert page_size == strategy.get_page_size() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_limit_paginator.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py similarity index 61% rename from airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_limit_paginator.py rename to airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py index 26d55a0276eea..f6ec277d0a836 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_limit_paginator.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py @@ -9,15 +9,15 @@ import requests from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean -from airbyte_cdk.sources.declarative.requesters.paginators.limit_paginator import LimitPaginator, RequestOption, RequestOptionType +from airbyte_cdk.sources.declarative.requesters.paginators.default_paginator import DefaultPaginator, RequestOption, RequestOptionType from airbyte_cdk.sources.declarative.requesters.paginators.strategies.cursor_pagination_strategy import CursorPaginationStrategy @pytest.mark.parametrize( - "test_name, page_token_request_option, stop_condition, expected_updated_path, expected_request_params, expected_headers, expected_body_data, expected_body_json, last_records, expected_next_page_token", + "test_name, page_token_request_option, stop_condition, expected_updated_path, expected_request_params, expected_headers, expected_body_data, expected_body_json, last_records, expected_next_page_token, limit", [ ( - "test_limit_paginator_path", + "test_default_paginator_path", RequestOption(inject_into=RequestOptionType.path, options={}), None, "/next_url", @@ -27,9 +27,10 @@ {}, [{"id": 0}, {"id": 1}], {"next_page_token": "https://airbyte.io/next_url"}, + 2, ), ( - "test_limit_paginator_request_param", + "test_default_paginator_request_param", RequestOption(inject_into=RequestOptionType.request_parameter, field_name="from", options={}), None, None, @@ -39,9 +40,10 @@ {}, [{"id": 0}, {"id": 1}], {"next_page_token": "https://airbyte.io/next_url"}, + 2, ), ( - "test_limit_paginator_no_token", + "test_default_paginator_no_token", RequestOption(inject_into=RequestOptionType.request_parameter, field_name="from", options={}), InterpolatedBoolean(condition="{{True}}", options={}), None, @@ -51,9 +53,10 @@ {}, [{"id": 0}, {"id": 1}], None, + 2, ), ( - "test_limit_paginator_cursor_header", + "test_default_paginator_cursor_header", RequestOption(inject_into=RequestOptionType.header, field_name="from", options={}), None, None, @@ -63,9 +66,10 @@ {}, [{"id": 0}, {"id": 1}], {"next_page_token": "https://airbyte.io/next_url"}, + 2, ), ( - "test_limit_paginator_cursor_body_data", + "test_default_paginator_cursor_body_data", RequestOption(inject_into=RequestOptionType.body_data, field_name="from", options={}), None, None, @@ -75,9 +79,10 @@ {}, [{"id": 0}, {"id": 1}], {"next_page_token": "https://airbyte.io/next_url"}, + 2, ), ( - "test_limit_paginator_cursor_body_json", + "test_default_paginator_cursor_body_json", RequestOption(inject_into=RequestOptionType.body_json, field_name="from", options={}), None, None, @@ -87,10 +92,11 @@ {"from": "https://airbyte.io/next_url"}, [{"id": 0}, {"id": 1}], {"next_page_token": "https://airbyte.io/next_url"}, + 2, ), ], ) -def test_limit_paginator( +def test_default_paginator_with_cursor( test_name, page_token_request_option, stop_condition, @@ -101,18 +107,23 @@ def test_limit_paginator( expected_body_json, last_records, expected_next_page_token, + limit, ): - limit_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", options={}) + page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", options={}) cursor_value = "{{ response.next }}" url_base = "https://airbyte.io" config = {} options = {} strategy = CursorPaginationStrategy( - cursor_value=cursor_value, stop_condition=stop_condition, decoder=JsonDecoder(options={}), config=config, options=options + page_size=limit, + cursor_value=cursor_value, + stop_condition=stop_condition, + decoder=JsonDecoder(options={}), + config=config, + options=options, ) - paginator = LimitPaginator( - page_size=2, - limit_option=limit_request_option, + paginator = DefaultPaginator( + page_size_option=page_size_request_option, page_token_option=page_token_request_option, pagination_strategy=strategy, config=config, @@ -140,17 +151,60 @@ def test_limit_paginator( def test_limit_cannot_be_set_in_path(): - limit_request_option = RequestOption(inject_into=RequestOptionType.path, options={}) + page_size_request_option = RequestOption(inject_into=RequestOptionType.path, options={}) + page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", options={}) + cursor_value = "{{ response.next }}" + url_base = "https://airbyte.io" + config = {} + options = {} + strategy = CursorPaginationStrategy(page_size=5, cursor_value=cursor_value, config=config, options=options) + try: + DefaultPaginator( + page_size_option=page_size_request_option, + page_token_option=page_token_request_option, + pagination_strategy=strategy, + config=config, + url_base=url_base, + options={}, + ) + assert False + except ValueError: + pass + + +def test_page_size_option_cannot_be_set_if_strategy_has_no_limit(): + page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="page_size", options={}) + page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", options={}) + cursor_value = "{{ response.next }}" + url_base = "https://airbyte.io" + config = {} + options = {} + strategy = CursorPaginationStrategy(page_size=None, cursor_value=cursor_value, config=config, options=options) + try: + DefaultPaginator( + page_size_option=page_size_request_option, + page_token_option=page_token_request_option, + pagination_strategy=strategy, + config=config, + url_base=url_base, + options={}, + ) + assert False + except ValueError: + pass + + +def test_page_size_option_must_be_set_if_strategy_has_limit(): + page_size_request_option = None page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", options={}) cursor_value = "{{ response.next }}" url_base = "https://airbyte.io" config = {} options = {} - strategy = CursorPaginationStrategy(cursor_value=cursor_value, config=config, options=options) + strategy = CursorPaginationStrategy(page_size=5, cursor_value=cursor_value, config=config, options=options) try: - LimitPaginator( - page_size=2, - limit_option=limit_request_option, + DefaultPaginator( + page_size_option=page_size_request_option, page_token_option=page_token_request_option, pagination_strategy=strategy, config=config, @@ -163,10 +217,10 @@ def test_limit_cannot_be_set_in_path(): def test_reset(): - limit_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", options={}) + page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", options={}) page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", options={}) url_base = "https://airbyte.io" config = {} strategy = MagicMock() - LimitPaginator(2, limit_request_option, page_token_request_option, strategy, config, url_base, options={}).reset() + DefaultPaginator(page_size_request_option, page_token_request_option, strategy, config, url_base, options={}).reset() assert strategy.reset.called diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index f6079fdaada48..e103aaaea85b2 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -27,7 +27,7 @@ from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import DefaultErrorHandler from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import HttpResponseFilter from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester -from airbyte_cdk.sources.declarative.requesters.paginators.limit_paginator import LimitPaginator +from airbyte_cdk.sources.declarative.requesters.paginators.default_paginator import DefaultPaginator from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider import ( InterpolatedRequestOptionsProvider, @@ -280,9 +280,8 @@ def test_full_config(): class_name: airbyte_cdk.sources.declarative.extractors.record_filter.RecordFilter condition: "{{ record['id'] > stream_state['id'] }}" metadata_paginator: - type: "LimitPaginator" - page_size: 10 - limit_option: + type: "DefaultPaginator" + page_size_option: inject_into: request_parameter field_name: page_size page_token_option: @@ -290,6 +289,7 @@ def test_full_config(): pagination_strategy: type: "CursorPagination" cursor_value: "{{ response._metadata.next }}" + page_size: 10 url_base: "https://api.sendgrid.com/v3/" next_page_url_from_token_partial: class_name: "airbyte_cdk.sources.declarative.interpolation.interpolated_string.InterpolatedString" @@ -476,9 +476,8 @@ def test_config_with_defaults(): file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" retriever: paginator: - type: "LimitPaginator" - page_size: 10 - limit_option: + type: "DefaultPaginator" + page_size_option: inject_into: request_parameter field_name: page_size page_token_option: @@ -486,6 +485,7 @@ def test_config_with_defaults(): pagination_strategy: type: "CursorPagination" cursor_value: "{{ response._metadata.next }}" + page_size: 10 requester: path: "/v3/marketing/lists" authenticator: @@ -515,25 +515,25 @@ def test_config_with_defaults(): assert stream.retriever.requester.authenticator._token.eval(input_config) == "verysecrettoken" assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_pointer] == ["result"] assert stream.schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.yaml" - assert isinstance(stream.retriever.paginator, LimitPaginator) + assert isinstance(stream.retriever.paginator, DefaultPaginator) assert stream.retriever.paginator.url_base.string == "https://api.sendgrid.com" - assert stream.retriever.paginator.page_size == 10 + assert stream.retriever.paginator.pagination_strategy.get_page_size() == 10 -def test_create_limit_paginator(): +def test_create_default_paginator(): content = """ paginator: - type: "LimitPaginator" - page_size: 10 + type: "DefaultPaginator" url_base: "https://airbyte.io" - limit_option: + page_size_option: inject_into: request_parameter field_name: page_size page_token_option: inject_into: path pagination_strategy: type: "CursorPagination" + page_size: 50 cursor_value: "{{ response._metadata.next }}" """ config = parser.parse(content) @@ -542,7 +542,7 @@ def test_create_limit_paginator(): paginator_config = config["paginator"] paginator = factory.create_component(paginator_config, input_config)() - assert isinstance(paginator, LimitPaginator) + assert isinstance(paginator, DefaultPaginator) page_token_option = paginator.page_token_option assert isinstance(page_token_option, RequestOption) assert page_token_option.inject_into == RequestOptionType.path @@ -679,10 +679,10 @@ def test_validation_type_missing_required_fields(): def test_validation_wrong_interface_type(): content = """ paginator: - type: "LimitPaginator" + type: "DefaultPaginator" page_size: 10 url_base: "https://airbyte.io" - limit_option: + page_size_option: inject_into: request_parameter field_name: page_size page_token_option: @@ -717,10 +717,10 @@ def test_validation_create_composite_error_handler(): def test_validation_wrong_object_type(): content = """ paginator: - type: "LimitPaginator" + type: "DefaultPaginator" page_size: 10 url_base: "https://airbyte.io" - limit_option: + page_size_option: inject_into: request_parameter field_name: page_size page_token_option: diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py index a9620eff6bb39..b804da15123e2 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py @@ -28,9 +28,9 @@ # file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" # retriever: # paginator: -# type: "LimitPaginator" +# type: "DefaultPaginator" # page_size: 10 -# limit_option: +# page_size_option: # inject_into: request_parameter # field_name: page_size # page_token_option: @@ -72,9 +72,9 @@ # file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" # retriever: # paginator: -# type: "LimitPaginator" +# type: "DefaultPaginator" # page_size: 10 -# limit_option: +# page_size_option: # inject_into: request_parameter # field_name: page_size # page_token_option: @@ -118,9 +118,9 @@ # file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" # retriever: # paginator: -# type: "LimitPaginator" +# type: "DefaultPaginator" # page_size: 10 -# limit_option: +# page_size_option: # inject_into: request_parameter # field_name: page_size # page_token_option: @@ -173,9 +173,9 @@ # file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" # retriever: # paginator: -# type: "LimitPaginator" +# type: "DefaultPaginator" # page_size: 10 -# limit_option: +# page_size_option: # inject_into: request_parameter # field_name: page_size # page_token_option: @@ -292,7 +292,7 @@ def test_generate_schema(): "anyOf" ] assert {"type": "string"} in declarative_stream["properties"]["primary_key"]["anyOf"] - assert {"$ref": "#/definitions/LimitPaginator"} in simple_retriever["properties"]["paginator"]["anyOf"] + assert {"$ref": "#/definitions/DefaultPaginator"} in simple_retriever["properties"]["paginator"]["anyOf"] assert {"$ref": "#/definitions/NoPagination"} in simple_retriever["properties"]["paginator"]["anyOf"] assert {"$ref": "#/definitions/CartesianProductStreamSlicer"} in simple_retriever["properties"]["stream_slicer"]["anyOf"] assert {"$ref": "#/definitions/DatetimeStreamSlicer"} in simple_retriever["properties"]["stream_slicer"]["anyOf"] @@ -330,17 +330,14 @@ def test_generate_schema(): assert default_error_handler["properties"]["max_retries"]["type"] == "integer" assert default_error_handler["properties"]["backoff_strategies"]["type"] == "array" - limit_paginator = schema["definitions"]["LimitPaginator"]["allOf"][1] - assert {"page_size", "limit_option", "page_token_option", "pagination_strategy", "config", "url_base"}.issubset( - limit_paginator["required"] - ) - assert limit_paginator["properties"]["page_size"]["type"] == "integer" - assert limit_paginator["properties"]["limit_option"]["$ref"] == "#/definitions/RequestOption" - assert limit_paginator["properties"]["page_token_option"]["$ref"] == "#/definitions/RequestOption" - assert {"$ref": "#/definitions/CursorPaginationStrategy"} in limit_paginator["properties"]["pagination_strategy"]["anyOf"] - assert {"$ref": "#/definitions/OffsetIncrement"} in limit_paginator["properties"]["pagination_strategy"]["anyOf"] - assert {"$ref": "#/definitions/PageIncrement"} in limit_paginator["properties"]["pagination_strategy"]["anyOf"] - assert limit_paginator["properties"]["decoder"]["$ref"] == "#/definitions/JsonDecoder" + default_paginator = schema["definitions"]["DefaultPaginator"]["allOf"][1] + assert {"page_token_option", "pagination_strategy", "config", "url_base"}.issubset(default_paginator["required"]) + assert default_paginator["properties"]["page_size_option"]["$ref"] == "#/definitions/RequestOption" + assert default_paginator["properties"]["page_token_option"]["$ref"] == "#/definitions/RequestOption" + assert {"$ref": "#/definitions/CursorPaginationStrategy"} in default_paginator["properties"]["pagination_strategy"]["anyOf"] + assert {"$ref": "#/definitions/OffsetIncrement"} in default_paginator["properties"]["pagination_strategy"]["anyOf"] + assert {"$ref": "#/definitions/PageIncrement"} in default_paginator["properties"]["pagination_strategy"]["anyOf"] + assert default_paginator["properties"]["decoder"]["$ref"] == "#/definitions/JsonDecoder" assert {"$ref": "#/definitions/InterpolatedString"} in http_requester["properties"]["url_base"]["anyOf"] assert {"type": "string"} in http_requester["properties"]["path"]["anyOf"] diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_transform.py b/airbyte-cdk/python/unit_tests/sources/utils/test_transform.py index 3d53f9ba024a8..308dc2475862c 100644 --- a/airbyte-cdk/python/unit_tests/sources/utils/test_transform.py +++ b/airbyte-cdk/python/unit_tests/sources/utils/test_transform.py @@ -197,6 +197,12 @@ {"value1": "value2"}, "Failed to transform value 'value2' of type 'string' to 'object', key path: '.value1'", ), + ( + {"type": "object", "properties": {"value": {"type": "array", "items": {"type": "object"}}}}, + {"value": ["one", "two"]}, + {"value": ["one", "two"]}, + "Failed to transform value 'one' of type 'string' to 'object', key path: '.value.0'", + ), ], ) def test_transform(schema, actual, expected, expected_warns, caplog): diff --git a/airbyte-commons-protocol/build.gradle b/airbyte-commons-protocol/build.gradle new file mode 100644 index 0000000000000..502c714ffd8e3 --- /dev/null +++ b/airbyte-commons-protocol/build.gradle @@ -0,0 +1,11 @@ +dependencies { + annotationProcessor libs.bundles.micronaut.annotation.processor + testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor + + implementation libs.bundles.micronaut.annotation + testImplementation libs.bundles.micronaut.test + + implementation project(':airbyte-protocol:protocol-models') +} + +Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java new file mode 100644 index 0000000000000..1b2c2a653024e --- /dev/null +++ b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol; + +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.protocol.migrations.AirbyteMessageMigration; +import io.airbyte.commons.version.AirbyteVersion; +import jakarta.annotation.PostConstruct; +import jakarta.inject.Singleton; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + * AirbyteProtocol Message Migrator + * + * This class is intended to apply the transformations required to go from one version of the + * AirbyteProtocol to another. + */ +@Singleton +public class AirbyteMessageMigrator { + + private final List> migrationsToRegister; + private final SortedMap> migrations = new TreeMap<>(); + private String mostRecentVersion = ""; + + public AirbyteMessageMigrator(List> migrations) { + migrationsToRegister = migrations; + } + + public AirbyteMessageMigrator() { + this(Collections.emptyList()); + } + + @PostConstruct + public void initialize() { + migrationsToRegister.forEach(this::registerMigration); + } + + /** + * Downgrade a message from the most recent version to the target version by chaining all the + * required migrations + */ + public PreviousVersion downgrade(final CurrentVersion message, final AirbyteVersion target) { + if (target.getMajorVersion().equals(mostRecentVersion)) { + return (PreviousVersion) message; + } + + Object result = message; + Object[] selectedMigrations = selectMigrations(target).toArray(); + for (int i = selectedMigrations.length; i > 0; --i) { + result = applyDowngrade((AirbyteMessageMigration) selectedMigrations[i - 1], result); + } + return (PreviousVersion) result; + } + + /** + * Upgrade a message from the source version to the most recent version by chaining all the required + * migrations + */ + public CurrentVersion upgrade(final PreviousVersion message, final AirbyteVersion source) { + if (source.getMajorVersion().equals(mostRecentVersion)) { + return (CurrentVersion) message; + } + + Object result = message; + for (var migration : selectMigrations(source)) { + result = applyUpgrade(migration, result); + } + return (CurrentVersion) result; + } + + private Collection> selectMigrations(final AirbyteVersion version) { + final Collection> results = migrations.tailMap(version.getMajorVersion()).values(); + if (results.isEmpty()) { + throw new RuntimeException("Unsupported migration version " + version.serialize()); + } + return results; + } + + // Helper function to work around type casting + private PreviousVersion applyDowngrade(final AirbyteMessageMigration migration, + final Object message) { + return migration.downgrade((CurrentVersion) message); + } + + // Helper function to work around type casting + private CurrentVersion applyUpgrade(final AirbyteMessageMigration migration, + final Object message) { + return migration.upgrade((PreviousVersion) message); + } + + /** + * Store migration in a sorted map key by the major of the lower version of the migration. + * + * The goal is to be able to retrieve the list of migrations to apply to get to/from a given + * version. We are only keying on the lower version because the right side (most recent version of + * the migration range) is always current version. + */ + @VisibleForTesting + void registerMigration(final AirbyteMessageMigration migration) { + final String key = migration.getPreviousVersion().getMajorVersion(); + if (!migrations.containsKey(key)) { + migrations.put(key, migration); + if (migration.getCurrentVersion().getMajorVersion().compareTo(mostRecentVersion) > 0) { + mostRecentVersion = migration.getCurrentVersion().getMajorVersion(); + } + } else { + throw new RuntimeException("Trying to register a duplicated migration " + migration.getClass().getName()); + } + } + + // Used for inspection of the injection + @VisibleForTesting + Set getMigrationKeys() { + return migrations.keySet(); + } + +} diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java new file mode 100644 index 0000000000000..829b11b696f49 --- /dev/null +++ b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol; + +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.protocol.serde.AirbyteMessageDeserializer; +import io.airbyte.commons.protocol.serde.AirbyteMessageSerializer; +import io.airbyte.commons.version.AirbyteVersion; +import jakarta.annotation.PostConstruct; +import jakarta.inject.Singleton; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * AirbyteProtocol Message Serializer/Deserializer provider + * + * This class is intended to help access the serializer/deserializer for a given version of the + * Airbyte Protocol. + */ +@Singleton +public class AirbyteMessageSerDeProvider { + + private final List> deserializersToRegister; + private final List> serializersToRegister; + + private final Map> deserializers = new HashMap<>(); + private final Map> serializers = new HashMap<>(); + + public AirbyteMessageSerDeProvider(final List> deserializers, + final List> serializers) { + deserializersToRegister = deserializers; + serializersToRegister = serializers; + } + + public AirbyteMessageSerDeProvider() { + this(Collections.emptyList(), Collections.emptyList()); + } + + @PostConstruct + public void initialize() { + deserializersToRegister.forEach(this::registerDeserializer); + serializersToRegister.forEach(this::registerSerializer); + } + + /** + * Returns the Deserializer for the version if known else empty + */ + public Optional> getDeserializer(final AirbyteVersion version) { + return Optional.ofNullable(deserializers.get(version.getMajorVersion())); + } + + /** + * Returns the Serializer for the version if known else empty + */ + public Optional> getSerializer(final AirbyteVersion version) { + return Optional.ofNullable(serializers.get(version.getMajorVersion())); + } + + @VisibleForTesting + void registerDeserializer(final AirbyteMessageDeserializer deserializer) { + final String key = deserializer.getTargetVersion().getMajorVersion(); + if (!deserializers.containsKey(key)) { + deserializers.put(key, deserializer); + } else { + throw new RuntimeException(String.format("Trying to register a deserializer for protocol version {} when {} already exists", + deserializer.getTargetVersion().serialize(), deserializers.get(key).getTargetVersion().serialize())); + } + } + + @VisibleForTesting + void registerSerializer(final AirbyteMessageSerializer serializer) { + final String key = serializer.getTargetVersion().getMajorVersion(); + if (!serializers.containsKey(key)) { + serializers.put(key, serializer); + } else { + throw new RuntimeException(String.format("Trying to register a serializer for protocol version {} when {} already exists", + serializer.getTargetVersion().serialize(), serializers.get(key).getTargetVersion().serialize())); + } + } + + // Used for inspection of the injection + @VisibleForTesting + Set getDeserializerKeys() { + return deserializers.keySet(); + } + + // Used for inspection of the injection + @VisibleForTesting + Set getSerializerKeys() { + return serializers.keySet(); + } + +} diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java new file mode 100644 index 0000000000000..8a615ea744e9f --- /dev/null +++ b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol.migrations; + +import io.airbyte.commons.version.AirbyteVersion; + +/** + * AirbyteProtocol message migration interface + * + * @param The Old AirbyteMessage type + * @param The New AirbyteMessage type + */ +public interface AirbyteMessageMigration { + + /** + * Downgrades a message to from the new version to the old version + * + * @param message: the message to downgrade + * @return the downgraded message + */ + PreviousVersion downgrade(final CurrentVersion message); + + /** + * Upgrades a message from the old version to the new version + * + * @param message: the message to upgrade + * @return the upgrade message + */ + CurrentVersion upgrade(final PreviousVersion message); + + /** + * The Old version, note that due to semver, the important piece of information is the Major. + */ + AirbyteVersion getPreviousVersion(); + + /** + * The New version, note that due to semver, the important piece of information is the Major. + */ + AirbyteVersion getCurrentVersion(); + +} diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigrationV0.java b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigrationV0.java new file mode 100644 index 0000000000000..8a48c7b75df18 --- /dev/null +++ b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigrationV0.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol.migrations; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.protocol.models.AirbyteMessage; + +/** + * Demo migration to illustrate the template. This should be deleted once we added the v0 to v1 + * migration. + */ +// NOTE, to actually wire this migration, uncomment the annotation +// @Singleton +public class AirbyteMessageMigrationV0 + implements AirbyteMessageMigration { + + @Override + public io.airbyte.protocol.models.v0.AirbyteMessage upgrade(final io.airbyte.protocol.models.AirbyteMessage oldMessage) { + final io.airbyte.protocol.models.v0.AirbyteMessage newMessage = + Jsons.object(Jsons.jsonNode(oldMessage), io.airbyte.protocol.models.v0.AirbyteMessage.class); + return newMessage; + } + + @Override + public io.airbyte.protocol.models.AirbyteMessage downgrade(final io.airbyte.protocol.models.v0.AirbyteMessage newMessage) { + final io.airbyte.protocol.models.AirbyteMessage oldMessage = + Jsons.object(Jsons.jsonNode(newMessage), io.airbyte.protocol.models.AirbyteMessage.class); + return oldMessage; + } + + @Override + public AirbyteVersion getPreviousVersion() { + return new AirbyteVersion("0.2.0"); + } + + @Override + public AirbyteVersion getCurrentVersion() { + return new AirbyteVersion("0.2.0"); + } + +} diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java new file mode 100644 index 0000000000000..9203f29cb3a77 --- /dev/null +++ b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol.serde; + +import io.airbyte.commons.version.AirbyteVersion; + +public interface AirbyteMessageDeserializer { + + T deserialize(final String json); + + AirbyteVersion getTargetVersion(); + +} diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java new file mode 100644 index 0000000000000..355235703be7d --- /dev/null +++ b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol.serde; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.version.AirbyteVersion; +import lombok.Getter; + +public class AirbyteMessageGenericDeserializer implements AirbyteMessageDeserializer { + + @Getter + final AirbyteVersion targetVersion; + final Class typeClass; + + public AirbyteMessageGenericDeserializer(final AirbyteVersion targetVersion, final Class typeClass) { + this.targetVersion = targetVersion; + this.typeClass = typeClass; + } + + @Override + public T deserialize(String json) { + return Jsons.deserialize(json, typeClass); + } + +} diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java new file mode 100644 index 0000000000000..d3e7b251c0d66 --- /dev/null +++ b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol.serde; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.version.AirbyteVersion; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +public class AirbyteMessageGenericSerializer implements AirbyteMessageSerializer { + + @Getter + private final AirbyteVersion targetVersion; + + @Override + public String serialize(T message) { + return Jsons.serialize(message); + } + +} diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java new file mode 100644 index 0000000000000..d3f42b27c689d --- /dev/null +++ b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol.serde; + +import io.airbyte.commons.version.AirbyteVersion; + +public interface AirbyteMessageSerializer { + + String serialize(final T message); + + AirbyteVersion getTargetVersion(); + +} diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java new file mode 100644 index 0000000000000..198fbff83cff0 --- /dev/null +++ b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol.serde; + +import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import jakarta.inject.Singleton; + +@Singleton +public class AirbyteMessageV0Deserializer extends AirbyteMessageGenericDeserializer { + + public AirbyteMessageV0Deserializer() { + super(new AirbyteVersion("0.3.0"), AirbyteMessage.class); + } + +} diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java new file mode 100644 index 0000000000000..d70442065cdb0 --- /dev/null +++ b/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol.serde; + +import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import jakarta.inject.Singleton; + +@Singleton +public class AirbyteMessageV0Serializer extends AirbyteMessageGenericSerializer { + + public AirbyteMessageV0Serializer() { + super(new AirbyteVersion("0.3.0")); + } + +} diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorMicronautTest.java b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorMicronautTest.java new file mode 100644 index 0000000000000..d5a899fcbd10a --- /dev/null +++ b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorMicronautTest.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import jakarta.inject.Inject; +import java.util.HashSet; +import org.junit.jupiter.api.Test; + +@MicronautTest +class AirbyteMessageMigratorMicronautTest { + + @Inject + AirbyteMessageMigrator messageMigrator; + + @Test + void testMigrationInjection() { + // This should contain the list of all the supported majors of the airbyte protocol except the most + // recent one since the migrations themselves are keyed on the lower version. + assertEquals(new HashSet<>(), messageMigrator.getMigrationKeys()); + } + +} diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java new file mode 100644 index 0000000000000..fd795f250800a --- /dev/null +++ b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import io.airbyte.commons.protocol.migrations.AirbyteMessageMigration; +import io.airbyte.commons.version.AirbyteVersion; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class AirbyteMessageMigratorTest { + + static final AirbyteVersion v0 = new AirbyteVersion("0.0.0"); + static final AirbyteVersion v1 = new AirbyteVersion("1.0.0"); + static final AirbyteVersion v2 = new AirbyteVersion("2.0.0"); + + record ObjectV0(String name0) {} + + record ObjectV1(String name1) {} + + record ObjectV2(String name2) {} + + static class Migrate0to1 implements AirbyteMessageMigration { + + @Override + public ObjectV0 downgrade(ObjectV1 message) { + return new ObjectV0(message.name1); + } + + @Override + public ObjectV1 upgrade(ObjectV0 message) { + return new ObjectV1(message.name0); + } + + @Override + public AirbyteVersion getPreviousVersion() { + return v0; + } + + @Override + public AirbyteVersion getCurrentVersion() { + return v1; + } + + } + + static class Migrate1to2 implements AirbyteMessageMigration { + + @Override + public ObjectV1 downgrade(ObjectV2 message) { + return new ObjectV1(message.name2); + } + + @Override + public ObjectV2 upgrade(ObjectV1 message) { + return new ObjectV2(message.name1); + } + + @Override + public AirbyteVersion getPreviousVersion() { + return v1; + } + + @Override + public AirbyteVersion getCurrentVersion() { + return v2; + } + + } + + AirbyteMessageMigrator migrator; + + @BeforeEach + void beforeEach() { + migrator = new AirbyteMessageMigrator(); + migrator.registerMigration(new Migrate0to1()); + migrator.registerMigration(new Migrate1to2()); + } + + @Test + void testDowngrade() { + final ObjectV2 obj = new ObjectV2("my name"); + + final ObjectV0 objDowngradedTo0 = migrator.downgrade(obj, v0); + assertEquals(obj.name2, objDowngradedTo0.name0); + + final ObjectV1 objDowngradedTo1 = migrator.downgrade(obj, v1); + assertEquals(obj.name2, objDowngradedTo1.name1); + + final ObjectV2 objDowngradedTo2 = migrator.downgrade(obj, v2); + assertEquals(obj.name2, objDowngradedTo2.name2); + } + + @Test + void testUpgrade() { + final ObjectV0 obj0 = new ObjectV0("my name 0"); + final ObjectV2 objUpgradedFrom0 = migrator.upgrade(obj0, v0); + assertEquals(obj0.name0, objUpgradedFrom0.name2); + + final ObjectV1 obj1 = new ObjectV1("my name 1"); + final ObjectV2 objUpgradedFrom1 = migrator.upgrade(obj1, v1); + assertEquals(obj1.name1, objUpgradedFrom1.name2); + + final ObjectV2 obj2 = new ObjectV2("my name 2"); + final ObjectV2 objUpgradedFrom2 = migrator.upgrade(obj2, v2); + assertEquals(obj2.name2, objUpgradedFrom2.name2); + } + + @Test + void testUnsupportedDowngradeShouldFailExplicitly() { + assertThrows(RuntimeException.class, () -> { + migrator.downgrade(new ObjectV2("woot"), new AirbyteVersion("5.0.0")); + }); + } + + @Test + void testUnsupportedUpgradeShouldFailExplicitly() { + assertThrows(RuntimeException.class, () -> { + migrator.upgrade(new ObjectV0("woot"), new AirbyteVersion("4.0.0")); + }); + } + + @Test + void testRegisterCollisionsShouldFail() { + assertThrows(RuntimeException.class, () -> { + migrator.registerMigration(new Migrate0to1()); + }); + } + +} diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderMicronautTest.java b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderMicronautTest.java new file mode 100644 index 0000000000000..ccab4503aa1db --- /dev/null +++ b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderMicronautTest.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import jakarta.inject.Inject; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; + +@MicronautTest +class AirbyteMessageSerDeProviderMicronautTest { + + @Inject + AirbyteMessageSerDeProvider serDeProvider; + + @Test + void testSerDeInjection() { + // This should contain the list of all the supported majors of the airbyte protocol + final Set expectedVersions = new HashSet<>(List.of("0")); + + assertEquals(expectedVersions, serDeProvider.getDeserializerKeys()); + assertEquals(expectedVersions, serDeProvider.getSerializerKeys()); + } + +} diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java new file mode 100644 index 0000000000000..20fd6339aae0b --- /dev/null +++ b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.airbyte.commons.protocol.serde.AirbyteMessageDeserializer; +import io.airbyte.commons.protocol.serde.AirbyteMessageSerializer; +import io.airbyte.commons.version.AirbyteVersion; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class AirbyteMessageSerDeProviderTest { + + AirbyteMessageSerDeProvider serDeProvider; + AirbyteMessageDeserializer deserV0; + AirbyteMessageDeserializer deserV1; + + AirbyteMessageSerializer serV0; + AirbyteMessageSerializer serV1; + + @BeforeEach + void beforeEach() { + serDeProvider = new AirbyteMessageSerDeProvider(); + + deserV0 = buildDeserializer(new AirbyteVersion("0.1.0")); + deserV1 = buildDeserializer(new AirbyteVersion("1.1.0")); + serDeProvider.registerDeserializer(deserV0); + serDeProvider.registerDeserializer(deserV1); + + serV0 = buildSerializer(new AirbyteVersion("0.2.0")); + serV1 = buildSerializer(new AirbyteVersion("1.0.0")); + serDeProvider.registerSerializer(serV0); + serDeProvider.registerSerializer(serV1); + } + + @Test + void testGetDeserializer() { + assertEquals(Optional.of(deserV0), serDeProvider.getDeserializer(new AirbyteVersion("0.1.0"))); + assertEquals(Optional.of(deserV0), serDeProvider.getDeserializer(new AirbyteVersion("0.2.0"))); + assertEquals(Optional.of(deserV1), serDeProvider.getDeserializer(new AirbyteVersion("1.1.0"))); + assertEquals(Optional.empty(), serDeProvider.getDeserializer(new AirbyteVersion("2.0.0"))); + } + + @Test + void testGetSerializer() { + assertEquals(Optional.of(serV0), serDeProvider.getSerializer(new AirbyteVersion("0.1.0"))); + assertEquals(Optional.of(serV1), serDeProvider.getSerializer(new AirbyteVersion("1.0.0"))); + assertEquals(Optional.empty(), serDeProvider.getSerializer(new AirbyteVersion("3.2.0"))); + } + + @Test + void testRegisterDeserializerShouldFailOnVersionCollision() { + AirbyteMessageDeserializer deser = buildDeserializer(new AirbyteVersion("0.2.0")); + assertThrows(RuntimeException.class, () -> { + serDeProvider.registerDeserializer(deser); + }); + } + + @Test + void testRegisterSerializerShouldFailOnVersionCollision() { + AirbyteMessageSerializer ser = buildSerializer(new AirbyteVersion("0.5.0")); + assertThrows(RuntimeException.class, () -> { + serDeProvider.registerSerializer(ser); + }); + } + + private AirbyteMessageDeserializer buildDeserializer(AirbyteVersion version) { + final AirbyteMessageDeserializer deser = mock(AirbyteMessageDeserializer.class); + when(deser.getTargetVersion()).thenReturn(version); + return deser; + } + + private AirbyteMessageSerializer buildSerializer(AirbyteVersion version) { + final AirbyteMessageSerializer ser = mock(AirbyteMessageSerializer.class); + when(ser.getTargetVersion()).thenReturn(version); + return ser; + } + +} diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/DefaultToV0MigrationTest.java b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/DefaultToV0MigrationTest.java new file mode 100644 index 0000000000000..a3b0d3eb58f23 --- /dev/null +++ b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/DefaultToV0MigrationTest.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol.migrations; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.net.URI; +import lombok.SneakyThrows; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class DefaultToV0MigrationTest { + + AirbyteMessageMigrationV0 v0migration; + + @BeforeEach + void beforeEach() { + v0migration = new AirbyteMessageMigrationV0(); + } + + @Test + void testVersionMetadata() { + assertEquals("0", v0migration.getPreviousVersion().getMajorVersion()); + assertEquals("0", v0migration.getCurrentVersion().getMajorVersion()); + } + + @Test + void testDowngrade() { + final io.airbyte.protocol.models.v0.AirbyteMessage v0Message = getV0Message(); + + final io.airbyte.protocol.models.AirbyteMessage downgradedMessage = v0migration.downgrade(v0Message); + final io.airbyte.protocol.models.AirbyteMessage expectedMessage = getUnversionedMessage(); + assertEquals(expectedMessage, downgradedMessage); + } + + @Test + void testUpgrade() { + final io.airbyte.protocol.models.AirbyteMessage unversionedMessage = getUnversionedMessage(); + + final io.airbyte.protocol.models.v0.AirbyteMessage upgradedMessage = v0migration.upgrade(unversionedMessage); + final io.airbyte.protocol.models.v0.AirbyteMessage expectedMessage = getV0Message(); + assertEquals(expectedMessage, upgradedMessage); + } + + @SneakyThrows + private io.airbyte.protocol.models.v0.AirbyteMessage getV0Message() { + return new io.airbyte.protocol.models.v0.AirbyteMessage() + .withType(io.airbyte.protocol.models.v0.AirbyteMessage.Type.SPEC) + .withSpec( + new io.airbyte.protocol.models.v0.ConnectorSpecification() + .withProtocolVersion("0.3.0") + .withDocumentationUrl(new URI("file:///tmp/doc"))); + } + + @SneakyThrows + private io.airbyte.protocol.models.AirbyteMessage getUnversionedMessage() { + return new io.airbyte.protocol.models.AirbyteMessage() + .withType(io.airbyte.protocol.models.AirbyteMessage.Type.SPEC) + .withSpec( + new io.airbyte.protocol.models.ConnectorSpecification() + .withProtocolVersion("0.3.0") + .withDocumentationUrl(new URI("file:///tmp/doc"))); + } + +} diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java new file mode 100644 index 0000000000000..3acdf31f35d7c --- /dev/null +++ b/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.protocol.serde; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.ConnectorSpecification; +import java.net.URI; +import java.net.URISyntaxException; +import org.junit.jupiter.api.Test; + +class AirbyteMessageV0SerDeTest { + + @Test + void v0SerDeRoundTripTest() throws URISyntaxException { + final AirbyteMessageV0Deserializer deser = new AirbyteMessageV0Deserializer(); + final AirbyteMessageV0Serializer ser = new AirbyteMessageV0Serializer(); + + final AirbyteMessage message = new AirbyteMessage() + .withType(Type.SPEC) + .withSpec( + new ConnectorSpecification() + .withProtocolVersion("0.3.0") + .withDocumentationUrl(new URI("file:///tmp/doc"))); + + final String serializedMessage = ser.serialize(message); + final AirbyteMessage deserializedMessage = deser.deserialize(serializedMessage); + + assertEquals(message, deserializedMessage); + } + +} diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalInitializationUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalInitializationUtils.java new file mode 100644 index 0000000000000..33f15d6f6f85e --- /dev/null +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalInitializationUtils.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.temporal; + +import io.grpc.StatusRuntimeException; +import io.micronaut.context.annotation.Value; +import io.micronaut.core.util.StringUtils; +import io.temporal.api.workflowservice.v1.DescribeNamespaceRequest; +import io.temporal.serviceclient.WorkflowServiceStubs; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; + +@Singleton +@Slf4j +public class TemporalInitializationUtils { + + @Inject + private WorkflowServiceStubs temporalService; + @Value("${temporal.cloud.namespace}") + private String temporalCloudNamespace; + + /** + * Blocks until the Temporal {@link TemporalUtils#DEFAULT_NAMESPACE} has been created. This is + * necessary to avoid issues related to + * https://community.temporal.io/t/running-into-an-issue-when-creating-namespace-programmatically/2783/8. + */ + public void waitForTemporalNamespace() { + boolean namespaceExists = false; + final String temporalNamespace = getTemporalNamespace(); + while (!namespaceExists) { + try { + // This is to allow the configured namespace to be available in the Temporal + // cache before continuing on with any additional configuration/bean creation. + temporalService.blockingStub().describeNamespace(DescribeNamespaceRequest.newBuilder().setNamespace(temporalNamespace).build()); + namespaceExists = true; + } catch (final StatusRuntimeException e) { + log.debug("Namespace '{}' does not exist yet. Re-checking...", temporalNamespace); + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(5)); + } catch (final InterruptedException ie) { + log.debug("Sleep interrupted. Exiting loop..."); + } + } + } + } + + /** + * Retrieve the Temporal namespace based on the configuration. + * + * @return The Temporal namespace. + */ + private String getTemporalNamespace() { + return StringUtils.isNotEmpty(temporalCloudNamespace) ? temporalCloudNamespace : TemporalUtils.DEFAULT_NAMESPACE; + } + +} diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java b/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java index f97598dbacb7f..81eb74b4d6426 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java @@ -21,11 +21,6 @@ public boolean autoDisablesFailingConnections() { return Boolean.parseBoolean(System.getenv("AUTO_DISABLE_FAILING_CONNECTIONS")); } - @Override - public boolean exposeSecretsInExport() { - return Boolean.parseBoolean(System.getenv("EXPOSE_SECRETS_IN_EXPORT")); - } - @Override public boolean forceSecretMigration() { return Boolean.parseBoolean(System.getenv("FORCE_MIGRATE_SECRET_STORE")); diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java b/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java index 0757f025f9369..eb3b765ceef80 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java @@ -12,8 +12,6 @@ public interface FeatureFlags { boolean autoDisablesFailingConnections(); - boolean exposeSecretsInExport(); - boolean forceSecretMigration(); boolean useStreamCapableState(); diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java b/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java index 339b4a92d3273..27543ba3cd2ce 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java @@ -6,13 +6,16 @@ public class AirbyteProtocolVersion { - public final static AirbyteVersion DEFAULT_AIRBYTE_PROTOCOL_VERSION = new AirbyteVersion("0.2.0"); + public final static Version DEFAULT_AIRBYTE_PROTOCOL_VERSION = new Version("0.2.0"); - public static AirbyteVersion getWithDefault(final String version) { + public final static String AIRBYTE_PROTOCOL_VERSION_MAX_KEY_NAME = "airbyte_protocol_version_max"; + public final static String AIRBYTE_PROTOCOL_VERSION_MIN_KEY_NAME = "airbyte_protocol_version_min"; + + public static Version getWithDefault(final String version) { if (version == null || version.isEmpty() || version.isBlank()) { return DEFAULT_AIRBYTE_PROTOCOL_VERSION; } else { - return new AirbyteVersion(version); + return new Version(version); } } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java b/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java index 078d40b487c29..7c5406fc1c3f5 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java @@ -4,146 +4,19 @@ package io.airbyte.commons.version; -import com.google.common.base.Preconditions; -import java.util.Objects; - /** * The AirbyteVersion identifies the version of the database used internally by Airbyte services. */ -@SuppressWarnings("PMD.ConstructorCallsOverridableMethod") -public class AirbyteVersion { +public class AirbyteVersion extends Version { - public static final String DEV_VERSION_PREFIX = "dev"; public static final String AIRBYTE_VERSION_KEY_NAME = "airbyte_version"; - private final String version; - private final String major; - private final String minor; - private final String patch; - public AirbyteVersion(final String version) { - Preconditions.checkNotNull(version); - this.version = version; - final String[] parsedVersion = version.replace("\n", "").strip().split("-")[0].split("\\."); - - if (isDev()) { - this.major = null; - this.minor = null; - this.patch = null; - } else { - Preconditions.checkArgument(parsedVersion.length >= 3, "Invalid version string: " + version); - this.major = parsedVersion[0]; - this.minor = parsedVersion[1]; - this.patch = parsedVersion[2]; - } + super(version); } public AirbyteVersion(final String major, final String minor, final String patch) { - this.version = String.format("%s.%s.%s", major, minor, patch); - this.major = major; - this.minor = minor; - this.patch = patch; - } - - public String serialize() { - return version; - } - - public String getMajorVersion() { - return major; - } - - public String getMinorVersion() { - return minor; - } - - public String getPatchVersion() { - return patch; - } - - /** - * Compares two Airbyte Version to check if they are equivalent. - * - * Only the major and minor part of the Version is taken into account. - */ - public int compatibleVersionCompareTo(final AirbyteVersion another) { - if (isDev() || another.isDev()) - return 0; - final int majorDiff = compareVersion(major, another.major); - if (majorDiff != 0) { - return majorDiff; - } - return compareVersion(minor, another.minor); - } - - /** - * @return true if this is greater than other. otherwise false. - */ - public boolean greaterThan(final AirbyteVersion other) { - return patchVersionCompareTo(other) > 0; - } - - /** - * @return true if this is greater than or equal toother. otherwise false. - */ - public boolean greaterThanOrEqualTo(final AirbyteVersion other) { - return patchVersionCompareTo(other) >= 0; - } - - /** - * @return true if this is less than other. otherwise false. - */ - public boolean lessThan(final AirbyteVersion other) { - return patchVersionCompareTo(other) < 0; - } - - /** - * Compares two Airbyte Version to check if they are equivalent (including patch version). - */ - public int patchVersionCompareTo(final AirbyteVersion another) { - if (isDev() || another.isDev()) { - return 0; - } - final int majorDiff = compareVersion(major, another.major); - if (majorDiff != 0) { - return majorDiff; - } - final int minorDiff = compareVersion(minor, another.minor); - if (minorDiff != 0) { - return minorDiff; - } - return compareVersion(patch, another.patch); - } - - /** - * Compares two Airbyte Version to check if only the patch version was updated. - */ - public boolean checkOnlyPatchVersionIsUpdatedComparedTo(final AirbyteVersion another) { - if (isDev() || another.isDev()) { - return false; - } - final int majorDiff = compareVersion(major, another.major); - if (majorDiff > 0) { - return false; - } - final int minorDiff = compareVersion(minor, another.minor); - if (minorDiff > 0) { - return false; - } - return compareVersion(patch, another.patch) > 0; - } - - public boolean isDev() { - return version.startsWith(DEV_VERSION_PREFIX); - } - - /** - * Version string needs to be converted to integer for comparison, because string comparison does - * not handle version string with different digits correctly. For example: - * {@code "11".compare("3") < 0}, while {@code Integer.compare(11, 3) > 0}. - */ - private static int compareVersion(final String v1, final String v2) { - return Integer.compare(Integer.parseInt(v1), Integer.parseInt(v2)); + super(major, minor, patch); } public static void assertIsCompatible(final AirbyteVersion version1, final AirbyteVersion version2) throws IllegalStateException { @@ -159,10 +32,6 @@ public static String getErrorMessage(final AirbyteVersion version1, final Airbyt version1.serialize(), version2.serialize()); } - public static boolean isCompatible(final AirbyteVersion v1, final AirbyteVersion v2) { - return v1.compatibleVersionCompareTo(v2) == 0; - } - @Override public String toString() { return "AirbyteVersion{" + @@ -186,22 +55,4 @@ public static AirbyteVersion versionWithoutPatch(final String airbyteVersion) { return versionWithoutPatch(new AirbyteVersion(airbyteVersion)); } - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final AirbyteVersion that = (AirbyteVersion) o; - return Objects.equals(version, that.version) && Objects.equals(major, that.major) && Objects.equals(minor, that.minor) - && Objects.equals(patch, that.patch); - } - - @Override - public int hashCode() { - return Objects.hash(version, major, minor, patch); - } - } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/version/Version.java b/airbyte-commons/src/main/java/io/airbyte/commons/version/Version.java new file mode 100644 index 0000000000000..49fe82e05c555 --- /dev/null +++ b/airbyte-commons/src/main/java/io/airbyte/commons/version/Version.java @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.version; + +import com.google.common.base.Preconditions; +import java.util.Objects; + +/** + * A semVer Version class that allows "dev" as a version. + */ +@SuppressWarnings({"PMD.AvoidFieldNameMatchingTypeName", "PMD.ConstructorCallsOverridableMethod"}) +public class Version { + + public static final String DEV_VERSION_PREFIX = "dev"; + + protected final String version; + protected final String major; + protected final String minor; + protected final String patch; + + public Version(final String version) { + Preconditions.checkNotNull(version); + this.version = version; + final String[] parsedVersion = version.replace("\n", "").strip().split("-")[0].split("\\."); + + if (isDev()) { + this.major = null; + this.minor = null; + this.patch = null; + } else { + Preconditions.checkArgument(parsedVersion.length >= 3, "Invalid version string: " + version); + this.major = parsedVersion[0]; + this.minor = parsedVersion[1]; + this.patch = parsedVersion[2]; + } + } + + public Version(final String major, final String minor, final String patch) { + this.version = String.format("%s.%s.%s", major, minor, patch); + this.major = major; + this.minor = minor; + this.patch = patch; + } + + public String serialize() { + return version; + } + + public String getMajorVersion() { + return major; + } + + public String getMinorVersion() { + return minor; + } + + public String getPatchVersion() { + return patch; + } + + /** + * Compares two Version to check if they are equivalent. + * + * Only the major and minor part of the Version is taken into account. + */ + public int compatibleVersionCompareTo(final Version another) { + if (isDev() || another.isDev()) + return 0; + final int majorDiff = compareVersion(major, another.major); + if (majorDiff != 0) { + return majorDiff; + } + return compareVersion(minor, another.minor); + } + + /** + * @return true if this is greater than other. otherwise false. + */ + public boolean greaterThan(final Version other) { + return patchVersionCompareTo(other) > 0; + } + + /** + * @return true if this is greater than or equal toother. otherwise false. + */ + public boolean greaterThanOrEqualTo(final Version other) { + return patchVersionCompareTo(other) >= 0; + } + + /** + * @return true if this is less than other. otherwise false. + */ + public boolean lessThan(final Version other) { + return patchVersionCompareTo(other) < 0; + } + + /** + * Compares two Version to check if they are equivalent (including patch version). + */ + public int patchVersionCompareTo(final Version another) { + if (isDev() || another.isDev()) { + return 0; + } + final int majorDiff = compareVersion(major, another.major); + if (majorDiff != 0) { + return majorDiff; + } + final int minorDiff = compareVersion(minor, another.minor); + if (minorDiff != 0) { + return minorDiff; + } + return compareVersion(patch, another.patch); + } + + /** + * Compares two Version to check if only the patch version was updated. + */ + public boolean checkOnlyPatchVersionIsUpdatedComparedTo(final Version another) { + if (isDev() || another.isDev()) { + return false; + } + final int majorDiff = compareVersion(major, another.major); + if (majorDiff > 0) { + return false; + } + final int minorDiff = compareVersion(minor, another.minor); + if (minorDiff > 0) { + return false; + } + return compareVersion(patch, another.patch) > 0; + } + + public boolean isDev() { + return version.startsWith(DEV_VERSION_PREFIX); + } + + /** + * Version string needs to be converted to integer for comparison, because string comparison does + * not handle version string with different digits correctly. For example: + * {@code "11".compare("3") < 0}, while {@code Integer.compare(11, 3) > 0}. + */ + private static int compareVersion(final String v1, final String v2) { + return Integer.compare(Integer.parseInt(v1), Integer.parseInt(v2)); + } + + public static boolean isCompatible(final Version v1, final Version v2) { + return v1.compatibleVersionCompareTo(v2) == 0; + } + + @Override + public String toString() { + return "Version{" + + "version='" + version + '\'' + + ", major='" + major + '\'' + + ", minor='" + minor + '\'' + + ", patch='" + patch + '\'' + + '}'; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final Version that = (Version) o; + return Objects.equals(version, that.version) && Objects.equals(major, that.major) && Objects.equals(minor, that.minor) + && Objects.equals(patch, that.patch); + } + + @Override + public int hashCode() { + return Objects.hash(version, major, minor, patch); + } + +} diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java index 5dac93a102c5c..6e05ed9e3cfb8 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java @@ -5,6 +5,7 @@ package io.airbyte.config; import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.commons.version.Version; import io.airbyte.config.helpers.LogConfigs; import io.airbyte.config.storage.CloudStorageConfigs; import java.net.URI; @@ -43,6 +44,16 @@ public interface Configs { */ AirbyteVersion getAirbyteVersion(); + /** + * Defines the max supported Airbyte Protocol Version + */ + Version getAirbyteProtocolVersionMax(); + + /** + * Defines the min supported Airbyte Protocol Version + */ + Version getAirbyteProtocolVersionMin(); + String getAirbyteVersionOrWarning(); /** @@ -568,15 +579,6 @@ public interface Configs { */ boolean shouldRunConnectionManagerWorkflows(); - /** - * Define if the worker is operating within Airbyte's Control Plane, or within an external Data - * Plane. - Workers in the Control Plane process tasks related to control-flow, like scheduling and - * routing, as well as data syncing tasks that are enqueued for the Control Plane's default task - * queue. - Workers in a Data Plane process only tasks related to data syncing that are specifically - * enqueued for that worker's particular Data Plane. - */ - WorkerPlane getWorkerPlane(); - // Worker - Control Plane configs /** @@ -717,9 +719,4 @@ enum SecretPersistenceType { VAULT } - enum WorkerPlane { - CONTROL_PLANE, - DATA_PLANE - } - } diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java index 10e4e751e0ab3..d680240607bcb 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -10,6 +10,7 @@ import io.airbyte.commons.lang.Exceptions; import io.airbyte.commons.map.MoreMaps; import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.commons.version.Version; import io.airbyte.config.helpers.LogClientSingleton; import io.airbyte.config.helpers.LogConfigs; import io.airbyte.config.storage.CloudStorageConfigs; @@ -41,6 +42,8 @@ public class EnvConfigs implements Configs { // env variable names public static final String AIRBYTE_ROLE = "AIRBYTE_ROLE"; public static final String AIRBYTE_VERSION = "AIRBYTE_VERSION"; + public static final String AIRBYTE_PROTOCOL_VERSION_MAX = "AIRBYTE_PROTOCOL_VERSION_MAX"; + public static final String AIRBYTE_PROTOCOL_VERSION_MIN = "AIRBYTE_PROTOCOL_VERSION_MIN"; public static final String INTERNAL_API_HOST = "INTERNAL_API_HOST"; public static final String AIRBYTE_API_AUTH_HEADER_NAME = "AIRBYTE_API_AUTH_HEADER_NAME"; public static final String AIRBYTE_API_AUTH_HEADER_VALUE = "AIRBYTE_API_AUTH_HEADER_VALUE"; @@ -131,7 +134,6 @@ public class EnvConfigs implements Configs { private static final String SHOULD_RUN_DISCOVER_WORKFLOWS = "SHOULD_RUN_DISCOVER_WORKFLOWS"; private static final String SHOULD_RUN_SYNC_WORKFLOWS = "SHOULD_RUN_SYNC_WORKFLOWS"; private static final String SHOULD_RUN_CONNECTION_MANAGER_WORKFLOWS = "SHOULD_RUN_CONNECTION_MANAGER_WORKFLOWS"; - private static final String WORKER_PLANE = "WORKER_PLANE"; // Worker - Control plane configs private static final String DEFAULT_DATA_SYNC_TASK_QUEUES = "SYNC"; // should match TemporalJobType.SYNC.name() @@ -229,8 +231,6 @@ public EnvConfigs(final Map envMap) { this.getAllEnvKeys = envMap::keySet; this.logConfigs = new LogConfigs(getLogConfiguration()); this.stateStorageCloudConfigs = getStateStorageConfiguration().orElse(null); - - validateSyncWorkflowConfigs(); } private Optional getLogConfiguration() { @@ -289,6 +289,16 @@ public AirbyteVersion getAirbyteVersion() { return new AirbyteVersion(getEnsureEnv(AIRBYTE_VERSION)); } + @Override + public Version getAirbyteProtocolVersionMax() { + return new Version(getEnvOrDefault(AIRBYTE_PROTOCOL_VERSION_MAX, "0.3.0")); + } + + @Override + public Version getAirbyteProtocolVersionMin() { + return new Version(getEnvOrDefault(AIRBYTE_PROTOCOL_VERSION_MIN, "0.0.0")); + } + @Override public String getAirbyteVersionOrWarning() { return Optional.ofNullable(getEnv(AIRBYTE_VERSION)).orElse("version not set"); @@ -931,11 +941,6 @@ public boolean shouldRunConnectionManagerWorkflows() { return getEnvOrDefault(SHOULD_RUN_CONNECTION_MANAGER_WORKFLOWS, true); } - @Override - public WorkerPlane getWorkerPlane() { - return getEnvOrDefault(WORKER_PLANE, WorkerPlane.CONTROL_PLANE, s -> WorkerPlane.valueOf(s.toUpperCase())); - } - // Worker - Control plane @Override @@ -973,22 +978,6 @@ public String getDataPlaneServiceAccountEmail() { return getEnvOrDefault(DATA_PLANE_SERVICE_ACCOUNT_EMAIL, ""); } - /** - * Ensures the user hasn't configured themselves into a corner by making sure that the worker is set - * up to properly process sync workflows. With sensible defaults, it should be hard to fail this - * validation, but this provides a safety net regardless. - */ - private void validateSyncWorkflowConfigs() { - if (shouldRunSyncWorkflows()) { - if (getWorkerPlane().equals(WorkerPlane.DATA_PLANE) && getDataSyncTaskQueues().isEmpty()) { - throw new IllegalArgumentException(String.format( - "When %s is true, the worker must either be configured as a Control Plane worker, or %s must be non-empty.", - SHOULD_RUN_SYNC_WORKFLOWS, - DATA_SYNC_TASK_QUEUES)); - } - } - } - @Override public Set getTemporalWorkerPorts() { final var ports = getEnvOrDefault(TEMPORAL_WORKER_PORTS, ""); diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessor.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessor.java index f97ce914bb1a4..f89cc8966df35 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessor.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessor.java @@ -27,10 +27,7 @@ public class JsonSecretsProcessor { @Builder.Default - final private boolean maskSecrets = true; - - @Builder.Default - final private boolean copySecrets = true; + final private Boolean copySecrets = false; protected static final JsonSchemaValidator VALIDATOR = new JsonSchemaValidator(); @@ -54,18 +51,14 @@ public class JsonSecretsProcessor { * @param obj Object containing potentially secret fields */ public JsonNode prepareSecretsForOutput(final JsonNode obj, final JsonNode schema) { - if (maskSecrets) { - // todo (cgardens) this is not safe. should throw. - // if schema is an object and has a properties field - if (!isValidJsonSchema(schema)) { - log.error("The schema is not valid, the secret can't be hidden"); - return obj; - } - - return maskAllSecrets(obj, schema); + // todo (cgardens) this is not safe. should throw. + // if schema is an object and has a properties field + if (!isValidJsonSchema(schema)) { + log.error("The schema is not valid, the secret can't be hidden"); + return obj; } - return obj; + return maskAllSecrets(obj, schema); } /** diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessorTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessorTest.java index 8302226800a1c..a522e4db4b554 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessorTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessorTest.java @@ -198,7 +198,6 @@ class JsonSecretsProcessorTest { public void setup() { processor = JsonSecretsProcessor.builder() .copySecrets(true) - .maskSecrets(true) .build(); } @@ -502,7 +501,6 @@ class NoOpTest { public void setup() { processor = JsonSecretsProcessor.builder() .copySecrets(false) - .maskSecrets(false) .build(); } @@ -568,7 +566,7 @@ void testSecretScenario(final String folder, final boolean partial) throws IOExc final InputStream inputIs = getClass().getClassLoader().getResourceAsStream(inputFilePath); final JsonNode input = objectMapper.readTree(inputIs); - final String expectedFilePath = folder + (partial ? "/partial_config.json" : "/full_config.json"); + final String expectedFilePath = folder + "/expected.json"; final InputStream expectedIs = getClass().getClassLoader().getResourceAsStream(expectedFilePath); final JsonNode expected = objectMapper.readTree(expectedIs); diff --git a/airbyte-config/init/src/main/resources/icons/cliskhouse.svg b/airbyte-config/init/src/main/resources/icons/clickhouse.svg similarity index 100% rename from airbyte-config/init/src/main/resources/icons/cliskhouse.svg rename to airbyte-config/init/src/main/resources/icons/clickhouse.svg diff --git a/airbyte-config/init/src/main/resources/icons/fauna.svg b/airbyte-config/init/src/main/resources/icons/fauna.svg new file mode 100644 index 0000000000000..167de812b0587 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/fauna.svg @@ -0,0 +1,12 @@ + + + + + + diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 39f14d29f419b..1b25393b2108d 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -67,7 +67,7 @@ - name: Clickhouse destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 dockerRepository: airbyte/destination-clickhouse - dockerImageTag: 0.1.12 + dockerImageTag: 0.2.0 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse releaseStage: alpha - name: Cloudflare R2 @@ -210,7 +210,7 @@ - name: Postgres destinationDefinitionId: 25c5221d-dce2-4163-ade9-739ef790f503 dockerRepository: airbyte/destination-postgres - dockerImageTag: 0.3.24 + dockerImageTag: 0.3.26 documentationUrl: https://docs.airbyte.io/integrations/destinations/postgres icon: postgresql.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 17e58c76911ba..a27d3317222aa 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -797,7 +797,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-clickhouse:0.1.12" +- dockerImage: "airbyte/destination-clickhouse:0.2.0" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" connectionSpecification: @@ -818,7 +818,7 @@ order: 0 port: title: "Port" - description: "JDBC port (not the native port) of the database." + description: "HTTP port of the database." type: "integer" minimum: 0 maximum: 65536 @@ -826,45 +826,35 @@ examples: - "8123" order: 1 - tcp-port: - title: "Native Port" - description: "Native port (not the JDBC) of the database." - type: "integer" - minimum: 0 - maximum: 65536 - default: 9000 - examples: - - "9000" - order: 2 database: title: "DB Name" description: "Name of the database." type: "string" - order: 3 + order: 2 username: title: "User" description: "Username to use to access the database." type: "string" - order: 4 + order: 3 password: title: "Password" description: "Password associated with the username." type: "string" airbyte_secret: true - order: 5 + order: 4 jdbc_url_params: description: "Additional properties to pass to the JDBC URL string when\ \ connecting to the database formatted as 'key=value' pairs separated\ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." title: "JDBC URL Params" type: "string" - order: 6 + order: 5 ssl: title: "SSL Connection" description: "Encrypt data using SSL." type: "boolean" default: false - order: 7 + order: 6 tunnel_method: type: "object" title: "SSH Tunnel Method" @@ -3588,7 +3578,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-postgres:0.3.24" +- dockerImage: "airbyte/destination-postgres:0.3.26" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/postgres" connectionSpecification: diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 521cb16cd010d..693d5f2273a22 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -21,7 +21,7 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/alloydb icon: alloydb.svg sourceType: database - releaseStage: alpha + releaseStage: generally_available - name: AWS CloudTrail sourceDefinitionId: 6ff047c0-f5d5-4ce5-8c81-204a830fa7e1 dockerRepository: airbyte/source-aws-cloudtrail @@ -33,7 +33,7 @@ - name: Amazon Ads sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 0.1.21 + dockerImageTag: 0.1.22 documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads icon: amazonads.svg sourceType: api @@ -41,7 +41,7 @@ - name: Amazon Seller Partner sourceDefinitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460 dockerRepository: airbyte/source-amazon-seller-partner - dockerImageTag: 0.2.25 + dockerImageTag: 0.2.27 sourceType: api documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-seller-partner icon: amazonsellerpartner.svg @@ -56,7 +56,7 @@ - name: Amplitude sourceDefinitionId: fa9f58c6-2d03-4237-aaa4-07d75e0c1396 dockerRepository: airbyte/source-amplitude - dockerImageTag: 0.1.13 + dockerImageTag: 0.1.14 documentationUrl: https://docs.airbyte.io/integrations/sources/amplitude icon: amplitude.svg sourceType: api @@ -128,7 +128,7 @@ - name: Bing Ads sourceDefinitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 dockerRepository: airbyte/source-bing-ads - dockerImageTag: 0.1.12 + dockerImageTag: 0.1.14 documentationUrl: https://docs.airbyte.io/integrations/sources/bing-ads icon: bingads.svg sourceType: api @@ -152,11 +152,11 @@ - name: Chargebee sourceDefinitionId: 686473f1-76d9-4994-9cc7-9b13da46147c dockerRepository: airbyte/source-chargebee - dockerImageTag: 0.1.14 + dockerImageTag: 0.1.15 documentationUrl: https://docs.airbyte.io/integrations/sources/chargebee icon: chargebee.svg sourceType: api - releaseStage: beta + releaseStage: generally_available - name: Chargify sourceDefinitionId: 9b2d3607-7222-4709-9fa2-c2abdebbdd88 dockerRepository: airbyte/source-chargify @@ -253,6 +253,13 @@ icon: drift.svg sourceType: api releaseStage: alpha +- name: DV 360 + sourceDefinitionId: 1356e1d9-977f-4057-ad4b-65f25329cf61 + dockerRepository: airbyte/source-dv-360 + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/dv-360 + sourceType: api + releaseStage: alpha - name: E2E Testing sourceDefinitionId: d53f9084-fa6b-4a5a-976c-5b8392f4ad8a dockerRepository: airbyte/source-e2e-test @@ -272,7 +279,7 @@ - name: Facebook Marketing sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c dockerRepository: airbyte/source-facebook-marketing - dockerImageTag: 0.2.64 + dockerImageTag: 0.2.66 documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing icon: facebook.svg sourceType: api @@ -292,10 +299,18 @@ documentationUrl: https://docs.airbyte.com/integrations/sources/faker sourceType: api releaseStage: alpha +- name: Fauna + sourceDefinitionId: 3825db3e-c94b-42ac-bd53-b5a9507ace2b + dockerRepository: airbyte/source-fauna + dockerImageTag: dev + documentationUrl: https://docs.airbyte.com/integrations/sources/fauna + icon: fauna.svg + sourceType: database + releaseStage: alpha - name: File sourceDefinitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 dockerRepository: airbyte/source-file - dockerImageTag: 0.2.22 + dockerImageTag: 0.2.23 documentationUrl: https://docs.airbyte.io/integrations/sources/file icon: file.svg sourceType: file @@ -315,11 +330,11 @@ - name: Freshdesk sourceDefinitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567 dockerRepository: airbyte/source-freshdesk - dockerImageTag: 0.3.5 + dockerImageTag: 0.3.6 documentationUrl: https://docs.airbyte.io/integrations/sources/freshdesk icon: freshdesk.svg sourceType: api - releaseStage: beta + releaseStage: generally_available - name: Freshsales sourceDefinitionId: eca08d79-7b92-4065-b7f3-79c14836ebe7 dockerRepository: airbyte/source-freshsales @@ -339,7 +354,7 @@ - name: GitHub sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e dockerRepository: airbyte/source-github - dockerImageTag: 0.3.1 + dockerImageTag: 0.3.3 documentationUrl: https://docs.airbyte.io/integrations/sources/github icon: github.svg sourceType: api @@ -363,7 +378,7 @@ - name: Google Ads sourceDefinitionId: 253487c0-2246-43ba-a21f-5116b20a2c50 dockerRepository: airbyte/source-google-ads - dockerImageTag: 0.2.0 + dockerImageTag: 0.2.1 documentationUrl: https://docs.airbyte.io/integrations/sources/google-ads icon: google-adwords.svg sourceType: api @@ -371,7 +386,7 @@ - name: Google Analytics (Universal Analytics) sourceDefinitionId: eff3616a-f9c3-11eb-9a03-0242ac130003 dockerRepository: airbyte/source-google-analytics-v4 - dockerImageTag: 0.1.25 + dockerImageTag: 0.1.26 documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-universal-analytics icon: google-analytics.svg sourceType: api @@ -395,7 +410,7 @@ - name: Google Search Console sourceDefinitionId: eb4c9e00-db83-4d63-a386-39cfa91012a8 dockerRepository: airbyte/source-google-search-console - dockerImageTag: 0.1.15 + dockerImageTag: 0.1.16 documentationUrl: https://docs.airbyte.io/integrations/sources/google-search-console icon: googlesearchconsole.svg sourceType: api @@ -403,7 +418,7 @@ - name: Google Sheets sourceDefinitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 dockerRepository: airbyte/source-google-sheets - dockerImageTag: 0.2.17 + dockerImageTag: 0.2.19 documentationUrl: https://docs.airbyte.io/integrations/sources/google-sheets icon: google-sheets.svg sourceType: file @@ -419,11 +434,11 @@ - name: Greenhouse sourceDefinitionId: 59f1e50a-331f-4f09-b3e8-2e8d4d355f44 dockerRepository: airbyte/source-greenhouse - dockerImageTag: 0.2.10 + dockerImageTag: 0.2.11 documentationUrl: https://docs.airbyte.io/integrations/sources/greenhouse icon: greenhouse.svg sourceType: api - releaseStage: beta + releaseStage: generally_available - name: Harness sourceDefinitionId: 6fe89830-d04d-401b-aad6-6552ffa5c4af dockerRepository: farosai/airbyte-harness-source @@ -435,7 +450,7 @@ - name: Harvest sourceDefinitionId: fe2b4084-3386-4d3b-9ad6-308f61a6f1e6 dockerRepository: airbyte/source-harvest - dockerImageTag: 0.1.10 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.io/integrations/sources/harvest icon: harvest.svg sourceType: api @@ -457,7 +472,7 @@ - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.2.0 + dockerImageTag: 0.2.1 documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot icon: hubspot.svg sourceType: api @@ -473,7 +488,7 @@ - name: Instagram sourceDefinitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 dockerRepository: airbyte/source-instagram - dockerImageTag: 0.1.11 + dockerImageTag: 1.0.0 documentationUrl: https://docs.airbyte.com/integrations/sources/instagram icon: instagram.svg sourceType: api @@ -481,7 +496,7 @@ - name: Intercom sourceDefinitionId: d8313939-3782-41b0-be29-b3ca20d8dd3a dockerRepository: airbyte/source-intercom - dockerImageTag: 0.1.26 + dockerImageTag: 0.1.27 documentationUrl: https://docs.airbyte.io/integrations/sources/intercom icon: intercom.svg sourceType: api @@ -521,7 +536,7 @@ - name: Klaviyo sourceDefinitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde dockerRepository: airbyte/source-klaviyo - dockerImageTag: 0.1.8 + dockerImageTag: 0.1.10 documentationUrl: https://docs.airbyte.io/integrations/sources/klaviyo icon: klaviyo.svg sourceType: api @@ -551,7 +566,7 @@ - name: LinkedIn Ads sourceDefinitionId: 137ece28-5434-455c-8f34-69dc3782f451 dockerRepository: airbyte/source-linkedin-ads - dockerImageTag: 0.1.9 + dockerImageTag: 0.1.10 documentationUrl: https://docs.airbyte.io/integrations/sources/linkedin-ads icon: linkedin.svg sourceType: api @@ -583,7 +598,7 @@ - name: Mailchimp sourceDefinitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002 dockerRepository: airbyte/source-mailchimp - dockerImageTag: 0.2.14 + dockerImageTag: 0.2.15 documentationUrl: https://docs.airbyte.io/integrations/sources/mailchimp icon: mailchimp.svg sourceType: api @@ -599,11 +614,11 @@ - name: Marketo sourceDefinitionId: 9e0556f4-69df-4522-a3fb-03264d36b348 dockerRepository: airbyte/source-marketo - dockerImageTag: 0.1.8 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.io/integrations/sources/marketo icon: marketo.svg sourceType: api - releaseStage: beta + releaseStage: generally_available - name: Metabase sourceDefinitionId: c7cb421b-942e-4468-99ee-e369bcabaec5 dockerRepository: airbyte/source-metabase @@ -631,11 +646,11 @@ - name: Mixpanel sourceDefinitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a dockerRepository: airbyte/source-mixpanel - dockerImageTag: 0.1.25 + dockerImageTag: 0.1.27 documentationUrl: https://docs.airbyte.io/integrations/sources/mixpanel icon: mixpanel.svg sourceType: api - releaseStage: beta + releaseStage: generally_available - name: Monday sourceDefinitionId: 80a54ea2-9959-4040-aac1-eee42423ec9b dockerRepository: airbyte/source-monday @@ -663,7 +678,7 @@ - name: MySQL sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad dockerRepository: airbyte/source-mysql - dockerImageTag: 0.6.14 + dockerImageTag: 1.0.1 documentationUrl: https://docs.airbyte.io/integrations/sources/mysql icon: mysql.svg sourceType: database @@ -671,7 +686,7 @@ - name: Netsuite sourceDefinitionId: 4f2f093d-ce44-4121-8118-9d13b7bfccd0 dockerRepository: airbyte/source-netsuite - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/netsuite # icon: notion.svg sourceType: api @@ -679,7 +694,7 @@ - name: Notion sourceDefinitionId: 6e00b415-b02e-4160-bf02-58176a0ae687 dockerRepository: airbyte/source-notion - dockerImageTag: 0.1.9 + dockerImageTag: 0.1.10 documentationUrl: https://docs.airbyte.io/integrations/sources/notion icon: notion.svg sourceType: api @@ -774,11 +789,11 @@ - name: Pinterest sourceDefinitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003 dockerRepository: airbyte/source-pinterest - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.7 documentationUrl: https://docs.airbyte.io/integrations/sources/pinterest icon: pinterest.svg sourceType: api - releaseStage: beta + releaseStage: generally_available - name: Pipedrive sourceDefinitionId: d8286229-c680-4063-8c59-23b9b391c700 dockerRepository: airbyte/source-pipedrive @@ -821,7 +836,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.11 + dockerImageTag: 1.0.13 documentationUrl: https://docs.airbyte.io/integrations/sources/postgres icon: postgresql.svg sourceType: database @@ -861,7 +876,7 @@ - name: Recharge sourceDefinitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e dockerRepository: airbyte/source-recharge - dockerImageTag: 0.2.1 + dockerImageTag: 0.2.2 documentationUrl: https://docs.airbyte.io/integrations/sources/recharge icon: recharge.svg sourceType: api @@ -900,7 +915,7 @@ - name: S3 sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 dockerRepository: airbyte/source-s3 - dockerImageTag: 0.1.21 + dockerImageTag: 0.1.22 documentationUrl: https://docs.airbyte.io/integrations/sources/s3 icon: s3.svg sourceType: file @@ -916,7 +931,7 @@ - name: Salesforce sourceDefinitionId: b117307c-14b6-41aa-9422-947e34922962 dockerRepository: airbyte/source-salesforce - dockerImageTag: 1.0.17 + dockerImageTag: 1.0.20 documentationUrl: https://docs.airbyte.io/integrations/sources/salesforce icon: salesforce.svg sourceType: api @@ -956,11 +971,11 @@ - name: Slack sourceDefinitionId: c2281cee-86f9-4a86-bb48-d23286b4c7bd dockerRepository: airbyte/source-slack - dockerImageTag: 0.1.17 + dockerImageTag: 0.1.18 documentationUrl: https://docs.airbyte.io/integrations/sources/slack icon: slack.svg sourceType: api - releaseStage: beta + releaseStage: generally_available - name: Smartsheets sourceDefinitionId: 374ebc65-6636-4ea0-925c-7d35999a8ffc dockerRepository: airbyte/source-smartsheets @@ -976,7 +991,7 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/snapchat-marketing icon: snapchat.svg sourceType: api - releaseStage: beta + releaseStage: generally_available - name: Snowflake sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2 dockerRepository: airbyte/source-snowflake @@ -1004,7 +1019,7 @@ - name: Stripe sourceDefinitionId: e094cb9a-26de-4645-8761-65c0c425d1de dockerRepository: airbyte/source-stripe - dockerImageTag: 0.1.38 + dockerImageTag: 0.1.39 documentationUrl: https://docs.airbyte.io/integrations/sources/stripe icon: stripe.svg sourceType: api @@ -1012,7 +1027,7 @@ - name: SurveyMonkey sourceDefinitionId: badc5925-0485-42be-8caa-b34096cb71b5 dockerRepository: airbyte/source-surveymonkey - dockerImageTag: 0.1.10 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.io/integrations/sources/surveymonkey icon: surveymonkey.svg sourceType: api @@ -1044,7 +1059,7 @@ - name: TikTok Marketing sourceDefinitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35 dockerRepository: airbyte/source-tiktok-marketing - dockerImageTag: 0.1.15 + dockerImageTag: 0.1.16 documentationUrl: https://docs.airbyte.io/integrations/sources/tiktok-marketing icon: tiktok.svg sourceType: api @@ -1068,7 +1083,7 @@ - name: Twilio sourceDefinitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1 dockerRepository: airbyte/source-twilio - dockerImageTag: 0.1.9 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.io/integrations/sources/twilio icon: twilio.svg sourceType: api @@ -1092,7 +1107,7 @@ - sourceDefinitionId: afa734e4-3571-11ec-991a-1e0031268139 name: YouTube Analytics dockerRepository: airbyte/source-youtube-analytics - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/youtube-analytics icon: youtube.svg sourceType: api @@ -1132,7 +1147,7 @@ - name: Zendesk Chat sourceDefinitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 dockerRepository: airbyte/source-zendesk-chat - dockerImageTag: 0.1.9 + dockerImageTag: 0.1.10 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-chat icon: zendesk.svg sourceType: api @@ -1148,7 +1163,7 @@ - name: Zendesk Support sourceDefinitionId: 79c1aa37-dae3-42ae-b333-d1c105477715 dockerRepository: airbyte/source-zendesk-support - dockerImageTag: 0.2.15 + dockerImageTag: 0.2.16 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-support icon: zendesk.svg sourceType: api @@ -1156,11 +1171,11 @@ - name: Zendesk Talk sourceDefinitionId: c8630570-086d-4a40-99ae-ea5b18673071 dockerRepository: airbyte/source-zendesk-talk - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-talk icon: zendesk.svg sourceType: api - releaseStage: beta + releaseStage: generally_available - name: Zenefits sourceDefinitionId: 8baba53d-2fe3-4e33-bc85-210d0eb62884 dockerRepository: airbyte/source-zenefits @@ -1172,18 +1187,18 @@ - name: Zenloop sourceDefinitionId: f1e4c7f6-db5c-4035-981f-d35ab4998794 dockerRepository: airbyte/source-zenloop - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/zenloop sourceType: api releaseStage: alpha - sourceDefinitionId: cdaf146a-9b75-49fd-9dd2-9d64a0bb4781 name: Sentry dockerRepository: airbyte/source-sentry - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.7 documentationUrl: https://docs.airbyte.io/integrations/sources/sentry icon: sentry.svg sourceType: api - releaseStage: beta + releaseStage: generally_available - name: Zoom sourceDefinitionId: aea2fd0d-377d-465e-86c0-4fdc4f688e51 dockerRepository: airbyte/source-zoom-singer diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index c2720b121b8b5..41ee97d0e1d6b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -647,7 +647,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-ads:0.1.21" +- dockerImage: "airbyte/source-amazon-ads:0.1.22" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads" connectionSpecification: @@ -771,7 +771,7 @@ type: "string" path_in_connector_config: - "client_secret" -- dockerImage: "airbyte/source-amazon-seller-partner:0.2.25" +- dockerImage: "airbyte/source-amazon-seller-partner:0.2.27" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/amazon-seller-partner" changelogUrl: "https://docs.airbyte.io/integrations/sources/amazon-seller-partner" @@ -1105,7 +1105,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amplitude:0.1.13" +- dockerImage: "airbyte/source-amplitude:0.1.14" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/amplitude" connectionSpecification: @@ -1460,7 +1460,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/source-bing-ads:0.1.12" +- dockerImage: "airbyte/source-bing-ads:0.1.14" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/bing-ads" connectionSpecification: @@ -1712,7 +1712,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-chargebee:0.1.14" +- dockerImage: "airbyte/source-chargebee:0.1.15" spec: documentationUrl: "https://apidocs.chargebee.com/docs/api" connectionSpecification: @@ -2340,6 +2340,76 @@ oauthFlowOutputParameters: - - "access_token" - - "refresh_token" +- dockerImage: "airbyte/source-dv-360:0.1.0" + spec: + documentationUrl: "https://docsurl.com" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Display & Video 360 Spec" + type: "object" + required: + - "credentials" + - "partner_id" + - "start_date" + additionalProperties: true + properties: + credentials: + type: "object" + description: "Oauth2 credentials" + order: 0 + required: + - "access_token" + - "refresh_token" + - "token_uri" + - "client_id" + - "client_secret" + properties: + access_token: + type: "string" + description: "Access token" + airbyte_secret: true + refresh_token: + type: "string" + description: "Refresh token" + airbyte_secret: true + token_uri: + type: "string" + description: "Token URI" + airbyte_secret: true + client_id: + type: "string" + description: "Client ID" + airbyte_secret: true + client_secret: + type: "string" + description: "Client secret" + airbyte_secret: true + partner_id: + type: "integer" + description: "Partner ID" + order: 1 + start_date: + type: "string" + description: "UTC date and time in the format 2017-01-25. Any data before\ + \ this date will not be replicated" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + order: 2 + end_date: + type: "string" + description: "UTC date and time in the format 2017-01-25. Any data after\ + \ this date will not be replicated." + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + order: 3 + filters: + type: "array" + description: "filters for the dimensions. each filter object had 2 keys:\ + \ 'type' for the name of the dimension to be used as. and 'value' for\ + \ the value of the filter" + default: [] + order: 4 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-e2e-test:2.1.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/e2e-test" @@ -2491,7 +2561,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-facebook-marketing:0.2.64" +- dockerImage: "airbyte/source-facebook-marketing:0.2.66" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" @@ -2925,7 +2995,104 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-file:0.2.22" +- dockerImage: "airbyte/source-fauna:dev" + spec: + documentationUrl: "https://github.com/fauna/airbyte/blob/source-fauna/docs/integrations/sources/fauna.md" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Fauna Spec" + type: "object" + required: + - "domain" + - "port" + - "scheme" + - "secret" + additionalProperties: true + properties: + domain: + order: 0 + type: "string" + title: "Domain" + description: "Domain of Fauna to query. Defaults db.fauna.com. See the\ + \ docs." + default: "db.fauna.com" + port: + order: 1 + type: "integer" + title: "Port" + description: "Endpoint port." + default: 443 + scheme: + order: 2 + type: "string" + title: "Scheme" + description: "URL scheme." + default: "https" + secret: + order: 3 + type: "string" + title: "Fauna Secret" + description: "Fauna secret, used when authenticating with the database." + airbyte_secret: true + collection: + order: 5 + type: "object" + title: "Collection" + description: "Settings for the Fauna Collection." + required: + - "page_size" + - "deletions" + properties: + page_size: + order: 4 + type: "integer" + title: "Page Size" + default: 64 + description: "The page size used when reading documents from the database.\ + \ The larger the page size, the faster the connector processes documents.\ + \ However, if a page is too large, the connector may fail.
\n\ + Choose your page size based on how large the documents are.
\n\ + See the docs." + deletions: + order: 5 + type: "object" + title: "Deletion Mode" + description: "This only applies to incremental syncs.
\n\ + Enabling deletion mode informs your destination of deleted documents.
\n\ + Disabled - Leave this feature disabled, and ignore deleted documents.
\n\ + Enabled - Enables this feature. When a document is deleted, the connector\ + \ exports a record with a \"deleted at\" column containing the time\ + \ that the document was deleted." + oneOf: + - title: "Disabled" + type: "object" + order: 0 + required: + - "deletion_mode" + properties: + deletion_mode: + type: "string" + const: "ignore" + - title: "Enabled" + type: "object" + order: 1 + required: + - "deletion_mode" + - "column" + properties: + deletion_mode: + type: "string" + const: "deleted_field" + column: + type: "string" + title: "Deleted At Column" + description: "Name of the \"deleted at\" column." + default: "deleted_at" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-file:0.2.23" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/file" connectionSpecification: @@ -3230,7 +3397,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-freshdesk:0.3.5" +- dockerImage: "airbyte/source-freshdesk:0.3.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/freshdesk" connectionSpecification: @@ -3336,7 +3503,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-github:0.3.1" +- dockerImage: "airbyte/source-github:0.3.3" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/github" connectionSpecification: @@ -3539,7 +3706,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-google-ads:0.2.0" +- dockerImage: "airbyte/source-google-ads:0.2.1" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/google-ads" connectionSpecification: @@ -3699,7 +3866,7 @@ oauthFlowOutputParameters: - - "access_token" - - "refresh_token" -- dockerImage: "airbyte/source-google-analytics-v4:0.1.25" +- dockerImage: "airbyte/source-google-analytics-v4:0.1.26" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/google-analytics-universal-analytics" connectionSpecification: @@ -4050,7 +4217,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-google-search-console:0.1.15" +- dockerImage: "airbyte/source-google-search-console:0.1.16" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/google-search-console" connectionSpecification: @@ -4187,12 +4354,12 @@ oauthFlowOutputParameters: - - "access_token" - - "refresh_token" -- dockerImage: "airbyte/source-google-sheets:0.2.17" +- dockerImage: "airbyte/source-google-sheets:0.2.19" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/google-sheets" connectionSpecification: $schema: "http://json-schema.org/draft-07/schema#" - title: "Stripe Source Spec" + title: "Google Sheets Source Spec" type: "object" required: - "spreadsheet_id" @@ -4308,7 +4475,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-greenhouse:0.2.10" +- dockerImage: "airbyte/source-greenhouse:0.2.11" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/greenhouse" connectionSpecification: @@ -4373,7 +4540,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-harvest:0.1.10" +- dockerImage: "airbyte/source-harvest:0.1.11" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/harvest" connectionSpecification: @@ -4548,7 +4715,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-hubspot:0.2.0" +- dockerImage: "airbyte/source-hubspot:0.2.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" connectionSpecification: @@ -4758,7 +4925,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-instagram:0.1.11" +- dockerImage: "airbyte/source-instagram:1.0.0" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/instagram" changelogUrl: "https://docs.airbyte.io/integrations/sources/instagram" @@ -4798,7 +4965,7 @@ oauthFlowInitParameters: [] oauthFlowOutputParameters: - - "access_token" -- dockerImage: "airbyte/source-intercom:0.1.26" +- dockerImage: "airbyte/source-intercom:0.1.27" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/intercom" connectionSpecification: @@ -5284,7 +5451,7 @@ supported_destination_sync_modes: [] supported_source_sync_modes: - "append" -- dockerImage: "airbyte/source-klaviyo:0.1.8" +- dockerImage: "airbyte/source-klaviyo:0.1.10" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/klaviyo" changelogUrl: "https://docs.airbyte.io/integrations/sources/klaviyo" @@ -5507,7 +5674,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-linkedin-ads:0.1.9" +- dockerImage: "airbyte/source-linkedin-ads:0.1.10" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/linkedin-ads" connectionSpecification: @@ -5749,7 +5916,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mailchimp:0.2.14" +- dockerImage: "airbyte/source-mailchimp:0.2.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mailchimp" connectionSpecification: @@ -5879,7 +6046,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-marketo:0.1.8" +- dockerImage: "airbyte/source-marketo:0.1.11" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/marketo" connectionSpecification: @@ -6393,7 +6560,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-mixpanel:0.1.25" +- dockerImage: "airbyte/source-mixpanel:0.1.27" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mixpanel" connectionSpecification: @@ -6798,7 +6965,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:0.6.14" +- dockerImage: "airbyte/source-mysql:1.0.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mysql" connectionSpecification: @@ -6846,8 +7013,10 @@ jdbc_url_params: description: "Additional properties to pass to the JDBC URL string when\ \ connecting to the database formatted as 'key=value' pairs separated\ - \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." - title: "JDBC URL Params" + \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For\ + \ more information read about JDBC URL parameters." + title: "JDBC URL Parameters (Advanced)" type: "string" order: 5 ssl: @@ -7121,7 +7290,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-netsuite:0.1.0" +- dockerImage: "airbyte/source-netsuite:0.1.1" spec: documentationUrl: "https://docsurl.com" connectionSpecification: @@ -7201,7 +7370,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-notion:0.1.9" +- dockerImage: "airbyte/source-notion:0.1.10" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/notion" connectionSpecification: @@ -8100,7 +8269,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-pinterest:0.1.5" +- dockerImage: "airbyte/source-pinterest:0.1.7" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/pinterest" connectionSpecification: @@ -8115,9 +8284,9 @@ type: "string" title: "Start Date" description: "A date in the format YYYY-MM-DD. If you have not set a date,\ - \ it would be defaulted to 2020-07-28." + \ it would be defaulted to latest allowed date by api (914 days from today)." examples: - - "2020-07-28" + - "2022-07-28" credentials: title: "Authorization Method" type: "object" @@ -8409,7 +8578,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.11" +- dockerImage: "airbyte/source-postgres:1.0.13" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: @@ -8987,7 +9156,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-recharge:0.2.1" +- dockerImage: "airbyte/source-recharge:0.2.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/recharge" connectionSpecification: @@ -9249,7 +9418,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-s3:0.1.21" +- dockerImage: "airbyte/source-s3:0.1.22" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/s3" changelogUrl: "https://docs.airbyte.io/integrations/sources/s3" @@ -9594,7 +9763,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-salesforce:1.0.17" +- dockerImage: "airbyte/source-salesforce:1.0.20" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/salesforce" connectionSpecification: @@ -9985,7 +10154,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-slack:0.1.17" +- dockerImage: "airbyte/source-slack:0.1.18" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/slack" connectionSpecification: @@ -10055,6 +10224,7 @@ type: "string" examples: - "slack-client-id-example" + airbyte_secret: true client_secret: title: "Client Secret" description: "Slack client_secret. See our docs\ \ for more information." airbyte_secret: true + start_date: + type: "string" + title: "Start Date" + order: 2 + description: "The date from which you'd like to replicate data for Zendesk\ + \ Talk API, in the format YYYY-MM-DDT00:00:00Z. All data generated after\ + \ this date will be replicated." + examples: + - "2020-10-15T00:00:00Z" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] @@ -12068,7 +12243,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-zenloop:0.1.2" +- dockerImage: "airbyte/source-zenloop:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/zenloop" connectionSpecification: @@ -12103,7 +12278,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-sentry:0.1.6" +- dockerImage: "airbyte/source-sentry:0.1.7" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/sentry" connectionSpecification: diff --git a/airbyte-config/init/src/test/resources/connector_catalog.json b/airbyte-config/init/src/test/resources/connector_catalog.json index a8da005c14793..c65a2d07cc359 100644 --- a/airbyte-config/init/src/test/resources/connector_catalog.json +++ b/airbyte-config/init/src/test/resources/connector_catalog.json @@ -791,7 +791,7 @@ }, "port": { "title": "Port", - "description": "JDBC port (not the native port) of the database.", + "description": "HTTP port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, @@ -799,16 +799,6 @@ "examples": ["8123"], "order": 1 }, - "tcp-port": { - "title": "Native Port", - "description": "Native port (not the JDBC) of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 9000, - "examples": ["9000"], - "order": 2 - }, "database": { "title": "DB Name", "description": "Name of the database.", @@ -6120,7 +6110,7 @@ "dockerRepository": "airbyte/source-clickhouse-strict-encrypt", "dockerImageTag": "0.1.8", "documentationUrl": "https://docs.airbyte.io/integrations/sources/clickhouse", - "icon": "cliskhouse.svg", + "icon": "clickhouse.svg", "sourceType": "database", "spec": { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/clickhouse", diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index 15a4ac37f5d27..94c55bd2ea381 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -11,7 +11,7 @@ RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/s && chmod +x kubectl && mv kubectl /usr/local/bin/ # Don't change this manually. Bump version expects to make moves based on this string -ARG VERSION=0.40.9 +ARG VERSION=0.40.10 ENV APPLICATION airbyte-container-orchestrator ENV VERSION=${VERSION} diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java index e363080543663..a46746b7a227c 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java @@ -131,6 +131,7 @@ private void runInternal(final DefaultAsyncStateManager asyncStateManager) { // required to kill clients with thread pools System.exit(0); } catch (final Throwable t) { + log.error("Killing orchestrator because of an Exception", t); asyncStateManager.write(kubePodInfo, AsyncKubePodStatus.FAILED); System.exit(1); } diff --git a/airbyte-cron/Dockerfile b/airbyte-cron/Dockerfile index eb1e7df9d4b70..b92a001aace5f 100644 --- a/airbyte-cron/Dockerfile +++ b/airbyte-cron/Dockerfile @@ -2,7 +2,7 @@ ARG JDK_VERSION=17.0.4 ARG JDK_IMAGE=amazoncorretto:${JDK_VERSION} FROM ${JDK_IMAGE} AS cron -ARG VERSION=0.40.9 +ARG VERSION=0.40.10 ENV APPLICATION airbyte-cron ENV VERSION ${VERSION} diff --git a/airbyte-cron/src/main/java/io/airbyte/cron/ApplicationInitializer.java b/airbyte-cron/src/main/java/io/airbyte/cron/ApplicationInitializer.java new file mode 100644 index 0000000000000..2158a72d6168f --- /dev/null +++ b/airbyte-cron/src/main/java/io/airbyte/cron/ApplicationInitializer.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cron; + +import io.airbyte.commons.temporal.TemporalInitializationUtils; +import io.micronaut.context.event.ApplicationEventListener; +import io.micronaut.discovery.event.ServiceReadyEvent; +import jakarta.inject.Inject; + +public class ApplicationInitializer implements ApplicationEventListener { + + @Inject + private TemporalInitializationUtils temporalInitializationUtils; + + @Override + public void onApplicationEvent(ServiceReadyEvent event) { + temporalInitializationUtils.waitForTemporalNamespace(); + } + +} diff --git a/airbyte-cron/src/main/java/io/airbyte/cron/selfhealing/WorkspaceCleaner.java b/airbyte-cron/src/main/java/io/airbyte/cron/selfhealing/WorkspaceCleaner.java index 156aef8cf2500..6af6af927b2d0 100644 --- a/airbyte-cron/src/main/java/io/airbyte/cron/selfhealing/WorkspaceCleaner.java +++ b/airbyte-cron/src/main/java/io/airbyte/cron/selfhealing/WorkspaceCleaner.java @@ -6,6 +6,8 @@ import io.airbyte.config.Configs; import io.airbyte.config.EnvConfigs; +import io.micronaut.context.annotation.Requires; +import io.micronaut.context.env.Environment; import io.micronaut.scheduling.annotation.Scheduled; import jakarta.inject.Singleton; import java.io.File; @@ -22,12 +24,15 @@ @Singleton @Slf4j +@Requires(notEnv = Environment.KUBERNETES) public class WorkspaceCleaner { private final Path workspaceRoot; private final long maxAgeFilesInDays; WorkspaceCleaner() { + log.info("Creating workspace cleaner"); + // TODO Configs should get injected through micronaut final Configs configs = new EnvConfigs(); diff --git a/airbyte-cron/src/main/resources/application-control.yml b/airbyte-cron/src/main/resources/application-control.yml deleted file mode 100644 index 752bc85a5f112..0000000000000 --- a/airbyte-cron/src/main/resources/application-control.yml +++ /dev/null @@ -1,40 +0,0 @@ -datasources: - config: - connection-test-query: SELECT 1 - connection-timeout: 30000 - idle-timeout: 600000 - maximum-pool-size: 10 - url: ${DATABASE_URL} - driverClassName: org.postgresql.Driver - username: ${DATABASE_USER} - password: ${DATABASE_PASSWORD} - jobs: - connection-test-query: SELECT 1 - connection-timeout: 30000 - idle-timeout: 600000 - maximum-pool-size: 10 - url: ${DATABASE_URL} - driverClassName: org.postgresql.Driver - username: ${DATABASE_USER} - password: ${DATABASE_PASSWORD} - -flyway: - enabled: true - datasources: - config: - enabled: false - locations: - - 'classpath:io/airbyte/db/instance/configs/migrations' - jobs: - enabled: false - locations: - - 'classpath:io/airbyte/db/instance/jobs/migrations' - -jooq: - datasources: - config: - jackson-converter-enabled: true - sql-dialect: POSTGRES - jobs: - jackson-converter-enabled: true - sql-dialect: POSTGRES \ No newline at end of file diff --git a/airbyte-cron/src/main/resources/application.yml b/airbyte-cron/src/main/resources/application.yml index 6961c8a3801c6..9e22a199f3553 100644 --- a/airbyte-cron/src/main/resources/application.yml +++ b/airbyte-cron/src/main/resources/application.yml @@ -1,6 +1,6 @@ micronaut: application: - name: airbyte-workers + name: airbyte-cron security: intercept-url-map: - pattern: /** @@ -11,182 +11,16 @@ micronaut: port: 9000 airbyte: - activity: - initial-delay: ${ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS:30} - max-attempts: ${ACTIVITY_MAX_ATTEMPT:5} - max-delay: ${ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS:600} - max-timeout: ${ACTIVITY_MAX_TIMEOUT_SECOND:120} - cloud: - storage: - logs: - type: ${WORKER_LOGS_STORAGE_TYPE:} - gcs: - application-credentials: ${GOOGLE_APPLICATION_CREDENTIALS:} - bucket: ${GCS_LOG_BUCKET:} - minio: - access-key: ${AWS_ACCESS_KEY_ID:} - bucket: ${S3_LOG_BUCKET:} - endpoint: ${S3_MINIO_ENDPOINT:} - secret-access-key: ${AWS_SECRET_ACCESS_KEY:} - s3: - access-key: ${AWS_ACCESS_KEY_ID:} - bucket: ${S3_LOG_BUCKET:} - region: ${S3_LOG_BUCKET_REGION:} - secret-access-key: ${AWS_SECRET_ACCESS_KEY:} - state: - type: ${WORKER_STATE_STORAGE_TYPE:} - gcs: - application-credentials: ${STATE_STORAGE_GCS_APPLICATION_CREDENTIALS:} - bucket: ${STATE_STORAGE_GCS_BUCKET_NAME:} - minio: - access-key: ${STATE_STORAGE_MINIO_ACCESS_KEY:} - bucket: ${STATE_STORAGE_MINIO_BUCKET_NAME:} - endpoint: ${STATE_STORAGE_MINIO_ENDPOINT:} - secret-access-key: ${STATE_STORAGE_MINIO_SECRET_ACCESS_KEY:} - s3: - access-key: ${STATE_STORAGE_S3_ACCESS_KEY:} - bucket: ${STATE_STORAGE_S3_BUCKET_NAME:} - region: ${STATE_STORAGE_S3_BUCKET_REGION:} - secret-access-key: ${STATE_STORAGE_S3_SECRET_ACCESS_KEY:} - connector: - specific-resource-defaults-enabled: ${CONNECTOR_SPECIFIC_RESOURCE_DEFAULTS_ENABLED:false} - container: - orchestrator: - enabled: ${CONTAINER_ORCHESTRATOR_ENABLED:false} - image: ${CONTAINER_ORCHESTRATOR_IMAGE:} - secret-mount-path: ${CONTAINER_ORCHESTRATOR_SECRET_MOUNT_PATH:} - secret-name: ${CONTAINER_ORCHESTRATOR_SECRET_NAME:} - control: - plane: - auth-endpoint: ${CONTROL_PLANE_AUTH_ENDPOINT:} - data: - sync: - task-queue: ${DATA_SYNC_TASK_QUEUES:SYNC} - plane: - connection-ids-mvp: ${CONNECTION_IDS_FOR_MVP_DATA_PLANE:} - service-account: - credentials-path: ${DATA_PLANE_SERVICE_ACCOUNT_CREDENTIALS_PATH:} - email: ${DATA_PLANE_SERVICE_ACCOUNT_EMAIL:} deployment-mode: ${DEPLOYMENT_MODE:OSS} - flyway: - configs: - initialization-timeout-ms: ${CONFIGS_DATABASE_INITIALIZATION_TIMEOUT_MS:60000} - minimum-migration-version: ${CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION} - jobs: - initialization-timeout-ms: ${JOBS_DATABASE_INITIALIZATION_TIMEOUT_MS:60000} - minimum-migration-version: ${JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION} - internal: - api: - auth-header: - name: ${AIRBYTE_API_AUTH_HEADER_NAME:} - value: ${AIRBYTE_API_AUTH_HEADER_VALUE:} - host: ${INTERNAL_API_HOST} local: docker-mount: ${LOCAL_DOCKER_MOUNT} root: ${LOCAL_ROOT} - worker: - env: ${WORKER_ENVIRONMENT:DOCKER} - check: - enabled: ${SHOULD_RUN_CHECK_CONNECTION_WORKFLOWS:true} - kube: - annotations: ${CHECK_JOB_KUBE_ANNOTATION:} - node-selectors: ${CHECK_JOB_KUBE_NODE_SELECTORS:} - max-workers: ${MAX_CHECK_WORKERS:5} - main: - container: - cpu: - limit: ${CHECK_JOB_MAIN_CONTAINER_CPU_LIMIT:} - request: ${CHECK_JOB_MAIN_CONTAINER_CPU_REQUEST:} - memory: - limit: ${CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT:} - request: ${CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST:} - connection: - enabled: ${SHOULD_RUN_CONNECTION_MANAGER_WORKFLOWS:true} - discover: - enabled: ${SHOULD_RUN_DISCOVER_WORKFLOWS:true} - kube: - annotations: ${DISCOVER_JOB_KUBE_ANNOTATIONS:} - node-selectors: ${DISCOVER_JOB_KUBE_NODE_SELECTORS:} - max-workers: ${MAX_DISCOVER_WORKERS:5} - job: - error-reporting: - sentry: - dsn: ${JOB_ERROR_REPORTING_SENTRY_DSN} - strategy: ${JOB_ERROR_REPORTING_STRATEGY:LOGGING} - failed: - max-days: ${MAX_DAYS_OF_ONLY_FAILED_JOBS_BEFORE_CONNECTION_DISABLE:14} - max-jobs: ${MAX_FAILED_JOBS_IN_A_ROW_BEFORE_CONNECTION_DISABLE:100} - kube: - annotations: ${JOB_KUBE_ANNOTATIONS:} - images: - busybox: ${JOB_KUBE_BUSYBOX_IMAGE:`busybox:1.28`} - curl: ${JOB_KUBE_CURL_IMAGE:`curlimages/curl:7.83.1`} - socat: ${JOB_KUBE_SOCAT_IMAGE:`alpine/socat:1.7.4.3-r0`} - main: - container: - image-pull-policy: ${JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY:IfNotPresent} - image-pull-secret: ${JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET:} - namespace: ${JOB_KUBE_NAMESPACE:default} - node-selectors: ${JOB_KUBE_NODE_SELECTORS:} - sidecar: - container: - image-pull-policy: ${JOB_KUBE_SIDECAR_CONTAINER_IMAGE_PULL_POLICY:IfNotPresent} - tolerations: ${JOB_KUBE_TOLERATIONS:} - main: - container: - cpu: - limit: ${JOB_MAIN_CONTAINER_CPU_LIMIT:} - request: ${JOB_MAIN_CONTAINER_CPU_REQUEST:} - memory: - limit: ${JOB_MAIN_CONTAINER_MEMORY_LIMIT:} - request: ${JOB_MAIN_CONTAINER_MEMORY_REQUEST:} - normalization: - main: - container: - cpu: - limit: ${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT:} - request: ${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST:} - memory: - limit: ${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT:} - request: ${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST:} - plane: ${WORKER_PLANE:CONTROL_PLANE} - replication: - orchestrator: - cpu: - limit: ${REPLICATION_ORCHESTRATOR_CPU_LIMIT:} - request: ${REPLICATION_ORCHESTRATOR_CPU_REQUEST:} - memory: - limit: ${REPLICATION_ORCHESTRATOR_MEMORY_LIMIT:} - request: ${REPLICATION_ORCHESTRATOR_MEMORY_REQUEST:} - spec: - enabled: ${SHOULD_RUN_GET_SPEC_WORKFLOWS:true} - kube: - annotations: ${SPEC_JOB_KUBE_ANNOTATIONS:} - node-selectors: ${SPEC_JOB_KUBE_NODE_SELECTORS:} - max-workers: ${MAX_SPEC_WORKERS:5} - sync: - enabled: ${SHOULD_RUN_SYNC_WORKFLOWS:true} - max-workers: ${MAX_SYNC_WORKERS:5} - max-attempts: ${SYNC_JOB_MAX_ATTEMPTS:3} - max-timeout: ${SYNC_JOB_MAX_TIMEOUT_DAYS:3} role: ${AIRBYTE_ROLE:} - secret: - persistence: ${SECRET_PERSISTENCE:TESTING_CONFIG_DB_TABLE} - store: - gcp: - credentials: ${SECRET_STORE_GCP_CREDENTIALS:} - project-id: ${SECRET_STORE_GCP_PROJECT_ID:} - vault: - address: ${VAULT_ADDRESS:} - prefix: ${VAULT_PREFIX:} - token: ${VAULT_AUTH_TOKEN:} temporal: worker: ports: ${TEMPORAL_WORKER_PORTS:} tracking-strategy: ${TRACKING_STRATEGY:LOGGING} version: ${AIRBYTE_VERSION} - web-app: - url: ${WEBAPP_URL:} workflow: failure: restart-delay: ${WORKFLOW_FAILURE_RESTART_DELAY_SECONDS:600} @@ -197,10 +31,6 @@ airbyte: docker: network: ${DOCKER_NETWORK:host} -endpoints: - all: - enabled: true - temporal: cloud: client: diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index 2e1aea25d215c..e359a2331889d 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -41,7 +41,7 @@ public class JdbcUtils { public static final String JDBC_URL_PARAMS_KEY = "jdbc_url_params"; public static final String PASSWORD_KEY = "password"; public static final String PORT_KEY = "port"; - public static final String TCP_PORT_KEY = "tcp-port"; + public static final List PORT_LIST_KEY = List.of("port"); public static final String SCHEMA_KEY = "schema"; // NOTE: this is the plural version of SCHEMA_KEY diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py index 43d50c67daa48..80726e45e7797 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py @@ -27,7 +27,6 @@ NORMALIZATION_TEST_MYSQL_DB_PORT = "NORMALIZATION_TEST_MYSQL_DB_PORT" NORMALIZATION_TEST_POSTGRES_DB_PORT = "NORMALIZATION_TEST_POSTGRES_DB_PORT" NORMALIZATION_TEST_CLICKHOUSE_DB_PORT = "NORMALIZATION_TEST_CLICKHOUSE_DB_PORT" -NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT = "NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT" NORMALIZATION_TEST_TIDB_DB_PORT = "NORMALIZATION_TEST_TIDB_DB_PORT" @@ -224,28 +223,20 @@ def setup_mssql_db(self): def setup_clickhouse_db(self): """ - ClickHouse official JDBC driver use HTTP port 8123, while Python ClickHouse - driver uses native port 9000, so we need to open both ports for destination - connector and dbt container respectively. + ClickHouse official JDBC driver uses HTTP port 8123. Ref: https://altinity.com/blog/2019/3/15/clickhouse-networking-part-1 """ start_db = True port = 8123 - tcp_port = 9000 if os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_PORT): port = int(os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_PORT)) start_db = False - if os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT): - tcp_port = int(os.getenv(NORMALIZATION_TEST_CLICKHOUSE_DB_TCP_PORT)) - start_db = False if start_db: port = self.find_free_port() - tcp_port = self.find_free_port() config = { "host": "localhost", "port": port, - "tcp-port": tcp_port, "database": self.target_schema, "username": "default", "password": "", @@ -263,8 +254,6 @@ def setup_clickhouse_db(self): "--ulimit", "nofile=262144:262144", "-p", - f"{config['tcp-port']}:9000", # Python clickhouse driver use native port - "-p", f"{config['port']}:8123", # clickhouse JDBC driver use HTTP port "-d", # so far, only the latest version ClickHouse server image turned on diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py index 756cdba77e9af..a762b39f1a452 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py @@ -318,7 +318,8 @@ def transform_clickhouse(config: Dict[str, Any]): # https://docs.getdbt.com/reference/warehouse-profiles/clickhouse-profile dbt_config = { "type": "clickhouse", - "driver": "native", + "driver": "http", + "verify": False, "host": config["host"], "port": config["port"], "schema": config["database"], @@ -327,8 +328,6 @@ def transform_clickhouse(config: Dict[str, Any]): } if "password" in config: dbt_config["password"] = config["password"] - if "tcp-port" in config: - dbt_config["port"] = config["tcp-port"] return dbt_config @staticmethod diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py index 337afaca4b12f..2adbb2f441cf7 100644 --- a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py +++ b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py @@ -440,7 +440,8 @@ def test_transform_clickhouse(self): actual = TransformConfig().transform_clickhouse(input) expected = { "type": "clickhouse", - "driver": "native", + "driver": "http", + "verify": False, "host": "airbyte.io", "port": 9440, "schema": "default", diff --git a/airbyte-integrations/bases/debezium-v1-9-2/build.gradle b/airbyte-integrations/bases/debezium-v1-9-6/build.gradle similarity index 81% rename from airbyte-integrations/bases/debezium-v1-9-2/build.gradle rename to airbyte-integrations/bases/debezium-v1-9-6/build.gradle index f91fcb2552fe5..870a3765b2340 100644 --- a/airbyte-integrations/bases/debezium-v1-9-2/build.gradle +++ b/airbyte-integrations/bases/debezium-v1-9-6/build.gradle @@ -9,11 +9,11 @@ dependencies { implementation project(':airbyte-protocol:protocol-models') implementation project(':airbyte-db:db-lib') - implementation 'io.debezium:debezium-api:1.9.2.Final' - implementation 'io.debezium:debezium-embedded:1.9.2.Final' + implementation 'io.debezium:debezium-api:1.9.6.Final' + implementation 'io.debezium:debezium-embedded:1.9.6.Final' // implementation 'io.debezium:debezium-connector-sqlserver:1.9.2.Final' - implementation 'io.debezium:debezium-connector-mysql:1.9.2.Final' - implementation 'io.debezium:debezium-connector-postgres:1.9.2.Final' + implementation 'io.debezium:debezium-connector-mysql:1.9.6.Final' + implementation 'io.debezium:debezium-connector-postgres:1.9.6.Final' implementation 'org.codehaus.plexus:plexus-utils:3.4.2' testFixturesImplementation project(':airbyte-db:db-lib') diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandler.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandler.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandler.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandler.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/CdcMetadataInjector.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/CdcMetadataInjector.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/CdcMetadataInjector.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/CdcMetadataInjector.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/CdcSavedInfoFetcher.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/CdcSavedInfoFetcher.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/CdcSavedInfoFetcher.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/CdcSavedInfoFetcher.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/CdcStateHandler.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/CdcStateHandler.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/CdcStateHandler.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/CdcStateHandler.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/CdcTargetPosition.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/CdcTargetPosition.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/CdcTargetPosition.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/CdcTargetPosition.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumEventUtils.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumEventUtils.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumEventUtils.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumEventUtils.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumPropertiesManager.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumPropertiesManager.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumPropertiesManager.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumPropertiesManager.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordIterator.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordIterator.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordIterator.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordIterator.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordPublisher.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordPublisher.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordPublisher.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordPublisher.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/MySQLDateTimeConverter.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MySQLDateTimeConverter.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/MySQLDateTimeConverter.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/MySQLDateTimeConverter.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/PostgresCustomLoader.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/PostgresCustomLoader.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/PostgresCustomLoader.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/PostgresCustomLoader.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtil.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtil.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtil.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtil.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/SnapshotMetadata.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/SnapshotMetadata.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/main/java/io/airbyte/integrations/debezium/internals/SnapshotMetadata.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/SnapshotMetadata.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/test/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandlerTest.java b/airbyte-integrations/bases/debezium-v1-9-6/src/test/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandlerTest.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/test/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandlerTest.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/test/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandlerTest.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/test/java/io/airbyte/integrations/debezium/AirbyteFileOffsetBackingStoreTest.java b/airbyte-integrations/bases/debezium-v1-9-6/src/test/java/io/airbyte/integrations/debezium/AirbyteFileOffsetBackingStoreTest.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/test/java/io/airbyte/integrations/debezium/AirbyteFileOffsetBackingStoreTest.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/test/java/io/airbyte/integrations/debezium/AirbyteFileOffsetBackingStoreTest.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/test/java/io/airbyte/integrations/debezium/DebeziumEventUtilsTest.java b/airbyte-integrations/bases/debezium-v1-9-6/src/test/java/io/airbyte/integrations/debezium/DebeziumEventUtilsTest.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/test/java/io/airbyte/integrations/debezium/DebeziumEventUtilsTest.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/test/java/io/airbyte/integrations/debezium/DebeziumEventUtilsTest.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/test/java/io/airbyte/integrations/debezium/DebeziumRecordPublisherTest.java b/airbyte-integrations/bases/debezium-v1-9-6/src/test/java/io/airbyte/integrations/debezium/DebeziumRecordPublisherTest.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/test/java/io/airbyte/integrations/debezium/DebeziumRecordPublisherTest.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/test/java/io/airbyte/integrations/debezium/DebeziumRecordPublisherTest.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtilsTest.java b/airbyte-integrations/bases/debezium-v1-9-6/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtilsTest.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtilsTest.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtilsTest.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/test/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java b/airbyte-integrations/bases/debezium-v1-9-6/src/test/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/test/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/test/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/delete_change_event.json b/airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/delete_change_event.json similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/delete_change_event.json rename to airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/delete_change_event.json diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/delete_message.json b/airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/delete_message.json similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/delete_message.json rename to airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/delete_message.json diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/insert_change_event.json b/airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/insert_change_event.json similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/insert_change_event.json rename to airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/insert_change_event.json diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/insert_message.json b/airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/insert_message.json similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/insert_message.json rename to airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/insert_message.json diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/test_debezium_offset.dat b/airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/test_debezium_offset.dat similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/test_debezium_offset.dat rename to airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/test_debezium_offset.dat diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/update_change_event.json b/airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/update_change_event.json similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/update_change_event.json rename to airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/update_change_event.json diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/update_message.json b/airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/update_message.json similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/test/resources/update_message.json rename to airbyte-integrations/bases/debezium-v1-9-6/src/test/resources/update_message.json diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java b/airbyte-integrations/bases/debezium-v1-9-6/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java similarity index 100% rename from airbyte-integrations/bases/debezium-v1-9-2/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java rename to airbyte-integrations/bases/debezium-v1-9-6/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 37e0f70e7e028..7af75b786c278 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -33,6 +33,7 @@ | End-to-End Testing | [![source-e2e-test](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-e2e-test%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-e2e-test) | | Exchange Rates API | [![source-exchange-rates](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-exchange-rates%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-exchange-rates) | | Facebook Marketing | [![source-facebook-marketing](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-facebook-marketing%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-facebook-marketing) | +| Fauna | [![source-fauna](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-fauna%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-fauna) | | Files | [![source-file](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-file%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-file) | | Flexport | [![source-file](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-flexport%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-flexport) | | Freshdesk | [![source-freshdesk](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-freshdesk%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-freshdesk) | diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile index 50a0831ce3fe7..8f27e4f6b615f 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index 0294dd1fecfbd..741bded1ba210 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -113,7 +113,6 @@ protected String getDefaultSchema(final JsonNode config) { protected JsonNode getConfig() { return Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveIpAddress(db)) - .put(JdbcUtils.TCP_PORT_KEY, NATIVE_SECURE_PORT) .put(JdbcUtils.PORT_KEY, HTTPS_PORT) .put(JdbcUtils.DATABASE_KEY, DB_NAME) .put(JdbcUtils.USERNAME_KEY, USER_NAME) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json index 2238b3870b70b..0cc91dd63bf10 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test/resources/expected_spec.json @@ -19,7 +19,7 @@ }, "port": { "title": "Port", - "description": "JDBC port (not the native port) of the database.", + "description": "HTTP port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, @@ -27,40 +27,30 @@ "examples": ["8123"], "order": 1 }, - "tcp-port": { - "title": "Native Port", - "description": "Native port (not the JDBC) of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 9000, - "examples": ["9000"], - "order": 2 - }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", - "order": 3 + "order": 2 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", - "order": 4 + "order": 3 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, - "order": 5 + "order": 4 }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", - "order": 6 + "order": 5 }, "tunnel_method": { "type": "object", diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile index f9eeeb4d8d7f2..0f5a4cf4c3501 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-clickhouse diff --git a/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md b/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md index 13d96b5951108..c728bde55a2ec 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md +++ b/airbyte-integrations/connectors/destination-clickhouse/bootstrap.md @@ -10,12 +10,8 @@ This destination connector uses ClickHouse official JDBC driver, which uses HTTP ## Quick Notes -- ClickHouse JDBC driver uses HTTP protocal (default 8123) but [dbt clickhouse adapter](https://github.com/silentsokolov/dbt-clickhouse) use TCP protocal (default 9000). - - This connector doesn't support nested streams and schema change yet. -- The community [dbt clickhouse adapter](https://github.com/silentsokolov/dbt-clickhouse) has some bugs haven't been fixed yet, for example [https://github.com/silentsokolov/dbt-clickhouse/issues/20](https://github.com/silentsokolov/dbt-clickhouse/issues/20), so the dbt test is based on a fork [https://github.com/burmecia/dbt-clickhouse](https://github.com/burmecia/dbt-clickhouse). - ## API Reference The ClickHouse reference documents: [https://clickhouse.com/docs/en/](https://clickhouse.com/docs/en/) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json index adf44db3857bb..94f5acffb296b 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/resources/spec.json @@ -19,7 +19,7 @@ }, "port": { "title": "Port", - "description": "JDBC port (not the native port) of the database.", + "description": "HTTP port of the database.", "type": "integer", "minimum": 0, "maximum": 65536, @@ -27,47 +27,37 @@ "examples": ["8123"], "order": 1 }, - "tcp-port": { - "title": "Native Port", - "description": "Native port (not the JDBC) of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 9000, - "examples": ["9000"], - "order": 2 - }, "database": { "title": "DB Name", "description": "Name of the database.", "type": "string", - "order": 3 + "order": 2 }, "username": { "title": "User", "description": "Username to use to access the database.", "type": "string", - "order": 4 + "order": 3 }, "password": { "title": "Password", "description": "Password associated with the username.", "type": "string", "airbyte_secret": true, - "order": 5 + "order": 4 }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "title": "JDBC URL Params", "type": "string", - "order": 6 + "order": 5 }, "ssl": { "title": "SSL Connection", "description": "Encrypt data using SSL.", "type": "boolean", "default": false, - "order": 7 + "order": 6 } } } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java index b2688949c7db4..0dfd1db487fb0 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java @@ -89,15 +89,9 @@ protected String getDefaultSchema(final JsonNode config) { @Override protected JsonNode getConfig() { - final Optional tcpPort = db.getExposedPorts().stream() - .map(exPort -> db.getMappedPort((Integer) exPort)) - .filter(el -> !db.getFirstMappedPort().equals(el)) - .findFirst(); - return Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) - .put(JdbcUtils.TCP_PORT_KEY, tcpPort.get()) .put(JdbcUtils.DATABASE_KEY, DB_NAME) .put(JdbcUtils.USERNAME_KEY, db.getUsername()) .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java index 9cb7aabac298b..a76e5983f8e1f 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java @@ -94,7 +94,7 @@ protected String getDefaultSchema(final JsonNode config) { @Override protected JsonNode getConfig() throws Exception { return bastion.getTunnelConfig(getTunnelMethod(), bastion.getBasicDbConfigBuider(db, DB_NAME) - .put("schema", DB_NAME).put(JdbcUtils.TCP_PORT_KEY, 9000)); + .put("schema", DB_NAME)); } @Override diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java index e935eff44730a..9577ff5db2758 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java @@ -34,7 +34,6 @@ public class ClickhouseDestinationSpecTest { + "\"username\" : \"clickhouse\", " + "\"database\" : \"clickhouse_db\", " + "\"port\" : 8123, " - + "\"tcp-port\" : 9000, " + "\"host\" : \"localhost\", " + "\"jdbc_url_params\" : \"property1=pValue1&property2=pValue2\", " + "\"ssl\" : true " diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile index 1299145f6af43..4962191e6f3da 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.25 +LABEL io.airbyte.version=0.3.26 LABEL io.airbyte.name=airbyte/destination-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.java index 372b021233fef..6fdb53da53c6a 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.java @@ -54,7 +54,7 @@ public AirbyteConnectionStatus check(final JsonNode config) throws Exception { //Fail in case SSL mode is disable, allow or prefer return new AirbyteConnectionStatus() .withStatus(Status.FAILED) - .withMessage("Unsecured connection not allowed"); + .withMessage("Unsecured connection not allowed. If no SSH Tunnel set up, please use one of the following SSL modes: require, verify-ca, verify-full"); } } } diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java index 5e08449027b7a..48f8dfd99fcf6 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java @@ -7,6 +7,7 @@ import static io.airbyte.db.PostgresUtils.getCertificate; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; @@ -181,7 +182,7 @@ void testStrictSSLUnsecuredNoTunnel() throws WorkerException { final var actual = runCheck(config); assertEquals(Status.FAILED, actual.getStatus()); - assertEquals("Unsecured connection not allowed", actual.getMessage()); + assertTrue(actual.getMessage().contains("Unsecured connection not allowed")); } @Test diff --git a/airbyte-integrations/connectors/destination-postgres/Dockerfile b/airbyte-integrations/connectors/destination-postgres/Dockerfile index 6a79208dd968a..0ba0237ece8cc 100644 --- a/airbyte-integrations/connectors/destination-postgres/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.24 +LABEL io.airbyte.version=0.3.26 LABEL io.airbyte.name=airbyte/destination-postgres diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index 6d610e830b1a1..21aba2f05dfa1 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.21 +LABEL io.airbyte.version=0.1.22 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/abnormal_state.json index cef25db0669ac..788886327afda 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/abnormal_state.json @@ -1,7 +1,9 @@ -{ - "sponsored_products_report_stream": { - "1861552880916640": { - "reportDate": "20990101" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "1861552880916640": { "reportDate": "20990101" } }, + "stream_descriptor": { "name": "sponsored_products_report_stream" } } } -} +] diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/sample_state.json index 57c0b677f1b61..0622e007bcbf6 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/sample_state.json @@ -1 +1,9 @@ -{ "display_report_stream": { "reportDate": "20210804" } } +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "reportDate": "20210804" }, + "stream_descriptor": { "name": "display_report_stream" } + } + } +] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile b/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile index f88b433a52d0e..e7b4197565c69 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.26 +LABEL io.airbyte.version=0.2.27 LABEL io.airbyte.name=airbyte/source-amazon-seller-partner diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml index dbefb426e98e4..13530f0ade72d 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml @@ -2,6 +2,8 @@ connector_image: airbyte/source-amazon-seller-partner:dev tests: spec: - spec_path: "integration_tests/spec.json" + backward_compatibility_tests_config: + disable_for_version: "0.2.26" connection: # - config_path: "secrets/config.json" # status: "succeed" @@ -11,6 +13,8 @@ tests: timeout_seconds: 60 discovery: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.2.26" # TODO: uncomment when at least one record exist # basic_read: # - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEventGroups.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEventGroups.json index d47ced5a7cab3..17833f54c1c20 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEventGroups.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEventGroups.json @@ -20,7 +20,7 @@ "type": ["null", "string"] }, "CurrencyAmount": { - "type": ["null", "integer"] + "type": ["null", "number"] } } }, @@ -31,7 +31,7 @@ "type": ["null", "string"] }, "CurrencyAmount": { - "type": ["null", "integer"] + "type": ["null", "number"] } } }, @@ -51,7 +51,7 @@ "type": ["null", "string"] }, "CurrencyAmount": { - "type": ["null", "integer"] + "type": ["null", "number"] } } }, diff --git a/airbyte-integrations/connectors/source-amplitude/Dockerfile b/airbyte-integrations/connectors/source-amplitude/Dockerfile index 97feb3d7ed779..b58dea27445d0 100644 --- a/airbyte-integrations/connectors/source-amplitude/Dockerfile +++ b/airbyte-integrations/connectors/source-amplitude/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=0.1.14 LABEL io.airbyte.name=airbyte/source-amplitude diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-amplitude/integration_tests/sample_state.json index cb907b53c2b9f..889de1cd67a88 100644 --- a/airbyte-integrations/connectors/source-amplitude/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/sample_state.json @@ -1,11 +1,23 @@ -{ - "events": { - "event_time": "2021-05-27 11:59:53.710000" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "event_time": "2021-05-27 11:59:53.710000" }, + "stream_descriptor": { "name": "events" } + } }, - "active_users": { - "date": "2021-05-27" + { + "type": "STREAM", + "stream": { + "stream_state": { "date": "2021-05-27" }, + "stream_descriptor": { "name": "active_users" } + } }, - "average_session_length": { - "date": "2021-05-27" + { + "type": "STREAM", + "stream": { + "stream_state": { "date": "2021-05-27" }, + "stream_descriptor": { "name": "average_session_length" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-bing-ads/Dockerfile b/airbyte-integrations/connectors/source-bing-ads/Dockerfile index 6e35e1ee00560..3c6bc2952b1ae 100644 --- a/airbyte-integrations/connectors/source-bing-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-bing-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.1.14 LABEL io.airbyte.name=airbyte/source-bing-ads diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json index 26f12335c9e30..ef99548a8bb29 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json @@ -1,22 +1,46 @@ -{ - "keyword_performance_report_hourly": { - "180278106": { - "TimePeriod": 1627820152 +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "180278106": { + "TimePeriod": 1627820152 + } + }, + "stream_descriptor": { "name": "keyword_performance_report_hourly" } } }, - "budget_summary_report_hourly": { - "180278106": { - "Date": 1627800152 + { + "type": "STREAM", + "stream": { + "stream_state": { + "180278106": { + "Date": 1627800152 + } + }, + "stream_descriptor": { "name": "budget_summary_report_hourly" } } }, - "ad_performance_report_hourly": { - "180278106": { - "TimePeriod": 1627795152 + { + "type": "STREAM", + "stream": { + "stream_state": { + "180278106": { + "TimePeriod": 1627795152 + } + }, + "stream_descriptor": { "name": "ad_performance_report_hourly" } } }, - "campaign_performance_report_hourly": { - "180278106": { - "TimePeriod": 1727810152 + { + "type": "STREAM", + "stream": { + "stream_state": { + "180278106": { + "TimePeriod": 1727810152 + } + }, + "stream_descriptor": { "name": "campaign_performance_report_hourly" } } } -} +] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index b6d06ca1a1db7..179610ce5ca59 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -161,7 +161,7 @@ def get_service( environment=self.environment, ) - @lru_cache(maxsize=None) + @lru_cache(maxsize=16) def _get_reporting_service( self, customer_id: Optional[str] = None, diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py index f62092e7d8f1f..c5420e1537862 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py @@ -233,7 +233,7 @@ def get_updated_state( current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any], ) -> Mapping[str, Any]: - account_id = latest_record["AccountId"] + account_id = str(latest_record["AccountId"]) current_stream_state[account_id] = current_stream_state.get(account_id, {}) current_stream_state[account_id][self.cursor_field] = max( self.get_report_record_timestamp(latest_record[self.cursor_field]), diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py index 5ec08a203f585..491dfd8f01a59 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py @@ -40,15 +40,15 @@ def test_get_column_value(): def test_get_updated_state_new_state(): test_report = TestReport() - stream_state = {123: {"Time": pendulum.parse("2020-01-01").timestamp()}} + stream_state = {"123": {"Time": pendulum.parse("2020-01-01").timestamp()}} latest_record = {"AccountId": 123, "Time": "2020-01-02"} new_state = test_report.get_updated_state(stream_state, latest_record) - assert new_state[123]["Time"] == pendulum.parse("2020-01-02").timestamp() + assert new_state["123"]["Time"] == pendulum.parse("2020-01-02").timestamp() -def test_get_updated_state_state_uncahanged(): +def test_get_updated_state_state_unchanged(): test_report = TestReport() - stream_state = {123: {"Time": pendulum.parse("2020-01-03").timestamp()}} + stream_state = {"123": {"Time": pendulum.parse("2020-01-03").timestamp()}} latest_record = {"AccountId": 123, "Time": "2020-01-02"} new_state = test_report.get_updated_state(copy.deepcopy(stream_state), latest_record) assert stream_state == new_state @@ -56,11 +56,11 @@ def test_get_updated_state_state_uncahanged(): def test_get_updated_state_state_new_account(): test_report = TestReport() - stream_state = {123: {"Time": pendulum.parse("2020-01-03").timestamp()}} + stream_state = {"123": {"Time": pendulum.parse("2020-01-03").timestamp()}} latest_record = {"AccountId": 234, "Time": "2020-01-02"} new_state = test_report.get_updated_state(stream_state, latest_record) - assert 234 in new_state and 123 in new_state - assert new_state[234]["Time"] == pendulum.parse("2020-01-02").timestamp() + assert "234" in new_state and "123" in new_state + assert new_state["234"]["Time"] == pendulum.parse("2020-01-02").timestamp() def test_get_report_record_timestamp_daily(): diff --git a/airbyte-integrations/connectors/source-chargebee/Dockerfile b/airbyte-integrations/connectors/source-chargebee/Dockerfile index c057552ff5f28..ed67f6ef3b9b0 100644 --- a/airbyte-integrations/connectors/source-chargebee/Dockerfile +++ b/airbyte-integrations/connectors/source-chargebee/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.14 +LABEL io.airbyte.version=0.1.15 LABEL io.airbyte.name=airbyte/source-chargebee diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json index 2d80c7b29ca0b..c4ccf17f795b4 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json @@ -1,34 +1,72 @@ -{ - "subscription": { - "updated_at": 2147483647 +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "subscription" } + } }, - "coupon": { - "updated_at": 2147483647 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "coupon" } + } }, - "customer": { - "updated_at": 2147483647 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "customer" } + } }, - "invoice": { - "updated_at": 2147483647 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "invoice" } + } }, - "order": { - "updated_at": 2147483647 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "order" } + } }, - "addon": { - "updated_at": 2147483647 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "addon" } + } }, - "plan": { - "updated_at": 2147483647 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "plan" } + } }, - "item": { - "updated_at": 2147483647 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "item" } + } }, - "item_price": { - "updated_at": 2147483647 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "item_price" } + } }, - "attached_item": { - "cbdemo_standard": { - "updated_at": 2147483647 + { + "type": "STREAM", + "stream": { + "stream_state": { "cbdemo_standard": { "updated_at": 2147483647 } }, + "stream_descriptor": { "name": "attached_item" } } } -} +] diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json index 1d641890ab917..0858153037e57 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json @@ -1,26 +1,58 @@ -{ - "subscription": { - "updated_at": 1624345058 +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1624345058 }, + "stream_descriptor": { "name": "subscription" } + } }, - "customer": { - "updated_at": 1624345056 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1624345056 }, + "stream_descriptor": { "name": "customer" } + } }, - "invoice": { - "updated_at": 1624345059 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1624345059 }, + "stream_descriptor": { "name": "invoice" } + } }, - "order": { - "updated_at": 1625596058 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1625596058 }, + "stream_descriptor": { "name": "order" } + } }, - "addon": { - "updated_at": 1625596058 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1625596058 }, + "stream_descriptor": { "name": "addon" } + } }, - "plan": { - "updated_at": 1625596058 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1625596058 }, + "stream_descriptor": { "name": "plan" } + } }, - "item": { - "updated_at": 2147483647 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "item" } + } }, - "item_price": { - "updated_at": 2147483647 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "item_price" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-dv-360/.dockerignore b/airbyte-integrations/connectors/source-dv-360/.dockerignore new file mode 100644 index 0000000000000..8a6afc671730c --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_dv_360 +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-dv-360/BOOTSTRAP.md b/airbyte-integrations/connectors/source-dv-360/BOOTSTRAP.md new file mode 100644 index 0000000000000..7951ef8f6eecc --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/BOOTSTRAP.md @@ -0,0 +1,17 @@ +# Display & Video 360 + +Google DoubleClick Bid Manager (DBM) is the API that enables developers to manage Queries and retrieve Reports from Display & Video 360. + +DoubleClick Bid Manager API `v1.1` is the latest available and recommended version. + +[Link](https://developers.google.com/bid-manager/v1.1) to the official documentation. + +[Getting started with the API](https://developers.google.com/bid-manager/guides/getting-started-api) + +**Workflow of the API**: +* In order to fetch data from the DBM API, it is necessary to first build a [query](https://developers.google.com/bid-manager/v1.1/queries) that gets created in the [user interface (UI)](https://www.google.com/ddm/bidmanager/). +* Once the query is created it can be executed, and the resulting [report](https://developers.google.com/bid-manager/v1.1/reports) can be found and downloaded in the UI. + +**Filters and Metrics**: Dimensions are referred to as Filters in DV360. All available dimensions metrics can be found [here](https://developers.google.com/bid-manager/v1.1/filters-metrics). + +**Note**: It is recommended in the reporting [best practices](https://developers.google.com/bid-manager/guides/scheduled-reports/best-practices) to first build the desired report in the UI to avoid any errors, since there are several limilations and requirements pertaining to reporting types, filters, dimensions, and metrics (such as valid combinations of metrics and dimensions). \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-dv-360/Dockerfile b/airbyte-integrations/connectors/source-dv-360/Dockerfile new file mode 100644 index 0000000000000..33d9e3f0e5d61 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.7.11-alpine3.14 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_dv_360 ./source_dv_360 + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-dv-360 diff --git a/airbyte-integrations/connectors/source-dv-360/README.md b/airbyte-integrations/connectors/source-dv-360/README.md new file mode 100644 index 0000000000000..5c2554eb9b049 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/README.md @@ -0,0 +1,129 @@ +# DISPLAY & VIDEO 360 Source + +This is the repository for the DISPLAY & VIDEO 360 source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/dv360). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-dv-360:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/dv360) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_dv_360/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source dv360 test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-dv-360:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-dv-360:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-dv-360:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dv-360:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dv-360:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-dv-360:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-dv-360:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-dv-360:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-dv-360/acceptance-test-config.yml b/airbyte-integrations/connectors/source-dv-360/acceptance-test-config.yml new file mode 100644 index 0000000000000..6462d2f5a5eab --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-dv-360:dev +tests: + spec: + - spec_path: "source_dv_360/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: # TODO if your connector does not implement incremental sync, remove this block + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-dv-360/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-dv-360/acceptance-test-docker.sh new file mode 100644 index 0000000000000..e4d8b1cef8961 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-dv-360/build.gradle b/airbyte-integrations/connectors/source-dv-360/build.gradle new file mode 100644 index 0000000000000..600c3e9884bc8 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_dv_360_singer' +} diff --git a/airbyte-integrations/connectors/source-dv-360/integration_tests/__init__.py b/airbyte-integrations/connectors/source-dv-360/integration_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-dv-360/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-dv-360/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..24ce726cb18d7 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "standard": { + "date": "2224-01-01" + } +} diff --git a/airbyte-integrations/connectors/source-dv-360/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-dv-360/integration_tests/acceptance.py new file mode 100644 index 0000000000000..34f2f625e15bb --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-dv-360/integration_tests/catalog.json b/airbyte-integrations/connectors/source-dv-360/integration_tests/catalog.json new file mode 100644 index 0000000000000..33819136327b3 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/integration_tests/catalog.json @@ -0,0 +1,2016 @@ +{ + "type": "CATALOG", + "catalog": { + "streams": [ + { + "name": "reach", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "advertiser": { + "type": ["null", "string"] + }, + "advertiser_id": { + "type": ["null", "string"] + }, + "advertiser_integration_code": { + "type": ["null", "string"] + }, + "advertiser_status": { + "type": ["null", "string"] + }, + "app_url": { + "type": ["null", "string"] + }, + "app_url_excluded": { + "type": ["null", "string"] + }, + "campaign": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "cm_placement_id": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "country_id": { + "type": ["null", "string"] + }, + "creative": { + "type": ["null", "string"] + }, + "creative_id": { + "type": ["null", "string"] + }, + "creative_integration_code": { + "type": ["null", "string"] + }, + "creative_source": { + "type": ["null", "string"] + }, + "creative_status": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"] + }, + "insertion_order": { + "type": ["null", "string"] + }, + "insertion_order_id": { + "type": ["null", "string"] + }, + "insertion_order_integration_code": { + "type": ["null", "string"] + }, + "insertion_order_status": { + "type": ["null", "string"] + }, + "inventory_source": { + "type": ["null", "string"] + }, + "line_item": { + "type": ["null", "string"] + }, + "line_item_id": { + "type": ["null", "string"] + }, + "line_item_integration_code": { + "type": ["null", "string"] + }, + "line_item_status": { + "type": ["null", "string"] + }, + "partner": { + "type": ["null", "string"] + }, + "partner_id": { + "type": ["null", "string"] + }, + "partner_status": { + "type": ["null", "string"] + }, + "targeted_data_providers": { + "type": ["null", "string"] + }, + "cookie_reach_average_impression_frequency": { + "type": ["null", "string"] + }, + "cookie_reach_impression_reach": { + "type": ["null", "string"] + }, + "unique_reach_average_impression_frequency": { + "type": ["null", "string"] + }, + "unique_reach_click_reach": { + "type": ["null", "string"] + }, + "unique_reach_impression_reach": { + "type": ["null", "string"] + }, + "unique_reach_total_reach": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + { + "name": "standard", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "active_view_custom_metric_id ": { + "type": ["null", "string"] + }, + "active_view_custom_metric_name ": { + "type": ["null", "string"] + }, + "ad_position ": { + "type": ["null", "string"] + }, + "ad_type ": { + "type": ["null", "string"] + }, + "advertiser ": { + "type": ["null", "string"] + }, + "advertiser_currency ": { + "type": ["null", "string"] + }, + "advertiser_id ": { + "type": ["null", "string"] + }, + "advertiser_integration_code ": { + "type": ["null", "string"] + }, + "advertiser_status ": { + "type": ["null", "string"] + }, + "advertiser_time_zone ": { + "type": ["null", "string"] + }, + "algorithm ": { + "type": ["null", "string"] + }, + "algorithm_id ": { + "type": ["null", "string"] + }, + "amp_page_request ": { + "type": ["null", "string"] + }, + "app_url ": { + "type": ["null", "string"] + }, + "app_url_excluded ": { + "type": ["null", "string"] + }, + "app_url_id ": { + "type": ["null", "string"] + }, + "attributed_userlist ": { + "type": ["null", "string"] + }, + "attributed_userlist_cost ": { + "type": ["null", "string"] + }, + "attributed_userlist_id ": { + "type": ["null", "string"] + }, + "attributed_userlist_type ": { + "type": ["null", "string"] + }, + "attribution_model ": { + "type": ["null", "string"] + }, + "audience_list ": { + "type": ["null", "string"] + }, + "audience_list_cost ": { + "type": ["null", "string"] + }, + "audience_list_id ": { + "type": ["null", "string"] + }, + "audience_list_type ": { + "type": ["null", "string"] + }, + "audience_name ": { + "type": ["null", "string"] + }, + "audience_type ": { + "type": ["null", "string"] + }, + "authorized_seller_state ": { + "type": ["null", "string"] + }, + "billable_outcome ": { + "type": ["null", "string"] + }, + "brand_lift_type ": { + "type": ["null", "string"] + }, + "browser_id ": { + "type": ["null", "string"] + }, + "budget_segment_description ": { + "type": ["null", "string"] + }, + "campaign ": { + "type": ["null", "string"] + }, + "campaign_id ": { + "type": ["null", "string"] + }, + "category ": { + "type": ["null", "string"] + }, + "channel ": { + "type": ["null", "string"] + }, + "channel_id ": { + "type": ["null", "string"] + }, + "channel_type ": { + "type": ["null", "string"] + }, + "city ": { + "type": ["null", "string"] + }, + "city_id ": { + "type": ["null", "string"] + }, + "cm_placement_id ": { + "type": ["null", "string"] + }, + "companion_creative ": { + "type": ["null", "string"] + }, + "companion_creative_id ": { + "type": ["null", "string"] + }, + "companion_creative_size ": { + "type": ["null", "string"] + }, + "country ": { + "type": ["null", "string"] + }, + "country_id ": { + "type": ["null", "string"] + }, + "creative ": { + "type": ["null", "string"] + }, + "creative_asset ": { + "type": ["null", "string"] + }, + "creative_attributes ": { + "type": ["null", "string"] + }, + "creative_height ": { + "type": ["null", "string"] + }, + "creative_id ": { + "type": ["null", "string"] + }, + "creative_integration_code ": { + "type": ["null", "string"] + }, + "creative_rendered_in_amp ": { + "type": ["null", "string"] + }, + "creative_size ": { + "type": ["null", "string"] + }, + "creative_source ": { + "type": ["null", "string"] + }, + "creative_status ": { + "type": ["null", "string"] + }, + "creative_type ": { + "type": ["null", "string"] + }, + "creative_width ": { + "type": ["null", "string"] + }, + "data_provider ": { + "type": ["null", "string"] + }, + "data_provider_id ": { + "type": ["null", "string"] + }, + "date ": { + "type": ["null", "string"] + }, + "day_of_week ": { + "type": ["null", "string"] + }, + "detailed_demographics ": { + "type": ["null", "string"] + }, + "detailed_demographics_id ": { + "type": ["null", "string"] + }, + "device ": { + "type": ["null", "string"] + }, + "device_make ": { + "type": ["null", "string"] + }, + "device_model ": { + "type": ["null", "string"] + }, + "device_type ": { + "type": ["null", "string"] + }, + "digital_content_label ": { + "type": ["null", "string"] + }, + "dma ": { + "type": ["null", "string"] + }, + "dma_code ": { + "type": ["null", "string"] + }, + "exchange ": { + "type": ["null", "string"] + }, + "exchange_code ": { + "type": ["null", "string"] + }, + "exchange_id ": { + "type": ["null", "string"] + }, + "extension ": { + "type": ["null", "string"] + }, + "extension_status ": { + "type": ["null", "string"] + }, + "extension_type ": { + "type": ["null", "string"] + }, + "floodlight_activity ": { + "type": ["null", "string"] + }, + "floodlight_activity_id ": { + "type": ["null", "string"] + }, + "format ": { + "type": ["null", "string"] + }, + "gmail_age ": { + "type": ["null", "string"] + }, + "gmail_city ": { + "type": ["null", "string"] + }, + "gmail_country ": { + "type": ["null", "string"] + }, + "gmail_device_type ": { + "type": ["null", "string"] + }, + "gmail_gender ": { + "type": ["null", "string"] + }, + "gmail_region ": { + "type": ["null", "string"] + }, + "gmail_remarketing_list ": { + "type": ["null", "string"] + }, + "household_income ": { + "type": ["null", "string"] + }, + "impression_counting_method ": { + "type": ["null", "string"] + }, + "insertion_order ": { + "type": ["null", "string"] + }, + "insertion_order_daily_frequency ": { + "type": ["null", "string"] + }, + "insertion_order_id ": { + "type": ["null", "string"] + }, + "insertion_order_integration_code ": { + "type": ["null", "string"] + }, + "insertion_order_status ": { + "type": ["null", "string"] + }, + "interest ": { + "type": ["null", "string"] + }, + "inventory_commitment_type ": { + "type": ["null", "string"] + }, + "inventory_delivery_method ": { + "type": ["null", "string"] + }, + "inventory_rate_type ": { + "type": ["null", "string"] + }, + "inventory_source ": { + "type": ["null", "string"] + }, + "inventory_source_group ": { + "type": ["null", "string"] + }, + "inventory_source_group_id ": { + "type": ["null", "string"] + }, + "inventory_source_id ": { + "type": ["null", "string"] + }, + "inventory_source_id_external ": { + "type": ["null", "string"] + }, + "inventory_source_type ": { + "type": ["null", "string"] + }, + "isp_or_carrier ": { + "type": ["null", "string"] + }, + "isp_or_carrier_id ": { + "type": ["null", "string"] + }, + "keyword ": { + "type": ["null", "string"] + }, + "life_event ": { + "type": ["null", "string"] + }, + "life_events ": { + "type": ["null", "string"] + }, + "line_item ": { + "type": ["null", "string"] + }, + "line_item_daily_frequency ": { + "type": ["null", "string"] + }, + "line_item_id ": { + "type": ["null", "string"] + }, + "line_item_integration_code ": { + "type": ["null", "string"] + }, + "line_item_lifetime_frequency ": { + "type": ["null", "string"] + }, + "line_item_status ": { + "type": ["null", "string"] + }, + "line_item_type ": { + "type": ["null", "string"] + }, + "max_video_duration ": { + "type": ["null", "string"] + }, + "measurement_source ": { + "type": ["null", "string"] + }, + "month ": { + "type": ["null", "string"] + }, + "operating_system ": { + "type": ["null", "string"] + }, + "partner ": { + "type": ["null", "string"] + }, + "partner_currency ": { + "type": ["null", "string"] + }, + "partner_id ": { + "type": ["null", "string"] + }, + "partner_status ": { + "type": ["null", "string"] + }, + "platform ": { + "type": ["null", "string"] + }, + "playback_method ": { + "type": ["null", "string"] + }, + "position_in_content ": { + "type": ["null", "string"] + }, + "public_inventory ": { + "type": ["null", "string"] + }, + "publisher_property ": { + "type": ["null", "string"] + }, + "publisher_property_id ": { + "type": ["null", "string"] + }, + "publisher_property_section ": { + "type": ["null", "string"] + }, + "publisher_property_section_id ": { + "type": ["null", "string"] + }, + "refund_reason ": { + "type": ["null", "string"] + }, + "region ": { + "type": ["null", "string"] + }, + "region_id ": { + "type": ["null", "string"] + }, + "rewarded ": { + "type": ["null", "string"] + }, + "sensitive_category ": { + "type": ["null", "string"] + }, + "served_pixel_density ": { + "type": ["null", "string"] + }, + "targeted_data_providers ": { + "type": ["null", "string"] + }, + "time_of_day ": { + "type": ["null", "string"] + }, + "time_to_conversion ": { + "type": ["null", "string"] + }, + "variant_id ": { + "type": ["null", "string"] + }, + "variant_name ": { + "type": ["null", "string"] + }, + "variant_version ": { + "type": ["null", "string"] + }, + "verification_video_player_size ": { + "type": ["null", "string"] + }, + "verification_video_position ": { + "type": ["null", "string"] + }, + "video_continuous_play ": { + "type": ["null", "string"] + }, + "video_player_size ": { + "type": ["null", "string"] + }, + "video_skippable_support ": { + "type": ["null", "string"] + }, + "week ": { + "type": ["null", "string"] + }, + "year ": { + "type": ["null", "string"] + }, + "zip_code ": { + "type": ["null", "string"] + }, + "zip_code_id ": { + "type": ["null", "string"] + }, + "pct_clicks_leading_to_conversions ": { + "type": ["null", "string"] + }, + "pct_impressions_leading_to_conversions ": { + "type": ["null", "string"] + }, + "pct_impressions_with_positive_custom_value ": { + "type": ["null", "string"] + }, + "active_view_pct_audible_and_visible_at_completion ": { + "type": ["null", "string"] + }, + "active_view_pct_audible_and_visible_at_first_quartile ": { + "type": ["null", "string"] + }, + "active_view_pct_audible_and_visible_at_midpoint ": { + "type": ["null", "string"] + }, + "active_view_pct_audible_and_visible_at_start ": { + "type": ["null", "string"] + }, + "active_view_pct_audible_and_visible_at_third_quartile ": { + "type": ["null", "string"] + }, + "active_view_pct_audible_impressions ": { + "type": ["null", "string"] + }, + "active_view_pct_full_screen ": { + "type": ["null", "string"] + }, + "active_view_pct_fully_on_screen_2_sec ": { + "type": ["null", "string"] + }, + "active_view_pct_in_background ": { + "type": ["null", "string"] + }, + "active_view_pct_measurable_impressions ": { + "type": ["null", "string"] + }, + "active_view_pct_of_ad_played ": { + "type": ["null", "string"] + }, + "active_view_pct_of_completed_impressions_audible_and_visible ": { + "type": ["null", "string"] + }, + "active_view_pct_of_completed_impressions_visible ": { + "type": ["null", "string"] + }, + "active_view_pct_of_first_quartile_impressions_audible_and_visible ": { + "type": ["null", "string"] + }, + "active_view_pct_of_first_quartile_impressions_visible ": { + "type": ["null", "string"] + }, + "active_view_pct_of_midpoint_impressions_audible_and_visible ": { + "type": ["null", "string"] + }, + "active_view_pct_of_midpoint_impressions_visible ": { + "type": ["null", "string"] + }, + "active_view_pct_of_third_quartile_impressions_audible_and_visible ": { + "type": ["null", "string"] + }, + "active_view_pct_of_third_quartile_impressions_visible ": { + "type": ["null", "string"] + }, + "active_view_pct_play_time_audible ": { + "type": ["null", "string"] + }, + "active_view_pct_play_time_audible_and_visible ": { + "type": ["null", "string"] + }, + "active_view_pct_play_time_visible ": { + "type": ["null", "string"] + }, + "active_view_pct_viewable_impressions ": { + "type": ["null", "string"] + }, + "active_view_pct_visible_10_seconds ": { + "type": ["null", "string"] + }, + "active_view_pct_visible_at_completion ": { + "type": ["null", "string"] + }, + "active_view_pct_visible_at_first_quartile ": { + "type": ["null", "string"] + }, + "active_view_pct_visible_at_midpoint ": { + "type": ["null", "string"] + }, + "active_view_pct_visible_at_start ": { + "type": ["null", "string"] + }, + "active_view_pct_visible_at_third_quartile ": { + "type": ["null", "string"] + }, + "active_view_audible_and_fully_on_screen_for_half_of_duration_15_sec_cap_impressions ": { + "type": ["null", "string"] + }, + "active_view_audible_and_fully_on_screen_for_half_of_duration_15_sec_cap_measurable_impressions ": { + "type": ["null", "string"] + }, + "active_view_audible_and_fully_on_screen_for_half_of_duration_15_sec_cap_rate ": { + "type": ["null", "string"] + }, + "active_view_audible_and_fully_on_screen_for_half_of_duration_trueview_impressions ": { + "type": ["null", "string"] + }, + "active_view_audible_and_fully_on_screen_for_half_of_duration_trueview_measurable_impressions ": { + "type": ["null", "string"] + }, + "active_view_audible_and_fully_on_screen_for_half_of_duration_trueview_rate ": { + "type": ["null", "string"] + }, + "active_view_average_viewable_time_seconds ": { + "type": ["null", "string"] + }, + "active_view_custom_metric_measurable_impressions ": { + "type": ["null", "string"] + }, + "active_view_custom_metric_viewable_impressions ": { + "type": ["null", "string"] + }, + "active_view_custom_metric_viewable_rate ": { + "type": ["null", "string"] + }, + "active_view_eligible_impressions ": { + "type": ["null", "string"] + }, + "active_view_impression_distribution_not_measurable ": { + "type": ["null", "string"] + }, + "active_view_impression_distribution_not_viewable ": { + "type": ["null", "string"] + }, + "active_view_impression_distribution_viewable ": { + "type": ["null", "string"] + }, + "active_view_impressions_audible_and_visible_at_completion ": { + "type": ["null", "string"] + }, + "active_view_impressions_visible_10_seconds ": { + "type": ["null", "string"] + }, + "active_view_measurable_impressions ": { + "type": ["null", "string"] + }, + "active_view_not_measurable_impressions ": { + "type": ["null", "string"] + }, + "active_view_not_viewable_impressions ": { + "type": ["null", "string"] + }, + "active_view_viewable_impressions ": { + "type": ["null", "string"] + }, + "adlingo_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "adloox_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "adloox_fee_partner_currency ": { + "type": ["null", "string"] + }, + "adloox_fee_usd ": { + "type": ["null", "string"] + }, + "adloox_pre_bid_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "adloox_pre_bid_fee_partner_currency ": { + "type": ["null", "string"] + }, + "adloox_pre_bid_fee_usd ": { + "type": ["null", "string"] + }, + "adsafe_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "adsafe_fee_partner_currency ": { + "type": ["null", "string"] + }, + "adsafe_fee_usd ": { + "type": ["null", "string"] + }, + "adxpose_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "adxpose_fee_partner_currency ": { + "type": ["null", "string"] + }, + "adxpose_fee_usd ": { + "type": ["null", "string"] + }, + "agency_trading_desk_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "agency_trading_desk_fee_partner_currency ": { + "type": ["null", "string"] + }, + "agency_trading_desk_fee_usd ": { + "type": ["null", "string"] + }, + "aggregate_knowledge_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "aggregate_knowledge_fee_partner_currency ": { + "type": ["null", "string"] + }, + "aggregate_knowledge_fee_usd ": { + "type": ["null", "string"] + }, + "audio_client_cost_ecpcl_advertiser_currency ": { + "type": ["null", "string"] + }, + "audio_media_cost_ecpcl_advertiser_currency ": { + "type": ["null", "string"] + }, + "audio_mutes_audio ": { + "type": ["null", "string"] + }, + "audio_mutes_video ": { + "type": ["null", "string"] + }, + "audio_revenue_ecpcl_advertiser_currency ": { + "type": ["null", "string"] + }, + "audio_unmutes_audio ": { + "type": ["null", "string"] + }, + "audio_unmutes_video ": { + "type": ["null", "string"] + }, + "average_display_time ": { + "type": ["null", "string"] + }, + "average_interaction_time ": { + "type": ["null", "string"] + }, + "begin_to_render_eligible_impressions ": { + "type": ["null", "string"] + }, + "begin_to_render_impressions ": { + "type": ["null", "string"] + }, + "billable_cost_advertiser_currency ": { + "type": ["null", "string"] + }, + "billable_cost_partner_currency ": { + "type": ["null", "string"] + }, + "billable_cost_usd ": { + "type": ["null", "string"] + }, + "billable_impressions ": { + "type": ["null", "string"] + }, + "click_rate_ctr ": { + "type": ["null", "string"] + }, + "clicks ": { + "type": ["null", "string"] + }, + "client_cost_advertiser_currency ": { + "type": ["null", "string"] + }, + "client_cost_ecpa_advertiser_currency ": { + "type": ["null", "string"] + }, + "client_cost_ecpa_pc_advertiser_currency ": { + "type": ["null", "string"] + }, + "client_cost_ecpa_pv_advertiser_currency ": { + "type": ["null", "string"] + }, + "client_cost_ecpc_advertiser_currency ": { + "type": ["null", "string"] + }, + "client_cost_ecpm_advertiser_currency ": { + "type": ["null", "string"] + }, + "client_cost_viewable_ecpm_advertiser_currency ": { + "type": ["null", "string"] + }, + "cm_post_click_revenue ": { + "type": ["null", "string"] + }, + "cm_post_click_revenue__cross_environment ": { + "type": ["null", "string"] + }, + "cm_post_view_revenue ": { + "type": ["null", "string"] + }, + "cm_post_view_revenue__cross_environment ": { + "type": ["null", "string"] + }, + "companion_clicks_audio ": { + "type": ["null", "string"] + }, + "companion_clicks_video ": { + "type": ["null", "string"] + }, + "companion_impressions_audio ": { + "type": ["null", "string"] + }, + "companion_impressions_video ": { + "type": ["null", "string"] + }, + "complete_listens_audio ": { + "type": ["null", "string"] + }, + "complete_views_video ": { + "type": ["null", "string"] + }, + "completion_rate_audio ": { + "type": ["null", "string"] + }, + "completion_rate_video ": { + "type": ["null", "string"] + }, + "comscore_vce_in_doubleclick_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "comscore_vce_in_doubleclick_fee_partner_currency ": { + "type": ["null", "string"] + }, + "comscore_vce_in_doubleclick_fee_usd ": { + "type": ["null", "string"] + }, + "conversions_per_1000_impressions ": { + "type": ["null", "string"] + }, + "cookie_unconsented_clicks ": { + "type": ["null", "string"] + }, + "counters ": { + "type": ["null", "string"] + }, + "cpm_fee_1_advertiser_currency ": { + "type": ["null", "string"] + }, + "cpm_fee_1_partner_currency ": { + "type": ["null", "string"] + }, + "cpm_fee_1_usd ": { + "type": ["null", "string"] + }, + "cpm_fee_2_advertiser_currency ": { + "type": ["null", "string"] + }, + "cpm_fee_2_partner_currency ": { + "type": ["null", "string"] + }, + "cpm_fee_2_usd ": { + "type": ["null", "string"] + }, + "cpm_fee_3_advertiser_currency ": { + "type": ["null", "string"] + }, + "cpm_fee_3_partner_currency ": { + "type": ["null", "string"] + }, + "cpm_fee_3_usd ": { + "type": ["null", "string"] + }, + "cpm_fee_4_advertiser_currency ": { + "type": ["null", "string"] + }, + "cpm_fee_4_partner_currency ": { + "type": ["null", "string"] + }, + "cpm_fee_4_usd ": { + "type": ["null", "string"] + }, + "cpm_fee_5_advertiser_currency ": { + "type": ["null", "string"] + }, + "cpm_fee_5_partner_currency ": { + "type": ["null", "string"] + }, + "cpm_fee_5_usd ": { + "type": ["null", "string"] + }, + "custom_fee_1_advertiser_currency ": { + "type": ["null", "string"] + }, + "custom_fee_2_advertiser_currency ": { + "type": ["null", "string"] + }, + "custom_fee_3_advertiser_currency ": { + "type": ["null", "string"] + }, + "custom_fee_4_advertiser_currency ": { + "type": ["null", "string"] + }, + "custom_fee_5_advertiser_currency ": { + "type": ["null", "string"] + }, + "data_fees_advertiser_currency ": { + "type": ["null", "string"] + }, + "data_fees_partner_currency ": { + "type": ["null", "string"] + }, + "data_fees_usd ": { + "type": ["null", "string"] + }, + "data_management_platform_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "data_management_platform_fee_partner_currency ": { + "type": ["null", "string"] + }, + "data_management_platform_fee_usd ": { + "type": ["null", "string"] + }, + "doubleverify_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "doubleverify_fee_partner_currency ": { + "type": ["null", "string"] + }, + "doubleverify_fee_usd ": { + "type": ["null", "string"] + }, + "doubleverify_pre_bid_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "doubleverify_pre_bid_fee_partner_currency ": { + "type": ["null", "string"] + }, + "doubleverify_pre_bid_fee_usd ": { + "type": ["null", "string"] + }, + "engagement_rate ": { + "type": ["null", "string"] + }, + "engagements ": { + "type": ["null", "string"] + }, + "estimated_cpm_for_impressions_with_custom_value_advertiser_currency ": { + "type": ["null", "string"] + }, + "estimated_total_cost_for_impressions_with_custom_value_advertiser_currency ": { + "type": ["null", "string"] + }, + "evidon_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "evidon_fee_partner_currency ": { + "type": ["null", "string"] + }, + "evidon_fee_usd ": { + "type": ["null", "string"] + }, + "exits ": { + "type": ["null", "string"] + }, + "expansions ": { + "type": ["null", "string"] + }, + "first_quartile_audio ": { + "type": ["null", "string"] + }, + "first_quartile_views_video ": { + "type": ["null", "string"] + }, + "fullscreens_video ": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_active_view_eligible_impressions ": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_active_view_measurable_impressions ": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_active_view_viewable_impressions ": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_begin_to_render_impressions ": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_clicks ": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_impressions ": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_tracked_ads ": { + "type": ["null", "string"] + }, + "gmail_conversions ": { + "type": ["null", "string"] + }, + "gmail_post_click_conversions ": { + "type": ["null", "string"] + }, + "gmail_post_view_conversions ": { + "type": ["null", "string"] + }, + "impression_custom_value_cost ": { + "type": ["null", "string"] + }, + "impressions ": { + "type": ["null", "string"] + }, + "impressions_with_custom_value ": { + "type": ["null", "string"] + }, + "impressions_with_positive_custom_value ": { + "type": ["null", "string"] + }, + "integral_ad_science_pre_bid_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "integral_ad_science_pre_bid_fee_partner_currency ": { + "type": ["null", "string"] + }, + "integral_ad_science_pre_bid_fee_usd ": { + "type": ["null", "string"] + }, + "integral_ad_science_video_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "integral_ad_science_video_fee_partner_currency ": { + "type": ["null", "string"] + }, + "integral_ad_science_video_fee_usd ": { + "type": ["null", "string"] + }, + "interactive_impressions ": { + "type": ["null", "string"] + }, + "invalid_active_view_eligible_impressions ": { + "type": ["null", "string"] + }, + "invalid_active_view_measurable_impressions ": { + "type": ["null", "string"] + }, + "invalid_active_view_viewable_impressions ": { + "type": ["null", "string"] + }, + "invalid_begin_to_render_impressions ": { + "type": ["null", "string"] + }, + "invalid_clicks ": { + "type": ["null", "string"] + }, + "invalid_impressions ": { + "type": ["null", "string"] + }, + "invalid_tracked_ads ": { + "type": ["null", "string"] + }, + "media_cost_advertiser_currency ": { + "type": ["null", "string"] + }, + "media_cost_partner_currency ": { + "type": ["null", "string"] + }, + "media_cost_usd ": { + "type": ["null", "string"] + }, + "media_cost_ecpa_advertiser_currency ": { + "type": ["null", "string"] + }, + "media_cost_ecpa_partner_currency ": { + "type": ["null", "string"] + }, + "media_cost_ecpa_pc_advertiser_currency ": { + "type": ["null", "string"] + }, + "media_cost_ecpa_pv_advertiser_currency ": { + "type": ["null", "string"] + }, + "media_cost_ecpa_usd ": { + "type": ["null", "string"] + }, + "media_cost_ecpc_advertiser_currency ": { + "type": ["null", "string"] + }, + "media_cost_ecpc_partner_currency ": { + "type": ["null", "string"] + }, + "media_cost_ecpc_pc_partner_currency ": { + "type": ["null", "string"] + }, + "media_cost_ecpc_pc_usd ": { + "type": ["null", "string"] + }, + "media_cost_ecpc_pv_partner_currency ": { + "type": ["null", "string"] + }, + "media_cost_ecpc_pv_usd ": { + "type": ["null", "string"] + }, + "media_cost_ecpc_usd ": { + "type": ["null", "string"] + }, + "media_cost_ecpm_advertiser_currency ": { + "type": ["null", "string"] + }, + "media_cost_ecpm_partner_currency ": { + "type": ["null", "string"] + }, + "media_cost_ecpm_usd ": { + "type": ["null", "string"] + }, + "media_cost_viewable_ecpm_advertiser_currency ": { + "type": ["null", "string"] + }, + "media_cost_viewable_ecpm_partner_currency ": { + "type": ["null", "string"] + }, + "media_cost_viewable_ecpm_usd ": { + "type": ["null", "string"] + }, + "media_fee_1_advertiser_currency ": { + "type": ["null", "string"] + }, + "media_fee_1_partner_currency ": { + "type": ["null", "string"] + }, + "media_fee_1_usd ": { + "type": ["null", "string"] + }, + "media_fee_2_advertiser_currency ": { + "type": ["null", "string"] + }, + "media_fee_2_partner_currency ": { + "type": ["null", "string"] + }, + "media_fee_2_usd ": { + "type": ["null", "string"] + }, + "media_fee_3_advertiser_currency ": { + "type": ["null", "string"] + }, + "media_fee_3_partner_currency ": { + "type": ["null", "string"] + }, + "media_fee_3_usd ": { + "type": ["null", "string"] + }, + "media_fee_4_advertiser_currency ": { + "type": ["null", "string"] + }, + "media_fee_4_partner_currency ": { + "type": ["null", "string"] + }, + "media_fee_4_usd ": { + "type": ["null", "string"] + }, + "media_fee_5_advertiser_currency ": { + "type": ["null", "string"] + }, + "media_fee_5_partner_currency ": { + "type": ["null", "string"] + }, + "media_fee_5_usd ": { + "type": ["null", "string"] + }, + "mediacost_data_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "mediacost_data_fee_partner_currency ": { + "type": ["null", "string"] + }, + "mediacost_data_fee_usd ": { + "type": ["null", "string"] + }, + "midpoint_audio ": { + "type": ["null", "string"] + }, + "midpoint_views_video ": { + "type": ["null", "string"] + }, + "moat_video_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "moat_video_fee_partner_currency ": { + "type": ["null", "string"] + }, + "moat_video_fee_usd ": { + "type": ["null", "string"] + }, + "nielsen_digital_ad_ratings_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "nielsen_digital_ad_ratings_fee_partner_currency ": { + "type": ["null", "string"] + }, + "nielsen_digital_ad_ratings_fee_usd ": { + "type": ["null", "string"] + }, + "pauses_audio ": { + "type": ["null", "string"] + }, + "pauses_video ": { + "type": ["null", "string"] + }, + "platform_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "platform_fee_partner_currency ": { + "type": ["null", "string"] + }, + "platform_fee_usd ": { + "type": ["null", "string"] + }, + "platform_fee_rate ": { + "type": ["null", "string"] + }, + "post_click_conversions ": { + "type": ["null", "string"] + }, + "post_view_conversions ": { + "type": ["null", "string"] + }, + "post_view_conversions__cross_environment ": { + "type": ["null", "string"] + }, + "premium_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "profit_advertiser_currency ": { + "type": ["null", "string"] + }, + "profit_partner_currency ": { + "type": ["null", "string"] + }, + "profit_usd ": { + "type": ["null", "string"] + }, + "profit_ecpm_advertiser_currency ": { + "type": ["null", "string"] + }, + "profit_ecpm_partner_currency ": { + "type": ["null", "string"] + }, + "profit_ecpm_usd ": { + "type": ["null", "string"] + }, + "profit_margin ": { + "type": ["null", "string"] + }, + "profit_viewable_ecpm_advertiser_currency ": { + "type": ["null", "string"] + }, + "profit_viewable_ecpm_partner_currency ": { + "type": ["null", "string"] + }, + "profit_viewable_ecpm_usd ": { + "type": ["null", "string"] + }, + "programmatic_guaranteed_impressions_passed_due_to_frequency ": { + "type": ["null", "string"] + }, + "programmatic_guaranteed_savings_re_invested_due_to_frequency_advertiser_currency ": { + "type": ["null", "string"] + }, + "refund_billable_cost_advertiser_currency ": { + "type": ["null", "string"] + }, + "refund_media_cost_advertiser_currency ": { + "type": ["null", "string"] + }, + "refund_platform_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "revenue_advertiser_currency ": { + "type": ["null", "string"] + }, + "revenue_partner_currency ": { + "type": ["null", "string"] + }, + "revenue_usd ": { + "type": ["null", "string"] + }, + "revenue_ecpa_advertiser_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpa_partner_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpa_pc_advertiser_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpa_pc_partner_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpa_pc_usd ": { + "type": ["null", "string"] + }, + "revenue_ecpa_pv_advertiser_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpa_pv_partner_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpa_pv_usd ": { + "type": ["null", "string"] + }, + "revenue_ecpa_usd ": { + "type": ["null", "string"] + }, + "revenue_ecpc_advertiser_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpc_partner_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpc_usd ": { + "type": ["null", "string"] + }, + "revenue_ecpe_advertiser_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpe_partner_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpe_usd ": { + "type": ["null", "string"] + }, + "revenue_ecpm_advertiser_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpm_partner_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpm_usd ": { + "type": ["null", "string"] + }, + "revenue_ecpv_advertiser_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpv_partner_currency ": { + "type": ["null", "string"] + }, + "revenue_ecpv_usd ": { + "type": ["null", "string"] + }, + "revenue_viewable_ecpm_advertiser_currency ": { + "type": ["null", "string"] + }, + "revenue_viewable_ecpm_partner_currency ": { + "type": ["null", "string"] + }, + "revenue_viewable_ecpm_usd ": { + "type": ["null", "string"] + }, + "rich_media_engagements ": { + "type": ["null", "string"] + }, + "scrolls ": { + "type": ["null", "string"] + }, + "shoplocal_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "shoplocal_fee_partner_currency ": { + "type": ["null", "string"] + }, + "shoplocal_fee_usd ": { + "type": ["null", "string"] + }, + "skips_video ": { + "type": ["null", "string"] + }, + "starts_audio ": { + "type": ["null", "string"] + }, + "starts_video ": { + "type": ["null", "string"] + }, + "stops_audio ": { + "type": ["null", "string"] + }, + "teracent_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "teracent_fee_partner_currency ": { + "type": ["null", "string"] + }, + "teracent_fee_usd ": { + "type": ["null", "string"] + }, + "third_party_ad_server_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "third_party_ad_server_fee_partner_currency ": { + "type": ["null", "string"] + }, + "third_party_ad_server_fee_usd ": { + "type": ["null", "string"] + }, + "third_quartile_audio ": { + "type": ["null", "string"] + }, + "third_quartile_views_video ": { + "type": ["null", "string"] + }, + "timers ": { + "type": ["null", "string"] + }, + "total_conversions ": { + "type": ["null", "string"] + }, + "total_conversions__cross_environment ": { + "type": ["null", "string"] + }, + "total_display_time ": { + "type": ["null", "string"] + }, + "total_impression_custom_value ": { + "type": ["null", "string"] + }, + "total_interaction_time ": { + "type": ["null", "string"] + }, + "total_media_cost_advertiser_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_partner_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_usd ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_advertiser_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_partner_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_pc_advertiser_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_pc_partner_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_pc_usd ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_pv_advertiser_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_pv_partner_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_pv_usd ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_usd ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpc_advertiser_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpc_partner_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpc_usd ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpm_advertiser_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpm_partner_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_ecpm_usd ": { + "type": ["null", "string"] + }, + "total_media_cost_viewable_ecpm_advertiser_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_viewable_ecpm_partner_currency ": { + "type": ["null", "string"] + }, + "total_media_cost_viewable_ecpm_usd ": { + "type": ["null", "string"] + }, + "total_video_media_cost_ecpcv_advertiser_currency ": { + "type": ["null", "string"] + }, + "total_video_media_cost_ecpcv_partner_currency ": { + "type": ["null", "string"] + }, + "total_video_media_cost_ecpcv_usd ": { + "type": ["null", "string"] + }, + "tracked_ads ": { + "type": ["null", "string"] + }, + "trueview_general_invalid_traffic_givt_views ": { + "type": ["null", "string"] + }, + "trueview_invalid_views ": { + "type": ["null", "string"] + }, + "trustmetrics_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "trustmetrics_fee_partner_currency ": { + "type": ["null", "string"] + }, + "trustmetrics_fee_usd ": { + "type": ["null", "string"] + }, + "verifiable_impressions ": { + "type": ["null", "string"] + }, + "video_client_cost_ecpcv_advertiser_currency ": { + "type": ["null", "string"] + }, + "video_media_cost_ecpcv_advertiser_currency ": { + "type": ["null", "string"] + }, + "video_media_cost_ecpcv_partner_currency ": { + "type": ["null", "string"] + }, + "video_media_cost_ecpcv_usd ": { + "type": ["null", "string"] + }, + "vizu_fee_advertiser_currency ": { + "type": ["null", "string"] + }, + "vizu_fee_partner_currency ": { + "type": ["null", "string"] + }, + "vizu_fee_usd ": { + "type": ["null", "string"] + }, + "youtube_view_rate ": { + "type": ["null", "string"] + }, + "youtube_views ": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + { + "name": "audience_composition", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "audience_list": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"] + }, + "eligible_cookies_on_first_party_audience_list": { + "type": ["null", "string"] + }, + "eligible_cookies_on_third_party_audience_list_and_interest": { + "type": ["null", "string"] + }, + "first_party_audience_list": { + "type": ["null", "string"] + }, + "first_party_audience_list_cost": { + "type": ["null", "string"] + }, + "first_party_audience_list_id": { + "type": ["null", "string"] + }, + "first_party_audience_list_type": { + "type": ["null", "string"] + }, + "match_ratio": { + "type": ["null", "string"] + }, + "third_party_audience_list": { + "type": ["null", "string"] + }, + "third_party_audience_list_cost": { + "type": ["null", "string"] + }, + "third_party_audience_list_id": { + "type": ["null", "string"] + }, + "third_party_audience_list_type": { + "type": ["null", "string"] + }, + "potential_impressions": { + "type": ["null", "string"] + }, + "unique_cookies_with_impressions": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + { + "name": "floodlight", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "advertiser": { + "type": ["null", "string"] + }, + "advertiser_currency": { + "type": ["null", "string"] + }, + "advertiser_id": { + "type": ["null", "string"] + }, + "advertiser_integration_code": { + "type": ["null", "string"] + }, + "advertiser_status": { + "type": ["null", "string"] + }, + "advertiser_time_zone": { + "type": ["null", "string"] + }, + "app_url": { + "type": ["null", "string"] + }, + "app_url_excluded": { + "type": ["null", "string"] + }, + "app_url_id": { + "type": ["null", "string"] + }, + "campaign": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "category": { + "type": ["null", "string"] + }, + "cm_placement_id": { + "type": ["null", "string"] + }, + "creative": { + "type": ["null", "string"] + }, + "creative_asset": { + "type": ["null", "string"] + }, + "creative_attributes": { + "type": ["null", "string"] + }, + "creative_height": { + "type": ["null", "string"] + }, + "creative_id": { + "type": ["null", "string"] + }, + "creative_integration_code": { + "type": ["null", "string"] + }, + "creative_rendered_in_amp": { + "type": ["null", "string"] + }, + "creative_size": { + "type": ["null", "string"] + }, + "creative_source": { + "type": ["null", "string"] + }, + "creative_status": { + "type": ["null", "string"] + }, + "creative_type": { + "type": ["null", "string"] + }, + "creative_width": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"] + }, + "day_of_week": { + "type": ["null", "string"] + }, + "exchange": { + "type": ["null", "string"] + }, + "exchange_code": { + "type": ["null", "string"] + }, + "exchange_id": { + "type": ["null", "string"] + }, + "floodlight_activity": { + "type": ["null", "string"] + }, + "floodlight_activity_id": { + "type": ["null", "string"] + }, + "insertion_order": { + "type": ["null", "string"] + }, + "insertion_order_integration_code": { + "type": ["null", "string"] + }, + "insertion_order_status": { + "type": ["null", "string"] + }, + "line_item": { + "type": ["null", "string"] + }, + "line_item_id": { + "type": ["null", "string"] + }, + "line_item_integration_code": { + "type": ["null", "string"] + }, + "line_item_status": { + "type": ["null", "string"] + }, + "line_item_type": { + "type": ["null", "string"] + }, + "month": { + "type": ["null", "string"] + }, + "order_id": { + "type": ["null", "string"] + }, + "partner": { + "type": ["null", "string"] + }, + "partner_currency": { + "type": ["null", "string"] + }, + "partner_id": { + "type": ["null", "string"] + }, + "partner_status": { + "type": ["null", "string"] + }, + "targeted_data_providers": { + "type": ["null", "string"] + }, + "year": { + "type": ["null", "string"] + }, + "cm_post_click_revenue": { + "type": ["null", "string"] + }, + "cm_post_view_revenue": { + "type": ["null", "string"] + }, + "cookie_consented_floodlight_impressions": { + "type": ["null", "string"] + }, + "cookie_unconsented_floodlight_impressions": { + "type": ["null", "string"] + }, + "duplicate_floodlight_impressions": { + "type": ["null", "string"] + }, + "floodlight_impressions": { + "type": ["null", "string"] + }, + "post_click_conversions": { + "type": ["null", "string"] + }, + "post_view_conversions": { + "type": ["null", "string"] + }, + "total_conversions": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + { + "name": "unique_reach_audience", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "advertiser ": { + "type": ["null", "string"] + }, + "advertiser_id ": { + "type": ["null", "string"] + }, + "age ": { + "type": ["null", "string"] + }, + "country ": { + "type": ["null", "string"] + }, + "gender ": { + "type": ["null", "string"] + }, + "date ": { + "type": ["null", "string"] + }, + "insertion_order_id ": { + "type": ["null", "string"] + }, + "insertion_order_integration_code ": { + "type": ["null", "string"] + }, + "insertion_order ": { + "type": ["null", "string"] + }, + "insertion_order_status ": { + "type": ["null", "string"] + }, + "line_item_id ": { + "type": ["null", "string"] + }, + "line_item_integration_code ": { + "type": ["null", "string"] + }, + "line_item ": { + "type": ["null", "string"] + }, + "line_item_status ": { + "type": ["null", "string"] + }, + "line_item_type ": { + "type": ["null", "string"] + }, + "device_type ": { + "type": ["null", "string"] + }, + "creative ": { + "type": ["null", "string"] + }, + "creative_height ": { + "type": ["null", "string"] + }, + "creative_id ": { + "type": ["null", "string"] + }, + "creative_size ": { + "type": ["null", "string"] + }, + "creative_source ": { + "type": ["null", "string"] + }, + "creative_status ": { + "type": ["null", "string"] + }, + "creative_type ": { + "type": ["null", "string"] + }, + "creative_width ": { + "type": ["null", "string"] + }, + "partner_id ": { + "type": ["null", "string"] + }, + "partner ": { + "type": ["null", "string"] + }, + "month ": { + "type": ["null", "string"] + }, + "campaign_id ": { + "type": ["null", "string"] + }, + "campaign ": { + "type": ["null", "string"] + }, + "pct_composition_impressions ": { + "type": ["null", "string"] + }, + "pct_composition_reach ": { + "type": ["null", "string"] + }, + "pct_population_reach ": { + "type": ["null", "string"] + }, + "clicks ": { + "type": ["null", "string"] + }, + "impressions ": { + "type": ["null", "string"] + }, + "population ": { + "type": ["null", "string"] + }, + "target_rating_points ": { + "type": ["null", "string"] + }, + "unique_reach_average_impression_frequency ": { + "type": ["null", "string"] + }, + "unique_reach_click_reach ": { + "type": ["null", "string"] + }, + "unique_reach_impression_reach ": { + "type": ["null", "string"] + }, + "unique_reach_viewable_impression_reach ": { + "type": ["null", "string"] + }, + "viewable_target_rating_points ": { + "type": ["null", "string"] + }, + "viewable_impressions ": { + "type": ["null", "string"] + }, + "pct_viewable_composition_impressions ": { + "type": ["null", "string"] + }, + "pct_viewable_composition_reach ": { + "type": ["null", "string"] + }, + "pct_viewable_population_reach ": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"] + } + ] + } +} diff --git a/airbyte-integrations/connectors/source-dv-360/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-dv-360/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..f36ba916f928c --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/integration_tests/configured_catalog.json @@ -0,0 +1,100 @@ +{ + "streams": [ + { + "stream": { + "name": "standard", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"] + }, + "partner_id": { + "type": ["null", "string"] + }, + "partner": { + "type": ["null", "string"] + }, + "clicks": { + "type": ["null", "string"] + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + } + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["date"] + }, + { + "stream": { + "name": "reach", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"] + }, + "partner_id": { + "type": ["null", "string"] + }, + "partner": { + "type": ["null", "string"] + }, + "cookie_reach_impression_reach": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["date"] + }, + { + "stream": { + "name": "unique_reach_audience", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"] + }, + "partner_id": { + "type": ["null", "string"] + }, + "partner": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "age": { + "type": ["null", "string"] + }, + "gender": { + "type": ["null", "string"] + }, + "viewable_impressions": { + "type": ["null", "string"] + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + } + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["date"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-dv-360/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-dv-360/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..cf2b62201433b --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/integration_tests/invalid_config.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "access_token": "access_token", + "refresh_token": "refresh_token", + "token_uri": "uri", + "client_id": "client_id", + "client_secret": "client_secret" + }, + "start_date": "2022-03-01", + "end_date": "2022-03-08", + "partner_id": 123 +} diff --git a/airbyte-integrations/connectors/source-dv-360/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-dv-360/integration_tests/sample_config.json new file mode 100644 index 0000000000000..cad699edff6a0 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/integration_tests/sample_config.json @@ -0,0 +1,13 @@ +{ + "credentials": { + "access_token": "access_token", + "refresh_token": "refresh_token", + "token_uri": "uri", + "client_id": "client_id", + "client_secret": "client_secret" + }, + "start_date": "2022-03-01", + "end_date": "2022-03-08", + "partner_id": 123, + "filters": [] +} diff --git a/airbyte-integrations/connectors/source-dv-360/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-dv-360/integration_tests/sample_state.json new file mode 100644 index 0000000000000..4b3085983f1f9 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "standard": { + "date": "2022-03-15" + } +} diff --git a/airbyte-integrations/connectors/source-dv-360/main.py b/airbyte-integrations/connectors/source-dv-360/main.py new file mode 100644 index 0000000000000..2f4395facd43a --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_dv_360 import SourceDV360 + +if __name__ == "__main__": + source = SourceDV360() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-dv-360/requirements.txt b/airbyte-integrations/connectors/source-dv-360/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-dv-360/setup.py b/airbyte-integrations/connectors/source-dv-360/setup.py new file mode 100644 index 0000000000000..bc2f7e3986eb9 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/setup.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "google-api-python-client"] + +TEST_REQUIREMENTS = ["pytest~=6.1", "source-acceptance-test", "pytest-mock"] + +setup( + name="source_dv_360", + description="Source implementation for Display & Video 360.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-dv-360/source_dv_360/__init__.py b/airbyte-integrations/connectors/source-dv-360/source_dv_360/__init__.py new file mode 100644 index 0000000000000..0b229354f45ac --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/source_dv_360/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceDV360 + +__all__ = ["SourceDV360"] diff --git a/airbyte-integrations/connectors/source-dv-360/source_dv_360/fields.py b/airbyte-integrations/connectors/source-dv-360/source_dv_360/fields.py new file mode 100644 index 0000000000000..f5d8dc12a2e93 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/source_dv_360/fields.py @@ -0,0 +1,557 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import re + + +def sanitize(s): + s = re.sub(r"[&]", "and", s) + s = re.sub(r"[%]", "pct", s) + s = re.sub(r"[\s/-]+", "_", s.strip()) + # Remove punctuation, which is anything that is not either a word or a whitespace character + s = re.sub(r"[^\w\s]+", "", s) + return s.lower() + + +# Mapping betwee naming in the connector and naming in the report builder +API_REPORT_BUILDER_MAPPING = { + "audience_list": "FILTER_AUDIENCE_LIST", + "date": "FILTER_DATE", + "eligible_cookies_on_first_party_audience_list": "FILTER_ELIGIBLE_COOKIES_ON_FIRST_PARTY_AUDIENCE_LIST", + "eligible_cookies_on_third_party_audience_list_and_interest": "FILTER_ELIGIBLE_COOKIES_ON_THIRD_PARTY_AUDIENCE_LIST_AND_INTEREST", + "first_party_audience_list": "FILTER_USER_LIST_FIRST_PARTY_NAME", + "first_party_audience_list_cost": "FILTER_FIRST_PARTY_AUDIENCE_LIST_COST", + "first_party_audience_list_id": "FILTER_USER_LIST_FIRST_PARTY", + "first_party_audience_list_type": "FILTER_FIRST_PARTY_AUDIENCE_LIST_TYPE", + "match_ratio": "FILTER_MATCH_RATIO", + "third_party_audience_list": "FILTER_USER_LIST_THIRD_PARTY_NAME", + "third_party_audience_list_cost": "FILTER_THIRD_PARTY_AUDIENCE_LIST_COST", + "third_party_audience_list_id": "FILTER_USER_LIST_THIRD_PARTY", + "third_party_audience_list_type": "FILTER_THIRD_PARTY_AUDIENCE_LIST_TYPE", + "advertiser": "FILTER_ADVERTISER_NAME", + "advertiser_currency": "FILTER_ADVERTISER_CURRENCY", + "advertiser_id": "FILTER_ADVERTISER", + "advertiser_integration_code": "FILTER_ADVERTISER_INTEGRATION_CODE", + "advertiser_status": "FILTER_ADVERTISER_INTEGRATION_STATUS", + "advertiser_time_zone": "FILTER_ADVERTISER_TIMEZONE", + "app_url": "FILTER_APP_URL", + "app_url_excluded": "FILTER_APP_URL_EXCLUDED", + "app_url_id": "FILTER_SITE_ID", + "campaign": "FILTER_MEDIA_PLAN_NAME", + "campaign_id": "FILTER_MEDIA_PLAN", + "category": "FILTER_PAGE_CATEGORY", + "cm_placement_id": "FILTER_CM_PLACEMENT_ID", + "creative": "FILTER_CREATIVE", + "creative_asset": "FILTER_CREATIVE_ASSET", + "creative_attributes": "FILTER_CREATIVE_ATTRIBUTE", + "creative_height": "FILTER_CREATIVE_HEIGHT", + "creative_id": "FILTER_CREATIVE_ID", + "creative_integration_code": "FILTER_CREATIVE_INTEGRATION_CODE", + "creative_rendered_in_amp": "FILTER_CREATIVE_RENDERED_IN_AMP", + "creative_size": "FILTER_CREATIVE_SIZE", + "creative_source": "FILTER_CREATIVE_SOURCE", + "creative_status": "FILTER_CREATIVE_STATUS", + "creative_type": "FILTER_CREATIVE_TYPE", + "creative_width": "FILTER_CREATIVE_WIDTH", + "day_of_week": "FILTER_DAY_OF_WEEK", + "exchange": "FILTER_EXCHANGE", + "exchange_code": "FILTER_EXCHANGE_CODE", + "exchange_id": "FILTER_EXCHANGE_ID", + "floodlight_activity": "FILTER_FLOODLIGHT_ACTIVITY", + "floodlight_activity_id": "FILTER_FLOODLIGHT_ACTIVITY_ID", + "insertion_order": "FILTER_INSERTION_ORDER_NAME", + "insertion_order_integration_code": "FILTER_INSERTION_ORDER_INTEGRATION_CODE", + "insertion_order_status": "FILTER_INSERTION_ORDER_STATUS", + "line_item": "FILTER_LINE_ITEM_NAME", + "line_item_id": "FILTER_LINE_ITEM", + "line_item_integration_code": "FILTER_LINE_ITEM_INTEGRATION_CODE", + "line_item_status": "FILTER_LINE_ITEM_STATUS", + "line_item_type": "FILTER_LINE_ITEM_TYPE", + "month": "FILTER_MONTH", + "order_id": "FILTER_ORDER_ID", + "partner": "FILTER_PARTNER_NAME", + "partner_currency": "FILTER_PARTNER_CURRENCY", + "partner_id": "FILTER_PARTNER", + "partner_status": "FILTER_PARTNER_STATUS", + "targeted_data_providers": "FILTER_TARGETED_DATA_PROVIDERS", + "year": "FILTER_YEAR", + "country": "FILTER_COUNTRY", + "country_id": "FILTER_COUNTRY_ID", + "insertion_order_id": "FILTER_INSERTION_ORDER", + "inventory_source": "FILTER_INVENTORY_SOURCE_NAME", + "active_view_custom_metric_id": "FILTER_ACTIVE_VIEW_CUSTOM_METRIC_ID", + "active_view_custom_metric_name": "FILTER_ACTIVE_VIEW_CUSTOM_METRIC_NAME", + "ad_position": "FILTER_AD_POSITION", + "ad_type": "FILTER_AD_TYPE", + "algorithm": "FILTER_ALGORITHM", + "algorithm_id": "FILTER_ALGORITHM_ID", + "amp_page_request": "FILTER_AMP_PAGE_REQUEST", + "attributed_userlist": "FILTER_ATTRIBUTED_USERLIST", + "attributed_userlist_cost": "FILTER_ATTRIBUTED_USERLIST_COST", + "attributed_userlist_id": "FILTER_TARGETED_USER_LIST", + "attributed_userlist_type": "FILTER_ATTRIBUTED_USERLIST_TYPE", + "attribution_model": "FILTER_ATTRIBUTION_MODEL", + "audience_list_cost": "FILTER_AUDIENCE_LIST_COST", + "audience_list_id": "FILTER_USER_LIST", + "audience_list_type": "FILTER_AUDIENCE_LIST_TYPE", + "audience_name": "FILTER_AUDIENCE_NAME", + "audience_type": "FILTER_AUDIENCE_TYPE", + "authorized_seller_state": "FILTER_AUTHORIZED_SELLER_STATE", + "billable_outcome": "FILTER_BILLABLE_OUTCOME", + "brand_lift_type": "FILTER_BRAND_LIFT_TYPE", + "browser_id": "FILTER_BROWSER", + "budget_segment_description": "FILTER_BUDGET_SEGMENT_DESCRIPTION", + "channel": "FILTER_CHANNEL_NAME", + "channel_id": "FILTER_CHANNEL_ID", + "channel_type": "FILTER_CHANNEL_TYPE", + "city": "FILTER_CITY_NAME", + "city_id": "FILTER_CITY", + "companion_creative": "FILTER_COMPANION_CREATIVE_NAME", + "companion_creative_id": "FILTER_COMPANION_CREATIVE_ID", + "companion_creative_size": "FILTER_COMPANION_CREATIVE_SIZE", + "data_provider": "FILTER_DATA_PROVIDER_NAME", + "data_provider_id": "FILTER_DATA_PROVIDER", + "detailed_demographics": "FILTER_DETAILED_DEMOGRAPHICS", + "detailed_demographics_id": "FILTER_DETAILED_DEMOGRAPHICS_ID", + "device": "FILTER_DEVICE", + "device_make": "FILTER_DEVICE_MAKE", + "device_model": "FILTER_DEVICE_MODEL", + "device_type": "FILTER_DEVICE_TYPE", + "digital_content_label": "FILTER_DIGITAL_CONTENT_LABEL", + "dma": "FILTER_DMA_NAME", + "dma_code": "FILTER_DMA", + "extension": "FILTER_EXTENSION", + "extension_status": "FILTER_EXTENSION_STATUS", + "extension_type": "FILTER_EXTENSION_TYPE", + "format": "FILTER_FORMAT", + "gmail_age": "FILTER_GMAIL_AGE", + "gmail_city": "FILTER_GMAIL_CITY", + "gmail_country": "FILTER_GMAIL_COUNTRY", + "gmail_device_type": "FILTER_GMAIL_DEVICE_TYPE", + "gmail_gender": "FILTER_GMAIL_GENDER", + "gmail_region": "FILTER_GMAIL_REGION", + "gmail_remarketing_list": "FILTER_GMAIL_REMARKETING_LIST", + "household_income": "FILTER_HOUSEHOLD_INCOME", + "impression_counting_method": "FILTER_IMPRESSION_COUNTING_METHOD", + "insertion_order_daily_frequency": "FILTER_CAMPAIGN_DAILY_FREQUENCY", + "interest": "FILTER_INTEREST", + "inventory_commitment_type": "FILTER_INVENTORY_COMMITMENT_TYPE", + "inventory_delivery_method": "FILTER_INVENTORY_DELIVERY_METHOD", + "inventory_rate_type": "FILTER_INVENTORY_RATE_TYPE", + "inventory_source_group": "FILTER_INVENTORY_SOURCE_GROUP", + "inventory_source_group_id": "FILTER_INVENTORY_SOURCE_GROUP_ID", + "inventory_source_id": "FILTER_INVENTORY_SOURCE_ID", + "inventory_source_id_external": "FILTER_INVENTORY_SOURCE_EXTERNAL_ID", + "inventory_source_type": "FILTER_INVENTORY_SOURCE_TYPE", + "isp_or_carrier": "FILTER_CARRIER_NAME", + "isp_or_carrier_id": "FILTER_CARRIER", + "keyword": "FILTER_KEYWORD", + "life_event": "FILTER_LIFE_EVENT", + "life_events": "FILTER_LIFE_EVENTS", + "line_item_daily_frequency": "FILTER_LINE_ITEM_DAILY_FREQUENCY", + "line_item_lifetime_frequency": "FILTER_LINE_ITEM_LIFETIME_FREQUENCY", + "max_video_duration": "FILTER_VIDEO_DURATION_SECONDS", + "measurement_source": "FILTER_MEASUREMENT_SOURCE", + "operating_system": "FILTER_OS", + "platform": "FILTER_PLATFORM", + "playback_method": "FILTER_PLAYBACK_METHOD", + "position_in_content": "FILTER_POSITION_IN_CONTENT", + "public_inventory": "FILTER_PUBLIC_INVENTORY", + "publisher_property": "FILTER_PUBLISHER_PROPERTY", + "publisher_property_id": "FILTER_PUBLISHER_PROPERTY_ID", + "publisher_property_section": "FILTER_PUBLISHER_PROPERTY_SECTION", + "publisher_property_section_id": "FILTER_PUBLISHER_PROPERTY_SECTION_ID", + "refund_reason": "FILTER_REFUND_REASON", + "region": "FILTER_REGION_NAME", + "region_id": "FILTER_REGION", + "rewarded": "FILTER_REWARDED", + "sensitive_category": "FILTER_SENSITIVE_CATEGORY", + "served_pixel_density": "FILTER_SERVED_PIXEL_DENSITY", + "time_of_day": "FILTER_TIME_OF_DAY", + "time_to_conversion": "FILTER_CONVERSION_DELAY", + "variant_id": "FILTER_VARIANT_ID", + "variant_name": "FILTER_VARIANT_NAME", + "variant_version": "FILTER_VARIANT_VERSION", + "verification_video_player_size": "FILTER_VERIFICATION_VIDEO_PLAYER_SIZE", + "verification_video_position": "FILTER_VERIFICATION_VIDEO_POSITION", + "video_continuous_play": "FILTER_VIDEO_CONTINUOUS_PLAY", + "video_player_size": "FILTER_VIDEO_PLAYER_SIZE", + "video_skippable_support": "FILTER_SKIPPABLE_SUPPORT", + "week": "FILTER_WEEK", + "zip_code": "FILTER_ZIP_POSTAL_CODE", + "zip_code_id": "FILTER_ZIP_CODE", + "age": "FILTER_AGE", + "gender": "FILTER_GENDER", + "potential_impressions": "METRIC_POTENTIAL_IMPRESSIONS", + "unique_cookies_with_impressions": "METRIC_UNIQUE_COOKIES_WITH_IMPRESSIONS", + "cm_post_click_revenue": "METRIC_CM_POST_CLICK_REVENUE", + "cm_post_view_revenue": "METRIC_CM_POST_VIEW_REVENUE", + "cookie_consented_floodlight_impressions": "METRIC_COOKIE_CONSENTED_FLOODLIGHT_IMPRESSIONS", + "cookie_unconsented_floodlight_impressions": "METRIC_COOKIE_UNCONSENTED_FLOODLIGHT_IMPRESSIONS", + "duplicate_floodlight_impressions": "METRIC_DUPLICATE_FLOODLIGHT_IMPRESSIONS", + "floodlight_impressions": "METRIC_FLOODLIGHT_IMPRESSIONS", + "post_click_conversions": "METRIC_LAST_CLICKS", + "post_view_conversions": "METRIC_LAST_IMPRESSIONS", + "total_conversions": "METRIC_TOTAL_CONVERSIONS", + "cookie_reach_average_impression_frequency": "METRIC_COOKIE_REACH_AVERAGE_IMPRESSION_FREQUENCY", + "cookie_reach_impression_reach": "METRIC_COOKIE_REACH_IMPRESSION_REACH", + "unique_reach_average_impression_frequency": "METRIC_UNIQUE_REACH_AVERAGE_IMPRESSION_FREQUENCY", + "unique_reach_click_reach": "METRIC_UNIQUE_REACH_CLICK_REACH", + "unique_reach_impression_reach": "METRIC_UNIQUE_REACH_IMPRESSION_REACH", + "unique_reach_total_reach": "METRIC_UNIQUE_REACH_TOTAL_REACH", + "pct_clicks_leading_to_conversions": "METRIC_CLICK_TO_POST_CLICK_CONVERSION_RATE", + "pct_impressions_leading_to_conversions": "METRIC_IMPRESSIONS_TO_CONVERSION_RATE", + "pct_impressions_with_positive_custom_value": "METRIC_PERCENT_IMPRESSIONS_WITH_POSITIVE_CUSTOM_VALUE", + "active_view_pct_audible_and_visible_at_completion": "METRIC_ACTIVE_VIEW_PERCENT_AUDIBLE_VISIBLE_ON_COMPLETE", + "active_view_pct_audible_and_visible_at_first_quartile": "METRIC_ACTIVE_VIEW_PERCENT_AUDIBLE_VISIBLE_FIRST_QUAR", + "active_view_pct_audible_and_visible_at_midpoint": "METRIC_ACTIVE_VIEW_PERCENT_AUDIBLE_VISIBLE_SECOND_QUAR", + "active_view_pct_audible_and_visible_at_start": "METRIC_ACTIVE_VIEW_PERCENT_AUDIBLE_VISIBLE_AT_START", + "active_view_pct_audible_and_visible_at_third_quartile": "METRIC_ACTIVE_VIEW_PERCENT_AUDIBLE_VISIBLE_THIRD_QUAR", + "active_view_pct_audible_impressions": "METRIC_ACTIVE_VIEW_PERCENT_AUDIBLE_IMPRESSIONS", + "active_view_pct_full_screen": "METRIC_ACTIVE_VIEW_PERCENT_FULL_SCREEN", + "active_view_pct_fully_on_screen_2_sec": "METRIC_ACTIVE_VIEW_PERCENT_FULLY_ON_SCREEN_2_SEC", + "active_view_pct_in_background": "METRIC_ACTIVE_VIEW_PERCENT_IN_BACKGROUND", + "active_view_pct_measurable_impressions": "METRIC_ACTIVE_VIEW_PCT_MEASURABLE_IMPRESSIONS", + "active_view_pct_of_ad_played": "METRIC_ACTIVE_VIEW_PERCENT_OF_AD_PLAYED", + "active_view_pct_of_completed_impressions_audible_and_visible": "METRIC_ACTIVE_VIEW_PERCENT_OF_COMPLETED_IMPRESSIONS_AUDIBLE_AND_VISIBLE", + "active_view_pct_of_completed_impressions_visible": "METRIC_ACTIVE_VIEW_PERCENT_OF_COMPLETED_IMPRESSIONS_VISIBLE", + "active_view_pct_of_first_quartile_impressions_audible_and_visible": "METRIC_ACTIVE_VIEW_PERCENT_OF_FIRST_QUARTILE_IMPRESSIONS_AUDIBLE_AND_VISIBLE", + "active_view_pct_of_first_quartile_impressions_visible": "METRIC_ACTIVE_VIEW_PERCENT_OF_FIRST_QUARTILE_IMPRESSIONS_VISIBLE", + "active_view_pct_of_midpoint_impressions_audible_and_visible": "METRIC_ACTIVE_VIEW_PERCENT_OF_MIDPOINT_IMPRESSIONS_AUDIBLE_AND_VISIBLE", + "active_view_pct_of_midpoint_impressions_visible": "METRIC_ACTIVE_VIEW_PERCENT_OF_MIDPOINT_IMPRESSIONS_VISIBLE", + "active_view_pct_of_third_quartile_impressions_audible_and_visible": "METRIC_ACTIVE_VIEW_PERCENT_OF_THIRD_QUARTILE_IMPRESSIONS_AUDIBLE_AND_VISIBLE", + "active_view_pct_of_third_quartile_impressions_visible": "METRIC_ACTIVE_VIEW_PERCENT_OF_THIRD_QUARTILE_IMPRESSIONS_VISIBLE", + "active_view_pct_play_time_audible": "METRIC_ACTIVE_VIEW_PERCENT_PLAY_TIME_AUDIBLE", + "active_view_pct_play_time_audible_and_visible": "METRIC_ACTIVE_VIEW_PERCENT_PLAY_TIME_AUDIBLE_AND_VISIBLE", + "active_view_pct_play_time_visible": "METRIC_ACTIVE_VIEW_PERCENT_PLAY_TIME_VISIBLE", + "active_view_pct_viewable_impressions": "METRIC_ACTIVE_VIEW_PCT_VIEWABLE_IMPRESSIONS", + "active_view_pct_visible_10_seconds": "METRIC_ACTIVE_VIEW_PERCENT_VIEWABLE_FOR_TIME_THRESHOLD", + "active_view_pct_visible_at_completion": "METRIC_ACTIVE_VIEW_PERCENT_VISIBLE_ON_COMPLETE", + "active_view_pct_visible_at_first_quartile": "METRIC_ACTIVE_VIEW_PERCENT_VISIBLE_FIRST_QUAR", + "active_view_pct_visible_at_midpoint": "METRIC_ACTIVE_VIEW_PERCENT_VISIBLE_SECOND_QUAR", + "active_view_pct_visible_at_start": "METRIC_ACTIVE_VIEW_PERCENT_VISIBLE_AT_START", + "active_view_pct_visible_at_third_quartile": "METRIC_ACTIVE_VIEW_PERCENT_VISIBLE_THIRD_QUAR", + "active_view_audible_and_fully_on_screen_for_half_of_duration_15_sec_cap_impressions": "METRIC_ACTIVE_VIEW_AUDIBLE_FULLY_ON_SCREEN_HALF_OF_DURATION_IMPRESSIONS", + "active_view_audible_and_fully_on_screen_for_half_of_duration_15_sec_cap_measurable_impressions": "METRIC_ACTIVE_VIEW_AUDIBLE_FULLY_ON_SCREEN_HALF_OF_DURATION_MEASURABLE_IMPRESSIONS", + "active_view_audible_and_fully_on_screen_for_half_of_duration_15_sec_cap_rate": "METRIC_ACTIVE_VIEW_AUDIBLE_FULLY_ON_SCREEN_HALF_OF_DURATION_RATE", + "active_view_audible_and_fully_on_screen_for_half_of_duration_trueview_impressions": "METRIC_ACTIVE_VIEW_AUDIBLE_FULLY_ON_SCREEN_HALF_OF_DURATION_TRUEVIEW_IMPRESSIONS", + "active_view_audible_and_fully_on_screen_for_half_of_duration_trueview_measurable_impressions": "METRIC_ACTIVE_VIEW_AUDIBLE_FULLY_ON_SCREEN_HALF_OF_DURATION_TRUEVIEW_MEASURABLE_IMPRESSIONS", + "active_view_audible_and_fully_on_screen_for_half_of_duration_trueview_rate": "METRIC_ACTIVE_VIEW_AUDIBLE_FULLY_ON_SCREEN_HALF_OF_DURATION_TRUEVIEW_RATE", + "active_view_average_viewable_time_seconds": "METRIC_ACTIVE_VIEW_AVERAGE_VIEWABLE_TIME", + "active_view_custom_metric_measurable_impressions": "METRIC_ACTIVE_VIEW_CUSTOM_METRIC_MEASURABLE_IMPRESSIONS", + "active_view_custom_metric_viewable_impressions": "METRIC_ACTIVE_VIEW_CUSTOM_METRIC_VIEWABLE_IMPRESSIONS", + "active_view_custom_metric_viewable_rate": "METRIC_ACTIVE_VIEW_CUSTOM_METRIC_VIEWABLE_RATE", + "active_view_eligible_impressions": "METRIC_ACTIVE_VIEW_ELIGIBLE_IMPRESSIONS", + "active_view_impression_distribution_not_measurable": "METRIC_ACTIVE_VIEW_DISTRIBUTION_UNMEASURABLE", + "active_view_impression_distribution_not_viewable": "METRIC_ACTIVE_VIEW_DISTRIBUTION_UNVIEWABLE", + "active_view_impression_distribution_viewable": "METRIC_ACTIVE_VIEW_DISTRIBUTION_VIEWABLE", + "active_view_impressions_audible_and_visible_at_completion": "METRIC_ACTIVE_VIEW_AUDIBLE_VISIBLE_ON_COMPLETE_IMPRESSIONS", + "active_view_impressions_visible_10_seconds": "METRIC_ACTIVE_VIEW_VIEWABLE_FOR_TIME_THRESHOLD", + "active_view_measurable_impressions": "METRIC_ACTIVE_VIEW_MEASURABLE_IMPRESSIONS", + "active_view_not_measurable_impressions": "METRIC_ACTIVE_VIEW_UNMEASURABLE_IMPRESSIONS", + "active_view_not_viewable_impressions": "METRIC_ACTIVE_VIEW_UNVIEWABLE_IMPRESSIONS", + "active_view_viewable_impressions": "METRIC_ACTIVE_VIEW_VIEWABLE_IMPRESSIONS", + "adlingo_fee_advertiser_currency": "METRIC_ADLINGO_FEE_ADVERTISER_CURRENCY", + "adloox_fee_advertiser_currency": "METRIC_FEE21_ADVERTISER", + "adloox_fee_partner_currency": "METRIC_FEE21_PARTNER", + "adloox_fee_usd": "METRIC_FEE21_USD", + "adloox_pre_bid_fee_advertiser_currency": "METRIC_FEE22_ADVERTISER", + "adloox_pre_bid_fee_partner_currency": "METRIC_FEE22_PARTNER", + "adloox_pre_bid_fee_usd": "METRIC_FEE22_USD", + "adsafe_fee_advertiser_currency": "METRIC_FEE4_ADVERTISER", + "adsafe_fee_partner_currency": "METRIC_FEE4_PARTNER", + "adsafe_fee_usd": "METRIC_FEE4_USD", + "adxpose_fee_advertiser_currency": "METRIC_FEE5_ADVERTISER", + "adxpose_fee_partner_currency": "METRIC_FEE5_PARTNER", + "adxpose_fee_usd": "METRIC_FEE5_USD", + "agency_trading_desk_fee_advertiser_currency": "METRIC_FEE10_ADVERTISER", + "agency_trading_desk_fee_partner_currency": "METRIC_FEE10_PARTNER", + "agency_trading_desk_fee_usd": "METRIC_FEE10_USD", + "aggregate_knowledge_fee_advertiser_currency": "METRIC_FEE7_ADVERTISER", + "aggregate_knowledge_fee_partner_currency": "METRIC_FEE7_PARTNER", + "aggregate_knowledge_fee_usd": "METRIC_FEE7_USD", + "audio_client_cost_ecpcl_advertiser_currency": "METRIC_AUDIO_CLIENT_COST_ECPCL_ADVERTISER_CURRENCY", + "audio_media_cost_ecpcl_advertiser_currency": "METRIC_AUDIO_MEDIA_COST_ECPCL_ADVERTISER_CURRENCY", + "audio_mutes_audio": "METRIC_AUDIO_MUTES_AUDIO", + "audio_mutes_video": "METRIC_RICH_MEDIA_VIDEO_MUTES", + "audio_revenue_ecpcl_advertiser_currency": "METRIC_AUDIO_REVENUE_ECPCL_ADVERTISER_CURRENCY", + "audio_unmutes_audio": "METRIC_AUDIO_UNMUTES_AUDIO", + "audio_unmutes_video": "METRIC_AUDIO_UNMUTES_VIDEO", + "average_display_time": "METRIC_AVERAGE_DISPLAY_TIME", + "average_interaction_time": "METRIC_AVERAGE_INTERACTION_TIME", + "begin_to_render_eligible_impressions": "METRIC_BEGIN_TO_RENDER_ELIGIBLE_IMPRESSIONS", + "begin_to_render_impressions": "METRIC_BEGIN_TO_RENDER_IMPRESSIONS", + "billable_cost_advertiser_currency": "METRIC_BILLABLE_COST_ADVERTISER", + "billable_cost_partner_currency": "METRIC_BILLABLE_COST_PARTNER", + "billable_cost_usd": "METRIC_BILLABLE_COST_USD", + "billable_impressions": "METRIC_BILLABLE_IMPRESSIONS", + "click_rate_ctr": "METRIC_CTR", + "clicks": "METRIC_CLICKS", + "client_cost_advertiser_currency": "METRIC_CLIENT_COST_ADVERTISER_CURRENCY", + "client_cost_ecpa_advertiser_currency": "METRIC_CLIENT_COST_ECPA_ADVERTISER_CURRENCY", + "client_cost_ecpa_pc_advertiser_currency": "METRIC_CLIENT_COST_ECPA_PC_ADVERTISER_CURRENCY", + "client_cost_ecpa_pv_advertiser_currency": "METRIC_CLIENT_COST_ECPA_PV_ADVERTISER_CURRENCY", + "client_cost_ecpc_advertiser_currency": "METRIC_CLIENT_COST_ECPC_ADVERTISER_CURRENCY", + "client_cost_ecpm_advertiser_currency": "METRIC_CLIENT_COST_ECPM_ADVERTISER_CURRENCY", + "client_cost_viewable_ecpm_advertiser_currency": "METRIC_CLIENT_COST_VIEWABLE_ECPM_ADVERTISER_CURRENCY", + "cm_post_click_revenue__cross_environment": "METRIC_CM_POST_CLICK_REVENUE_CROSS_ENVIRONMENT", + "cm_post_view_revenue__cross_environment": "METRIC_CM_POST_VIEW_REVENUE_CROSS_ENVIRONMENT", + "companion_clicks_audio": "METRIC_COMPANION_CLICKS_AUDIO", + "companion_clicks_video": "METRIC_VIDEO_COMPANION_CLICKS", + "companion_impressions_audio": "METRIC_COMPANION_IMPRESSIONS_AUDIO", + "companion_impressions_video": "METRIC_VIDEO_COMPANION_IMPRESSIONS", + "complete_listens_audio": "METRIC_COMPLETE_LISTENS_AUDIO", + "complete_views_video": "METRIC_RICH_MEDIA_VIDEO_COMPLETIONS", + "completion_rate_audio": "METRIC_COMPLETION_RATE_AUDIO", + "completion_rate_video": "METRIC_VIDEO_COMPLETION_RATE", + "comscore_vce_in_doubleclick_fee_advertiser_currency": "METRIC_FEE20_ADVERTISER", + "comscore_vce_in_doubleclick_fee_partner_currency": "METRIC_FEE20_PARTNER", + "comscore_vce_in_doubleclick_fee_usd": "METRIC_FEE20_USD", + "conversions_per_1000_impressions": "METRIC_CONVERSIONS_PER_MILLE", + "cookie_unconsented_clicks": "METRIC_TRACKING_UNCONSENTED_CLICKS", + "counters": "METRIC_COUNTERS", + "cpm_fee_1_advertiser_currency": "METRIC_CPM_FEE1_ADVERTISER", + "cpm_fee_1_partner_currency": "METRIC_CPM_FEE1_PARTNER", + "cpm_fee_1_usd": "METRIC_CPM_FEE1_USD", + "cpm_fee_2_advertiser_currency": "METRIC_CPM_FEE2_ADVERTISER", + "cpm_fee_2_partner_currency": "METRIC_CPM_FEE2_PARTNER", + "cpm_fee_2_usd": "METRIC_CPM_FEE2_USD", + "cpm_fee_3_advertiser_currency": "METRIC_CPM_FEE3_ADVERTISER", + "cpm_fee_3_partner_currency": "METRIC_CPM_FEE3_PARTNER", + "cpm_fee_3_usd": "METRIC_CPM_FEE3_USD", + "cpm_fee_4_advertiser_currency": "METRIC_CPM_FEE4_ADVERTISER", + "cpm_fee_4_partner_currency": "METRIC_CPM_FEE4_PARTNER", + "cpm_fee_4_usd": "METRIC_CPM_FEE4_USD", + "cpm_fee_5_advertiser_currency": "METRIC_CPM_FEE5_ADVERTISER", + "cpm_fee_5_partner_currency": "METRIC_CPM_FEE5_PARTNER", + "cpm_fee_5_usd": "METRIC_CPM_FEE5_USD", + "custom_fee_1_advertiser_currency": "METRIC_CUSTOM_FEE_1_ADVERTISER_CURRENCY", + "custom_fee_2_advertiser_currency": "METRIC_CUSTOM_FEE_2_ADVERTISER_CURRENCY", + "custom_fee_3_advertiser_currency": "METRIC_CUSTOM_FEE_3_ADVERTISER_CURRENCY", + "custom_fee_4_advertiser_currency": "METRIC_CUSTOM_FEE_4_ADVERTISER_CURRENCY", + "custom_fee_5_advertiser_currency": "METRIC_CUSTOM_FEE_5_ADVERTISER_CURRENCY", + "data_fees_advertiser_currency": "METRIC_DATA_COST_ADVERTISER", + "data_fees_partner_currency": "METRIC_DATA_COST_PARTNER", + "data_fees_usd": "METRIC_DATA_COST_USD", + "data_management_platform_fee_advertiser_currency": "METRIC_FEE11_ADVERTISER", + "data_management_platform_fee_partner_currency": "METRIC_FEE11_PARTNER", + "data_management_platform_fee_usd": "METRIC_FEE11_USD", + "doubleverify_fee_advertiser_currency": "METRIC_FEE3_ADVERTISER", + "doubleverify_fee_partner_currency": "METRIC_FEE3_PARTNER", + "doubleverify_fee_usd": "METRIC_FEE3_USD", + "doubleverify_pre_bid_fee_advertiser_currency": "METRIC_FEE13_ADVERTISER", + "doubleverify_pre_bid_fee_partner_currency": "METRIC_FEE13_PARTNER", + "doubleverify_pre_bid_fee_usd": "METRIC_FEE13_USD", + "engagement_rate": "METRIC_DBM_ENGAGEMENT_RATE", + "engagements": "METRIC_ENGAGEMENTS", + "estimated_cpm_for_impressions_with_custom_value_advertiser_currency": "METRIC_ESTIMATED_CPM_FOR_IMPRESSIONS_WITH_CUSTOM_VALUE_ADVERTISER_CURRENCY", + "estimated_total_cost_for_impressions_with_custom_value_advertiser_currency": "METRIC_ESTIMATED_TOTAL_COST_FOR_IMPRESSIONS_WITH_CUSTOM_VALUE_ADVERTISER_CURRENCY", + "evidon_fee_advertiser_currency": "METRIC_FEE9_ADVERTISER", + "evidon_fee_partner_currency": "METRIC_FEE9_PARTNER", + "evidon_fee_usd": "METRIC_FEE9_USD", + "exits": "METRIC_EXITS", + "expansions": "METRIC_EXPANSIONS", + "first_quartile_audio": "METRIC_FIRST_QUARTILE_AUDIO", + "first_quartile_views_video": "METRIC_RICH_MEDIA_VIDEO_FIRST_QUARTILE_COMPLETES", + "fullscreens_video": "METRIC_RICH_MEDIA_VIDEO_FULL_SCREENS", + "general_invalid_traffic_givt_active_view_eligible_impressions": "METRIC_GIVT_ACTIVE_VIEW_ELIGIBLE_IMPRESSIONS", + "general_invalid_traffic_givt_active_view_measurable_impressions": "METRIC_GIVT_ACTIVE_VIEW_MEASURABLE_IMPRESSIONS", + "general_invalid_traffic_givt_active_view_viewable_impressions": "METRIC_GIVT_ACTIVE_VIEW_VIEWABLE_IMPRESSIONS", + "general_invalid_traffic_givt_begin_to_render_impressions": "METRIC_GIVT_BEGIN_TO_RENDER_IMPRESSIONS", + "general_invalid_traffic_givt_clicks": "METRIC_GIVT_CLICKS", + "general_invalid_traffic_givt_impressions": "METRIC_GENERAL_INVALID_TRAFFIC_GIVT_IMPRESSIONS", + "general_invalid_traffic_givt_tracked_ads": "METRIC_GENERAL_INVALID_TRAFFIC_GIVT_TRACKED_ADS", + "gmail_conversions": "METRIC_GMAIL_CONVERSIONS", + "gmail_post_click_conversions": "METRIC_GMAIL_POST_CLICK_CONVERSIONS", + "gmail_post_view_conversions": "METRIC_GMAIL_POST_VIEW_CONVERSIONS", + "impression_custom_value_cost": "METRIC_IMPRESSION_CUSTOM_VALUE_COST", + "impressions": "METRIC_IMPRESSIONS", + "impressions_with_custom_value": "METRIC_IMPRESSIONS_WITH_CUSTOM_VALUE", + "impressions_with_positive_custom_value": "METRIC_IMPRESSIONS_WITH_POSITIVE_CUSTOM_VALUE", + "integral_ad_science_pre_bid_fee_advertiser_currency": "METRIC_FEE12_ADVERTISER", + "integral_ad_science_pre_bid_fee_partner_currency": "METRIC_FEE12_PARTNER", + "integral_ad_science_pre_bid_fee_usd": "METRIC_FEE12_USD", + "integral_ad_science_video_fee_advertiser_currency": "METRIC_FEE17_ADVERTISER", + "integral_ad_science_video_fee_partner_currency": "METRIC_FEE17_PARTNER", + "integral_ad_science_video_fee_usd": "METRIC_FEE17_USD", + "interactive_impressions": "METRIC_INTERACTIVE_IMPRESSIONS", + "invalid_active_view_eligible_impressions": "METRIC_INVALID_ACTIVE_VIEW_ELIGIBLE_IMPRESSIONS", + "invalid_active_view_measurable_impressions": "METRIC_INVALID_ACTIVE_VIEW_MEASURABLE_IMPRESSIONS", + "invalid_active_view_viewable_impressions": "METRIC_INVALID_ACTIVE_VIEW_VIEWABLE_IMPRESSIONS", + "invalid_begin_to_render_impressions": "METRIC_INVALID_BEGIN_TO_RENDER_IMPRESSIONS", + "invalid_clicks": "METRIC_INVALID_CLICKS", + "invalid_impressions": "METRIC_INVALID_IMPRESSIONS", + "invalid_tracked_ads": "METRIC_INVALID_TRACKED_ADS", + "media_cost_advertiser_currency": "METRIC_MEDIA_COST_ADVERTISER", + "media_cost_partner_currency": "METRIC_MEDIA_COST_PARTNER", + "media_cost_usd": "METRIC_MEDIA_COST_USD", + "media_cost_ecpa_advertiser_currency": "METRIC_MEDIA_COST_ECPA_ADVERTISER", + "media_cost_ecpa_partner_currency": "METRIC_MEDIA_COST_ECPA_PARTNER", + "media_cost_ecpa_pc_advertiser_currency": "METRIC_MEDIA_COST_ECPAPC_ADVERTISER", + "media_cost_ecpa_pv_advertiser_currency": "METRIC_MEDIA_COST_ECPAPV_ADVERTISER", + "media_cost_ecpa_usd": "METRIC_MEDIA_COST_ECPA_USD", + "media_cost_ecpc_advertiser_currency": "METRIC_MEDIA_COST_ECPC_ADVERTISER", + "media_cost_ecpc_partner_currency": "METRIC_MEDIA_COST_ECPC_PARTNER", + "media_cost_ecpc_pc_partner_currency": "METRIC_MEDIA_COST_ECPAPC_PARTNER", + "media_cost_ecpc_pc_usd": "METRIC_MEDIA_COST_ECPAPC_USD", + "media_cost_ecpc_pv_partner_currency": "METRIC_MEDIA_COST_ECPAPV_PARTNER", + "media_cost_ecpc_pv_usd": "METRIC_MEDIA_COST_ECPAPV_USD", + "media_cost_ecpc_usd": "METRIC_MEDIA_COST_ECPC_USD", + "media_cost_ecpm_advertiser_currency": "METRIC_MEDIA_COST_ECPM_ADVERTISER", + "media_cost_ecpm_partner_currency": "METRIC_MEDIA_COST_ECPM_PARTNER", + "media_cost_ecpm_usd": "METRIC_MEDIA_COST_ECPM_USD", + "media_cost_viewable_ecpm_advertiser_currency": "METRIC_MEDIA_COST_VIEWABLE_ECPM_ADVERTISER", + "media_cost_viewable_ecpm_partner_currency": "METRIC_MEDIA_COST_VIEWABLE_ECPM_PARTNER", + "media_cost_viewable_ecpm_usd": "METRIC_MEDIA_COST_VIEWABLE_ECPM_USD", + "media_fee_1_advertiser_currency": "METRIC_MEDIA_FEE1_ADVERTISER", + "media_fee_1_partner_currency": "METRIC_MEDIA_FEE1_PARTNER", + "media_fee_1_usd": "METRIC_MEDIA_FEE1_USD", + "media_fee_2_advertiser_currency": "METRIC_MEDIA_FEE2_ADVERTISER", + "media_fee_2_partner_currency": "METRIC_MEDIA_FEE2_PARTNER", + "media_fee_2_usd": "METRIC_MEDIA_FEE2_USD", + "media_fee_3_advertiser_currency": "METRIC_MEDIA_FEE3_ADVERTISER", + "media_fee_3_partner_currency": "METRIC_MEDIA_FEE3_PARTNER", + "media_fee_3_usd": "METRIC_MEDIA_FEE3_USD", + "media_fee_4_advertiser_currency": "METRIC_MEDIA_FEE4_ADVERTISER", + "media_fee_4_partner_currency": "METRIC_MEDIA_FEE4_PARTNER", + "media_fee_4_usd": "METRIC_MEDIA_FEE4_USD", + "media_fee_5_advertiser_currency": "METRIC_MEDIA_FEE5_ADVERTISER", + "media_fee_5_partner_currency": "METRIC_MEDIA_FEE5_PARTNER", + "media_fee_5_usd": "METRIC_MEDIA_FEE5_USD", + "mediacost_data_fee_advertiser_currency": "METRIC_FEE16_ADVERTISER", + "mediacost_data_fee_partner_currency": "METRIC_FEE16_PARTNER", + "mediacost_data_fee_usd": "METRIC_FEE16_USD", + "midpoint_audio": "METRIC_MIDPOINT_AUDIO", + "midpoint_views_video": "METRIC_RICH_MEDIA_VIDEO_MIDPOINTS", + "moat_video_fee_advertiser_currency": "METRIC_FEE18_ADVERTISER", + "moat_video_fee_partner_currency": "METRIC_FEE18_PARTNER", + "moat_video_fee_usd": "METRIC_FEE18_USD", + "nielsen_digital_ad_ratings_fee_advertiser_currency": "METRIC_FEE19_ADVERTISER", + "nielsen_digital_ad_ratings_fee_partner_currency": "METRIC_FEE19_PARTNER", + "nielsen_digital_ad_ratings_fee_usd": "METRIC_FEE19_USD", + "pauses_audio": "METRIC_PAUSES_AUDIO", + "pauses_video": "METRIC_RICH_MEDIA_VIDEO_PAUSES", + "platform_fee_advertiser_currency": "METRIC_PLATFORM_FEE_ADVERTISER", + "platform_fee_partner_currency": "METRIC_PLATFORM_FEE_PARTNER", + "platform_fee_usd": "METRIC_PLATFORM_FEE_USD", + "platform_fee_rate": "METRIC_PLATFORM_FEE_RATE", + "post_view_conversions__cross_environment": "METRIC_POST_VIEW_CONVERSIONS_CROSS_ENVIRONMENT", + "premium_fee_advertiser_currency": "METRIC_PREMIUM_FEE_ADVERTISER_CURRENCY", + "profit_advertiser_currency": "METRIC_PROFIT_ADVERTISER", + "profit_partner_currency": "METRIC_PROFIT_PARTNER", + "profit_usd": "METRIC_PROFIT_USD", + "profit_ecpm_advertiser_currency": "METRIC_PROFIT_ECPM_ADVERTISER", + "profit_ecpm_partner_currency": "METRIC_PROFIT_ECPM_PARTNER", + "profit_ecpm_usd": "METRIC_PROFIT_ECPM_USD", + "profit_margin": "METRIC_PROFIT_MARGIN", + "profit_viewable_ecpm_advertiser_currency": "METRIC_PROFIT_VIEWABLE_ECPM_ADVERTISER", + "profit_viewable_ecpm_partner_currency": "METRIC_PROFIT_VIEWABLE_ECPM_PARTNER", + "profit_viewable_ecpm_usd": "METRIC_PROFIT_VIEWABLE_ECPM_USD", + "programmatic_guaranteed_impressions_passed_due_to_frequency": "METRIC_PROGRAMMATIC_GUARANTEED_IMPRESSIONS_PASSED_DUE_TO_FREQUENCY", + "programmatic_guaranteed_savings_re_invested_due_to_frequency_advertiser_currency": "METRIC_PROGRAMMATIC_GUARANTEED_SAVINGS_RE_INVESTED_DUE_TO_FREQUENCY_ADVERTISER_CURRENCY", + "refund_billable_cost_advertiser_currency": "METRIC_REFUND_BILLABLE_COST_ADVERTISER_CURRENCY", + "refund_media_cost_advertiser_currency": "METRIC_REFUND_MEDIA_COST_ADVERTISER_CURRENCY", + "refund_platform_fee_advertiser_currency": "METRIC_REFUND_PLATFORM_FEE_ADVERTISER_CURRENCY", + "revenue_advertiser_currency": "METRIC_REVENUE_ADVERTISER", + "revenue_partner_currency": "METRIC_REVENUE_PARTNER", + "revenue_usd": "METRIC_REVENUE_USD", + "revenue_ecpa_advertiser_currency": "METRIC_REVENUE_ECPA_ADVERTISER", + "revenue_ecpa_partner_currency": "METRIC_REVENUE_ECPA_PARTNER", + "revenue_ecpa_pc_advertiser_currency": "METRIC_REVENUE_ECPAPC_ADVERTISER", + "revenue_ecpa_pc_partner_currency": "METRIC_REVENUE_ECPAPC_PARTNER", + "revenue_ecpa_pc_usd": "METRIC_REVENUE_ECPAPC_USD", + "revenue_ecpa_pv_advertiser_currency": "METRIC_REVENUE_ECPAPV_ADVERTISER", + "revenue_ecpa_pv_partner_currency": "METRIC_REVENUE_ECPAPV_PARTNER", + "revenue_ecpa_pv_usd": "METRIC_REVENUE_ECPAPV_USD", + "revenue_ecpa_usd": "METRIC_REVENUE_ECPA_USD", + "revenue_ecpc_advertiser_currency": "METRIC_REVENUE_ECPC_ADVERTISER", + "revenue_ecpc_partner_currency": "METRIC_REVENUE_ECPC_PARTNER", + "revenue_ecpc_usd": "METRIC_REVENUE_ECPC_USD", + "revenue_ecpe_advertiser_currency": "METRIC_TRUEVIEW_AVERAGE_CPE_ADVERTISER", + "revenue_ecpe_partner_currency": "METRIC_TRUEVIEW_AVERAGE_CPE_PARTNER", + "revenue_ecpe_usd": "METRIC_TRUEVIEW_AVERAGE_CPE_USD", + "revenue_ecpm_advertiser_currency": "METRIC_REVENUE_ECPM_ADVERTISER", + "revenue_ecpm_partner_currency": "METRIC_REVENUE_ECPM_PARTNER", + "revenue_ecpm_usd": "METRIC_REVENUE_ECPM_USD", + "revenue_ecpv_advertiser_currency": "METRIC_TRUEVIEW_CPV_ADVERTISER", + "revenue_ecpv_partner_currency": "METRIC_TRUEVIEW_CPV_PARTNER", + "revenue_ecpv_usd": "METRIC_TRUEVIEW_CPV_USD", + "revenue_viewable_ecpm_advertiser_currency": "METRIC_REVENUE_VIEWABLE_ECPM_ADVERTISER", + "revenue_viewable_ecpm_partner_currency": "METRIC_REVENUE_VIEWABLE_ECPM_PARTNER", + "revenue_viewable_ecpm_usd": "METRIC_REVENUE_VIEWABLE_ECPM_USD", + "rich_media_engagements": "METRIC_RICH_MEDIA_ENGAGEMENTS", + "scrolls": "METRIC_RICH_MEDIA_SCROLLS", + "shoplocal_fee_advertiser_currency": "METRIC_FEE14_ADVERTISER", + "shoplocal_fee_partner_currency": "METRIC_FEE14_PARTNER", + "shoplocal_fee_usd": "METRIC_FEE14_USD", + "skips_video": "METRIC_RICH_MEDIA_VIDEO_SKIPS", + "starts_audio": "METRIC_STARTS_AUDIO", + "starts_video": "METRIC_RICH_MEDIA_VIDEO_PLAYS", + "stops_audio": "METRIC_STOPS_AUDIO", + "teracent_fee_advertiser_currency": "METRIC_FEE8_ADVERTISER", + "teracent_fee_partner_currency": "METRIC_FEE8_PARTNER", + "teracent_fee_usd": "METRIC_FEE8_USD", + "third_party_ad_server_fee_advertiser_currency": "METRIC_FEE2_ADVERTISER", + "third_party_ad_server_fee_partner_currency": "METRIC_FEE2_PARTNER", + "third_party_ad_server_fee_usd": "METRIC_FEE2_USD", + "third_quartile_audio": "METRIC_THIRD_QUARTILE_AUDIO", + "third_quartile_views_video": "METRIC_RICH_MEDIA_VIDEO_THIRD_QUARTILE_COMPLETES", + "timers": "METRIC_TIMERS", + "total_conversions__cross_environment": "METRIC_TOTAL_CONVERSIONS_CROSS_ENVIRONMENT", + "total_display_time": "METRIC_TOTAL_DISPLAY_TIME", + "total_impression_custom_value": "METRIC_TOTAL_IMPRESSION_CUSTOM_VALUE", + "total_interaction_time": "METRIC_TOTAL_INTERACTION_TIME", + "total_media_cost_advertiser_currency": "METRIC_TOTAL_MEDIA_COST_ADVERTISER", + "total_media_cost_partner_currency": "METRIC_TOTAL_MEDIA_COST_PARTNER", + "total_media_cost_usd": "METRIC_TOTAL_MEDIA_COST_USD", + "total_media_cost_ecpa_advertiser_currency": "METRIC_TOTAL_MEDIA_COST_ECPA_ADVERTISER", + "total_media_cost_ecpa_partner_currency": "METRIC_TOTAL_MEDIA_COST_ECPA_PARTNER", + "total_media_cost_ecpa_pc_advertiser_currency": "METRIC_TOTAL_MEDIA_COST_ECPAPC_ADVERTISER", + "total_media_cost_ecpa_pc_partner_currency": "METRIC_TOTAL_MEDIA_COST_ECPAPC_PARTNER", + "total_media_cost_ecpa_pc_usd": "METRIC_TOTAL_MEDIA_COST_ECPAPC_USD", + "total_media_cost_ecpa_pv_advertiser_currency": "METRIC_TOTAL_MEDIA_COST_ECPAPV_ADVERTISER", + "total_media_cost_ecpa_pv_partner_currency": "METRIC_TOTAL_MEDIA_COST_ECPAPV_PARTNER", + "total_media_cost_ecpa_pv_usd": "METRIC_TOTAL_MEDIA_COST_ECPAPV_USD", + "total_media_cost_ecpa_usd": "METRIC_TOTAL_MEDIA_COST_ECPA_USD", + "total_media_cost_ecpc_advertiser_currency": "METRIC_TOTAL_MEDIA_COST_ECPC_ADVERTISER", + "total_media_cost_ecpc_partner_currency": "METRIC_TOTAL_MEDIA_COST_ECPC_PARTNER", + "total_media_cost_ecpc_usd": "METRIC_TOTAL_MEDIA_COST_ECPC_USD", + "total_media_cost_ecpm_advertiser_currency": "METRIC_TOTAL_MEDIA_COST_ECPM_ADVERTISER", + "total_media_cost_ecpm_partner_currency": "METRIC_TOTAL_MEDIA_COST_ECPM_PARTNER", + "total_media_cost_ecpm_usd": "METRIC_TOTAL_MEDIA_COST_ECPM_USD", + "total_media_cost_viewable_ecpm_advertiser_currency": "METRIC_TOTAL_MEDIA_COST_VIEWABLE_ECPM_ADVERTISER", + "total_media_cost_viewable_ecpm_partner_currency": "METRIC_TOTAL_MEDIA_COST_VIEWABLE_ECPM_PARTNER", + "total_media_cost_viewable_ecpm_usd": "METRIC_TOTAL_MEDIA_COST_VIEWABLE_ECPM_USD", + "total_video_media_cost_ecpcv_advertiser_currency": "METRIC_TOTAL_MEDIA_COST_ECPCV_ADVERTISER", + "total_video_media_cost_ecpcv_partner_currency": "METRIC_TOTAL_MEDIA_COST_ECPCV_PARTNER", + "total_video_media_cost_ecpcv_usd": "METRIC_TOTAL_MEDIA_COST_ECPCV_USD", + "tracked_ads": "METRIC_TRACKED_ADS", + "trueview_general_invalid_traffic_givt_views": "METRIC_TRUEVIEW_GENERAL_INVALID_TRAFFIC_GIVT_VIEWS", + "trueview_invalid_views": "METRIC_TRUEVIEW_INVALID_VIEWS", + "trustmetrics_fee_advertiser_currency": "METRIC_FEE15_ADVERTISER", + "trustmetrics_fee_partner_currency": "METRIC_FEE15_PARTNER", + "trustmetrics_fee_usd": "METRIC_FEE15_USD", + "verifiable_impressions": "METRIC_VERIFIABLE_IMPRESSIONS", + "video_client_cost_ecpcv_advertiser_currency": "METRIC_VIDEO_CLIENT_COST_ECPCV_ADVERTISER_CURRENCY", + "video_media_cost_ecpcv_advertiser_currency": "METRIC_MEDIA_COST_ECPCV_ADVERTISER", + "video_media_cost_ecpcv_partner_currency": "METRIC_MEDIA_COST_ECPCV_PARTNER", + "video_media_cost_ecpcv_usd": "METRIC_MEDIA_COST_ECPCV_USD", + "vizu_fee_advertiser_currency": "METRIC_FEE6_ADVERTISER", + "vizu_fee_partner_currency": "METRIC_FEE6_PARTNER", + "vizu_fee_usd": "METRIC_FEE6_USD", + "youtube_view_rate": "METRIC_TRUEVIEW_VIEW_RATE", + "youtube_views": "METRIC_TRUEVIEW_VIEWS", + "pct_composition_impressions": "METRIC_DEMO_COMPOSITION_IMPRESSION", + "pct_composition_reach": "METRIC_VIRTUAL_PEOPLE_IMPRESSION_REACH_SHARE_PERCENT", + "pct_population_reach": "METRIC_VIRTUAL_PEOPLE_IMPRESSION_REACH_PERCENT", + "population": "METRIC_DEMO_POPULATION", + "target_rating_points": "METRIC_TARGET_RATING_POINTS", + "unique_reach_viewable_impression_reach": "METRIC_VIRTUAL_PEOPLE_VIEWABLE_IMPRESSION_REACH_BY_DEMO", + "viewable_target_rating_points": "METRIC_VIEWABLE_GROSS_RATING_POINTS", + "viewable_impressions": "METRIC_GRP_CORRECTED_VIEWABLE_IMPRESSIONS", + "pct_viewable_composition_impressions": "METRIC_GRP_CORRECTED_VIEWABLE_IMPRESSIONS_SHARE_PERCENT", + "pct_viewable_composition_reach": "METRIC_VIRTUAL_PEOPLE_VIEWABLE_IMPRESSION_REACH_SHARE_PERCENT", + "pct_viewable_population_reach": "METRIC_VIRTUAL_PEOPLE_VIEWABLE_IMPRESSION_REACH_PERCENT", +} diff --git a/airbyte-integrations/connectors/source-dv-360/source_dv_360/queries/query_template.json b/airbyte-integrations/connectors/source-dv-360/source_dv_360/queries/query_template.json new file mode 100644 index 0000000000000..4268df6b9cabe --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/source_dv_360/queries/query_template.json @@ -0,0 +1,33 @@ +{ + "kind": "doubleclickbidmanager#query", + "queryId": "0", + "metadata": { + "title": "", + "dataRange": "CUSTOM_DATES", + "format": "CSV", + "running": false, + "googleCloudStoragePathForLatestReport": "", + "latestReportRunTimeMs": "0", + "sendNotification": false + }, + "params": { + "type": "", + "groupBys": [], + "filters": [ + { + "type": "FILTER_PARTNER", + "value": "" + } + ], + "metrics": [], + "options": { + "includeOnlyTargetedUserLists": false + } + }, + "schedule": { + "frequency": "ONE_TIME" + }, + "reportDataStartTimeMs": "", + "reportDataEndTimeMs": "", + "timezoneCode": "UTC" +} diff --git a/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/audience_composition.json b/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/audience_composition.json new file mode 100644 index 0000000000000..aea493ba027d1 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/audience_composition.json @@ -0,0 +1,51 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "audience_list": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"] + }, + "eligible_cookies_on_first_party_audience_list": { + "type": ["null", "string"] + }, + "eligible_cookies_on_third_party_audience_list_and_interest": { + "type": ["null", "string"] + }, + "first_party_audience_list": { + "type": ["null", "string"] + }, + "first_party_audience_list_cost": { + "type": ["null", "string"] + }, + "first_party_audience_list_id": { + "type": ["null", "string"] + }, + "first_party_audience_list_type": { + "type": ["null", "string"] + }, + "match_ratio": { + "type": ["null", "string"] + }, + "third_party_audience_list": { + "type": ["null", "string"] + }, + "third_party_audience_list_cost": { + "type": ["null", "string"] + }, + "third_party_audience_list_id": { + "type": ["null", "string"] + }, + "third_party_audience_list_type": { + "type": ["null", "string"] + }, + "potential_impressions": { + "type": ["null", "string"] + }, + "unique_cookies_with_impressions": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/floodlight.json b/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/floodlight.json new file mode 100644 index 0000000000000..537336600b16f --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/floodlight.json @@ -0,0 +1,177 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "advertiser": { + "type": ["null", "string"] + }, + "advertiser_currency": { + "type": ["null", "string"] + }, + "advertiser_id": { + "type": ["null", "string"] + }, + "advertiser_integration_code": { + "type": ["null", "string"] + }, + "advertiser_status": { + "type": ["null", "string"] + }, + "advertiser_time_zone": { + "type": ["null", "string"] + }, + "app_url": { + "type": ["null", "string"] + }, + "app_url_excluded": { + "type": ["null", "string"] + }, + "app_url_id": { + "type": ["null", "string"] + }, + "campaign": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "category": { + "type": ["null", "string"] + }, + "cm_placement_id": { + "type": ["null", "string"] + }, + "creative": { + "type": ["null", "string"] + }, + "creative_asset": { + "type": ["null", "string"] + }, + "creative_attributes": { + "type": ["null", "string"] + }, + "creative_height": { + "type": ["null", "string"] + }, + "creative_id": { + "type": ["null", "string"] + }, + "creative_integration_code": { + "type": ["null", "string"] + }, + "creative_rendered_in_amp": { + "type": ["null", "string"] + }, + "creative_size": { + "type": ["null", "string"] + }, + "creative_source": { + "type": ["null", "string"] + }, + "creative_status": { + "type": ["null", "string"] + }, + "creative_type": { + "type": ["null", "string"] + }, + "creative_width": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"] + }, + "day_of_week": { + "type": ["null", "string"] + }, + "exchange": { + "type": ["null", "string"] + }, + "exchange_code": { + "type": ["null", "string"] + }, + "exchange_id": { + "type": ["null", "string"] + }, + "floodlight_activity": { + "type": ["null", "string"] + }, + "floodlight_activity_id": { + "type": ["null", "string"] + }, + "insertion_order": { + "type": ["null", "string"] + }, + "insertion_order_integration_code": { + "type": ["null", "string"] + }, + "insertion_order_status": { + "type": ["null", "string"] + }, + "line_item": { + "type": ["null", "string"] + }, + "line_item_id": { + "type": ["null", "string"] + }, + "line_item_integration_code": { + "type": ["null", "string"] + }, + "line_item_status": { + "type": ["null", "string"] + }, + "line_item_type": { + "type": ["null", "string"] + }, + "month": { + "type": ["null", "string"] + }, + "order_id": { + "type": ["null", "string"] + }, + "partner": { + "type": ["null", "string"] + }, + "partner_currency": { + "type": ["null", "string"] + }, + "partner_id": { + "type": ["null", "string"] + }, + "partner_status": { + "type": ["null", "string"] + }, + "targeted_data_providers": { + "type": ["null", "string"] + }, + "year": { + "type": ["null", "string"] + }, + "cm_post_click_revenue": { + "type": ["null", "string"] + }, + "cm_post_view_revenue": { + "type": ["null", "string"] + }, + "cookie_consented_floodlight_impressions": { + "type": ["null", "string"] + }, + "cookie_unconsented_floodlight_impressions": { + "type": ["null", "string"] + }, + "duplicate_floodlight_impressions": { + "type": ["null", "string"] + }, + "floodlight_impressions": { + "type": ["null", "string"] + }, + "post_click_conversions": { + "type": ["null", "string"] + }, + "post_view_conversions": { + "type": ["null", "string"] + }, + "total_conversions": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/reach.json b/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/reach.json new file mode 100644 index 0000000000000..a63c8efc86cf2 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/reach.json @@ -0,0 +1,114 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "advertiser": { + "type": ["null", "string"] + }, + "advertiser_id": { + "type": ["null", "string"] + }, + "advertiser_integration_code": { + "type": ["null", "string"] + }, + "advertiser_status": { + "type": ["null", "string"] + }, + "app_url": { + "type": ["null", "string"] + }, + "app_url_excluded": { + "type": ["null", "string"] + }, + "campaign": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "cm_placement_id": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "country_id": { + "type": ["null", "string"] + }, + "creative": { + "type": ["null", "string"] + }, + "creative_id": { + "type": ["null", "string"] + }, + "creative_integration_code": { + "type": ["null", "string"] + }, + "creative_source": { + "type": ["null", "string"] + }, + "creative_status": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"] + }, + "insertion_order": { + "type": ["null", "string"] + }, + "insertion_order_id": { + "type": ["null", "string"] + }, + "insertion_order_integration_code": { + "type": ["null", "string"] + }, + "insertion_order_status": { + "type": ["null", "string"] + }, + "inventory_source": { + "type": ["null", "string"] + }, + "line_item": { + "type": ["null", "string"] + }, + "line_item_id": { + "type": ["null", "string"] + }, + "line_item_integration_code": { + "type": ["null", "string"] + }, + "line_item_status": { + "type": ["null", "string"] + }, + "partner": { + "type": ["null", "string"] + }, + "partner_id": { + "type": ["null", "string"] + }, + "partner_status": { + "type": ["null", "string"] + }, + "targeted_data_providers": { + "type": ["null", "string"] + }, + "cookie_reach_average_impression_frequency": { + "type": ["null", "string"] + }, + "cookie_reach_impression_reach": { + "type": ["null", "string"] + }, + "unique_reach_average_impression_frequency": { + "type": ["null", "string"] + }, + "unique_reach_click_reach": { + "type": ["null", "string"] + }, + "unique_reach_impression_reach": { + "type": ["null", "string"] + }, + "unique_reach_total_reach": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/standard.json b/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/standard.json new file mode 100644 index 0000000000000..4c96e3c339eb7 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/standard.json @@ -0,0 +1,1506 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "active_view_custom_metric_id": { + "type": ["null", "string"] + }, + "active_view_custom_metric_name": { + "type": ["null", "string"] + }, + "ad_position": { + "type": ["null", "string"] + }, + "ad_type": { + "type": ["null", "string"] + }, + "advertiser": { + "type": ["null", "string"] + }, + "advertiser_currency": { + "type": ["null", "string"] + }, + "advertiser_id": { + "type": ["null", "string"] + }, + "advertiser_integration_code": { + "type": ["null", "string"] + }, + "advertiser_status": { + "type": ["null", "string"] + }, + "advertiser_time_zone": { + "type": ["null", "string"] + }, + "algorithm": { + "type": ["null", "string"] + }, + "algorithm_id": { + "type": ["null", "string"] + }, + "amp_page_request": { + "type": ["null", "string"] + }, + "app_url": { + "type": ["null", "string"] + }, + "app_url_excluded": { + "type": ["null", "string"] + }, + "app_url_id": { + "type": ["null", "string"] + }, + "attributed_userlist": { + "type": ["null", "string"] + }, + "attributed_userlist_cost": { + "type": ["null", "string"] + }, + "attributed_userlist_id": { + "type": ["null", "string"] + }, + "attributed_userlist_type": { + "type": ["null", "string"] + }, + "attribution_model": { + "type": ["null", "string"] + }, + "audience_list": { + "type": ["null", "string"] + }, + "audience_list_cost": { + "type": ["null", "string"] + }, + "audience_list_id": { + "type": ["null", "string"] + }, + "audience_list_type": { + "type": ["null", "string"] + }, + "audience_name": { + "type": ["null", "string"] + }, + "audience_type": { + "type": ["null", "string"] + }, + "authorized_seller_state": { + "type": ["null", "string"] + }, + "billable_outcome": { + "type": ["null", "string"] + }, + "brand_lift_type": { + "type": ["null", "string"] + }, + "browser_id": { + "type": ["null", "string"] + }, + "budget_segment_description": { + "type": ["null", "string"] + }, + "campaign": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "category": { + "type": ["null", "string"] + }, + "channel": { + "type": ["null", "string"] + }, + "channel_id": { + "type": ["null", "string"] + }, + "channel_type": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "city_id": { + "type": ["null", "string"] + }, + "cm_placement_id": { + "type": ["null", "string"] + }, + "companion_creative": { + "type": ["null", "string"] + }, + "companion_creative_id": { + "type": ["null", "string"] + }, + "companion_creative_size": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "country_id": { + "type": ["null", "string"] + }, + "creative": { + "type": ["null", "string"] + }, + "creative_asset": { + "type": ["null", "string"] + }, + "creative_attributes": { + "type": ["null", "string"] + }, + "creative_height": { + "type": ["null", "string"] + }, + "creative_id": { + "type": ["null", "string"] + }, + "creative_integration_code": { + "type": ["null", "string"] + }, + "creative_rendered_in_amp": { + "type": ["null", "string"] + }, + "creative_size": { + "type": ["null", "string"] + }, + "creative_source": { + "type": ["null", "string"] + }, + "creative_status": { + "type": ["null", "string"] + }, + "creative_type": { + "type": ["null", "string"] + }, + "creative_width": { + "type": ["null", "string"] + }, + "data_provider": { + "type": ["null", "string"] + }, + "data_provider_id": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"] + }, + "day_of_week": { + "type": ["null", "string"] + }, + "detailed_demographics": { + "type": ["null", "string"] + }, + "detailed_demographics_id": { + "type": ["null", "string"] + }, + "device": { + "type": ["null", "string"] + }, + "device_make": { + "type": ["null", "string"] + }, + "device_model": { + "type": ["null", "string"] + }, + "device_type": { + "type": ["null", "string"] + }, + "digital_content_label": { + "type": ["null", "string"] + }, + "dma": { + "type": ["null", "string"] + }, + "dma_code": { + "type": ["null", "string"] + }, + "exchange": { + "type": ["null", "string"] + }, + "exchange_code": { + "type": ["null", "string"] + }, + "exchange_id": { + "type": ["null", "string"] + }, + "extension": { + "type": ["null", "string"] + }, + "extension_status": { + "type": ["null", "string"] + }, + "extension_type": { + "type": ["null", "string"] + }, + "floodlight_activity": { + "type": ["null", "string"] + }, + "floodlight_activity_id": { + "type": ["null", "string"] + }, + "format": { + "type": ["null", "string"] + }, + "gmail_age": { + "type": ["null", "string"] + }, + "gmail_city": { + "type": ["null", "string"] + }, + "gmail_country": { + "type": ["null", "string"] + }, + "gmail_device_type": { + "type": ["null", "string"] + }, + "gmail_gender": { + "type": ["null", "string"] + }, + "gmail_region": { + "type": ["null", "string"] + }, + "gmail_remarketing_list": { + "type": ["null", "string"] + }, + "household_income": { + "type": ["null", "string"] + }, + "impression_counting_method": { + "type": ["null", "string"] + }, + "insertion_order": { + "type": ["null", "string"] + }, + "insertion_order_daily_frequency": { + "type": ["null", "string"] + }, + "insertion_order_id": { + "type": ["null", "string"] + }, + "insertion_order_integration_code": { + "type": ["null", "string"] + }, + "insertion_order_status": { + "type": ["null", "string"] + }, + "interest": { + "type": ["null", "string"] + }, + "inventory_commitment_type": { + "type": ["null", "string"] + }, + "inventory_delivery_method": { + "type": ["null", "string"] + }, + "inventory_rate_type": { + "type": ["null", "string"] + }, + "inventory_source": { + "type": ["null", "string"] + }, + "inventory_source_group": { + "type": ["null", "string"] + }, + "inventory_source_group_id": { + "type": ["null", "string"] + }, + "inventory_source_id": { + "type": ["null", "string"] + }, + "inventory_source_id_external": { + "type": ["null", "string"] + }, + "inventory_source_type": { + "type": ["null", "string"] + }, + "isp_or_carrier": { + "type": ["null", "string"] + }, + "isp_or_carrier_id": { + "type": ["null", "string"] + }, + "keyword": { + "type": ["null", "string"] + }, + "life_event": { + "type": ["null", "string"] + }, + "life_events": { + "type": ["null", "string"] + }, + "line_item": { + "type": ["null", "string"] + }, + "line_item_daily_frequency": { + "type": ["null", "string"] + }, + "line_item_id": { + "type": ["null", "string"] + }, + "line_item_integration_code": { + "type": ["null", "string"] + }, + "line_item_lifetime_frequency": { + "type": ["null", "string"] + }, + "line_item_status": { + "type": ["null", "string"] + }, + "line_item_type": { + "type": ["null", "string"] + }, + "max_video_duration": { + "type": ["null", "string"] + }, + "measurement_source": { + "type": ["null", "string"] + }, + "month": { + "type": ["null", "string"] + }, + "operating_system": { + "type": ["null", "string"] + }, + "partner": { + "type": ["null", "string"] + }, + "partner_currency": { + "type": ["null", "string"] + }, + "partner_id": { + "type": ["null", "string"] + }, + "partner_status": { + "type": ["null", "string"] + }, + "platform": { + "type": ["null", "string"] + }, + "playback_method": { + "type": ["null", "string"] + }, + "position_in_content": { + "type": ["null", "string"] + }, + "public_inventory": { + "type": ["null", "string"] + }, + "publisher_property": { + "type": ["null", "string"] + }, + "publisher_property_id": { + "type": ["null", "string"] + }, + "publisher_property_section": { + "type": ["null", "string"] + }, + "publisher_property_section_id": { + "type": ["null", "string"] + }, + "refund_reason": { + "type": ["null", "string"] + }, + "region": { + "type": ["null", "string"] + }, + "region_id": { + "type": ["null", "string"] + }, + "rewarded": { + "type": ["null", "string"] + }, + "sensitive_category": { + "type": ["null", "string"] + }, + "served_pixel_density": { + "type": ["null", "string"] + }, + "targeted_data_providers": { + "type": ["null", "string"] + }, + "time_of_day": { + "type": ["null", "string"] + }, + "time_to_conversion": { + "type": ["null", "string"] + }, + "variant_id": { + "type": ["null", "string"] + }, + "variant_name": { + "type": ["null", "string"] + }, + "variant_version": { + "type": ["null", "string"] + }, + "verification_video_player_size": { + "type": ["null", "string"] + }, + "verification_video_position": { + "type": ["null", "string"] + }, + "video_continuous_play": { + "type": ["null", "string"] + }, + "video_player_size": { + "type": ["null", "string"] + }, + "video_skippable_support": { + "type": ["null", "string"] + }, + "week": { + "type": ["null", "string"] + }, + "year": { + "type": ["null", "string"] + }, + "zip_code": { + "type": ["null", "string"] + }, + "zip_code_id": { + "type": ["null", "string"] + }, + "pct_clicks_leading_to_conversions": { + "type": ["null", "string"] + }, + "pct_impressions_leading_to_conversions": { + "type": ["null", "string"] + }, + "pct_impressions_with_positive_custom_value": { + "type": ["null", "string"] + }, + "active_view_pct_audible_and_visible_at_completion": { + "type": ["null", "string"] + }, + "active_view_pct_audible_and_visible_at_first_quartile": { + "type": ["null", "string"] + }, + "active_view_pct_audible_and_visible_at_midpoint": { + "type": ["null", "string"] + }, + "active_view_pct_audible_and_visible_at_start": { + "type": ["null", "string"] + }, + "active_view_pct_audible_and_visible_at_third_quartile": { + "type": ["null", "string"] + }, + "active_view_pct_audible_impressions": { + "type": ["null", "string"] + }, + "active_view_pct_full_screen": { + "type": ["null", "string"] + }, + "active_view_pct_fully_on_screen_2_sec": { + "type": ["null", "string"] + }, + "active_view_pct_in_background": { + "type": ["null", "string"] + }, + "active_view_pct_measurable_impressions": { + "type": ["null", "string"] + }, + "active_view_pct_of_ad_played": { + "type": ["null", "string"] + }, + "active_view_pct_of_completed_impressions_audible_and_visible": { + "type": ["null", "string"] + }, + "active_view_pct_of_completed_impressions_visible": { + "type": ["null", "string"] + }, + "active_view_pct_of_first_quartile_impressions_audible_and_visible": { + "type": ["null", "string"] + }, + "active_view_pct_of_first_quartile_impressions_visible": { + "type": ["null", "string"] + }, + "active_view_pct_of_midpoint_impressions_audible_and_visible": { + "type": ["null", "string"] + }, + "active_view_pct_of_midpoint_impressions_visible": { + "type": ["null", "string"] + }, + "active_view_pct_of_third_quartile_impressions_audible_and_visible": { + "type": ["null", "string"] + }, + "active_view_pct_of_third_quartile_impressions_visible": { + "type": ["null", "string"] + }, + "active_view_pct_play_time_audible": { + "type": ["null", "string"] + }, + "active_view_pct_play_time_audible_and_visible": { + "type": ["null", "string"] + }, + "active_view_pct_play_time_visible": { + "type": ["null", "string"] + }, + "active_view_pct_viewable_impressions": { + "type": ["null", "string"] + }, + "active_view_pct_visible_10_seconds": { + "type": ["null", "string"] + }, + "active_view_pct_visible_at_completion": { + "type": ["null", "string"] + }, + "active_view_pct_visible_at_first_quartile": { + "type": ["null", "string"] + }, + "active_view_pct_visible_at_midpoint": { + "type": ["null", "string"] + }, + "active_view_pct_visible_at_start": { + "type": ["null", "string"] + }, + "active_view_pct_visible_at_third_quartile": { + "type": ["null", "string"] + }, + "active_view_audible_and_fully_on_screen_for_half_of_duration_15_sec_cap_impressions": { + "type": ["null", "string"] + }, + "active_view_audible_and_fully_on_screen_for_half_of_duration_15_sec_cap_measurable_impressions": { + "type": ["null", "string"] + }, + "active_view_audible_and_fully_on_screen_for_half_of_duration_15_sec_cap_rate": { + "type": ["null", "string"] + }, + "active_view_audible_and_fully_on_screen_for_half_of_duration_trueview_impressions": { + "type": ["null", "string"] + }, + "active_view_audible_and_fully_on_screen_for_half_of_duration_trueview_measurable_impressions": { + "type": ["null", "string"] + }, + "active_view_audible_and_fully_on_screen_for_half_of_duration_trueview_rate": { + "type": ["null", "string"] + }, + "active_view_average_viewable_time_seconds": { + "type": ["null", "string"] + }, + "active_view_custom_metric_measurable_impressions": { + "type": ["null", "string"] + }, + "active_view_custom_metric_viewable_impressions": { + "type": ["null", "string"] + }, + "active_view_custom_metric_viewable_rate": { + "type": ["null", "string"] + }, + "active_view_eligible_impressions": { + "type": ["null", "string"] + }, + "active_view_impression_distribution_not_measurable": { + "type": ["null", "string"] + }, + "active_view_impression_distribution_not_viewable": { + "type": ["null", "string"] + }, + "active_view_impression_distribution_viewable": { + "type": ["null", "string"] + }, + "active_view_impressions_audible_and_visible_at_completion": { + "type": ["null", "string"] + }, + "active_view_impressions_visible_10_seconds": { + "type": ["null", "string"] + }, + "active_view_measurable_impressions": { + "type": ["null", "string"] + }, + "active_view_not_measurable_impressions": { + "type": ["null", "string"] + }, + "active_view_not_viewable_impressions": { + "type": ["null", "string"] + }, + "active_view_viewable_impressions": { + "type": ["null", "string"] + }, + "adlingo_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "adloox_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "adloox_fee_partner_currency": { + "type": ["null", "string"] + }, + "adloox_fee_usd": { + "type": ["null", "string"] + }, + "adloox_pre_bid_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "adloox_pre_bid_fee_partner_currency": { + "type": ["null", "string"] + }, + "adloox_pre_bid_fee_usd": { + "type": ["null", "string"] + }, + "adsafe_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "adsafe_fee_partner_currency": { + "type": ["null", "string"] + }, + "adsafe_fee_usd": { + "type": ["null", "string"] + }, + "adxpose_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "adxpose_fee_partner_currency": { + "type": ["null", "string"] + }, + "adxpose_fee_usd": { + "type": ["null", "string"] + }, + "agency_trading_desk_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "agency_trading_desk_fee_partner_currency": { + "type": ["null", "string"] + }, + "agency_trading_desk_fee_usd": { + "type": ["null", "string"] + }, + "aggregate_knowledge_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "aggregate_knowledge_fee_partner_currency": { + "type": ["null", "string"] + }, + "aggregate_knowledge_fee_usd": { + "type": ["null", "string"] + }, + "audio_client_cost_ecpcl_advertiser_currency": { + "type": ["null", "string"] + }, + "audio_media_cost_ecpcl_advertiser_currency": { + "type": ["null", "string"] + }, + "audio_mutes_audio": { + "type": ["null", "string"] + }, + "audio_mutes_video": { + "type": ["null", "string"] + }, + "audio_revenue_ecpcl_advertiser_currency": { + "type": ["null", "string"] + }, + "audio_unmutes_audio": { + "type": ["null", "string"] + }, + "audio_unmutes_video": { + "type": ["null", "string"] + }, + "average_display_time": { + "type": ["null", "string"] + }, + "average_interaction_time": { + "type": ["null", "string"] + }, + "begin_to_render_eligible_impressions": { + "type": ["null", "string"] + }, + "begin_to_render_impressions": { + "type": ["null", "string"] + }, + "billable_cost_advertiser_currency": { + "type": ["null", "string"] + }, + "billable_cost_partner_currency": { + "type": ["null", "string"] + }, + "billable_cost_usd": { + "type": ["null", "string"] + }, + "billable_impressions": { + "type": ["null", "string"] + }, + "click_rate_ctr": { + "type": ["null", "string"] + }, + "clicks": { + "type": ["null", "string"] + }, + "client_cost_advertiser_currency": { + "type": ["null", "string"] + }, + "client_cost_ecpa_advertiser_currency": { + "type": ["null", "string"] + }, + "client_cost_ecpa_pc_advertiser_currency": { + "type": ["null", "string"] + }, + "client_cost_ecpa_pv_advertiser_currency": { + "type": ["null", "string"] + }, + "client_cost_ecpc_advertiser_currency": { + "type": ["null", "string"] + }, + "client_cost_ecpm_advertiser_currency": { + "type": ["null", "string"] + }, + "client_cost_viewable_ecpm_advertiser_currency": { + "type": ["null", "string"] + }, + "cm_post_click_revenue": { + "type": ["null", "string"] + }, + "cm_post_click_revenue__cross_environment": { + "type": ["null", "string"] + }, + "cm_post_view_revenue": { + "type": ["null", "string"] + }, + "cm_post_view_revenue__cross_environment": { + "type": ["null", "string"] + }, + "companion_clicks_audio": { + "type": ["null", "string"] + }, + "companion_clicks_video": { + "type": ["null", "string"] + }, + "companion_impressions_audio": { + "type": ["null", "string"] + }, + "companion_impressions_video": { + "type": ["null", "string"] + }, + "complete_listens_audio": { + "type": ["null", "string"] + }, + "complete_views_video": { + "type": ["null", "string"] + }, + "completion_rate_audio": { + "type": ["null", "string"] + }, + "completion_rate_video": { + "type": ["null", "string"] + }, + "comscore_vce_in_doubleclick_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "comscore_vce_in_doubleclick_fee_partner_currency": { + "type": ["null", "string"] + }, + "comscore_vce_in_doubleclick_fee_usd": { + "type": ["null", "string"] + }, + "conversions_per_1000_impressions": { + "type": ["null", "string"] + }, + "cookie_unconsented_clicks": { + "type": ["null", "string"] + }, + "counters": { + "type": ["null", "string"] + }, + "cpm_fee_1_advertiser_currency": { + "type": ["null", "string"] + }, + "cpm_fee_1_partner_currency": { + "type": ["null", "string"] + }, + "cpm_fee_1_usd": { + "type": ["null", "string"] + }, + "cpm_fee_2_advertiser_currency": { + "type": ["null", "string"] + }, + "cpm_fee_2_partner_currency": { + "type": ["null", "string"] + }, + "cpm_fee_2_usd": { + "type": ["null", "string"] + }, + "cpm_fee_3_advertiser_currency": { + "type": ["null", "string"] + }, + "cpm_fee_3_partner_currency": { + "type": ["null", "string"] + }, + "cpm_fee_3_usd": { + "type": ["null", "string"] + }, + "cpm_fee_4_advertiser_currency": { + "type": ["null", "string"] + }, + "cpm_fee_4_partner_currency": { + "type": ["null", "string"] + }, + "cpm_fee_4_usd": { + "type": ["null", "string"] + }, + "cpm_fee_5_advertiser_currency": { + "type": ["null", "string"] + }, + "cpm_fee_5_partner_currency": { + "type": ["null", "string"] + }, + "cpm_fee_5_usd": { + "type": ["null", "string"] + }, + "custom_fee_1_advertiser_currency": { + "type": ["null", "string"] + }, + "custom_fee_2_advertiser_currency": { + "type": ["null", "string"] + }, + "custom_fee_3_advertiser_currency": { + "type": ["null", "string"] + }, + "custom_fee_4_advertiser_currency": { + "type": ["null", "string"] + }, + "custom_fee_5_advertiser_currency": { + "type": ["null", "string"] + }, + "data_fees_advertiser_currency": { + "type": ["null", "string"] + }, + "data_fees_partner_currency": { + "type": ["null", "string"] + }, + "data_fees_usd": { + "type": ["null", "string"] + }, + "data_management_platform_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "data_management_platform_fee_partner_currency": { + "type": ["null", "string"] + }, + "data_management_platform_fee_usd": { + "type": ["null", "string"] + }, + "doubleverify_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "doubleverify_fee_partner_currency": { + "type": ["null", "string"] + }, + "doubleverify_fee_usd": { + "type": ["null", "string"] + }, + "doubleverify_pre_bid_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "doubleverify_pre_bid_fee_partner_currency": { + "type": ["null", "string"] + }, + "doubleverify_pre_bid_fee_usd": { + "type": ["null", "string"] + }, + "engagement_rate": { + "type": ["null", "string"] + }, + "engagements": { + "type": ["null", "string"] + }, + "estimated_cpm_for_impressions_with_custom_value_advertiser_currency": { + "type": ["null", "string"] + }, + "estimated_total_cost_for_impressions_with_custom_value_advertiser_currency": { + "type": ["null", "string"] + }, + "evidon_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "evidon_fee_partner_currency": { + "type": ["null", "string"] + }, + "evidon_fee_usd": { + "type": ["null", "string"] + }, + "exits": { + "type": ["null", "string"] + }, + "expansions": { + "type": ["null", "string"] + }, + "first_quartile_audio": { + "type": ["null", "string"] + }, + "first_quartile_views_video": { + "type": ["null", "string"] + }, + "fullscreens_video": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_active_view_eligible_impressions": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_active_view_measurable_impressions": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_active_view_viewable_impressions": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_begin_to_render_impressions": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_clicks": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_impressions": { + "type": ["null", "string"] + }, + "general_invalid_traffic_givt_tracked_ads": { + "type": ["null", "string"] + }, + "gmail_conversions": { + "type": ["null", "string"] + }, + "gmail_post_click_conversions": { + "type": ["null", "string"] + }, + "gmail_post_view_conversions": { + "type": ["null", "string"] + }, + "impression_custom_value_cost": { + "type": ["null", "string"] + }, + "impressions": { + "type": ["null", "string"] + }, + "impressions_with_custom_value": { + "type": ["null", "string"] + }, + "impressions_with_positive_custom_value": { + "type": ["null", "string"] + }, + "integral_ad_science_pre_bid_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "integral_ad_science_pre_bid_fee_partner_currency": { + "type": ["null", "string"] + }, + "integral_ad_science_pre_bid_fee_usd": { + "type": ["null", "string"] + }, + "integral_ad_science_video_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "integral_ad_science_video_fee_partner_currency": { + "type": ["null", "string"] + }, + "integral_ad_science_video_fee_usd": { + "type": ["null", "string"] + }, + "interactive_impressions": { + "type": ["null", "string"] + }, + "invalid_active_view_eligible_impressions": { + "type": ["null", "string"] + }, + "invalid_active_view_measurable_impressions": { + "type": ["null", "string"] + }, + "invalid_active_view_viewable_impressions": { + "type": ["null", "string"] + }, + "invalid_begin_to_render_impressions": { + "type": ["null", "string"] + }, + "invalid_clicks": { + "type": ["null", "string"] + }, + "invalid_impressions": { + "type": ["null", "string"] + }, + "invalid_tracked_ads": { + "type": ["null", "string"] + }, + "media_cost_advertiser_currency": { + "type": ["null", "string"] + }, + "media_cost_partner_currency": { + "type": ["null", "string"] + }, + "media_cost_usd": { + "type": ["null", "string"] + }, + "media_cost_ecpa_advertiser_currency": { + "type": ["null", "string"] + }, + "media_cost_ecpa_partner_currency": { + "type": ["null", "string"] + }, + "media_cost_ecpa_pc_advertiser_currency": { + "type": ["null", "string"] + }, + "media_cost_ecpa_pv_advertiser_currency": { + "type": ["null", "string"] + }, + "media_cost_ecpa_usd": { + "type": ["null", "string"] + }, + "media_cost_ecpc_advertiser_currency": { + "type": ["null", "string"] + }, + "media_cost_ecpc_partner_currency": { + "type": ["null", "string"] + }, + "media_cost_ecpc_pc_partner_currency": { + "type": ["null", "string"] + }, + "media_cost_ecpc_pc_usd": { + "type": ["null", "string"] + }, + "media_cost_ecpc_pv_partner_currency": { + "type": ["null", "string"] + }, + "media_cost_ecpc_pv_usd": { + "type": ["null", "string"] + }, + "media_cost_ecpc_usd": { + "type": ["null", "string"] + }, + "media_cost_ecpm_advertiser_currency": { + "type": ["null", "string"] + }, + "media_cost_ecpm_partner_currency": { + "type": ["null", "string"] + }, + "media_cost_ecpm_usd": { + "type": ["null", "string"] + }, + "media_cost_viewable_ecpm_advertiser_currency": { + "type": ["null", "string"] + }, + "media_cost_viewable_ecpm_partner_currency": { + "type": ["null", "string"] + }, + "media_cost_viewable_ecpm_usd": { + "type": ["null", "string"] + }, + "media_fee_1_advertiser_currency": { + "type": ["null", "string"] + }, + "media_fee_1_partner_currency": { + "type": ["null", "string"] + }, + "media_fee_1_usd": { + "type": ["null", "string"] + }, + "media_fee_2_advertiser_currency": { + "type": ["null", "string"] + }, + "media_fee_2_partner_currency": { + "type": ["null", "string"] + }, + "media_fee_2_usd": { + "type": ["null", "string"] + }, + "media_fee_3_advertiser_currency": { + "type": ["null", "string"] + }, + "media_fee_3_partner_currency": { + "type": ["null", "string"] + }, + "media_fee_3_usd": { + "type": ["null", "string"] + }, + "media_fee_4_advertiser_currency": { + "type": ["null", "string"] + }, + "media_fee_4_partner_currency": { + "type": ["null", "string"] + }, + "media_fee_4_usd": { + "type": ["null", "string"] + }, + "media_fee_5_advertiser_currency": { + "type": ["null", "string"] + }, + "media_fee_5_partner_currency": { + "type": ["null", "string"] + }, + "media_fee_5_usd": { + "type": ["null", "string"] + }, + "mediacost_data_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "mediacost_data_fee_partner_currency": { + "type": ["null", "string"] + }, + "mediacost_data_fee_usd": { + "type": ["null", "string"] + }, + "midpoint_audio": { + "type": ["null", "string"] + }, + "midpoint_views_video": { + "type": ["null", "string"] + }, + "moat_video_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "moat_video_fee_partner_currency": { + "type": ["null", "string"] + }, + "moat_video_fee_usd": { + "type": ["null", "string"] + }, + "nielsen_digital_ad_ratings_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "nielsen_digital_ad_ratings_fee_partner_currency": { + "type": ["null", "string"] + }, + "nielsen_digital_ad_ratings_fee_usd": { + "type": ["null", "string"] + }, + "pauses_audio": { + "type": ["null", "string"] + }, + "pauses_video": { + "type": ["null", "string"] + }, + "platform_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "platform_fee_partner_currency": { + "type": ["null", "string"] + }, + "platform_fee_usd": { + "type": ["null", "string"] + }, + "platform_fee_rate": { + "type": ["null", "string"] + }, + "post_click_conversions": { + "type": ["null", "string"] + }, + "post_view_conversions": { + "type": ["null", "string"] + }, + "post_view_conversions__cross_environment": { + "type": ["null", "string"] + }, + "premium_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "profit_advertiser_currency": { + "type": ["null", "string"] + }, + "profit_partner_currency": { + "type": ["null", "string"] + }, + "profit_usd": { + "type": ["null", "string"] + }, + "profit_ecpm_advertiser_currency": { + "type": ["null", "string"] + }, + "profit_ecpm_partner_currency": { + "type": ["null", "string"] + }, + "profit_ecpm_usd": { + "type": ["null", "string"] + }, + "profit_margin": { + "type": ["null", "string"] + }, + "profit_viewable_ecpm_advertiser_currency": { + "type": ["null", "string"] + }, + "profit_viewable_ecpm_partner_currency": { + "type": ["null", "string"] + }, + "profit_viewable_ecpm_usd": { + "type": ["null", "string"] + }, + "programmatic_guaranteed_impressions_passed_due_to_frequency": { + "type": ["null", "string"] + }, + "programmatic_guaranteed_savings_re_invested_due_to_frequency_advertiser_currency": { + "type": ["null", "string"] + }, + "refund_billable_cost_advertiser_currency": { + "type": ["null", "string"] + }, + "refund_media_cost_advertiser_currency": { + "type": ["null", "string"] + }, + "refund_platform_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "revenue_advertiser_currency": { + "type": ["null", "string"] + }, + "revenue_partner_currency": { + "type": ["null", "string"] + }, + "revenue_usd": { + "type": ["null", "string"] + }, + "revenue_ecpa_advertiser_currency": { + "type": ["null", "string"] + }, + "revenue_ecpa_partner_currency": { + "type": ["null", "string"] + }, + "revenue_ecpa_pc_advertiser_currency": { + "type": ["null", "string"] + }, + "revenue_ecpa_pc_partner_currency": { + "type": ["null", "string"] + }, + "revenue_ecpa_pc_usd": { + "type": ["null", "string"] + }, + "revenue_ecpa_pv_advertiser_currency": { + "type": ["null", "string"] + }, + "revenue_ecpa_pv_partner_currency": { + "type": ["null", "string"] + }, + "revenue_ecpa_pv_usd": { + "type": ["null", "string"] + }, + "revenue_ecpa_usd": { + "type": ["null", "string"] + }, + "revenue_ecpc_advertiser_currency": { + "type": ["null", "string"] + }, + "revenue_ecpc_partner_currency": { + "type": ["null", "string"] + }, + "revenue_ecpc_usd": { + "type": ["null", "string"] + }, + "revenue_ecpe_advertiser_currency": { + "type": ["null", "string"] + }, + "revenue_ecpe_partner_currency": { + "type": ["null", "string"] + }, + "revenue_ecpe_usd": { + "type": ["null", "string"] + }, + "revenue_ecpm_advertiser_currency": { + "type": ["null", "string"] + }, + "revenue_ecpm_partner_currency": { + "type": ["null", "string"] + }, + "revenue_ecpm_usd": { + "type": ["null", "string"] + }, + "revenue_ecpv_advertiser_currency": { + "type": ["null", "string"] + }, + "revenue_ecpv_partner_currency": { + "type": ["null", "string"] + }, + "revenue_ecpv_usd": { + "type": ["null", "string"] + }, + "revenue_viewable_ecpm_advertiser_currency": { + "type": ["null", "string"] + }, + "revenue_viewable_ecpm_partner_currency": { + "type": ["null", "string"] + }, + "revenue_viewable_ecpm_usd": { + "type": ["null", "string"] + }, + "rich_media_engagements": { + "type": ["null", "string"] + }, + "scrolls": { + "type": ["null", "string"] + }, + "shoplocal_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "shoplocal_fee_partner_currency": { + "type": ["null", "string"] + }, + "shoplocal_fee_usd": { + "type": ["null", "string"] + }, + "skips_video": { + "type": ["null", "string"] + }, + "starts_audio": { + "type": ["null", "string"] + }, + "starts_video": { + "type": ["null", "string"] + }, + "stops_audio": { + "type": ["null", "string"] + }, + "teracent_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "teracent_fee_partner_currency": { + "type": ["null", "string"] + }, + "teracent_fee_usd": { + "type": ["null", "string"] + }, + "third_party_ad_server_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "third_party_ad_server_fee_partner_currency": { + "type": ["null", "string"] + }, + "third_party_ad_server_fee_usd": { + "type": ["null", "string"] + }, + "third_quartile_audio": { + "type": ["null", "string"] + }, + "third_quartile_views_video": { + "type": ["null", "string"] + }, + "timers": { + "type": ["null", "string"] + }, + "total_conversions": { + "type": ["null", "string"] + }, + "total_conversions__cross_environment": { + "type": ["null", "string"] + }, + "total_display_time": { + "type": ["null", "string"] + }, + "total_impression_custom_value": { + "type": ["null", "string"] + }, + "total_interaction_time": { + "type": ["null", "string"] + }, + "total_media_cost_advertiser_currency": { + "type": ["null", "string"] + }, + "total_media_cost_partner_currency": { + "type": ["null", "string"] + }, + "total_media_cost_usd": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_advertiser_currency": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_partner_currency": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_pc_advertiser_currency": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_pc_partner_currency": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_pc_usd": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_pv_advertiser_currency": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_pv_partner_currency": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_pv_usd": { + "type": ["null", "string"] + }, + "total_media_cost_ecpa_usd": { + "type": ["null", "string"] + }, + "total_media_cost_ecpc_advertiser_currency": { + "type": ["null", "string"] + }, + "total_media_cost_ecpc_partner_currency": { + "type": ["null", "string"] + }, + "total_media_cost_ecpc_usd": { + "type": ["null", "string"] + }, + "total_media_cost_ecpm_advertiser_currency": { + "type": ["null", "string"] + }, + "total_media_cost_ecpm_partner_currency": { + "type": ["null", "string"] + }, + "total_media_cost_ecpm_usd": { + "type": ["null", "string"] + }, + "total_media_cost_viewable_ecpm_advertiser_currency": { + "type": ["null", "string"] + }, + "total_media_cost_viewable_ecpm_partner_currency": { + "type": ["null", "string"] + }, + "total_media_cost_viewable_ecpm_usd": { + "type": ["null", "string"] + }, + "total_video_media_cost_ecpcv_advertiser_currency": { + "type": ["null", "string"] + }, + "total_video_media_cost_ecpcv_partner_currency": { + "type": ["null", "string"] + }, + "total_video_media_cost_ecpcv_usd": { + "type": ["null", "string"] + }, + "tracked_ads": { + "type": ["null", "string"] + }, + "trueview_general_invalid_traffic_givt_views": { + "type": ["null", "string"] + }, + "trueview_invalid_views": { + "type": ["null", "string"] + }, + "trustmetrics_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "trustmetrics_fee_partner_currency": { + "type": ["null", "string"] + }, + "trustmetrics_fee_usd": { + "type": ["null", "string"] + }, + "verifiable_impressions": { + "type": ["null", "string"] + }, + "video_client_cost_ecpcv_advertiser_currency": { + "type": ["null", "string"] + }, + "video_media_cost_ecpcv_advertiser_currency": { + "type": ["null", "string"] + }, + "video_media_cost_ecpcv_partner_currency": { + "type": ["null", "string"] + }, + "video_media_cost_ecpcv_usd": { + "type": ["null", "string"] + }, + "vizu_fee_advertiser_currency": { + "type": ["null", "string"] + }, + "vizu_fee_partner_currency": { + "type": ["null", "string"] + }, + "vizu_fee_usd": { + "type": ["null", "string"] + }, + "youtube_view_rate": { + "type": ["null", "string"] + }, + "youtube_views": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/unique_reach_audience.json b/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/unique_reach_audience.json new file mode 100644 index 0000000000000..0aefa4f72a595 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/source_dv_360/schemas/unique_reach_audience.json @@ -0,0 +1,144 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "advertiser": { + "type": ["null", "string"] + }, + "advertiser_id": { + "type": ["null", "string"] + }, + "age": { + "description": "required", + "type": ["null", "string"] + }, + "country": { + "description": "required", + "type": ["null", "string"] + }, + "gender": { + "description": "required", + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"] + }, + "insertion_order_id": { + "type": ["null", "string"] + }, + "insertion_order_integration_code": { + "type": ["null", "string"] + }, + "insertion_order": { + "type": ["null", "string"] + }, + "insertion_order_status": { + "type": ["null", "string"] + }, + "line_item_id": { + "type": ["null", "string"] + }, + "line_item_integration_code": { + "type": ["null", "string"] + }, + "line_item": { + "type": ["null", "string"] + }, + "line_item_status": { + "type": ["null", "string"] + }, + "line_item_type": { + "type": ["null", "string"] + }, + "device_type": { + "type": ["null", "string"] + }, + "creative": { + "type": ["null", "string"] + }, + "creative_height": { + "type": ["null", "string"] + }, + "creative_id": { + "type": ["null", "string"] + }, + "creative_size": { + "type": ["null", "string"] + }, + "creative_source": { + "type": ["null", "string"] + }, + "creative_status": { + "type": ["null", "string"] + }, + "creative_type": { + "type": ["null", "string"] + }, + "creative_width": { + "type": ["null", "string"] + }, + "partner_id": { + "type": ["null", "string"] + }, + "partner": { + "type": ["null", "string"] + }, + "month": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "campaign": { + "type": ["null", "string"] + }, + "pct_composition_impressions": { + "type": ["null", "string"] + }, + "pct_composition_reach": { + "type": ["null", "string"] + }, + "pct_population_reach": { + "type": ["null", "string"] + }, + "clicks": { + "type": ["null", "string"] + }, + "impressions": { + "type": ["null", "string"] + }, + "population": { + "type": ["null", "string"] + }, + "target_rating_points": { + "type": ["null", "string"] + }, + "unique_reach_average_impression_frequency": { + "type": ["null", "string"] + }, + "unique_reach_click_reach": { + "type": ["null", "string"] + }, + "unique_reach_impression_reach": { + "type": ["null", "string"] + }, + "unique_reach_viewable_impression_reach": { + "type": ["null", "string"] + }, + "viewable_target_rating_points": { + "type": ["null", "string"] + }, + "viewable_impressions": { + "type": ["null", "string"] + }, + "pct_viewable_composition_impressions": { + "type": ["null", "string"] + }, + "pct_viewable_composition_reach": { + "type": ["null", "string"] + }, + "pct_viewable_population_reach": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-dv-360/source_dv_360/source.py b/airbyte-integrations/connectors/source-dv-360/source_dv_360/source.py new file mode 100644 index 0000000000000..a4e90cac21539 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/source_dv_360/source.py @@ -0,0 +1,140 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import json +from datetime import datetime +from typing import Any, Generator, List, Mapping, MutableMapping, Tuple + +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode, Type +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from google.oauth2.credentials import Credentials +from googleapiclient.discovery import build + +from .streams import AudienceComposition, Floodlight, Reach, Standard, UniqueReachAudience + + +class SourceDV360(AbstractSource): + def get_credentials(self, config: json) -> Credentials: + """ + Get the credentials from the config file and returns them as a Credentials object + """ + cred_json = config.get("credentials") + creds = Credentials( + token=cred_json.get("access_token"), + refresh_token=cred_json.get("refresh_token"), + token_uri=cred_json.get("token_uri"), + client_id=cred_json.get("client_id"), + client_secret=cred_json.get("client_secret"), + ) + return creds + + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: + """ + Tests if the input configuration can be used to successfully connect to the integration + e.g: if a provided Stripe API token can be used to connect to the Stripe API. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this source, content of this json is as specified in + the properties of the spec.json file + + :return: AirbyteConnectionStatus indicating a Success or Failure + """ + try: + dbm_service = build("doubleclickbidmanager", "v1.1", credentials=self.get_credentials(config)) + request = dbm_service.queries().listqueries().execute() + if request: + return True, None + except Exception as err: + return False, f"Unable to connect to Google Ads API with the provided credentials - {repr(err)}" + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + :param config: The user-provided configuration as specified by the source's spec. + Any stream construction related operation should happen here. + :return: A list of the streams in this source connector. + """ + args = dict( + credentials=self.get_credentials(config), + partner_id=config.get("partner_id"), + start_date=config.get("start_date"), + end_date=config.get("end_date"), + filters=config.get("filters"), + ) + + streams = [ + Reach(**args), + Standard(**args), + AudienceComposition(**args), + Floodlight(**args), + UniqueReachAudience(**args), + ] + return streams + + def read( + self, logger: AirbyteLogger, config: json, catalog: ConfiguredAirbyteCatalog, state: MutableMapping[str, Any] + ) -> Generator[AirbyteMessage, None, None]: + """ + Returns a generator of the AirbyteMessages generated by reading the source with the given configuration, + catalog, and state. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this source, content of this json is as specified in + the properties of the spec.json file + :param catalog: The input catalog is a ConfiguredAirbyteCatalog which is almost the same as AirbyteCatalog + returned by discover(), but + in addition, it's been configured in the UI! For each particular stream and field, there may have been provided + with extra modifications such as: filtering streams and/or columns out, renaming some entities, etc + :param state: When a Airbyte reads data from a source, it might need to keep a checkpoint cursor to resume + replication in the future from that saved checkpoint. + This is the object that is provided with state from previous runs and avoid replicating the entire set of + data everytime. + + :return: A generator that produces a stream of AirbyteRecordMessage contained in AirbyteMessage object. + """ + stream_instances = {s.name: s for s in self.streams(config)} + for configured_stream in catalog.streams: + stream_name = configured_stream.stream.name + stream_instance = stream_instances.get(stream_name) + if not stream_instance: + raise KeyError( + f"The requested stream {stream_name} was not found in the source." f" Available streams: {stream_instances.keys()}" + ) + stream_state = state.get(stream_name, {}) + # if stream_state and "state" in dir(stream_instance): + stream_instance.state = stream_state + logger.info(f"Syncing {stream_name} stream") + logger.info(f"Setting state of {stream_name} stream to {stream_state}") + yield AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=state)) + try: + config_catalog_fields = configured_stream.stream.json_schema.get("properties").keys() + slices = stream_instance.stream_slices( + cursor_field=configured_stream.cursor_field, + sync_mode=SyncMode.incremental, + stream_state=stream_state, + ) + for _slice in slices: + data = stream_instance.read_records( + sync_mode=SyncMode.incremental, + catalog_fields=config_catalog_fields, + stream_slice=_slice, + stream_state=stream_state, + cursor_field=configured_stream.cursor_field or None, + ) + + # data= stream_instance.read_records(catalog_fields= config_catalog_fields, sync_mode= SyncMode.incremental, stream_slice= _slice) + for row in data: + yield AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage(stream=stream_name, data=row, emitted_at=int(datetime.now().timestamp()) * 1000), + ) + + yield self._checkpoint_state(stream_instance, stream_state, state) + + logger.info(f"Finished syncing {stream_name} stream") + except Exception as e: + logger.error("Failed to read the data: " + repr(e)) diff --git a/airbyte-integrations/connectors/source-dv-360/source_dv_360/spec.json b/airbyte-integrations/connectors/source-dv-360/source_dv_360/spec.json new file mode 100644 index 0000000000000..8b3e147b08837 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/source_dv_360/spec.json @@ -0,0 +1,74 @@ +{ + "documentationUrl": "https://docsurl.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Display & Video 360 Spec", + "type": "object", + "required": ["credentials", "partner_id", "start_date"], + "additionalProperties": true, + "properties": { + "credentials": { + "type": "object", + "description": "Oauth2 credentials", + "order": 0, + "required": [ + "access_token", + "refresh_token", + "token_uri", + "client_id", + "client_secret" + ], + "properties": { + "access_token": { + "type": "string", + "description": "Access token", + "airbyte_secret": true + }, + "refresh_token": { + "type": "string", + "description": "Refresh token", + "airbyte_secret": true + }, + "token_uri": { + "type": "string", + "description": "Token URI", + "airbyte_secret": true + }, + "client_id": { + "type": "string", + "description": "Client ID", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "description": "Client secret", + "airbyte_secret": true + } + } + }, + "partner_id": { + "type": "integer", + "description": "Partner ID", + "order": 1 + }, + "start_date": { + "type": "string", + "description": "UTC date and time in the format 2017-01-25. Any data before this date will not be replicated", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "order": 2 + }, + "end_date": { + "type": "string", + "description": "UTC date and time in the format 2017-01-25. Any data after this date will not be replicated.", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "order": 3 + }, + "filters": { + "type": "array", + "description": "filters for the dimensions. each filter object had 2 keys: 'type' for the name of the dimension to be used as. and 'value' for the value of the filter", + "default": [], + "order": 4 + } + } + } +} diff --git a/airbyte-integrations/connectors/source-dv-360/source_dv_360/streams.py b/airbyte-integrations/connectors/source-dv-360/source_dv_360/streams.py new file mode 100644 index 0000000000000..f16c4c7f5e059 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/source_dv_360/streams.py @@ -0,0 +1,398 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import csv +import io +import json +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import pendulum +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import Stream +from google.oauth2.credentials import Credentials +from googleapiclient.discovery import build + +from .fields import API_REPORT_BUILDER_MAPPING, sanitize + +# Mapping between the schema names and the report types in the report builder +REPORT_TYPE_MAPPING = { + "audience_composition": "TYPE_AUDIENCE_COMPOSITION", + "reach": "TYPE_REACH_AND_FREQUENCY", + "floodlight": "FLOODLIGHT", + "standard": "TYPE_GENERAL", + "unique_reach_audience": "TYPE_REACH_AUDIENCE", +} + + +def chunk_date_range( + start_date: str, + field: str, + end_date: str = None, + range_days: int = None, +) -> Iterable[Mapping[str, any]]: + """ + Passing optional parameter end_date for testing + Returns a list of the beginning and ending timestamps of each `range_days` between the start date and now. + The return value is a list of dicts {'date': str} which can be used directly with the Slack API + """ + intervals = [] + end_date = pendulum.parse(end_date) if end_date else pendulum.yesterday() + start_date = pendulum.parse(start_date) + + # to return some state when state is abnormal + if start_date > end_date: + start_date = end_date + + while start_date < end_date: + intervals.append( + { + "start_date": start_date.to_date_string(), + "end_date": end_date.to_date_string(), + } + ) + start_date = start_date.add(days=range_days) + return intervals + + +class DBM: + QUERY_TEMPLATE_PATH = "source_dv_360/queries/query_template.json" # Template for creating the query object + DBM_SCOPE = "doubleclickbidmanager" # Scope required to fetch data + + def __init__(self, credentials: Credentials, partner_id: str, scope: str = DBM_SCOPE, version: str = "v1.1"): + self.service = build(scope, version, credentials=credentials) # build a service with scope dbm + self.partner_id = partner_id + + @staticmethod + def get_date_params_ms(start_date: str, end_date: str = None) -> Tuple[str, str]: + """ + Returns `start_date` and `end_date` in milliseconds + """ + start_date = pendulum.parse(start_date) + # if end_date is null, take date until yesterday + end_date = pendulum.parse(end_date) if end_date else pendulum.yesterday() + + # check if start date is after end date + if start_date > end_date: + start_date = end_date + + start_date_ms = str(int(start_date.timestamp() * 1000)) + end_date_ms = str(int(end_date.timestamp() * 1000)) + + return start_date_ms, end_date_ms + + @staticmethod + def get_fields_from_schema(schema: Mapping[str, Any], catalog_fields: List[str]) -> List[str]: + """ + Get list of fields in a given schema + :param schema: the list of fields to be converted + :param catalog_fields: the list of fields to be converted + + :return: A list of fields + """ + schema_fields = schema.get("properties").keys() + fields = [field for field in schema_fields if field in catalog_fields] + return fields + + @staticmethod + def convert_fields(fields: List[str]) -> List[str]: + """ + Convert a list of fields into the API naming + :param fields: the list of fields to be converted + + :return: A list of converted fields + """ + return [API_REPORT_BUILDER_MAPPING[key] for key in fields] + + @staticmethod + def get_dimensions_from_fields(fields: List[str]) -> List[str]: + """ + Get a list of dimensions from a list of fields. Dimensions start with FILTER_ + :param fields: A list of fields from the stream + + :return: A list of dimensions in the naming form of the API + """ + conv_fields = DBM.convert_fields(fields) + dimensions = [field for field in conv_fields if field.startswith("FILTER")] + return dimensions + + @staticmethod + def get_metrics_from_fields(fields: List[str]) -> List[str]: + """ + Get a list of metrics from from a list of fields. Metrics start with METRIC_ + :param fields: A list of fields from the stream + + :return: A list of metrics in the naming form of the API + """ + conv_fields = DBM.convert_fields(fields) + metrics = [field for field in conv_fields if field.startswith("METRIC")] + return metrics + + @staticmethod + def set_partner_filter(query: Mapping[str, Any], partner_id: str): + """ + set the partner id filter to the partner id in the config + :param query: the query object where the filter is to be set + """ + filters = query.get("params").get("filters") + if filters: + partner_filter_index = next( + (index for (index, filter) in enumerate(filters) if filter["type"] == "FILTER_PARTNER"), None + ) # get the index of the partner filter + if partner_filter_index is not None: + query["params"]["filters"][partner_filter_index]["value"] = partner_id # set filter to the partner id in the config + + @staticmethod + def create_query_object( + report_name: str, + dimensions: List[str], + metrics: List[str], + partner_id: str, + start_date: str, + end_date: str, + filters: List[dict] = [], + ) -> Mapping[str, Any]: + """ + Create a query object using the query template and a list of parameter for the query + :param report_name: Name of the report + :param dimensions: List of dimensions + :param metrics: list of metrics + :param start_date: Start date of the report, in the same form of the date in the config, as specified in the spec + :param end_date: End date of the report, in the same form of the date in the config, as specified in the spec + :param filters: additional filters to be set + + :return the query object created according to the template + """ + with open(DBM.QUERY_TEMPLATE_PATH, "r") as template: + query_body = json.loads(template.read()) + + # get dates in ms + start_date_ms, end_date_ms = DBM.get_date_params_ms(start_date, end_date) + + DBM.set_partner_filter(query_body, partner_id) # Set partner Id in the filter + query_body["metadata"]["title"] = report_name + query_body["params"]["type"] = REPORT_TYPE_MAPPING[report_name] # get the report type from the mapping + query_body["params"]["groupBys"] = dimensions # dimensions are put in the groupBy section of the query + query_body["params"]["filters"].extend(filters) # Add additional filters if needed + query_body["params"]["metrics"] = metrics + query_body["reportDataStartTimeMs"] = start_date_ms + query_body["reportDataEndTimeMs"] = end_date_ms + return query_body + + def convert_schema_into_query( + self, + schema: Mapping[str, Any], + report_name: str, + catalog_fields: List[str], + partner_id: str, + filters: List[dict], + start_date: str, + end_date: str, + ) -> str: + """ + Create and run a query from the given schema + :param report_name: Name of the report + :param catalog_fields: List of fields which names are sanitized + :param start_date: Start date of the report, in the same form of the date in the config, as specified in the spec + :param end_date: End date of the report, in the same form of the date in the config, as specified in the spec + :param filters: additional filters to be set + + :return the query object created according to the template + """ + fields = self.get_fields_from_schema(schema, catalog_fields) + query = self.create_query_object( + report_name=report_name, + dimensions=self.get_dimensions_from_fields(fields), + metrics=self.get_metrics_from_fields(fields), + start_date=start_date, + end_date=end_date, + partner_id=partner_id, + filters=filters or [], + ) + create_query = self.service.queries().createquery(body=query).execute() # Create query + get_query = ( + self.service.queries().getquery(queryId=create_query.get("queryId")).execute() + ) # get the query which will include the report url + return get_query + + +class DBMStream(Stream, ABC): + """ + Base stream class + """ + + primary_key = None + + def __init__(self, credentials: Credentials, partner_id: str, filters: List[dict], start_date: str, end_date: str = None): + self.dbm = DBM(credentials=credentials, partner_id=partner_id) + self._start_date = start_date + self._end_date = end_date + self._partner_id = partner_id + self._filters = filters + + def get_query(self, catalog_fields: List[str], stream_slice: Mapping[str, Any]) -> Iterable[Mapping]: + """ + Create and run a query from the datastream schema and parameters, and a list of fields provided in the configured catalog + :param catalog_fields: A list of fields provided in the configured catalog + + :return the created query + """ + query = self.dbm.convert_schema_into_query( + schema=self.get_json_schema(), + catalog_fields=catalog_fields, + filters=self._filters, + report_name=self.name, + start_date=self._start_date, + end_date=self._end_date, + partner_id=self._partner_id, + ) + return query + + def read_records(self, catalog_fields: List[str], stream_slice: Mapping[str, Any] = None, sync_mode=None): + """ + Get the report from the url specified in the created query. The report is in csv form, with + additional meta data below the data that need to be remove. + :param catalog_fields: A list of fields provided in the configured catalog to create the query + + :return a generator of dict rows from the file + """ + query = self.get_query(catalog_fields=catalog_fields, stream_slice=stream_slice) # create and run the query + report_url = query["metadata"]["googleCloudStoragePathForLatestReport"] # Take the url of the generated report + with io.StringIO(requests.get(report_url).text) as csv_response: + header = csv_response.readline().split(",") # get the header of the file + header = [sanitize(field) for field in header] # sanitize the field names + data = self.buffer_reader(csv_response) # Remove the unnecessary rows that do not have data + reader = csv.DictReader(data, fieldnames=header) # convert csv data into dict rows to be yielded by the generator + report_type = query["params"]["type"] + list_reader = list(reader) + nb_rows = len(list_reader) + for index, row in enumerate(list_reader): + # In the case of the standard report, we are getting an additional summary row, therefore we need to exclude it. + if not (report_type == "TYPE_GENERAL" and index > nb_rows - 2): + yield row + + def buffer_reader(self, buffer: io.StringIO): + """ + Yield all lines from a file text buffer until the empty line is reached + + :return a generator of dict rows from the file + """ + for line in buffer.readlines(): + if line != "\n": # NB: the last non empty line contains the sum of the metrics in the data + yield line + else: + break + + +class DBMIncrementalStream(DBMStream, ABC): + cursor_field = "date" + primary_key = None + range_days = 30 # range of stream slice + + def __init__(self, credentials: Credentials, partner_id: str, filters: List[dict], start_date: str, end_date: str = None): + super().__init__(credentials, partner_id, filters, start_date, end_date) + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Update stream state from latest record + """ + current_stream_state = current_stream_state or {} + record_value = latest_record[self.cursor_field] + state_value = current_stream_state.get(self.cursor_field) or record_value + max_cursor = max(pendulum.parse(state_value), pendulum.parse(record_value)) + toreturn = { + self.cursor_field: max_cursor.to_date_string(), + } + return toreturn + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + """ + Slice the stream by date periods. + """ + stream_state = stream_state or {} + start_date = stream_state.get(self.cursor_field) or self._start_date + date_chunks = chunk_date_range( + start_date=start_date, + end_date=self._end_date, + field=self.cursor_field, + range_days=self.range_days, + ) + for chunk in date_chunks: + yield chunk + + def read_records( + self, + sync_mode: SyncMode, + catalog_fields: List[str], + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + """ + This method is overridden to update `start_date` key in the `stream_slice` with the latest read record's cursor value. + """ + records = super().read_records(catalog_fields=catalog_fields, sync_mode=sync_mode, stream_slice=stream_slice) + for record in records: + self.state = self.get_updated_state(self.state, record) + yield record + + def get_query(self, catalog_fields: List[str], stream_slice: Mapping[str, Any]) -> Iterable[Mapping]: + """ + Create and run a query from the datastream schema and parameters, and a list of fields provided in the configured catalog + :param catalog_fields: A list of fields provided in the configured catalog + + :return the created query + """ + query = self.dbm.convert_schema_into_query( + schema=self.get_json_schema(), + catalog_fields=catalog_fields, + filters=self._filters, + report_name=self.name, + start_date=stream_slice.get("start_date"), + end_date=stream_slice.get("end_date"), + partner_id=self._partner_id, + ) + return query + + +class AudienceComposition(DBMIncrementalStream): + """ + Audience Composition stream + """ + + primary_key = None + + +class Floodlight(DBMIncrementalStream): + """ + Floodlight stream + """ + + primary_key = None + + +class Standard(DBMIncrementalStream): + """ + Standard stream + """ + + primary_key = None + + +class UniqueReachAudience(DBMIncrementalStream): + """ + Unique Reach Audience stream + """ + + primary_key = None + + +class Reach(DBMIncrementalStream): + """ + Reach stream + """ + + primary_key = None diff --git a/airbyte-integrations/connectors/source-dv-360/unit_tests/conftest.py b/airbyte-integrations/connectors/source-dv-360/unit_tests/conftest.py new file mode 100644 index 0000000000000..c40a4656127df --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/unit_tests/conftest.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import json + +import pytest + + +@pytest.fixture(scope="session", name="config") +def config_fixture(): + with open("secrets/config.json", "r") as config_file: + return json.load(config_file) diff --git a/airbyte-integrations/connectors/source-dv-360/unit_tests/test_fields.py b/airbyte-integrations/connectors/source-dv-360/unit_tests/test_fields.py new file mode 100644 index 0000000000000..90427c9c63e97 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/unit_tests/test_fields.py @@ -0,0 +1,53 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from source_dv_360.fields import sanitize + + +def test_sanitize_with_pct(): + string = "% tesT string:" + sanitized_string = sanitize(string) + expected_result = "pct_test_string" + + assert sanitized_string == expected_result + + +def test_sanitize_trailing_space(): + string = "% tesT string: " + sanitized_string = sanitize(string) + expected_result = "pct_test_string" + + assert sanitized_string == expected_result + + +def test_sanitize_leading_space(): + string = " % tesT string:" + sanitized_string = sanitize(string) + expected_result = "pct_test_string" + + assert sanitized_string == expected_result + + +def test_sanitize_punctuation(): + string = "% tesT string:,;()#$" + sanitized_string = sanitize(string) + expected_result = "pct_test_string" + + assert sanitized_string == expected_result + + +def test_sanitize_slash(): + string = "% tesT string:/test" + sanitized_string = sanitize(string) + expected_result = "pct_test_string_test" + + assert sanitized_string == expected_result + + +def test_sanitize_and(): + string = "% tesT string & test" + sanitized_string = sanitize(string) + expected_result = "pct_test_string_and_test" + + assert sanitized_string == expected_result diff --git a/airbyte-integrations/connectors/source-dv-360/unit_tests/test_source.py b/airbyte-integrations/connectors/source-dv-360/unit_tests/test_source.py new file mode 100644 index 0000000000000..6eaf416209165 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/unit_tests/test_source.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from source_dv_360.source import SourceDV360 + +SAMPLE_CONFIG = { + "credentials": { + "access_token": "access_token", + "refresh_token": "refresh_token", + "token_uri": "uri", + "client_id": "client_id", + "client_secret": "client_secret", + }, + "start_date": "2022-03-01", + "end_date": "2022-03-08", + "partner_id": 123, + "filters": [], +} + + +EXPECTED_CRED = { + "access_token": "access_token", + "refresh_token": "refresh_token", + "token_uri": "uri", + "client_id": "client_id", + "client_secret": "client_secret", +} + + +def test_get_credentials(): + client = SourceDV360() + credentials = client.get_credentials(SAMPLE_CONFIG) + + assert credentials.token == "access_token" + assert credentials.refresh_token == "refresh_token" + assert credentials.token_uri == "uri" + assert credentials.client_id == "client_id" + assert credentials.client_secret == "client_secret" diff --git a/airbyte-integrations/connectors/source-dv-360/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-dv-360/unit_tests/test_streams.py new file mode 100644 index 0000000000000..4f4edd144882d --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/unit_tests/test_streams.py @@ -0,0 +1,234 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import json + +from source_dv_360.streams import DBM + + +def test_convert_fields(): + fields = ["app_url_id", "cm_placement_id", "pct_clicks_leading_to_conversions", "region_id", "date"] + converted_fields = DBM.convert_fields(fields) + expected_fields = [ + "FILTER_SITE_ID", + "FILTER_CM_PLACEMENT_ID", + "METRIC_CLICK_TO_POST_CLICK_CONVERSION_RATE", + "FILTER_REGION", + "FILTER_DATE", + ] + + assert converted_fields == expected_fields + + +with open("source_dv_360/schemas/reach.json") as FILE: + SCHEMA = json.loads(FILE.read()) + +CATALOG_FIELDS = [ + "advertiser", + "advertiser_id", + "advertiser_integration_code", + "advertiser_status", + "app_url", + "campaign", + "campaign_id", + "creative", + "creative_id", + "creative_source", + "date", + "insertion_order", + "insertion_order_id", + "insertion_order_integration_code", + "insertion_order_status", + "inventory_source", + "line_item", + "line_item_id", + "line_item_status", + "partner", + "partner_id", + "partner_status", + "targeted_data_providers", + "cookie_reach_average_impression_frequency", + "cookie_reach_impression_reach", + "unique_reach_average_impression_frequency", + "unique_reach_click_reach", + "unique_reach_impression_reach", +] + + +def test_get_fields_from_schema(): + fields = DBM.get_fields_from_schema(SCHEMA, CATALOG_FIELDS) + expected_fields = [ + "advertiser", + "advertiser_id", + "advertiser_integration_code", + "advertiser_status", + "app_url", + "campaign", + "campaign_id", + "creative", + "creative_id", + "creative_source", + "date", + "insertion_order", + "insertion_order_id", + "insertion_order_integration_code", + "insertion_order_status", + "inventory_source", + "line_item", + "line_item_id", + "line_item_status", + "partner", + "partner_id", + "partner_status", + "targeted_data_providers", + "cookie_reach_average_impression_frequency", + "cookie_reach_impression_reach", + "unique_reach_average_impression_frequency", + "unique_reach_click_reach", + "unique_reach_impression_reach", + ] + assert expected_fields == fields + + +def test_get_dimensions_from_fields(): + fields = DBM.get_fields_from_schema(SCHEMA, CATALOG_FIELDS) + diemsions = DBM.get_dimensions_from_fields(fields) + expected_diemsions = [ + "FILTER_ADVERTISER_NAME", + "FILTER_ADVERTISER", + "FILTER_ADVERTISER_INTEGRATION_CODE", + "FILTER_ADVERTISER_INTEGRATION_STATUS", + "FILTER_APP_URL", + "FILTER_MEDIA_PLAN_NAME", + "FILTER_MEDIA_PLAN", + "FILTER_CREATIVE", + "FILTER_CREATIVE_ID", + "FILTER_CREATIVE_SOURCE", + "FILTER_DATE", + "FILTER_INSERTION_ORDER_NAME", + "FILTER_INSERTION_ORDER", + "FILTER_INSERTION_ORDER_INTEGRATION_CODE", + "FILTER_INSERTION_ORDER_STATUS", + "FILTER_INVENTORY_SOURCE_NAME", + "FILTER_LINE_ITEM_NAME", + "FILTER_LINE_ITEM", + "FILTER_LINE_ITEM_STATUS", + "FILTER_PARTNER_NAME", + "FILTER_PARTNER", + "FILTER_PARTNER_STATUS", + "FILTER_TARGETED_DATA_PROVIDERS", + ] + assert expected_diemsions == diemsions + + +def test_get_metrics_from_fields(): + fields = DBM.get_fields_from_schema(SCHEMA, CATALOG_FIELDS) + metrics = DBM.get_metrics_from_fields(fields) + expected_metrics = [ + "METRIC_COOKIE_REACH_AVERAGE_IMPRESSION_FREQUENCY", + "METRIC_COOKIE_REACH_IMPRESSION_REACH", + "METRIC_UNIQUE_REACH_AVERAGE_IMPRESSION_FREQUENCY", + "METRIC_UNIQUE_REACH_CLICK_REACH", + "METRIC_UNIQUE_REACH_IMPRESSION_REACH", + ] + assert expected_metrics == metrics + + +EXPECTED_QUERY = { + "kind": "doubleclickbidmanager#query", + "queryId": "0", + "metadata": { + "title": "reach", + "dataRange": "CUSTOM_DATES", + "format": "CSV", + "running": False, + "googleCloudStoragePathForLatestReport": "", + "latestReportRunTimeMs": "0", + "sendNotification": False, + }, + "params": { + "type": "TYPE_REACH_AND_FREQUENCY", + "groupBys": [ + "FILTER_ADVERTISER_NAME", + "FILTER_ADVERTISER", + "FILTER_ADVERTISER_INTEGRATION_CODE", + "FILTER_ADVERTISER_INTEGRATION_STATUS", + "FILTER_APP_URL", + "FILTER_MEDIA_PLAN_NAME", + "FILTER_MEDIA_PLAN", + "FILTER_CREATIVE", + "FILTER_CREATIVE_ID", + "FILTER_CREATIVE_SOURCE", + "FILTER_DATE", + "FILTER_INSERTION_ORDER_NAME", + "FILTER_INSERTION_ORDER", + "FILTER_INSERTION_ORDER_INTEGRATION_CODE", + "FILTER_INSERTION_ORDER_STATUS", + "FILTER_INVENTORY_SOURCE_NAME", + "FILTER_LINE_ITEM_NAME", + "FILTER_LINE_ITEM", + "FILTER_LINE_ITEM_STATUS", + "FILTER_PARTNER_NAME", + "FILTER_PARTNER", + "FILTER_PARTNER_STATUS", + "FILTER_TARGETED_DATA_PROVIDERS", + ], + "filters": [{"type": "FILTER_PARTNER", "value": "123"}, {"type": "FILTER_LINE_ITEM", "value": 55}], + "metrics": [ + "METRIC_COOKIE_REACH_AVERAGE_IMPRESSION_FREQUENCY", + "METRIC_COOKIE_REACH_IMPRESSION_REACH", + "METRIC_UNIQUE_REACH_AVERAGE_IMPRESSION_FREQUENCY", + "METRIC_UNIQUE_REACH_CLICK_REACH", + "METRIC_UNIQUE_REACH_IMPRESSION_REACH", + ], + "options": {"includeOnlyTargetedUserLists": False}, + }, + "schedule": {"frequency": "ONE_TIME"}, + "reportDataStartTimeMs": "1646092800000", + "reportDataEndTimeMs": "1646697600000", + "timezoneCode": "UTC", +} + + +def test_create_query_object(): + query = DBM.create_query_object( + report_name="reach", + dimensions=[ + "FILTER_ADVERTISER_NAME", + "FILTER_ADVERTISER", + "FILTER_ADVERTISER_INTEGRATION_CODE", + "FILTER_ADVERTISER_INTEGRATION_STATUS", + "FILTER_APP_URL", + "FILTER_MEDIA_PLAN_NAME", + "FILTER_MEDIA_PLAN", + "FILTER_CREATIVE", + "FILTER_CREATIVE_ID", + "FILTER_CREATIVE_SOURCE", + "FILTER_DATE", + "FILTER_INSERTION_ORDER_NAME", + "FILTER_INSERTION_ORDER", + "FILTER_INSERTION_ORDER_INTEGRATION_CODE", + "FILTER_INSERTION_ORDER_STATUS", + "FILTER_INVENTORY_SOURCE_NAME", + "FILTER_LINE_ITEM_NAME", + "FILTER_LINE_ITEM", + "FILTER_LINE_ITEM_STATUS", + "FILTER_PARTNER_NAME", + "FILTER_PARTNER", + "FILTER_PARTNER_STATUS", + "FILTER_TARGETED_DATA_PROVIDERS", + ], + metrics=[ + "METRIC_COOKIE_REACH_AVERAGE_IMPRESSION_FREQUENCY", + "METRIC_COOKIE_REACH_IMPRESSION_REACH", + "METRIC_UNIQUE_REACH_AVERAGE_IMPRESSION_FREQUENCY", + "METRIC_UNIQUE_REACH_CLICK_REACH", + "METRIC_UNIQUE_REACH_IMPRESSION_REACH", + ], + start_date="2022-03-01", + end_date="2022-03-08", + partner_id="123", + filters=[{"type": "FILTER_LINE_ITEM", "value": 55}], + ) + assert query == EXPECTED_QUERY diff --git a/airbyte-integrations/connectors/source-dv-360/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-dv-360/unit_tests/unit_test.py new file mode 100644 index 0000000000000..dddaea0060fa1 --- /dev/null +++ b/airbyte-integrations/connectors/source-dv-360/unit_tests/unit_test.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +def test_example_method(): + assert True diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index bc72d60b29a03..2c0323f5826eb 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.64 +LABEL io.airbyte.version=0.2.66 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json index f8b048b18ffdf..b722670d563b9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json @@ -1,58 +1,142 @@ -{ - "activities": { - "event_time": "2121-07-25T13:34:26Z", - "include_deleted": true - }, - "campaigns": { - "updated_time": "2121-07-25T13:34:26Z", - "include_deleted": true - }, - "images": { - "updated_time": "2121-07-25T13:34:26Z", - "include_deleted": true - }, - "ad_creatives": { - "updated_time": "2121-07-25T13:34:26Z", - "include_deleted": true - }, - "ad_sets": { - "updated_time": "2121-07-25T13:34:26Z", - "include_deleted": true - }, - "ads": { - "updated_time": "2121-07-25T13:34:26Z", - "include_deleted": true - }, - "ads_insights": { - "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true - }, - "ads_insights_age_and_gender": { - "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true - }, - "ads_insights_country": { - "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true - }, - "ads_insights_dma": { - "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true - }, - "ads_insights_platform_and_device": { - "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true - }, - "ads_insights_region": { - "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true - }, - "ads_insights_action_type": { - "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true - }, - "custommy_custom_insights": { - "date_start": "2121-07-25T13:34:26Z", - "include_deleted": true +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "event_time": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "activities" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_time": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "campaigns" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_time": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "images" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_time": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "ad_creatives" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_time": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "ad_sets" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_time": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "ads" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_start": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "ads_insights" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_start": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "ads_insights_age_and_gender" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_start": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "ads_insights_country" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_start": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "ads_insights_dma" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_start": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "ads_insights_platform_and_device" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_start": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "ads_insights_region" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_start": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "ads_insights_action_type" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_start": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { "name": "custommy_custom_insights" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py index 6787de8203eb5..52da27d9bae88 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py @@ -73,6 +73,7 @@ class CustomConversions(FBMarketingStream): """doc: https://developers.facebook.com/docs/marketing-api/reference/custom-conversion""" entity_prefix = "customconversion" + enable_deleted = False def list_objects(self, params: Mapping[str, Any]) -> Iterable: return self._api.account.get_custom_conversions(params=params) diff --git a/airbyte-integrations/connectors/source-fauna/.dockerignore b/airbyte-integrations/connectors/source-fauna/.dockerignore new file mode 100644 index 0000000000000..7c867662e5255 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_fauna +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-fauna/.gitignore b/airbyte-integrations/connectors/source-fauna/.gitignore new file mode 100644 index 0000000000000..b38d9d24aba29 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/.gitignore @@ -0,0 +1,6 @@ +# Python version tools +.tool-versions +../../../.tool-versions +# emacs auto-save files +*~ +*# \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-fauna/Dockerfile b/airbyte-integrations/connectors/source-fauna/Dockerfile new file mode 100644 index 0000000000000..e200d1ebb1d7f --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_fauna ./source_fauna + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=dev +LABEL io.airbyte.name=airbyte/source-fauna diff --git a/airbyte-integrations/connectors/source-fauna/README.md b/airbyte-integrations/connectors/source-fauna/README.md new file mode 100644 index 0000000000000..c0fbf8ad0ba22 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/README.md @@ -0,0 +1,188 @@ +# New Readers + +If you know how Airbyte works, read [bootstrap.md](bootstrap.md) for a quick introduction to this source. If you haven't +used airbyte before, read [overview.md](overview.md) for a longer overview about what this connector is and how to use +it. + +# For Fauna Developers + +## Running locally + +First, start a local fauna container: +``` +docker run --rm --name faunadb -p 8443:8443 fauna/faunadb +``` + +In another terminal, cd into the connector directory: +``` +cd airbyte-integrations/connectors/source-fauna +``` + +Once started the container is up, setup the database: +``` +fauna eval "$(cat examples/setup_database.fql)" --domain localhost --port 8443 --scheme http --secret secret +``` + +Finally, run the connector: +``` +python main.py spec +python main.py check --config examples/config_localhost.json +python main.py discover --config examples/config_localhost.json +python main.py read --config examples/config_localhost.json --catalog examples/configured_catalog.json +``` + +To pick up a partial failure you need to pass in a state file. To test via example induce a crash via bad data (e.g. a missing required field), update `examples/sample_state_full_sync.json` to contain your emitted state and then run: + +``` +python main.py read --config examples/config_localhost.json --catalog examples/configured_catalog.json --state examples/sample_state_full_sync.json +``` + +## Running the intergration tests + +First, cd into the connector directory: +``` +cd airbyte-integrations/connectors/source-fauna +``` + +The integration tests require a secret config.json. Ping me on slack to get this file. +Once you have this file, put it in `secrets/config.json`. A sample of this file can be +found at `examples/secret_config.json`. Once the file is created, build the connector: +``` +docker build . -t airbyte/source-fauna:dev +``` + +Now, run the integration tests: +``` +python -m pytest -p integration_tests.acceptance +``` + + +# Fauna Source + +This is the repository for the Fauna source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/fauna). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-fauna:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/fauna) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_fauna/spec.yaml` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `examples/secret_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source fauna test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-fauna:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-fauna:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-fauna:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-fauna:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-fauna:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-fauna:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-fauna:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-fauna:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-fauna/acceptance-test-config.yml b/airbyte-integrations/connectors/source-fauna/acceptance-test-config.yml new file mode 100644 index 0000000000000..d8c5c2e6887af --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/acceptance-test-config.yml @@ -0,0 +1,45 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-fauna:dev +tests: + spec: + - spec_path: "source_fauna/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config-deletions.json" + status: "succeed" + - config_path: "integration_tests/config/invalid.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + - config_path: "secrets/config-deletions.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + expect_records: + path: "integration_tests/expected_records.txt" + extra_fields: no + exact_order: yes + extra_records: no + - config_path: "secrets/config-deletions.json" + configured_catalog_path: "integration_tests/configured_catalog_incremental.json" + empty_streams: [] + expect_records: + path: "integration_tests/expected_deletions_records.txt" + extra_fields: no + exact_order: yes + extra_records: no + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + # Note that the time in this file was generated with this fql: + # ToMicros(ToTime(Date("9999-01-01"))) + future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config-deletions.json" + configured_catalog_path: "integration_tests/configured_catalog_incremental.json" + future_state_path: "integration_tests/abnormal_deletions_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-fauna/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-fauna/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-fauna/bootstrap.md b/airbyte-integrations/connectors/source-fauna/bootstrap.md new file mode 100644 index 0000000000000..50e11fe32be8f --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/bootstrap.md @@ -0,0 +1,56 @@ +# Fauna Source + +[Fauna](https://fauna.com/) is a serverless "document-relational" database that user's interact with via APIs. This connector delivers Fauna as an airbyte source. + +This source is implemented in the [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). +It also uses the [Fauna Python Driver](https://docs.fauna.com/fauna/current/drivers/python), which +allows the connector to build FQL queries in python. This driver is what queries the Fauna database. + +Fauna has collections (similar to tables) and documents (similar to rows). + +Every document has at least 3 fields: `ref`, `ts` and `data`. The `ref` is a unique string identifier +for every document. The `ts` is a timestamp, which is the time that the document was last modified. +The `data` is arbitrary json data. Because there is no shape to this data, we also allow users of +airbyte to specify which fields of the document they want to export as top-level columns. + +Users can also choose to export the `data` field itself in the raw and in the case of incremental syncs, metadata regarding when a document was deleted. + +We currently only provide a single stream, which is the collection the user has chosen. This is +because to support incremental syncs we need an index with every collection, so it ends up being easier to just have the user +setup the index and tell us the collection and index name they wish to use. + +## Full sync + +This source will simply call the following [FQL](https://docs.fauna.com/fauna/current/api/fql/): `Paginate(Documents(Collection("collection-name")))`. +This queries all documents in the database in a paginated manner. The source then iterates over all the results from that query to export data from the connector. + +Docs: +[Paginate](https://docs.fauna.com/fauna/current/api/fql/functions/paginate?lang=python). +[Documents](https://docs.fauna.com/fauna/current/api/fql/functions/documents?lang=python). +[Collection](https://docs.fauna.com/fauna/current/api/fql/functions/collection?lang=python). + +## Incremental sync + +### Updates (uses an index over ts) + +The source will call FQL similar to this: `Paginate(Range(Match(Index("index-name")), , []))`. +The index we match against has the values `ts` and `ref`, so it will sort by the time since the document +has been modified. The Range() will limit the query to just pull the documents that have been modified +since the last query. + +Docs: +[Range](https://docs.fauna.com/fauna/current/api/fql/functions/range?lang=python). +[Match](https://docs.fauna.com/fauna/current/api/fql/functions/match?lang=python). +[Index](https://docs.fauna.com/fauna/current/api/fql/functions/iindex?lang=python). + +### Deletes (uses the events API) + +If the users wants deletes, we have a seperate query for that: +`Paginate(Events(Documents(Collection("collection-name"))))`. This will paginate over all the events +in the documents of the collection. We also filter this to only give us the events since the recently +modified documents. Using these events, we can produce a record with the "deleted at" field set, so +that users know the document has been deleted. + +Docs: +[Events](https://docs.fauna.com/fauna/current/api/fql/functions/events?lang=python). + diff --git a/airbyte-integrations/connectors/source-fauna/build.gradle b/airbyte-integrations/connectors/source-fauna/build.gradle new file mode 100644 index 0000000000000..b31dfb90368d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_fauna_singer' +} diff --git a/airbyte-integrations/connectors/source-fauna/examples/README.md b/airbyte-integrations/connectors/source-fauna/examples/README.md new file mode 100644 index 0000000000000..086eaa0eb258d --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/examples/README.md @@ -0,0 +1,41 @@ +This directory contains files for manual usage, and example files for setting up the Fauna Airbyte +Connector. + +- `config_localhost.json`: A config file, which will connect to a local Fauna container, and read + documents from it. +- `config_sample.json`: A sample config file. This demonstrates the format of the config, and + shows an example of a completed configuration. +- `sample_state_full_sync.json`: A sample state file for picking up a partially failed full sync. +- `configured_catalog.json`: A configured Airbyte catalog, which will run a single full sync. +- `setup_database.fql`: Evaling this file with the fauna shell will setup the local database for + testing. See below for instructions on running the connector locally. +- `secret_config.json`: This is the config that should be placed in `secrets/config.json`, and + will produce the records at `integration_tests/expected_records.txt`. Note that if you run this + yourself, you will need to manually setup the database, and the `ts` field will be incorrect. + +# Running locally + +These examples use the Fauna Shell, which can be downloaded here: https://github.com/fauna/fauna-shell + +First, start a local fauna container: +``` +docker run --rm --name faunadb -p 8443:8443 fauna/faunadb +``` + +In another terminal, cd into the connector directory: +``` +cd airbyte-integrations/connectors/source-fauna +``` + +Once started the container is up, setup the database: +``` +fauna eval "$(cat examples/setup_database.fql)" --domain localhost --port 8443 --scheme http --secret secret +``` + +Finally, run the connector: +``` +python main.py spec +python main.py check --config examples/config_localhost.json +python main.py discover --config examples/config_localhost.json +python main.py read --config examples/config_localhost.json --catalog examples/configured_catalog.json +``` diff --git a/airbyte-integrations/connectors/source-fauna/examples/config_localhost.json b/airbyte-integrations/connectors/source-fauna/examples/config_localhost.json new file mode 100644 index 0000000000000..7e67295b54643 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/examples/config_localhost.json @@ -0,0 +1,13 @@ +{ + "secret": "secret", + "domain": "localhost", + "port": 8443, + "scheme": "http", + "collection": { + "page_size": 2, + "deletions": { + "deletion_mode": "deleted_field", + "column": "deleted_at" + } + } +} diff --git a/airbyte-integrations/connectors/source-fauna/examples/config_sample.json b/airbyte-integrations/connectors/source-fauna/examples/config_sample.json new file mode 100644 index 0000000000000..a82d1ecfd216e --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/examples/config_sample.json @@ -0,0 +1,13 @@ +{ + "secret": "SECRET", + "domain": "db.fauna.com", + "port": 443, + "scheme": "https", + "collection": { + "page_size": 64, + "deletions": { + "deletion_mode": "deleted_field", + "column": "deleted_at" + } + } +} diff --git a/airbyte-integrations/connectors/source-fauna/examples/configured_catalog.json b/airbyte-integrations/connectors/source-fauna/examples/configured_catalog.json new file mode 100644 index 0000000000000..2914ce1f0431c --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/examples/configured_catalog.json @@ -0,0 +1,19 @@ +{ + "streams": [ + { + "stream": { + "name": "foo", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["column_name"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-fauna/examples/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-fauna/examples/configured_catalog_incremental.json new file mode 100644 index 0000000000000..a1155bf9aa00b --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/examples/configured_catalog_incremental.json @@ -0,0 +1,19 @@ +{ + "streams": [ + { + "stream": { + "name": "foo", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["column_name"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-fauna/examples/sample_state.json b/airbyte-integrations/connectors/source-fauna/examples/sample_state.json new file mode 100644 index 0000000000000..1b7644b31ec91 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/examples/sample_state.json @@ -0,0 +1,8 @@ +{ + "foo": { + "full_sync_cursor": { + "ts": 1657642473953000, + "after": "[{\"@ref\":{\"id\":\"336995084811633152\",\"collection\":{\"@ref\":{\"id\":\"foo\",\"collection\":{\"@ref\":{\"id\":\"collections\"}}}}}}]" + } + } +} diff --git a/airbyte-integrations/connectors/source-fauna/examples/sample_state_full_sync.json b/airbyte-integrations/connectors/source-fauna/examples/sample_state_full_sync.json new file mode 100644 index 0000000000000..0086887d1bc95 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/examples/sample_state_full_sync.json @@ -0,0 +1,8 @@ +{ + "foo": { + "full_sync_cursor": { + "ts": 1658504456881000, + "after": "[{\"@ref\":{\"id\":\"337898800783819264\",\"collection\":{\"@ref\":{\"id\":\"foo\",\"collection\":{\"@ref\":{\"id\":\"collections\"}}}}}}]" + } + } +} diff --git a/airbyte-integrations/connectors/source-fauna/examples/sample_state_incremental_adds.json b/airbyte-integrations/connectors/source-fauna/examples/sample_state_incremental_adds.json new file mode 100644 index 0000000000000..f5e4f25226acc --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/examples/sample_state_incremental_adds.json @@ -0,0 +1,8 @@ +{ + "foo": { + "add_remove_cursor": { + "ts": 1657661332900000, + "after": "{\"ts\":1657661332900000,\"action\":\"add\",\"document\":{\"@ref\":{\"id\":\"337014929909350912\",\"collection\":{\"@ref\":{\"id\":\"foo\",\"collection\":{\"@ref\":{\"id\":\"collections\"}}}}}}}" + } + } +} diff --git a/airbyte-integrations/connectors/source-fauna/examples/sample_state_incremental_update.json b/airbyte-integrations/connectors/source-fauna/examples/sample_state_incremental_update.json new file mode 100644 index 0000000000000..ad37db97366fb --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/examples/sample_state_incremental_update.json @@ -0,0 +1,8 @@ +{ + "foo": { + "updates_cursor": { + "now": 1657672406227000, + "after": "[1657661332900000,{\"@ref\":{\"id\":\"337014929909350912\",\"collection\":{\"@ref\":{\"id\":\"foo\",\"collection\":{\"@ref\":{\"id\":\"collections\"}}}}}},{\"@ref\":{\"id\":\"337014929909350912\",\"collection\":{\"@ref\":{\"id\":\"foo\",\"collection\":{\"@ref\":{\"id\":\"collections\"}}}}}}]" + } + } +} diff --git a/airbyte-integrations/connectors/source-fauna/examples/secret_config.json b/airbyte-integrations/connectors/source-fauna/examples/secret_config.json new file mode 100644 index 0000000000000..3aebda5580136 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/examples/secret_config.json @@ -0,0 +1,10 @@ +{ + "secret": "SECRET", + "domain": "db.us.fauna.com", + "port": 443, + "scheme": "https", + "collection": { + "page_size": 2, + "deletions": { "deletion_mode": "ignore" } + } +} diff --git a/airbyte-integrations/connectors/source-fauna/examples/setup_database.fql b/airbyte-integrations/connectors/source-fauna/examples/setup_database.fql new file mode 100644 index 0000000000000..d6fe290fe5be1 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/examples/setup_database.fql @@ -0,0 +1,79 @@ +If( + Exists(Collection("foo")), + "OK", + CreateCollection({ + name: "foo", + }) +) + +If( + Exists(Collection("other")), + "OK", + CreateCollection({ + name: "other", + }) +) + +If( + Exists(Ref(Collection("other"), "123")), + "OK", + Create(Ref(Collection("other"), "123")) +) + +Do( + Create(Collection("foo"), { + data: { + a: 5, + nested: { + value: 15, + }, + }, + }), + Create(Collection("foo"), { + data: { + a: 6, + nested: { + value: 20, + }, + }, + }), + Create(Collection("foo"), { + data: { + a: 7, + nested: { + value: 25, + }, + }, + }), + Create(Collection("foo"), { + data: { + a: 8, + nested: { + value: 30, + }, + }, + }), + Create(Collection("foo"), { + data: { + a: 9, + nested: { + value: 30, + }, + "my-ref": Ref(Collection("other"), "123"), + string: "cat", + timestamp: Now(), + number: 12, + date: Date('2012-02-01') + }, + }), +) + +CreateIndex({ + name: "ts", + source: Collection("foo"), + terms: [], + values: [ + { field: "ts" }, + { field: "ref" }, + ], +}) diff --git a/airbyte-integrations/connectors/source-fauna/integration_tests/__init__.py b/airbyte-integrations/connectors/source-fauna/integration_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-fauna/integration_tests/abnormal_deletions_state.json b/airbyte-integrations/connectors/source-fauna/integration_tests/abnormal_deletions_state.json new file mode 100644 index 0000000000000..689121a8a8b5f --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/integration_tests/abnormal_deletions_state.json @@ -0,0 +1,13 @@ +{ + "_comment": "Generated with ToMicros(ToTime(Date('9999-01-01')))", + "deletions-data": { + "updates_cursor": { + "ts": 253370764800000000, + "ref": "10" + }, + "remove_cursor": { + "ts": 253370764800000000, + "ref": "10" + } + } +} diff --git a/airbyte-integrations/connectors/source-fauna/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-fauna/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..bf30bb2846c63 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/integration_tests/abnormal_state.json @@ -0,0 +1,9 @@ +{ + "_comment": "Generated with ToMicros(ToTime(Date('9999-01-01')))", + "sample-data": { + "updates_cursor": { + "ts": 253370764800000000, + "ref": "10" + } + } +} diff --git a/airbyte-integrations/connectors/source-fauna/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-fauna/integration_tests/acceptance.py new file mode 100644 index 0000000000000..950b53b59d416 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-fauna/integration_tests/catalog.json b/airbyte-integrations/connectors/source-fauna/integration_tests/catalog.json new file mode 100644 index 0000000000000..3f26ebc411a11 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/integration_tests/catalog.json @@ -0,0 +1,25 @@ +{ + "streams": [ + { + "name": "foo", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "ts", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ref": { + "type": "string" + }, + "ts": { + "type": "integer" + }, + "data": { + "type": "object" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-fauna/integration_tests/config/invalid.json b/airbyte-integrations/connectors/source-fauna/integration_tests/config/invalid.json new file mode 100644 index 0000000000000..b116cc79ad830 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/integration_tests/config/invalid.json @@ -0,0 +1,17 @@ +{ + "secret": "invalid secret", + "domain": "127.0.0.1", + "port": 8443, + "scheme": "http", + "collection": { + "name": "foo", + "page_size": 64, + "data_column": true, + "updates": { + "update_mode": "disabled" + }, + "deletions": { + "deletion_mode": "ignore" + } + } +} diff --git a/airbyte-integrations/connectors/source-fauna/integration_tests/config/localhost_updates.json b/airbyte-integrations/connectors/source-fauna/integration_tests/config/localhost_updates.json new file mode 100644 index 0000000000000..17be249daa566 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/integration_tests/config/localhost_updates.json @@ -0,0 +1,25 @@ +{ + "secret": "secret", + "domain": "localhost", + "port": 8443, + "scheme": "http", + "collection": { + "name": "foo", + "data_column": true, + "additional_columns": [ + { + "name": "a", + "path": ["data", "a"], + "type": "integer" + } + ], + "page_size": 64, + "updates": { + "update_mode": "enabled", + "index": "foo_ts" + }, + "deletions": { + "deletion_mode": "ignore" + } + } +} diff --git a/airbyte-integrations/connectors/source-fauna/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-fauna/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..0bce16c96e3ae --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/integration_tests/configured_catalog.json @@ -0,0 +1,29 @@ +{ + "streams": [ + { + "stream": { + "name": "sample-data", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ref": { + "type": "string" + }, + "ts": { + "type": "integer" + }, + "data": { + "type": "object" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["ts"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-fauna/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-fauna/integration_tests/configured_catalog_incremental.json new file mode 100644 index 0000000000000..15200eb8a8232 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/integration_tests/configured_catalog_incremental.json @@ -0,0 +1,29 @@ +{ + "streams": [ + { + "stream": { + "name": "deletions-data", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ref": { + "type": "string" + }, + "ts": { + "type": "integer" + }, + "data": { + "type": "object" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["ts"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup" + } + ] +} diff --git a/airbyte-integrations/connectors/source-fauna/integration_tests/expected_deletions_records.txt b/airbyte-integrations/connectors/source-fauna/integration_tests/expected_deletions_records.txt new file mode 100644 index 0000000000000..3603505d5a5e4 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/integration_tests/expected_deletions_records.txt @@ -0,0 +1,5 @@ +{ "stream": "deletions-data", "emitted_at": "1", "data": { "ref": "338836293305763911", "ts": 1659398359360000, "deleted_at": "2022-08-01T23:59:19.360000" } } +{ "stream": "deletions-data", "emitted_at": "2", "data": { "ref": "338836293305761863", "ts": 1659398366330000, "deleted_at": "2022-08-01T23:59:26.330000" } } +{ "stream": "deletions-data", "emitted_at": "3", "data": { "ref": "338836293305765959", "ts": 1659398371330000, "deleted_at": "2022-08-01T23:59:31.330000" } } +{ "stream": "deletions-data", "emitted_at": "5", "data": { "ref": "338836293305762887", "ts": 1659398320430000, "data": { "a": 6, "nested": { "value": 20 } }, "ttl": null } } +{ "stream": "deletions-data", "emitted_at": "7", "data": { "ref": "338836293305764935", "ts": 1659398320430000, "data": { "a": 8, "nested": { "value": 30 } }, "ttl": null } } diff --git a/airbyte-integrations/connectors/source-fauna/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-fauna/integration_tests/expected_records.txt new file mode 100644 index 0000000000000..87368af19f3a2 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/integration_tests/expected_records.txt @@ -0,0 +1,4 @@ +{ "stream": "sample-data", "emitted_at": "1", "data": { "ref": "337567897171787849", "ts": 1658188683585000, "data": { "a": 5, "nested": { "value": 15 } }, "ttl": null } } +{ "stream": "sample-data", "emitted_at": "2", "data": { "ref": "337567897172836425", "ts": 1658271973660000, "data": { "a": 6, "nested": { "value": 20 }, "name": "hello world" }, "ttl": null } } +{ "stream": "sample-data", "emitted_at": "3", "data": { "ref": "337567897172837449", "ts": 1658188683585000, "data": { "a": 7, "nested": { "value": 25 } }, "ttl": null } } +{ "stream": "sample-data", "emitted_at": "4", "data": { "ref": "337567897173885001", "ts": 1658188683585000, "data": { "a": 8, "nested": { "value": 30 } }, "ttl": null } } diff --git a/airbyte-integrations/connectors/source-fauna/main.py b/airbyte-integrations/connectors/source-fauna/main.py new file mode 100644 index 0000000000000..fd004eadfed4f --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_fauna import SourceFauna + +if __name__ == "__main__": + source = SourceFauna() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-fauna/overview.md b/airbyte-integrations/connectors/source-fauna/overview.md new file mode 100644 index 0000000000000..08d2b392e690d --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/overview.md @@ -0,0 +1,112 @@ +# Fauna Airbyte Source + +This is a source for Airbyte, a tool which allows users to export data from Fauna into a number of destinations, such as +Snowflake, Big Query, Redshift, and more. Airbyte allows this source to export to any destination, as Airbyte provides a +common data format that all destinations must use. This means that this connector can support all Airbyte destinations, +listed here. + +The Source can be configured to export a single collection. To export another collection, you must setup multiple +sources within Airbyte. When I say “Fauna documents,” I really mean the documents in the collection. + +# Sync Modes + +The Sync Mode is how data should be extracted from the Fauna Source and how it should be written to the destination. + +The Fauna Source provides support for 2 export modes, Full Sync and Incremental Sync. Destinations provide support for 3 +import modes, Overwrite, Append, and Append Deduped. Only specific combinations of these modes are valid, and are listed +below. + +## Full Sync - Overwrite + +This imports all data from fauna, and clears the destination. This is the simplest mode, and is the slowest. + +This is useful when you want to keep the destination synced with fauna at all times. This doesn’t provide any method of +finding the state of Fauna in the past. + +## Full Sync - Append + +This imports all data from fauna, and appends it to the destination. This is slightly more complex, and stores the most +data. + +This allows for easy queries to lookup the state of the whole Fauna database at a specific date. + +## Incremental Sync - Append + +This pulls all new records from fauna, and appends them to the destination. This provides a list of all documents over +all time, but doesn’t have any notion of when an old document has been replaced. + +For example, this is useful when you want to export a list of logs, and you only care about the new ones each day. + +## Incremental Sync - Append Deduped + +This pulls all new records from fauna, and appends them to the destination. This also uses a primary key, so it knows +which documents have become out of date. This allows for a query which can lookup the state of the whole Fauna database +at a specific date, and stores only the new data each sync. + +This mode is slower to query, but stores the least about of data, and is the most useful. + +# Record Format + +Each document in Fauna is converted to an Airbyte record. Records are essentially rows, and they have a pre-defined list +of columns. Because Fauna doesn’t support a specific shape of data, we rely on the user to specify their data format +before any data can be exported. + +## Required Columns + +The resulting record will always have at least 2 columns, named ref and ts. The ref is a string, which is the document +ID. This can be used as a primary key in the destination, as it is a unique identifier for each document. The ts is an +integer, which is the time since the document was last modified, stored in microseconds since the Unix Epoch. + +The record has 1 optional column, named data. If this is enabled by the user, then the resulting record will contain all +of the fauna document data within a column. This is most useful when you simply wish to dump all of your data in the +destination, and you don’t need to worry about re-shaping it. + +## Additional Columns + +The remaining columns are all user-configurable. When the user is setting up the connector, they can specify any number +of “Additional Columns.” Each column has a name, a type, and a path. All of these fields are specified by the user. + +Additional columns are implemented to provide an easy way to flatten fauna data into columns. This is because Airbyte +doesn’t have another easy-to-use method of reshaping records, so we implemented this as part of the Fauna Source. + +The name of the additional column is the name that it will have in the destination. Additional columns must have unique +names, and cannot be named ref, ts, or data, as that would conflict with the required columns. + +The type of the additional column is the type in the destination. This is used so that destinations like Snowflake can +know the type of the column before any data is sent. + +The path of the additional column is the path within each Fauna Document for this data. This allows you to pick out a +single field, even if it is nested in fauna, and store it in a column in the destination. + +# Deleted Documents + +If a document is deleted in Fauna, some users would like a record that within their destination. However, in the +destination, they would like to know that it existed for some time, and then was removed at a specific date. + +To support this, we allow for an optional deleted_at column. This column will be null for all present documents, and is +set to a date after a document is deleted. + +This deleted_at column is only supported in incremental syncs. If you combine this with the incremental append deduped +mode, you can easily query for documents that are present at a certain time. + +# Data Serialization + +Fauna documents have a lot of extra types. These types need to be converted into the Airbyte JSON format. Below is an +exhaustive list of how all fauna documents are converted. + + +| Fauna Type | Format | Note | +| ------------- | ------------------------------------------------------------------- | -------------------------------------------------- | +| Document Ref | `{ id: "id", "collection": "collection-name", "type": "document" }` | | +| Other Ref | `{ id: "id", "type": "ref-type" }` | This includes collection refs, database refs, etc. | +| Byte Array | base64 url formatted string | | +| Timestamp | date-time, or an iso-format timestamp | | +| Query, SetRef | a string containing the wire protocol of this value | The wire protocol is not documented. | + +## Ref Types + +Every ref is serialized as a JSON object with 2 or 3 fields, as listed above. The type field will be a string, which is +the type of the reference. For example, a document ref would have the type document, and a collection reference would +have the type collection. + +For all other refs (for example if you stored the result of Collections()), the type will be "unknown". diff --git a/airbyte-integrations/connectors/source-fauna/requirements.txt b/airbyte-integrations/connectors/source-fauna/requirements.txt new file mode 100644 index 0000000000000..7be17a56d745d --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/requirements.txt @@ -0,0 +1,3 @@ +# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-fauna/setup.py b/airbyte-integrations/connectors/source-fauna/setup.py new file mode 100644 index 0000000000000..1353ad7c429db --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1.56", + "faunadb~=4.2", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "source-acceptance-test", +] + +setup( + name="source_fauna", + description="Source implementation for Fauna.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-fauna/source_fauna/__init__.py b/airbyte-integrations/connectors/source-fauna/source_fauna/__init__.py new file mode 100644 index 0000000000000..0c05999c36538 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/source_fauna/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceFauna + +__all__ = ["SourceFauna"] diff --git a/airbyte-integrations/connectors/source-fauna/source_fauna/serialize.py b/airbyte-integrations/connectors/source-fauna/source_fauna/serialize.py new file mode 100644 index 0000000000000..c10e515b017e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/source_fauna/serialize.py @@ -0,0 +1,108 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +# Handles serializing any fauna document into an airbyte record + +import base64 +from datetime import date + +from faunadb import _json +from faunadb.objects import FaunaTime, Query, Ref, SetRef + + +def fauna_doc_to_airbyte(doc: dict) -> dict: + """ + Converts a full fauna document into the airbyte representation. + + This will mutate and return the input `doc`. If you don't want this behavior, deep copy + the dict before passing it in. + """ + for k, v in doc.items(): + doc[k] = _fauna_value_to_airbyte(v) + return doc + + +def _fauna_value_to_airbyte(value: any) -> any: + """ + Converts a fauna document to an airbyte-serializable document. This will simply replace + all FaunaTime, Ref, dates, and byte arrays with native json objects. + + This will mutate `value` (if possible), and return the new `value`. + """ + if isinstance(value, dict): + for k, v in value.items(): + value[k] = _fauna_value_to_airbyte(v) + return value + elif isinstance(value, list): + for i, v in enumerate(value): + value[i] = _fauna_value_to_airbyte(v) + return value + elif isinstance(value, Ref): + # serialize this however we feel like + return ref_to_airbyte(value) + elif isinstance(value, (Query, SetRef)): + # for these we give up :P + return _json.to_json(value) + elif isinstance(value, FaunaTime): + # this matches the airbyte `date-time` spec + return value.value + elif isinstance(value, date): + # this matches the airbyte `date` spec + return value.isoformat() + elif isinstance(value, (bytes, bytearray)): + # airbyte has no byte arrays, so this is just a string + return base64.urlsafe_b64encode(value).decode("utf-8") + else: + # if its anything else, we don't mutate it, and let the json + # serializer deal with it. + return value + + +def ref_to_airbyte(ref) -> dict: + # Note that the ref.database() field is never set, so we ignore it. + if ref.collection() is None: + # We have no nesting on this ref. Therefore, it is invalid, so + # we return an unknown type. + return {"id": ref.id(), "type": "unknown"} + elif ref.collection().collection() is None: + # We have a singly nested ref. + # Example: Ref("my_collection", Ref("collections")) + # or: Ref("my_index", Ref("indexes")) + collection_names = { + "collections": "collection", + "databases": "database", + "indexes": "index", + "functions": "function", + "roles": "role", + "access_providers": "access_provider", + "keys": "key", + "tokens": "token", + "credentials": "credential", + } + return { + "id": ref.id(), + "type": collection_names.get( + # Use the collection id as the key in the above map + ref.collection().id(), + # If that fails, we have an invalid ref, so we fallback to this id. + ref.collection().id(), + ), + } + elif (ref.collection().collection().collection() is None) and (ref.collection().collection().id() == "collections"): + # This is a document. + # + # Example: Ref("1234", Ref("collection_name", Ref("collections"))) + return { + "id": ref.id(), + "collection": ref.collection().id(), + "type": "document", + } + else: + # We have a tripply nested ref, so we are in undefined behavior. + # + # Just try our best, and produce `type: unknown` + return { + "id": ref.id(), + "type": "unknown", + } diff --git a/airbyte-integrations/connectors/source-fauna/source_fauna/source.py b/airbyte-integrations/connectors/source-fauna/source_fauna/source.py new file mode 100644 index 0000000000000..3a2d03ff3e2c3 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/source_fauna/source.py @@ -0,0 +1,700 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import json +import time +from datetime import datetime +from typing import Dict, Generator, Optional + +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import ( + AirbyteCatalog, + AirbyteConnectionStatus, + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + Status, + SyncMode, + Type, +) +from airbyte_cdk.sources import Source +from faunadb import _json +from faunadb import query as q +from faunadb.client import FaunaClient +from faunadb.errors import FaunaError, Unauthorized +from faunadb.objects import Ref +from source_fauna.serialize import fauna_doc_to_airbyte + + +# All config fields. These match the shape of spec.yaml. +class Config: + def __init__(self, conf): + # Domain of Fauna connection (localhost, db.fauna.com). + self.domain = conf["domain"] + # Port of Fauna connection (8443, 443). + self.port = conf["port"] + # Scheme of Fauna connection (https, http). + self.scheme = conf["scheme"] + # Secret of a Fauna DB (my-secret). + self.secret = conf["secret"] + self.collection = CollectionConfig(conf["collection"]) + + +class CollectionConfig: + def __init__(self, conf): + # The page size, used in all Paginate() calls. + self.page_size = conf["page_size"] + + # Configs for how deletions are handled + self.deletions = DeletionsConfig(conf["deletions"]) + + +class DeletionsConfig: + def __init__(self, conf): + self.mode = conf["deletion_mode"] + self.column = conf.get("column") + + +def expand_column_query(conf: CollectionConfig, value): + """ + Returns a query on Fauna document producing an object according to the user's configuraiton. + + Using the given CollectionConfig, this will add every additional column that is listed into + the resulting object. This will also add ref and ts, and data if conf.data_column is + enabled. + + :param conf: the CollectionConfig + :param value: a Fauna expression, which will produce a Ref to the document in question + :return: a Fauna expression for extracting the object + """ + doc = q.var("document") + obj = { + "ref": q.select(["ref", "id"], doc), + "ts": q.select("ts", doc), + "data": q.select("data", doc, {}), + "ttl": q.select("ttl", doc, None), + } + return q.let( + {"document": q.get(value)}, + obj, + ) + + +class SourceFauna(Source): + def _setup_client(self, config): + self.client = FaunaClient( + secret=config.secret, + domain=config.domain, + port=config.port, + scheme=config.scheme, + ) + + def check(self, logger: AirbyteLogger, config: json) -> AirbyteConnectionStatus: + """ + Tests if the input configuration can be used to successfully connect to Fauna. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this source, content of this json is as specified in + the properties of the spec.yaml file + + :return: AirbyteConnectionStatus indicating a Success or Failure + """ + + config = Config(config) + + def fail(message: str) -> AirbyteConnectionStatus: + return AirbyteConnectionStatus( + status=Status.FAILED, + message=message, + ) + + try: + self._setup_client(config) + + # Validate everything else after making sure the database is up. + try: + self.client.query(q.now()) + except Exception as e: + if type(e) is Unauthorized: + return fail("Failed to connect to database: Unauthorized") + else: + return fail(f"Failed to connect to database: {e}") + + # Validate our permissions + try: + self.client.query(q.paginate(q.collections())) + except FaunaError: + return fail("No permissions to list collections") + try: + self.client.query(q.paginate(q.indexes())) + except FaunaError: + return fail("No permissions to list indexes") + + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + except Exception as e: + return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {e}") + + def _validate_index(self, collection: str, index: str) -> Optional[str]: + # Validate that the index exists + if not self.client.query(q.exists(q.index(index))): + return f"Index '{index}' does not exist" + + # Validate the index source + actual_source = self.client.query(q.select("source", q.get(q.index(index)))) + expected_source = Ref(collection, Ref("collections")) + if actual_source != expected_source: + return f"Index '{index}' should have source '{collection}', but it has source '{actual_source.id()}'" + + # If the index has no values, we return `[]` + actual_values = self.client.query(q.select("values", q.get(q.index(index)), [])) + expected_values = [ + {"field": "ts"}, + {"field": "ref"}, + ] + # If the index has extra values, that is fine. We just need the first two values to + # be `ts` and `ref`. Also note that python will not crash if 2 is out of range, + # instead [:2] will just return an empty list. The first two values must match to + # guarantee the expected sort order. + if actual_values[:2] != expected_values: + return f"Index should have values {expected_values}, but it has values {actual_values}" + + # All above checks passed, so it's valid. + return None + + def find_index_for_stream(self, collection: str) -> str: + """ + Finds the index for the given collection name. This will iterate over all indexes, and find + one that has the correct source, values, and terms. + + :param collection: The name of the collection to search for. + """ + page = self.client.query(q.paginate(q.indexes())) + while True: + for id in page["data"]: + try: + index = self.client.query(q.get(id)) + except Unauthorized: + # If we don't have permissions to read this index, we ignore it. + continue + source = index["source"] + # Source can be an array, in which case we want to skip this index + if ( + type(source) is Ref + and source.collection() == Ref("collections") + and source.id() == collection + # Index must have 2 values and no terms + and len(index["values"]) == 2 + and len(index["terms"]) == 0 + # Index values must be ts and ref + and index["values"][0] == {"field": "ts"} + and index["values"][1] == {"field": "ref"} + ): + return index["name"] + if "after" in page: + page = self.client.query(q.paginate(q.indexes(), after=page["after"])) + else: + break + raise ValueError(f"Could not find index for stream '{collection}'") + + def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: + """ + Returns an AirbyteCatalog representing the available streams and fields in the user's connection to Fauna. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this source, content of this json is as specified in + the properties of the spec.yaml file + + :return: AirbyteCatalog is an object describing a list of all available streams in this Fauna source. + A stream is an AirbyteStream object that includes: + - its stream name (or collection name in the case of Fauna) + - json_schema providing the specifications of expected schema for this stream (a list of fields described + by their names and types) + """ + + config = Config(config) + streams = [] + + try: + self._setup_client(config) + + # Map all the indexes with the correct values to their collection. + collections_to_indexes = {} + page = self.client.query(q.paginate(q.indexes())) + while True: + for id in page["data"]: + try: + index = self.client.query(q.get(id)) + except Unauthorized: + # If we don't have permissions to read this index, we ignore it. + continue + source = index["source"] + # Source can be an array, in which case we want to skip this index + if ( + type(source) is Ref + and source.collection() == Ref("collections") + # Index must have 2 values and no terms + and len(index["values"]) == 2 + and len(index["terms"]) == 0 + # Index values must be ts and ref + and index["values"][0] == {"field": "ts"} + and index["values"][1] == {"field": "ref"} + ): + collections_to_indexes[source.id()] = index + if "after" in page: + page = self.client.query(q.paginate(q.indexes(), after=page["after"])) + else: + break + + page = self.client.query(q.paginate(q.collections())) + while True: + for collection in page["data"]: + stream_name = collection.id() + json_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ref": { + "type": "string", + }, + "ts": { + "type": "integer", + }, + "data": { + "type": "object", + }, + "ttl": { + "type": ["null", "integer"], + }, + }, + } + supported_sync_modes = ["full_refresh"] + if stream_name in collections_to_indexes: + supported_sync_modes.append("incremental") + streams.append( + AirbyteStream( + name=stream_name, + json_schema=json_schema, + supported_sync_modes=supported_sync_modes, + source_defined_cursor=True, + default_cursor_field=["ts"], + ) + ) + if "after" in page: + page = self.client.query(q.paginate(q.collections(), after=page["after"])) + else: + break + except Exception as e: + logger.error(f"error in discover: {e}") + return AirbyteCatalog(streams=[]) + + return AirbyteCatalog(streams=streams) + + def find_ts(self, config) -> int: + # TODO: Config-defined timestamp here + return self.client.query(q.to_micros(q.now())) + + def find_emitted_at(self) -> int: + # Returns now, in microseconds. This is a seperate function, so that it is easy + # to replace in unit tests. + return int(time.time() * 1000) + + def read_removes( + self, logger, stream: ConfiguredAirbyteStream, conf: CollectionConfig, state: dict[str, any], deletion_column: str + ) -> Generator[any, None, None]: + """ + This handles all additions and deletions, not updates. + + :param logger: The Airbyte logger. + :param stream: The configured airbyte stream, which is only used in logging. + :param conf: The configured collection options. This is used for page size and expand_column_query. + :param state: The state of this stream. This should be a black-box to the caller. + :param deletion_column: The column to put the 'deleted_at' field in. + :return: A generator which will produce a number data fields for an AirbyteRecordMessage. + """ + + stream_name = stream.stream.name + logger.info(f"reading add/removes for stream {stream_name}") + + if "after" in state: + # If the last query failed, we will have stored the after token used there, so we can + # resume more reliably. + after = _json.parse_json(state["after"]) + else: + # If the last query succeeded, then we will fallback to the ts of the most recent + # document. Here we are creating an after token! However, because we are passing it + # to Paginate over Events, the docs specifically call out that we can pass a timestamp + # for the 'after' value, and that will give us the events after a certain time. + after = { + "ts": state.get("ts", 0), + "action": "remove", + } + if "ref" in state and state["ref"] != "": + after["resource"] = q.ref(q.collection(stream_name), state["ref"]) + + def setup_query(after): + paginate_section = q.paginate( + q.documents(q.collection(stream_name)), + events=True, + after=after, + size=conf.page_size, + ) + # Filter for only removes + paginate_section = q.filter_( + q.lambda_("x", q.equals(q.select(["action"], q.var("x")), "remove")), + paginate_section, + ) + return q.map_( + q.lambda_( + "x", + { + "ref": q.select("document", q.var("x")), + "ts": q.select("ts", q.var("x")), + }, + ), + paginate_section, + ) + + events = self.client.query(setup_query(after=after)) + # These are the new state values. It will be written to the state after we are done emitting + # documents. + # + # These are inclusive, so we will skip a document with this ref and ts if we find it next + # time. + new_ts = state.get("ts", 0) + new_ref = state.get("ref", "") + while True: + if "after" not in events and len(events["data"]) > 0: + new_ref = events["data"][-1]["ref"].id() + new_ts = events["data"][-1]["ts"] + for event in events["data"]: + ref = event["ref"].id() + ts = event["ts"] + + # We don't want duplicate documents, so we skip this if it was the one we emitted + # last time. + if ref == state.get("ref") and ts == state.get("ts"): + continue + + # Crash after a specific document. Used in manual testing. + # if int(ref) > 337014929908302336: + # raise ValueError("ahh") + + data_obj = { + "ref": ref, + "ts": ts, + # Put a timestamp in this column, to show that the document has been deleted + deletion_column: datetime.utcfromtimestamp(ts / 1_000_000).isoformat(), + } + logger.info(f"emitting object {data_obj}") + yield data_obj + + if "after" in events: + # Set this before running the query, so that if it fails, we can retry this same + # query next time. + state["after"] = _json.to_json(events["after"]) + events = self.client.query(setup_query(after=events["after"])) + else: + # Make sure we don't try to use this after token, as we've read to the end of this + # Paginate. + if "after" in state: + del state["after"] + # Now that we are done, write the new ts field. If we were to fail while `yield`ing + # above, then our state wouldn't be updated, so we won't skip documents. + state["ts"] = new_ts + state["ref"] = new_ref + break + + def read_updates( + self, logger, stream: ConfiguredAirbyteStream, conf: CollectionConfig, state: Dict[str, any], index: str, page_size: int + ) -> Generator[any, None, None]: + """ + This handles all document creations/updates. It does not handle document deletions. + + The state has 3 optional fields: + `ts`: + This is the `ts` of the last document emitted from the previous query. + `ref`: + This is the `ref` id of the last document emitted from the previous query. + `after`: + This is a wire-protocol serialized after token from Paginate(). This + will only be set if the last query failed. This is passed to Paginate() + in order to resume at the correct location. + + In the happy case, only `ts` and `ref` will be set. + """ + stream_name = stream.stream.name + logger.info(f"reading document updates for stream {stream_name}") + + if "after" in state: + # If the last query failed, we will have stored the after token used there, so we can + # resume more reliably. + after = _json.parse_json(state["after"]) + else: + # If there is no after token, the last query was successful. + after = None + + # If we have a broken state, or an incomplete state, this will build the correct range min. + range_min = [state.get("ts", 0)] + if "ref" in state: + range_min.append(q.ref(q.collection(stream_name), state.get("ref", ""))) + + def get_event_values(expr: q._Expr) -> q._Expr: + return q.map_( + q.lambda_("x", expand_column_query(conf, q.select(1, q.var("x")))), + expr, + ) + + modified_documents = self.client.query( + get_event_values( + q.paginate( + q.range( + q.match(q.index(index)), + range_min, # use the min we got above + [], # no max + ), + size=page_size, + after=after, + ) + ) + ) + # These are the new state values. It will be written to the state after we are done emitting + # documents. + # + # These are inclusive, so we will skip a document with this ref and ts if we find it next + # time. + new_ts = state.get("ts", 0) + new_ref = state.get("ref", "") + while True: + if "after" not in modified_documents and len(modified_documents["data"]) > 0: + new_ref = modified_documents["data"][-1]["ref"] + new_ts = modified_documents["data"][-1]["ts"] + for doc in modified_documents["data"]: + # We don't want duplicate documents, so we skip this if it was the one we emitted + # last time. + if doc["ref"] == state.get("ref") and doc["ts"] == state.get("ts"): + continue + yield doc + if "after" in modified_documents: + state["after"] = _json.to_json(modified_documents["after"]) + modified_documents = self.client.query( + get_event_values( + q.paginate( + q.range( + q.match(q.index(index)), + range_min, + [], + ), + size=page_size, + after=modified_documents["after"], + ) + ) + ) + else: + # Completed successfully, so we remove the after token, and update the ts. + if "after" in state: + del state["after"] + # Now that we are done, write the new ts field. If we were to fail while `yield`ing + # above, then our state wouldn't be updated, so we won't skip documents. + state["ts"] = new_ts + state["ref"] = new_ref + break + + def read_all(self, logger, stream: ConfiguredAirbyteStream, conf: CollectionConfig, state: dict) -> Generator[any, None, None]: + """ + Reads all documents. The `state` must have a field of 'full_sync_cursor', which is a dict + containing elements: `ts` and `ref`. + + The `ts` field must always be present. It is the value use in `At` for the whole query. + + The `ref` field is optional. If present, it will be used to resume a paginate. This should + only be present when resuming a failed sync. If not present, the `Paginate` will list every + document. + """ + # This handles fetching all documents. Used in full sync. + stream_name = stream.stream.name + + after = state["full_sync_cursor"].get("after") + if after is not None: + # Deserialize the after token from the wire protocol + after = _json.parse_json(after) + logger.info(f"using after token {after}") + else: + logger.info("no after token, starting from beginning") + + ts = state["full_sync_cursor"]["ts"] + + def get_event_values(expr: q._Expr) -> q._Expr: + return q.at( + ts, + q.map_( + q.lambda_("x", expand_column_query(conf, q.var("x"))), + expr, + ), + ) + + all_documents = self.client.query( + get_event_values( + q.paginate( + q.documents(q.collection(stream_name)), + size=conf.page_size, + after=after, + ) + ) + ) + while True: + yield from all_documents["data"] + if "after" in all_documents: + # Serialize the after token to the wire protocol + state["full_sync_cursor"]["after"] = _json.to_json(all_documents["after"]) + + # if this query crashes, the state will have this after token stored. + # therefore, on the next retry, this same query will be performend, + # which is what we want to have happened. + all_documents = self.client.query( + get_event_values( + q.paginate( + q.documents(q.collection(stream_name)), + size=conf.page_size, + after=all_documents["after"], + ) + ) + ) + else: + break + + def read( + self, logger: AirbyteLogger, config: json, catalog: ConfiguredAirbyteCatalog, state: Dict[str, any] + ) -> Generator[AirbyteMessage, None, None]: + """ + Returns a generator of the AirbyteMessages generated by reading the source with the given configuration, + catalog, and state. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this source, content of this json is as specified in + the properties of the spec.yaml file + :param catalog: The input catalog is a ConfiguredAirbyteCatalog which is almost the same as AirbyteCatalog + returned by discover(), but + in addition, it's been configured in the UI! For each particular stream and field, there may have been provided + with extra modifications such as: filtering streams and/or columns out, renaming some entities, etc + :param state: When a Airbyte reads data from a source, it might need to keep a checkpoint cursor to resume + replication in the future from that saved checkpoint. + This is the object that is provided with state from previous runs and avoid replicating the entire set of + data everytime. + + :return: A generator that produces a stream of AirbyteRecordMessage contained in AirbyteMessage object. + """ + + config = Config(config) + logger.info(f"state: {state}") + + def make_message(stream_name, data_obj) -> AirbyteMessage: + return AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + stream=stream_name, + data=fauna_doc_to_airbyte(data_obj), + emitted_at=self.find_emitted_at(), + ), + ) + + try: + self._setup_client(config) + + for stream in catalog.streams: + stream_name = stream.stream.name + if stream.sync_mode == SyncMode.full_refresh: + logger.info(f"syncing stream '{stream_name}' with full_refresh") + + if state[stream_name].get("full_sync_cursor", {}).get("ts") is None: + # No sync yet, so determine `ts` + logger.info("this is the start of a sync (no cursor has been set)") + state[stream_name]["full_sync_cursor"] = {"ts": self.find_ts(config)} + else: + logger.info("this is the middle of a sync (a cursor has been set)") + + logger.info(f"syncing at ts {state[stream_name]['full_sync_cursor']['ts']}") + + # Now, if we crash, we will emit this state. Airbyte will retry the sync, and + # use the state that was first passed to this function. This means the retry + # will not be able to resume correctly, and it may choose a new `ts` field. + # We cannot do anything about this behavior. + # + # If the user manually tries to sync again, after the sync failed, then this + # state will be used, and we can resume. + for data_obj in self.read_all(logger, stream, config.collection, state=state[stream_name]): + yield make_message(stream_name, data_obj) + + # We finished, so we clear the state, so that the next sync will start at the + # beginning. + del state[stream_name]["full_sync_cursor"] + yield AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + data=state, + emitted_at=self.find_emitted_at(), + ), + ) + elif stream.sync_mode == SyncMode.incremental: + logger.info(f"syncing stream '{stream_name}' with incremental") + if stream_name not in state: + state[stream_name] = {} + + index = self.find_index_for_stream(stream_name) + logger.info(f"found index '{index}', which will be used to sync '{stream_name}'") + read_deletions = config.collection.deletions.mode == "deleted_field" + + # Read removals + if read_deletions: + if "remove_cursor" not in state[stream_name]: + state[stream_name]["remove_cursor"] = {} + for data_obj in self.read_removes( + logger, + stream, + config.collection, + state[stream_name]["remove_cursor"], + deletion_column=config.collection.deletions.column, + ): + yield make_message(stream_name, data_obj) + else: + logger.info("skipping collection events (no deletions needed)") + # Read adds/updates + if "updates_cursor" not in state[stream_name]: + state[stream_name]["updates_cursor"] = {} + for data_obj in self.read_updates( + logger, + stream, + config.collection, + state[stream_name]["updates_cursor"], + index, + config.collection.page_size, + ): + yield make_message(stream_name, data_obj) + # Yield our state + yield AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + data=state, + emitted_at=self.find_emitted_at(), + ), + ) + else: + logger.error(f"could not sync stream '{stream.stream.name}', invalid sync_mode: {stream.sync_mode}") + + except Exception as e: + yield AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + data=state, + emitted_at=self.find_emitted_at(), + ), + ) + logger.error(e) + raise diff --git a/airbyte-integrations/connectors/source-fauna/source_fauna/spec.yaml b/airbyte-integrations/connectors/source-fauna/source_fauna/spec.yaml new file mode 100644 index 0000000000000..0a16fdecb9672 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/source_fauna/spec.yaml @@ -0,0 +1,102 @@ +documentationUrl: https://github.com/fauna/airbyte/blob/source-fauna/docs/integrations/sources/fauna.md +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Fauna Spec + type: object + required: + - domain + - port + - scheme + - secret + additionalProperties: true + properties: + domain: + order: 0 + type: string + title: Domain + description: >- + Domain of Fauna to query. Defaults db.fauna.com. + See the docs. + default: "db.fauna.com" + port: + order: 1 + type: integer + title: Port + description: Endpoint port. + default: 443 + scheme: + order: 2 + type: string + title: Scheme + description: URL scheme. + default: "https" + secret: + order: 3 + type: string + title: Fauna Secret + description: Fauna secret, used when authenticating with the database. + airbyte_secret: true + collection: + order: 5 + type: object + title: Collection + description: Settings for the Fauna Collection. + required: + - page_size + - deletions + properties: + page_size: + order: 4 + type: integer + title: Page Size + default: 64 + description: >- + The page size used when reading documents from the database. The larger the + page size, the faster the connector processes documents. However, if a + page is too large, the connector may fail. +
+ + Choose your page size based on how large the documents are. +
+ + See the docs. + deletions: + order: 5 + type: object + title: Deletion Mode + description: >- + This only applies to incremental syncs. +
+ + Enabling deletion mode informs your destination of deleted documents.
+ + Disabled - Leave this feature disabled, and ignore deleted documents.
+ + Enabled - Enables this feature. When a document is deleted, the connector + exports a record with a "deleted at" column containing the time that the + document was deleted. + oneOf: + - title: Disabled + type: object + order: 0 + required: + - deletion_mode + properties: + deletion_mode: + type: string + const: ignore + - title: Enabled + type: object + order: 1 + required: + - deletion_mode + - column + properties: + deletion_mode: + type: string + const: deleted_field + column: + type: string + title: Deleted At Column + description: Name of the "deleted at" column. + default: "deleted_at" diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/check_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/check_test.py new file mode 100644 index 0000000000000..b6fbf064acbd3 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/check_test.py @@ -0,0 +1,78 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock, Mock + +from airbyte_cdk.models import Status +from faunadb import query as q +from faunadb.errors import Unauthorized +from faunadb.objects import Ref +from source_fauna import SourceFauna +from test_util import config, mock_logger + + +def query_hardcoded(expr): + print(expr) + if expr == q.now(): + return 0 + elif expr == q.paginate(q.collections()): + return [{"ref": Ref("foo", Ref("collections"))}] + elif expr == q.paginate(q.indexes()): + return [ + { + "source": Ref("foo", Ref("collections")), + "values": [ + {"field": "ts"}, + {"field": "ref"}, + ], + "terms": [], + } + ] + else: + raise ValueError(f"invalid query {expr}") + + +# Asserts that the client is setup, and that the client is used to make sure the database is up. +def test_valid_query(): + source = SourceFauna() + source._setup_client = Mock() + source.client = MagicMock() + source.client.query = query_hardcoded + + logger = mock_logger() + result = source.check(logger, config=config({})) + print(result) + assert result.status == Status.SUCCEEDED + + assert source._setup_client.called + assert not logger.error.called + + +def test_invalid_check(): + source = SourceFauna() + source._setup_client = Mock() + source.client = MagicMock() + source.client.query = query_hardcoded + + request_result = MagicMock() + request_result.response_content = {"errors": [{"code": "403", "description": "Unauthorized"}]} + source.client.query = Mock(side_effect=Unauthorized(request_result)) + + print(source.client) + + logger = mock_logger() + result = source.check( + logger, + config=config( + { + "secret": "some invalid secret", + } + ), + ) + assert result.status == Status.FAILED + # We should get an unauthorized when there is a valid database, but an invalid key + assert result.message == "Failed to connect to database: Unauthorized" + + assert source._setup_client.called + assert not logger.error.called diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py new file mode 100644 index 0000000000000..0db45365f874e --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py @@ -0,0 +1,501 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +# This file contains the longest unit tests. This spawns a local fauna container and +# tests against that. These tests are used to make sure we don't skip documents on +# certain edge cases. + +import subprocess +import time +from datetime import datetime + +import docker +from airbyte_cdk.models import ( + AirbyteConnectionStatus, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + Status, + SyncMode, + Type, +) +from faunadb import query as q +from source_fauna import SourceFauna +from test_util import CollectionConfig, DeletionsConfig, FullConfig, config, mock_logger, ref + + +def setup_database(source: SourceFauna): + print("Setting up database...") + source.client.query( + q.create_collection( + { + "name": "foo", + } + ) + ) + # All these documents will have the same `ts`, so we need to make sure + # that we don't skip any of these. + db_results = source.client.query( + q.do( + [ + q.create( + ref(101, "foo"), + { + "data": { + "a": 5, + }, + }, + ), + q.create( + ref(102, "foo"), + { + "data": { + "a": 6, + }, + }, + ), + q.create( + ref(103, "foo"), + { + "data": { + "a": 7, + }, + }, + ), + q.create( + ref(104, "foo"), + { + "data": { + "a": 8, + }, + }, + ), + ] + ) + ) + # Do this seperately, so that the above documents get added to this index. + source.client.query( + q.create_index( + { + "name": "foo_ts", + "source": q.collection("foo"), + "terms": [], + "values": [ + {"field": "ts"}, + {"field": "ref"}, + ], + } + ), + ) + print("Database is setup!") + + # Store all the refs and ts of the documents we created, so that we can validate them + # below. + db_data = { + "ref": [], + "ts": [], + } + for create_result in db_results: + db_data["ref"].append(create_result["ref"]) + db_data["ts"].append(create_result["ts"]) + return db_data + + +def stop_container(container): + print("Stopping FaunaDB container...") + container.stop() + print("Stopped FaunaDB container") + + +def setup_container(): + """Starts and stops a local fauna container""" + client = docker.from_env() + # Bind to port 9000, so that we can run these tests without stopping a local container + container = client.containers.run( + "fauna/faunadb", + remove=True, + ports={8443: 9000}, + detach=True, + ) + print("Waiting for FaunaDB to start...") + i = 0 + while i < 100: + res = subprocess.run( + [ + "curl", + "-m", + "1", + "--output", + "/dev/null", + "--silent", + "--head", + "http://127.0.0.1:9000", + ] + ) + if res.returncode == 0: + print("") + break + time.sleep(1) + print(".", flush=True, end="") + i += 1 + print("FaunaDB is ready! Starting tests") + + try: + source = SourceFauna() + # Port 9000, bound above + source._setup_client( + FullConfig( + secret="secret", + port=9000, + domain="localhost", + scheme="http", + ) + ) + db_data = setup_database(source) + return container, db_data, source + except Exception: + stop_container(container) + raise + + +def run_add_removes_test(source: SourceFauna, logger, stream: ConfiguredAirbyteStream): + source._setup_client(FullConfig.localhost()) + source.client.query(q.create(ref(105, "foo"), {"data": {"a": 10}})) + deleted_ts = ( + source.client.query( + q.do( + q.delete(ref(105, "foo")), + q.now(), + ) + ) + .to_datetime() + .timestamp() + * 1_000_000 + ) + + conf = CollectionConfig( + deletions=DeletionsConfig.ignore(), + ) + results = list(source.read_removes(logger, stream, conf, state={}, deletion_column="my_deletion_col")) + assert len(results) == 1 + assert results[0]["ref"] == "105" + assert results[0]["ts"] >= deleted_ts + assert datetime.fromisoformat(results[0]["my_deletion_col"]).timestamp() * 1_000_000 >= deleted_ts + + +def run_removes_order_test(source: SourceFauna, logger, stream: ConfiguredAirbyteStream): + source._setup_client(FullConfig.localhost()) + + start = source.client.query(q.to_micros(q.now())) + + ref1 = source.client.query(q.select("ref", q.create(q.collection("foo"), {"data": {}}))) + ref2 = source.client.query(q.select("ref", q.create(q.collection("foo"), {"data": {}}))) + ref3 = source.client.query(q.select("ref", q.create(q.collection("foo"), {"data": {}}))) + + # Delete in a different order than created + source.client.query(q.delete(ref1)) + source.client.query(q.delete(ref3)) + source.client.query(q.delete(ref2)) + + print(ref1, ref2, ref3) + + conf = CollectionConfig( + deletions=DeletionsConfig.ignore(), + page_size=2, + ) + results = list( + source.read_removes( + logger, + stream, + conf, + state={ + "ts": start - 1, + }, + deletion_column="my_deletion_col", + ) + ) + assert len(results) == 3 + # The order received should be the order deleted + assert results[0]["ref"] == ref1.id() + assert results[1]["ref"] == ref3.id() + assert results[2]["ref"] == ref2.id() + # Make sure the newest event is last in the list. + assert results[0]["ts"] < results[1]["ts"] + assert results[1]["ts"] < results[2]["ts"] + + +def run_general_remove_test(source: SourceFauna, logger): + stream = ConfiguredAirbyteStream( + stream=AirbyteStream(name="deletions_test", json_schema={}), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append_dedup, + ) + catalog = ConfiguredAirbyteCatalog(streams=[stream]) + source.client.query( + q.create_collection( + { + "name": "deletions_test", + } + ) + ) + db_data = source.client.query( + [ + q.create( + ref(101, "deletions_test"), + { + "data": { + "a": 5, + }, + }, + ), + q.create( + ref(102, "deletions_test"), + { + "data": { + "a": 6, + }, + }, + ), + q.create( + ref(103, "deletions_test"), + { + "data": { + "a": 7, + }, + }, + ), + q.create( + ref(104, "deletions_test"), + { + "data": { + "a": 8, + }, + }, + ), + ] + ) + # Do this seperately, so that the above documents get added to this index. + source.client.query( + q.create_index( + { + "name": "deletions_test_ts", + "source": q.collection("deletions_test"), + "terms": [], + "values": [ + {"field": "ts"}, + {"field": "ref"}, + ], + } + ), + ) + conf = config( + { + "port": 9000, + "collection": { + "deletions": {"deletion_mode": "deleted_field", "column": "deleted_at"}, + }, + } + ) + print("=== check: make sure we read the initial state") + documents, state = read_records(source.read(logger, conf, catalog, {}), "deletions_test") + assert documents == [ + { + "ref": "101", + "ts": db_data[0]["ts"], + "data": {"a": 5}, + "ttl": None, + }, + { + "ref": "102", + "ts": db_data[1]["ts"], + "data": {"a": 6}, + "ttl": None, + }, + { + "ref": "103", + "ts": db_data[2]["ts"], + "data": {"a": 7}, + "ttl": None, + }, + { + "ref": "104", + "ts": db_data[3]["ts"], + "data": {"a": 8}, + "ttl": None, + }, + ] + + print("=== check: make sure we don't produce more records when nothing changed") + documents, state = read_records(source.read(logger, conf, catalog, state), "deletions_test") + assert documents == [] + + source.client.query(q.delete(ref(101, "deletions_test"))) + source.client.query(q.delete(ref(103, "deletions_test"))) + + print("=== check: make sure deleted documents produce records") + documents, state = read_records(source.read(logger, conf, catalog, state), "deletions_test") + assert len(documents) == 2 + + assert documents[0]["ref"] == "101" + assert documents[0]["ts"] > db_data[0]["ts"] + assert "data" not in documents[0] + assert datetime.fromisoformat(documents[0]["deleted_at"]).timestamp() * 1_000_000 > db_data[0]["ts"] + + assert documents[1]["ref"] == "103" + assert documents[1]["ts"] > db_data[2]["ts"] + assert "data" not in documents[1] + assert datetime.fromisoformat(documents[1]["deleted_at"]).timestamp() * 1_000_000 > db_data[2]["ts"] + + print("=== check: make sure we don't produce more deleted documents when nothing changed") + documents, state = read_records(source.read(logger, conf, catalog, state), "deletions_test") + assert documents == [] + + source.client.query(q.delete(ref(102, "deletions_test"))) + + print("=== check: make sure another deleted document produces one more record") + documents, state = read_records(source.read(logger, conf, catalog, state), "deletions_test") + assert len(documents) == 1 + + assert documents[0]["ref"] == "102" + assert documents[0]["ts"] > db_data[1]["ts"] + assert "data" not in documents[0] + assert datetime.fromisoformat(documents[0]["deleted_at"]).timestamp() * 1_000_000 > db_data[1]["ts"] + + print("=== check: make sure we don't produce more deleted documents when nothing changed") + documents, state = read_records(source.read(logger, conf, catalog, state), "deletions_test") + assert documents == [] + + +def handle_check(result: AirbyteConnectionStatus): + if result.status == Status.FAILED: + print("======================") + print("CHECK FAILED:", result.message) + print("======================") + raise ValueError("check failed") + + +def read_records(generator, collection_name): + state = None + records = [] + for message in generator: + if message.type == Type.RECORD: + assert message.record.stream == collection_name + records.append(message.record.data) + elif message.type == Type.STATE: + if state is not None: + raise ValueError("two state messages") + state = message.state.data + if state is None: + raise ValueError("no state message") + return records, state + + +def run_updates_test(db_data, source: SourceFauna, logger, catalog: ConfiguredAirbyteCatalog): + conf = config( + { + "port": 9000, + "collection": {}, + } + ) + handle_check(source.check(logger, conf)) + state = {} + print("=== check: make sure we read the initial state") + documents, state = read_records(source.read(logger, conf, catalog, state=state), "foo") + assert documents == [ + { + "ref": db_data["ref"][0].id(), + "ts": db_data["ts"][0], + "data": {"a": 5}, + "ttl": None, + }, + { + "ref": db_data["ref"][1].id(), + "ts": db_data["ts"][1], + "data": {"a": 6}, + "ttl": None, + }, + { + "ref": db_data["ref"][2].id(), + "ts": db_data["ts"][2], + "data": {"a": 7}, + "ttl": None, + }, + { + "ref": db_data["ref"][3].id(), + "ts": db_data["ts"][3], + "data": {"a": 8}, + "ttl": None, + }, + ] + print("=== check: make sure the state resumes") + documents, state = read_records(source.read(logger, conf, catalog, state=state), "foo") + assert documents == [] + + print("=== check: make sure that updates are actually read") + update_result = source.client.query( + q.update( + db_data["ref"][1], + { + "data": {"a": 10}, + }, + ) + ) + create_result = source.client.query( + q.create( + ref(200, "foo"), + { + "data": {"a": 10000}, + }, + ) + ) + documents, state = read_records(source.read(logger, conf, catalog, state=state), "foo") + assert documents == [ + { + # Same ref + "ref": db_data["ref"][1].id(), + # New ts + "ts": update_result["ts"], + # New data + "data": {"a": 10}, + # Same ttl + "ttl": None, + }, + { + # New ref + "ref": "200", + # New ts + "ts": create_result["ts"], + # New data + "data": {"a": 10000}, + # Same ttl + "ttl": None, + }, + ] + + +def run_test(db_data, source: SourceFauna): + logger = mock_logger() + stream = ConfiguredAirbyteStream( + stream=AirbyteStream(name="foo", json_schema={}), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append_dedup, + ) + run_add_removes_test(source, logger, stream) + run_removes_order_test(source, logger, stream) + catalog = ConfiguredAirbyteCatalog(streams=[stream]) + run_updates_test(db_data, source, logger, catalog) + run_general_remove_test(source, logger) + + +def test_incremental_reads(): + container, db_data, source = setup_container() + + try: + run_test(db_data, source) + except Exception as e: + print(f"ERROR IN TEST: {e}") + stop_container(container) + raise + stop_container(container) diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/discover_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/discover_test.py new file mode 100644 index 0000000000000..ff08e85c7a552 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/discover_test.py @@ -0,0 +1,123 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock, Mock + +from airbyte_cdk.models import AirbyteStream +from faunadb import query as q +from faunadb.objects import Ref +from source_fauna import SourceFauna +from test_util import config, mock_logger + + +def mock_source() -> SourceFauna: + source = SourceFauna() + source._setup_client = Mock() + source.client = MagicMock() + return source + + +def schema(properties) -> dict: + return { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": properties, + } + + +def query_hardcoded(expr): + print(expr) + if expr == q.now(): + return 0 + elif expr == q.paginate(q.collections()): + return {"data": [Ref("foo", Ref("collections")), Ref("bar", Ref("collections"))]} + elif expr == q.paginate(q.indexes()): + return { + "data": [ + Ref("ts", Ref("indexes")), + ] + } + elif expr == q.get(Ref("ts", Ref("indexes"))): + return { + "source": Ref("foo", Ref("collections")), + "name": "ts", + "values": [ + {"field": "ts"}, + {"field": "ref"}, + ], + "terms": [], + } + else: + raise ValueError(f"invalid query {expr}") + + +def test_simple_discover(): + source = SourceFauna() + source._setup_client = Mock() + source.client = MagicMock() + source.client.query = query_hardcoded + + logger = mock_logger() + result = source.discover( + logger, + config=config({}), + ) + assert result.streams == [ + AirbyteStream( + name="foo", + json_schema={ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "data": { + "type": "object", + }, + "ref": { + "type": "string", + }, + "ts": { + "type": "integer", + }, + "ttl": { + "type": ["null", "integer"], + }, + }, + }, + supported_sync_modes=["full_refresh", "incremental"], + source_defined_cursor=True, + default_cursor_field=["ts"], + source_defined_primary_key=None, + namespace=None, + ), + AirbyteStream( + name="bar", + json_schema={ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "data": { + "type": "object", + }, + "ref": { + "type": "string", + }, + "ts": { + "type": "integer", + }, + "ttl": { + "type": ["null", "integer"], + }, + }, + }, + supported_sync_modes=["full_refresh"], + source_defined_cursor=True, + default_cursor_field=["ts"], + source_defined_primary_key=None, + namespace=None, + ), + ] + assert not logger.info.called + assert not logger.error.called + + assert source._setup_client.called diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/full_refresh_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/full_refresh_test.py new file mode 100644 index 0000000000000..da534ae70ce21 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/full_refresh_test.py @@ -0,0 +1,363 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock, Mock + +from faunadb import _json +from faunadb import query as q +from source_fauna import SourceFauna +from test_util import CollectionConfig, expand_columns_query, mock_logger + + +def results(modified, after): + modified_obj = {"data": modified} + if after is not None: + modified_obj["after"] = after + return modified_obj + + +# Tests to make sure the read_all() function handles the pagination cursor correctly. +def test_read_all(): + TS = 12342134 + PAGE_SIZE = 12344315 + FIRST_AFTER_TOKEN = ["some magical", 3, "data"] + + current_query = 0 + QUERIES = [ + q.at( + TS, + q.map_( + q.lambda_("x", expand_columns_query(q.var("x"))), + q.paginate( + q.documents(q.collection("my_stream_name")), + size=PAGE_SIZE, + ), + ), + ), + q.at( + TS, + q.map_( + q.lambda_("x", expand_columns_query(q.var("x"))), + q.paginate( + q.documents(q.collection("my_stream_name")), + size=PAGE_SIZE, + after=FIRST_AFTER_TOKEN, + ), + ), + ), + ] + QUERY_RESULTS = [ + results( + [ + { + "ref": "3", + "ts": 12345, + "data": {"foo": "bar"}, + } + ], + after=FIRST_AFTER_TOKEN, + ), + results( + [ + { + "ref": "5", + "ts": 9999999, + "data": {"more": "data here"}, + } + ], + after=None, + ), + ] + + def query_hardcoded(expr): + nonlocal current_query + assert expr == QUERIES[current_query] + result = QUERY_RESULTS[current_query] + current_query += 1 + return result + + source = SourceFauna() + source._setup_client = Mock() + source.client = MagicMock() + source.client.query = query_hardcoded + + logger = mock_logger() + stream = Mock() + stream.stream.name = "my_stream_name" + # ts should be "now", which is whatever we want + # ref must not be present, as we are not resuming + result = list(source.read_all(logger, stream, conf=CollectionConfig(page_size=PAGE_SIZE), state={"full_sync_cursor": {"ts": TS}})) + assert result == [ + { + "ref": "3", + "ts": 12345, + "data": { + "foo": "bar", + }, + }, + { + "ref": "5", + "ts": 9999999, + "data": { + "more": "data here", + }, + }, + ] + + assert not source._setup_client.called + assert current_query == 2 + assert not logger.error.called + + +# Tests to make sure the read_all() function handles the pagination cursor correctly. +def test_read_all_extra_columns(): + def expand_columns_query_with_extra(ref): + doc = q.var("document") + return q.let( + { + "document": q.get(ref), + }, + { + "ref": q.select(["ref", "id"], doc), + "ts": q.select("ts", doc), + "data": q.select("data", doc, {}), + "ttl": q.select("ttl", doc, None), + }, + ) + + TS = 12342134 + PAGE_SIZE = 12344315 + + current_query = 0 + QUERIES = [ + q.at( + TS, + q.map_( + q.lambda_("x", expand_columns_query_with_extra(q.var("x"))), + q.paginate( + q.documents(q.collection("my_stream_name")), + size=PAGE_SIZE, + ), + ), + ), + ] + QUERY_RESULTS = [ + results( + [ + { + "ref": "3", + "ts": 12345, + "data": { + "my_column": "fancy string here", + "optional_data": 3, + }, + }, + { + "ref": "5", + "ts": 123459, + "data": {"my_column": "another fancy string here", "optional_data": 5}, + }, + { + "ref": "7", + "ts": 1234599, + "data": { + "my_column": "even more fancy string here", + "optional_data": None, + }, + }, + ], + after=None, + ), + ] + + def query_hardcoded(expr): + nonlocal current_query + assert expr == QUERIES[current_query] + result = QUERY_RESULTS[current_query] + current_query += 1 + return result + + source = SourceFauna() + source._setup_client = Mock() + source.client = MagicMock() + source.client.query = query_hardcoded + + logger = mock_logger() + stream = Mock() + stream.stream.name = "my_stream_name" + # ts should be "now", which is whatever we want + # ref must not be present, as we are not resuming + result = list( + source.read_all( + logger, + stream, + conf=CollectionConfig(page_size=PAGE_SIZE), + state={"full_sync_cursor": {"ts": TS}}, + ) + ) + assert result == [ + { + "ref": "3", + "ts": 12345, + "data": { + "my_column": "fancy string here", + "optional_data": 3, + }, + }, + { + "ref": "5", + "ts": 123459, + "data": { + "my_column": "another fancy string here", + "optional_data": 5, + }, + }, + { + "ref": "7", + "ts": 1234599, + "data": { + "my_column": "even more fancy string here", + "optional_data": None, + }, + }, + ] + + assert not source._setup_client.called + assert current_query == 1 + assert not logger.error.called + + +# After a failure, the source should emit the state, which we should pass back in, +# and then it should resume correctly. +def test_read_all_resume(): + TS = 12342134 + PAGE_SIZE = 12344315 + FIRST_AFTER_TOKEN = ["some magical", 3, "data"] + SECOND_AFTER_TOKEN = ["even more magical", 3, "data"] + + def make_query(after): + return q.at( + TS, + q.map_( + q.lambda_("x", expand_columns_query(q.var("x"))), + q.paginate( + q.documents(q.collection("foo")), + size=PAGE_SIZE, + after=after, + ), + ), + ) + + current_query = 0 + QUERIES = [ + make_query(after=None), + make_query(after=FIRST_AFTER_TOKEN), + make_query(after=SECOND_AFTER_TOKEN), + ] + QUERY_RESULTS = [ + results( + [ + { + "ref": "3", + "ts": 12345, + "data": {"foo": "bar"}, + } + ], + after=FIRST_AFTER_TOKEN, + ), + results( + [ + { + "ref": "5", + "ts": 9999999, + "data": {"more": "data here"}, + } + ], + after=SECOND_AFTER_TOKEN, + ), + results( + [ + { + "ref": "100", + "ts": 92321341234, + "data": {"last data": "some data"}, + } + ], + after=None, + ), + ] + + failed_yet = False + + def query_hardcoded(expr): + nonlocal current_query + nonlocal failed_yet + assert expr == QUERIES[current_query] + result = QUERY_RESULTS[current_query] + if current_query == 2 and not failed_yet: + failed_yet = True + raise ValueError("something has gone terribly wrong") + current_query += 1 + return result + + source = SourceFauna() + source._setup_client = Mock() + source.client = MagicMock() + source.client.query = query_hardcoded + + logger = mock_logger() + stream = Mock() + stream.stream.name = "foo" + # ts should be "now", which is whatever we want + # ref must not be present, as we are not resuming + state = {"full_sync_cursor": {"ts": TS}} + config = CollectionConfig(page_size=PAGE_SIZE) + outputs = [] + try: + for output in source.read_all(logger, stream, config, state): + outputs.append(output) + except ValueError: + # This means we caught the error thrown above + pass + # We should get the first 2 documents. + assert outputs == [ + { + "ref": "3", + "ts": 12345, + "data": { + "foo": "bar", + }, + }, + { + "ref": "5", + "ts": 9999999, + "data": { + "more": "data here", + }, + }, + ] + # Now we make sure our after token was serialized to json, + # and that it was stored within the state. + assert state == { + "full_sync_cursor": { + "ts": TS, + "after": _json.to_json(SECOND_AFTER_TOKEN), + } + } + + # Pass that state back in to resume. + result = list(source.read_all(logger, stream, config, state)) + # We should only get the remaining document (no duplicates). + assert result == [ + { + "ref": "100", + "ts": 92321341234, + "data": {"last data": "some data"}, + } + ] + + assert not source._setup_client.called + assert current_query == 3 + assert failed_yet + assert not logger.error.called diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py new file mode 100644 index 0000000000000..16747e438de01 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py @@ -0,0 +1,912 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from datetime import datetime, timezone +from typing import Dict, Generator +from unittest.mock import MagicMock, Mock + +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + SyncMode, + Type, +) +from faunadb import _json +from faunadb import query as q +from source_fauna import SourceFauna +from test_util import CollectionConfig, config, expand_columns_query, mock_logger, ref + +NOW = 1234512987 + + +def results(modified, after): + modified_obj = {"data": modified} + if after is not None: + modified_obj["after"] = after + return modified_obj + + +def record(stream: str, data: dict[str, any]) -> AirbyteMessage: + return AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + data=data, + stream=stream, + emitted_at=NOW, + ), + ) + + +def state(data: dict[str, any]) -> AirbyteMessage: + return AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + data=data, + emitted_at=NOW, + ), + ) + + +# Tests to make sure the read() function handles the various config combinations of +# updates/deletions correctly. +def test_read_no_updates_or_creates_but_removes_present(): + def find_index_for_stream(collection: str) -> str: + return "ts" + + def read_updates_hardcoded( + logger, stream: ConfiguredAirbyteStream, conf: CollectionConfig, state: Dict[str, any], index: str, page_size: int + ) -> Generator[any, None, None]: + return [] + + def read_removes_hardcoded( + logger, + stream: ConfiguredAirbyteStream, + conf, + state, + deletion_column: str, + ) -> Generator[any, None, None]: + yield { + "ref": "555", + "ts": 5, + "my_deleted_column": 5, + } + yield { + "ref": "123", + "ts": 3, + "my_deleted_column": 3, + } + + source = SourceFauna() + source._setup_client = Mock() + source.read_all = Mock() + source.find_index_for_stream = find_index_for_stream + source.read_updates = read_updates_hardcoded + source.read_removes = read_removes_hardcoded + source.client = MagicMock() + source.find_emitted_at = Mock(return_value=NOW) + + logger = mock_logger() + # Simplest query. Here we should only query Events(), and only track adds. + result = list( + source.read( + logger, + config( + { + "collection": { + "name": "my_stream_name", + "deletions": { + "deletion_mode": "deleted_field", + "column": "my_deleted_column", + }, + } + } + ), + ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append_dedup, + stream=AirbyteStream( + name="my_stream_name", + json_schema={}, + ), + ) + ] + ), + state={}, + ) + ) + # read_removes should update the state, so we should see a state message in the output. + assert result == [ + record( + "my_stream_name", + { + "ref": "555", + "ts": 5, + "my_deleted_column": 5, + }, + ), + record( + "my_stream_name", + { + "ref": "123", + "ts": 3, + "my_deleted_column": 3, + }, + ), + state( + { + "my_stream_name": { + "remove_cursor": {}, + "updates_cursor": {}, + } + } + ), + ] + + assert source._setup_client.called + assert not source.read_all.called + assert not logger.error.called + + +# Test to make sure read() calls read_updates() correctly. +def test_read_updates_ignore_deletes(): + was_called = False + + def find_index_for_stream(collection: str) -> str: + return "my_stream_name_ts" + + def read_updates_hardcoded( + logger, stream: ConfiguredAirbyteStream, conf, state: dict[str, any], index: str, page_size: int + ) -> Generator[any, None, None]: + yield { + "some_document": "data_here", + "ts": 5, + } + yield { + "more_document": "data_here", + "ts": 3, + } + + def read_removes_hardcoded( + logger, + stream: ConfiguredAirbyteStream, + conf, + state, + deletion_column: str, + ) -> Generator[any, None, None]: + nonlocal was_called + was_called = True + yield { + "ref": "555", + "ts": 5, + "my_deleted_column": 5, + } + yield { + "ref": "123", + "ts": 3, + "my_deleted_column": 3, + } + + source = SourceFauna() + source._setup_client = Mock() + source.read_all = Mock() + source.find_index_for_stream = find_index_for_stream + source.read_updates = read_updates_hardcoded + source.read_removes = read_removes_hardcoded + source.client = MagicMock() + source.find_emitted_at = Mock(return_value=NOW) + + logger = mock_logger() + # Here we want updates and adds (no deletions), so Events() should be skipped. + result = list( + source.read( + logger, + config( + { + "collection": { + "name": "my_stream_name", + "deletions": { + "deletion_mode": "ignore", + }, + } + } + ), + ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append_dedup, + stream=AirbyteStream( + name="my_stream_name", + json_schema={}, + ), + ) + ] + ), + state={}, + ) + ) + # Here we also validate that the cursor will stay on the latest 'ts' value. + assert result == [ + record( + "my_stream_name", + { + "some_document": "data_here", + "ts": 5, + }, + ), + record( + "my_stream_name", + { + "more_document": "data_here", + "ts": 3, + }, + ), + state( + { + "my_stream_name": { + "updates_cursor": {}, + } + } + ), + ] + + assert source._setup_client.called + assert not was_called + assert not source.read_all.called + assert not logger.error.called + + +# After a failure, the source should emit the state, which we should pass back in, +# and then it should resume correctly. +def test_read_removes_resume_from_partial_failure(): + PAGE_SIZE = 12344315 + FIRST_AFTER_TOKEN = ["some magical", 3, "data"] + SECOND_AFTER_TOKEN = ["even more magical", 3, "data"] + + def make_query(after): + return q.map_( + q.lambda_( + "x", + { + "ref": q.select("document", q.var("x")), + "ts": q.select("ts", q.var("x")), + }, + ), + q.filter_( + q.lambda_("x", q.equals(q.select(["action"], q.var("x")), "remove")), + q.paginate( + q.documents(q.collection("foo")), + events=True, + size=PAGE_SIZE, + after=after, + ), + ), + ) + + current_query = 0 + QUERIES = [ + make_query( + after={ + "ts": 0, + "action": "remove", + } + ), + make_query(after=FIRST_AFTER_TOKEN), + make_query(after=SECOND_AFTER_TOKEN), + make_query(after={"ts": 12345, "action": "remove", "resource": q.ref(q.collection("foo"), "3")}), + ] + QUERY_RESULTS = [ + results( + # Newest event + [ + { + "ref": ref(100), + "ts": 99, + } + ], + after=FIRST_AFTER_TOKEN, + ), + results( + [ + { + "ref": ref(5), + "ts": 999, + } + ], + after=SECOND_AFTER_TOKEN, + ), + results( + # Oldest event + [ + { + "ref": ref(3), + "ts": 12345, + } + ], + after=None, + ), + results( + # Oldest event + [ + { + "ref": ref(3), + "ts": 12345, + } + ], + after=None, + ), + ] + + failed_yet = False + + def find_index_for_stream(collection: str) -> str: + return "foo_ts" + + def query_hardcoded(expr): + nonlocal current_query + nonlocal failed_yet + assert expr == QUERIES[current_query] + result = QUERY_RESULTS[current_query] + if current_query == 2 and not failed_yet: + failed_yet = True + raise ValueError("something has gone terribly wrong") + current_query += 1 + return result + + source = SourceFauna() + source._setup_client = Mock() + source.client = MagicMock() + source.find_index_for_stream = find_index_for_stream + source.client.query = query_hardcoded + + logger = mock_logger() + stream = Mock() + stream.stream.name = "foo" + # ts should be "now", which is whatever we want + # ref must not be present, as we are not resuming + state = {} + config = CollectionConfig(page_size=PAGE_SIZE) + outputs = [] + try: + for output in source.read_removes(logger, stream, config, state, deletion_column="deletes_here"): + outputs.append(output) + except ValueError: + # This means we caught the error thrown above + pass + # We should get the first 2 documents. + assert outputs == [ + { + "ref": "100", + "ts": 99, + "deletes_here": datetime.utcfromtimestamp(99 / 1_000_000).isoformat(), + }, + { + "ref": "5", + "ts": 999, + "deletes_here": datetime.utcfromtimestamp(999 / 1_000_000).isoformat(), + }, + ] + # Now we make sure our after token was serialized to json, + # and that it was stored within the state. + assert state == { + "after": _json.to_json(SECOND_AFTER_TOKEN), + } + + # Pass that state back in to resume. + result = list(source.read_removes(logger, stream, config, state, deletion_column="deletes_here")) + # We should only get the remaining document (no duplicates). + assert result == [ + { + "ref": "3", + "ts": 12345, + "deletes_here": datetime.utcfromtimestamp(12345 / 1_000_000).isoformat(), + } + ] + assert state == { + "ts": 12345, + "ref": "3", + } + + result = list(source.read_removes(logger, stream, config, state, deletion_column="deletes_here")) + # We should skip the one result as it matches the state + assert result == [] + assert state == { + "ts": 12345, + "ref": "3", + } + + assert not source._setup_client.called + assert current_query == 4 + assert failed_yet + assert not logger.error.called + + +# Make sure we get deleted events when we need them. +def test_read_remove_deletions(): + DATE = datetime(2022, 4, 3).replace(tzinfo=timezone.utc) + # This is a timestamp in microseconds sync epoch + TS = DATE.timestamp() * 1_000_000 + PAGE_SIZE = 12344315 + + def make_query(after): + return q.map_( + q.lambda_( + "x", + { + "ref": q.select("document", q.var("x")), + "ts": q.select("ts", q.var("x")), + }, + ), + q.filter_( + q.lambda_("x", q.equals(q.select(["action"], q.var("x")), "remove")), + q.paginate( + q.documents(q.collection("foo")), + events=True, + size=PAGE_SIZE, + after=after, + ), + ), + ) + + current_query = 0 + QUERIES = [ + make_query( + after={ + "ts": 0, + "action": "remove", + } + ), + make_query(after={"ts": TS, "action": "remove", "resource": q.ref(q.collection("foo"), "100")}), + make_query(after={"ts": TS, "action": "remove", "resource": q.ref(q.collection("foo"), "100")}), + ] + QUERY_RESULTS = [ + results( + [ + { + "ref": ref(100), + "ts": TS, + } + ], + after=None, + ), + results( + [ + { + "ref": ref(100), + "ts": TS, + } + ], + after=None, + ), + results( + [ + { + "ref": ref(100), + "ts": TS, + }, + { + "ref": ref(300), + "ts": TS + 1_000_000, + }, + ], + after=None, + ), + ] + + def find_index_for_stream(collection: str) -> str: + return "foo_ts" + + def query_hardcoded(expr): + nonlocal current_query + assert expr == QUERIES[current_query] + result = QUERY_RESULTS[current_query] + current_query += 1 + return result + + source = SourceFauna() + source._setup_client = Mock() + source.client = MagicMock() + source.find_index_for_stream = find_index_for_stream + source.client.query = query_hardcoded + + logger = mock_logger() + stream = Mock() + stream.stream.name = "foo" + # ts should be "now", which is whatever we want + # ref must not be present, as we are not resuming + state = {} + config = CollectionConfig(page_size=PAGE_SIZE) + outputs = list(source.read_removes(logger, stream, config, state, deletion_column="my_deleted_column")) + # We should get the first document + assert outputs == [ + { + "ref": "100", + "ts": TS, + "my_deleted_column": "2022-04-03T00:00:00", + }, + ] + # Now we make sure our after token was serialized to json, + # and that it was stored within the state. + assert state == { + "ts": TS, + "ref": "100", + } + + outputs = list(source.read_removes(logger, stream, config, state, deletion_column="my_deleted_column")) + # We should get the first document again, but not emit it + assert outputs == [] + # State should be the same + assert state == { + "ts": TS, + "ref": "100", + } + + outputs = list(source.read_removes(logger, stream, config, state, deletion_column="my_deleted_column")) + # We should get the first and second document but only emit the second + assert outputs == [ + { + "ref": "300", + "ts": TS + 1_000_000, + "my_deleted_column": "2022-04-03T00:00:01", + }, + ] + # Now we make sure our after token was serialized to json, + # and that it was stored within the state. + assert state == { + "ts": TS + 1_000_000, + "ref": "300", + } + + assert not source._setup_client.called + assert current_query == 3 + assert not logger.error.called + + +def test_read_updates_query(): + """ + Validates that read_updates() queries the database correctly. + """ + + PAGE_SIZE = 12344315 + INDEX = "my_index_name" + FIRST_AFTER_TOKEN = ["some magical", 3, "data"] + SECOND_AFTER_TOKEN = ["even more magical", 3, "data"] + state = {} + + def make_query(after, start=[0]): + return q.map_( + q.lambda_("x", expand_columns_query(q.select(1, q.var("x")))), + q.paginate( + q.range(q.match(q.index(INDEX)), start, []), + after=after, + size=PAGE_SIZE, + ), + ) + + current_query = 0 + QUERIES = [ + make_query(after=None), + make_query(after=FIRST_AFTER_TOKEN), + make_query(after=SECOND_AFTER_TOKEN), + make_query(after=None, start=[999, q.ref(q.collection("my_stream_name"), "10")]), + make_query(after=None, start=[999, q.ref(q.collection("my_stream_name"), "10")]), + ] + # These results come from the query, so they will already be transformed + # into the columns the user expects. Therefore, we aren't testing much + # more than the query contents here. + QUERY_RESULTS = [ + results( + # Oldest value in index + [ + { + "ref": "3", + "ts": 99, + } + ], + after=FIRST_AFTER_TOKEN, + ), + results( + [ + { + "ref": "5", + "ts": 123, + } + ], + after=SECOND_AFTER_TOKEN, + ), + results( + # Newest value in index + [ + { + "ref": "10", + "ts": 999, + } + ], + after=None, + ), + results( + # Newest value in index + [ + { + "ref": "10", + "ts": 999, + } + ], + after=None, + ), + results( + # Newest value in index + [ + { + "ref": "10", + "ts": 999, + }, + { + "ref": "11", + "ts": 1000, + }, + ], + after=None, + ), + ] + + def query_hardcoded(expr): + nonlocal current_query + assert expr == QUERIES[current_query] + result = QUERY_RESULTS[current_query] + current_query += 1 + return result + + source = SourceFauna() + source._setup_client = Mock() + source.client = MagicMock() + source.find_index_for_stream = Mock() + source.client.query = query_hardcoded + source.find_emitted_at = Mock(return_value=NOW) + + logger = mock_logger() + # Here we want updates and adds (no deletions), so Events() should be skipped. + result = list( + source.read_updates( + logger, + ConfiguredAirbyteStream( + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append_dedup, + stream=AirbyteStream( + name="my_stream_name", + json_schema={}, + ), + ), + CollectionConfig(page_size=PAGE_SIZE), + state=state, + index=INDEX, + page_size=PAGE_SIZE, + ) + ) + # Here we also validate that the cursor will stay on the latest 'ts' value. + assert result == [ + { + "ref": "3", + "ts": 99, + }, + { + "ref": "5", + "ts": 123, + }, + { + "ref": "10", + "ts": 999, + }, + ] + + assert state == {"ref": "10", "ts": 999} + + # Call again with the emitted state but no new data, we should get no results + result = list( + source.read_updates( + logger, + ConfiguredAirbyteStream( + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append_dedup, + stream=AirbyteStream( + name="my_stream_name", + json_schema={}, + ), + ), + CollectionConfig(page_size=PAGE_SIZE), + state=state, + index=INDEX, + page_size=PAGE_SIZE, + ) + ) + # Here we also validate that the cursor will stay on the latest 'ts' value. + assert result == [] + assert state == {"ref": "10", "ts": 999} + + # Call again - we should skip the record in the state again but emit the match + result = list( + source.read_updates( + logger, + ConfiguredAirbyteStream( + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append_dedup, + stream=AirbyteStream( + name="my_stream_name", + json_schema={}, + ), + ), + CollectionConfig(page_size=PAGE_SIZE), + state=state, + index=INDEX, + page_size=PAGE_SIZE, + ) + ) + # Here we also validate that the cursor will stay on the latest 'ts' value. + assert result == [{"ref": "11", "ts": 1000}] + assert state == {"ref": "11", "ts": 1000} + + assert not source._setup_client.called + assert not source.find_index_for_stream.called + assert not logger.error.called + assert current_query == 5 + + +def test_read_updates_resume(): + """ + Validates that read_updates() queries the database correctly, and resumes + a failed query correctly. + """ + + PAGE_SIZE = 12344315 + INDEX = "my_index_name" + FIRST_AFTER_TOKEN = ["some magical", 3, "data"] + SECOND_AFTER_TOKEN = ["even more magical", 3, "data"] + + def make_query(after): + return q.map_( + q.lambda_("x", expand_columns_query(q.select(1, q.var("x")))), + q.paginate( + q.range(q.match(q.index(INDEX)), [0], []), + after=after, + size=PAGE_SIZE, + ), + ) + + current_query = 0 + QUERIES = [ + make_query(after=None), + make_query(after=FIRST_AFTER_TOKEN), + make_query(after=SECOND_AFTER_TOKEN), + ] + # These results come from the query, so they will already be transformed + # into the columns the user expects. Therefore, we aren't testing much + # more than the query contents here. + QUERY_RESULTS = [ + results( + # Oldest value in index + [ + { + "ref": "3", + "ts": 99, + } + ], + after=FIRST_AFTER_TOKEN, + ), + results( + [ + { + "ref": "5", + "ts": 123, + } + ], + after=SECOND_AFTER_TOKEN, + ), + results( + # Newest value in index + [ + { + "ref": "10", + "ts": 999, + } + ], + after=None, + ), + ] + failed_yet = False + + def query_hardcoded(expr): + nonlocal current_query + nonlocal failed_yet + assert expr == QUERIES[current_query] + result = QUERY_RESULTS[current_query] + if current_query == 1 and not failed_yet: + failed_yet = True + raise ValueError("oh no something went wrong") + current_query += 1 + return result + + source = SourceFauna() + source._setup_client = Mock() + source.client = MagicMock() + source.find_index_for_stream = Mock() + source.client.query = query_hardcoded + source.find_emitted_at = Mock(return_value=NOW) + + state = {} + logger = mock_logger() + # Here we want updates and adds (no deletions), so Events() should be skipped. + result = [] + got_error = False + try: + for record in source.read_updates( + logger, + ConfiguredAirbyteStream( + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append_dedup, + stream=AirbyteStream( + name="my_stream_name", + json_schema={}, + ), + ), + CollectionConfig(page_size=PAGE_SIZE), + state=state, + index=INDEX, + page_size=PAGE_SIZE, + ): + result.append(record) + except ValueError: + got_error = True + assert "ts" not in state # This is set after we finish reading + assert "ref" not in state # This is set after we finish reading + assert "after" in state # This is some after token, serialized to json + assert got_error + assert current_query == 1 + # Here we also validate that the cursor will stay on the latest 'ts' value. + assert result == [ + { + "ref": "3", + "ts": 99, + }, + ] + assert list( + source.read_updates( + logger, + ConfiguredAirbyteStream( + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append_dedup, + stream=AirbyteStream( + name="my_stream_name", + json_schema={}, + ), + ), + CollectionConfig(page_size=PAGE_SIZE), + state=state, + index=INDEX, + page_size=PAGE_SIZE, + ) + ) == [ + { + "ref": "5", + "ts": 123, + }, + { + "ref": "10", + "ts": 999, + }, + ] + + assert state["ts"] == 999 # This is set after we finish reading + assert state["ref"] == "10" # This is set after we finish reading + assert "after" not in state # This is some after token, serialized to json + assert not source._setup_client.called + assert not source.find_index_for_stream.called + assert not logger.error.called + assert current_query == 3 diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/serialize_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/serialize_test.py new file mode 100644 index 0000000000000..8fb25b0e0fffe --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/serialize_test.py @@ -0,0 +1,64 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from datetime import date + +from faunadb.objects import FaunaTime, Ref +from source_fauna.serialize import _fauna_value_to_airbyte + + +def check_value(fauna, airbyte): + assert _fauna_value_to_airbyte(fauna) == airbyte + + +def test_date(): + check_value(date(2022, 3, 4), "2022-03-04") + check_value(FaunaTime("2022-03-04T12:00:30Z"), "2022-03-04T12:00:30Z") + + +def test_fauna_time(): + check_value(FaunaTime("2022-03-04"), "2022-03-04") + + +def test_bytes(): + check_value(bytes("hello world", "utf-8"), "aGVsbG8gd29ybGQ=") + check_value(bytearray("hello world", "utf-8"), "aGVsbG8gd29ybGQ=") + + +def test_ref(): + # Valid refs + check_value(Ref("1234", Ref("foo", Ref("collections"))), {"id": "1234", "collection": "foo", "type": "document"}) + check_value(Ref("1234", Ref("foo", Ref("nopes"))), {"id": "1234", "type": "unknown"}) + check_value(Ref("foo", Ref("collections")), {"id": "foo", "type": "collection"}) + check_value(Ref("my_db", Ref("databases")), {"id": "my_db", "type": "database"}) + check_value(Ref("ts", Ref("indexes")), {"id": "ts", "type": "index"}) + + check_value(Ref("value", Ref("keys")), {"id": "value", "type": "key"}) + check_value(Ref("value", Ref("credentials")), {"id": "value", "type": "credential"}) + check_value(Ref("value", Ref("tokens")), {"id": "value", "type": "token"}) + + # Failure cases (we should never crash, but we will produce undefined data) + check_value(Ref("1234"), {"id": "1234", "type": "unknown"}) + check_value(Ref("ts", Ref("indexes_typoed")), {"id": "ts", "type": "indexes_typoed"}) + check_value( + Ref("ref_id?", Ref("or_am_i_ref_id?", Ref("bar", Ref("collections")))), + { + "id": "ref_id?", + "type": "unknown", + }, + ) + + +def test_recursive(): + check_value({"nested_ref": Ref("3", Ref("collections"))}, {"nested_ref": {"id": "3", "type": "collection"}}) + check_value({"nested_date": date(2022, 3, 4)}, {"nested_date": "2022-03-04"}) + check_value({"nested_dict": {"nested_date": date(2022, 3, 4)}}, {"nested_dict": {"nested_date": "2022-03-04"}}) + check_value( + {"array": [date(2022, 3, 4), Ref("3", Ref("collections"))]}, + {"array": ["2022-03-04", {"id": "3", "type": "collection"}]}, + ) + check_value( + {"nested_array": {"value": [date(2022, 3, 4)]}}, + {"nested_array": {"value": ["2022-03-04"]}}, + ) diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/test_util.py b/airbyte-integrations/connectors/source-fauna/unit_tests/test_util.py new file mode 100644 index 0000000000000..f660ff55a8f08 --- /dev/null +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/test_util.py @@ -0,0 +1,113 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import Mock + +from faunadb import query as q +from faunadb.objects import Ref + + +def ref(id: int, collection="foo") -> Ref: + return Ref(str(id), cls=Ref(collection, cls=Ref("collections"))) + + +def mock_logger(): + def mock_log(level: str): + def perform_mock_log(msg: str): + print(f"[{level}]: {msg}") + + return Mock(side_effect=perform_mock_log) + + logger = Mock() + logger.info = mock_log("info") + logger.error = mock_log("error") + return logger + + +class DeletionsConfig: + def __init__(self, mode: str, column=""): + self.mode = mode + self.column = column + + @staticmethod + def ignore() -> "DeletionsConfig": + return DeletionsConfig(mode="ignore") + + @staticmethod + def deleted_field(column: str) -> "DeletionsConfig": + return DeletionsConfig(mode="deleted_field", column=column) + + +class CollectionConfig: + def __init__( + self, + page_size=64, + deletions=DeletionsConfig.ignore(), + ): + self.page_size = page_size + self.deletions = deletions + + +class DiscoverConfig: + """ + A limited version of FullConfig, storing only the values needed for discover() + """ + + def __init__(self, collection: CollectionConfig): + self.collection = collection + + +class FullConfig: + def __init__(self, domain: str, port: int, scheme: str, secret: str, collection=CollectionConfig()): + self.domain = domain + self.port = port + self.scheme = scheme + self.secret = secret + self.collection = collection + + @staticmethod + def localhost(collection=CollectionConfig()) -> "FullConfig": + # 9000 is our testing db, that we spawn in database_test.py + return FullConfig(domain="127.0.0.1", port=9000, scheme="http", secret="secret", collection=collection) + + +def partial_overwrite(obj: dict, new: dict) -> dict: + """ + Recursively replaces the values in obj with the values in new. + """ + for k, v in new.items(): + if type(v) is dict: + partial_overwrite(obj[k], v) + else: + obj[k] = v + return obj + + +def config(extra: dict[str, any]) -> dict[str, any]: + obj = { + "domain": "127.0.0.1", + "port": 8443, + "scheme": "http", + "secret": "secret", + "collection": { + "page_size": 64, + "deletions": {"deletion_mode": "ignore"}, + }, + } + return partial_overwrite(obj, extra) + + +def expand_columns_query(ref): + doc = q.var("document") + return q.let( + { + "document": q.get(ref), + }, + { + "ref": q.select(["ref", "id"], doc), + "ts": q.select("ts", doc), + "data": q.select("data", doc, {}), + "ttl": q.select("ttl", doc, None), + }, + ) diff --git a/airbyte-integrations/connectors/source-file-secure/Dockerfile b/airbyte-integrations/connectors/source-file-secure/Dockerfile index bff4cdeb671d1..9d47045b98f35 100644 --- a/airbyte-integrations/connectors/source-file-secure/Dockerfile +++ b/airbyte-integrations/connectors/source-file-secure/Dockerfile @@ -9,5 +9,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.22 +LABEL io.airbyte.version=0.2.23 LABEL io.airbyte.name=airbyte/source-file-secure diff --git a/airbyte-integrations/connectors/source-file/Dockerfile b/airbyte-integrations/connectors/source-file/Dockerfile index b54de04b0c435..c3546538f2153 100644 --- a/airbyte-integrations/connectors/source-file/Dockerfile +++ b/airbyte-integrations/connectors/source-file/Dockerfile @@ -17,5 +17,5 @@ COPY source_file ./source_file ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.22 +LABEL io.airbyte.version=0.2.23 LABEL io.airbyte.name=airbyte/source-file diff --git a/airbyte-integrations/connectors/source-freshdesk/Dockerfile b/airbyte-integrations/connectors/source-freshdesk/Dockerfile index 7f5e70b3e0436..8f5537ca25ad4 100644 --- a/airbyte-integrations/connectors/source-freshdesk/Dockerfile +++ b/airbyte-integrations/connectors/source-freshdesk/Dockerfile @@ -34,5 +34,5 @@ COPY source_freshdesk ./source_freshdesk ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.5 +LABEL io.airbyte.version=0.3.6 LABEL io.airbyte.name=airbyte/source-freshdesk diff --git a/airbyte-integrations/connectors/source-freshdesk/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-freshdesk/integration_tests/abnormal_state.json index c50753b2ac811..fd35724d0aff5 100644 --- a/airbyte-integrations/connectors/source-freshdesk/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-freshdesk/integration_tests/abnormal_state.json @@ -1,35 +1,123 @@ -{ - "agents": { - "updated_at": "2121-11-01T00:00:00Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2121-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "agents" + } + } }, - "companies": { - "updated_at": "2121-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2121-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "companies" + } + } }, - "contacts": { - "updated_at": "2121-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2121-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "contacts" + } + } }, - "conversations": { - "updated_at": "2121-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2121-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "conversations" + } + } }, - "groups": { - "updated_at": "2121-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2121-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "groups" + } + } }, - "roles": { - "updated_at": "2121-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2121-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "roles" + } + } }, - "satisfaction_ratings": { - "updated_at": "2121-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2121-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "satisfaction_ratings" + } + } }, - "skills": { - "updated_at": "2121-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2121-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "skills" + } + } }, - "surveys": { - "updated_at": "2121-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2121-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "surveys" + } + } }, - "tickets": { - "updated_at": "2121-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2121-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "tickets" + } + } }, - "time_entries": { - "updated_at": "2121-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2121-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "time_entries" + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-freshdesk/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-freshdesk/integration_tests/sample_state.json index b2fb05e31c307..6e1d23b66dc43 100644 --- a/airbyte-integrations/connectors/source-freshdesk/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-freshdesk/integration_tests/sample_state.json @@ -1,35 +1,123 @@ -{ - "agents": { - "updated_at": "2021-01-01T00:00:00Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "agents" + } + } }, - "companies": { - "updated_at": "2021-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2021-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "companies" + } + } }, - "contacts": { - "updated_at": "2021-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2021-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "contacts" + } + } }, - "conversations": { - "updated_at": "2021-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2021-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "conversations" + } + } }, - "groups": { - "updated_at": "2021-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2021-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "groups" + } + } }, - "roles": { - "updated_at": "2021-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2021-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "roles" + } + } }, - "satisfaction_ratings": { - "updated_at": "2021-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2021-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "satisfaction_ratings" + } + } }, - "skills": { - "updated_at": "2021-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2021-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "skills" + } + } }, - "surveys": { - "updated_at": "2021-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2021-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "surveys" + } + } }, - "tickets": { - "updated_at": "2021-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2021-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "tickets" + } + } }, - "time_entries": { - "updated_at": "2021-11-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": "2021-11-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "time_entries" + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile index 72d021ed68727..112569d346609 100644 --- a/airbyte-integrations/connectors/source-github/Dockerfile +++ b/airbyte-integrations/connectors/source-github/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.1 +LABEL io.airbyte.version=0.3.3 LABEL io.airbyte.name=airbyte/source-github diff --git a/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json index db9341eaf0837..7fdd31d4ab2a4 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json @@ -1,145 +1,237 @@ -{ - "comments": { - "airbytehq/integration-test": { - "updated_at": "2121-06-30T10:22:10Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2121-06-30T10:22:10Z" } + }, + "stream_descriptor": { "name": "comments" } } }, - "commit_comment_reactions": { - "airbytehq/integration-test": { - "55538825": { - "created_at": "2121-12-31T23:59:59Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { + "55538825": { "created_at": "2121-12-31T23:59:59Z" }, + "55538840": { "created_at": "2121-12-31T23:59:59Z" } + } }, - "55538840": { - "created_at": "2121-12-31T23:59:59Z" - } + "stream_descriptor": { "name": "commit_comment_reactions" } } }, - "commit_comments": { - "airbytehq/integration-test": { - "updated_at": "2121-04-30T20:36:17Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2121-04-30T20:36:17Z" } + }, + "stream_descriptor": { "name": "commit_comments" } } }, - "commits": { - "airbytehq/integration-test": { - "created_at": "2121-06-30T10:04:41Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "created_at": "2121-06-30T10:04:41Z" } + }, + "stream_descriptor": { "name": "commits" } } }, - "deployments": { - "airbytehq/integration-test": { - "updated_at": "2121-06-30T10:04:41Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2121-06-30T10:04:41Z" } + }, + "stream_descriptor": { "name": "deployments" } } }, - "events": { - "airbytehq/integration-test": { - "created_at": "2121-06-29T03:44:45Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "created_at": "2121-06-29T03:44:45Z" } + }, + "stream_descriptor": { "name": "events" } } }, - "issue_comment_reactions": { - "airbytehq/integration-test": { - "907296275": { - "created_at": "2121-12-31T23:59:59Z" - } + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { + "907296275": { "created_at": "2121-12-31T23:59:59Z" } + } + }, + "stream_descriptor": { "name": "issue_comment_reactions" } } }, - "issue_events": { - "airbytehq/integration-test": { - "created_at": "2121-06-29T01:49:42Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "created_at": "2121-06-29T01:49:42Z" } + }, + "stream_descriptor": { "name": "issue_events" } } }, - "issue_milestones": { - "airbytehq/integration-test": { - "updated_at": "2121-06-25T22:28:33Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2121-06-25T22:28:33Z" } + }, + "stream_descriptor": { "name": "issue_milestones" } } }, - "issue_reactions": { - "airbytehq/integration-test": { - "11": { - "created_at": "2121-12-31T23:59:59Z" - } + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { + "11": { "created_at": "2121-12-31T23:59:59Z" } + } + }, + "stream_descriptor": { "name": "issue_reactions" } } }, - "issues": { - "airbytehq/integration-test": { - "updated_at": "2121-06-30T06:44:42Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2121-06-30T06:44:42Z" } + }, + "stream_descriptor": { "name": "issues" } } }, - "project_cards": { - "airbytehq/integration-test": { - "13167124": { - "17807006": { - "updated_at": "2121-06-29T02:04:57Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { + "13167124": { "17807006": { "updated_at": "2121-06-29T02:04:57Z" } } } - } + }, + "stream_descriptor": { "name": "project_cards" } } }, - "project_columns": { - "airbytehq/integration-test": { - "13167122": { - "updated_at": "2121-06-29T02:04:57Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { + "13167122": { "updated_at": "2121-06-29T02:04:57Z" }, + "13167124": { "updated_at": "2121-06-29T02:04:57Z" } + } }, - "13167124": { - "updated_at": "2121-06-29T02:04:57Z" - } + "stream_descriptor": { "name": "project_columns" } } }, - "projects": { - "airbytehq/integration-test": { - "updated_at": "2121-06-28T17:24:51Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2121-06-28T17:24:51Z" } + }, + "stream_descriptor": { "name": "projects" } } }, - "pull_request_comment_reactions": { - "airbytehq/integration-test": { - "created_at": "2121-12-31T23:59:59Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "created_at": "2121-12-31T23:59:59Z" } + }, + "stream_descriptor": { "name": "pull_request_comment_reactions" } } }, - "pull_request_stats": { - "airbytehq/integration-test": { - "updated_at": "2121-06-29T02:04:57Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2121-06-29T02:04:57Z" } + }, + "stream_descriptor": { "name": "pull_request_stats" } } }, - "pull_requests": { - "airbytehq/integration-test": { - "updated_at": "2121-06-28T23:36:35Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2121-06-28T23:36:35Z" } + }, + "stream_descriptor": { "name": "pull_requests" } } }, - "releases": { - "airbytehq/integration-test": { - "created_at": "2121-06-23T23:57:07Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "created_at": "2121-06-23T23:57:07Z" } + }, + "stream_descriptor": { "name": "releases" } } }, - "repositories": { - "airbytehq": { - "updated_at": "2121-12-31T23:59:59Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "airbytehq": { "updated_at": "2121-12-31T23:59:59Z" } }, + "stream_descriptor": { "name": "repositories" } } }, - "review_comments": { - "airbytehq/integration-test": { - "updated_at": "2121-06-23T23:57:07Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2121-06-23T23:57:07Z" } + }, + "stream_descriptor": { "name": "review_comments" } } }, - "reviews": { - "airbytehq/integration-test": { - "updated_at": "2121-06-29T02:04:57Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2121-06-29T02:04:57Z" } + }, + "stream_descriptor": { "name": "reviews" } } }, - "stargazers": { - "airbytehq/integration-test": { - "starred_at": "2121-06-29T02:04:57Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "starred_at": "2121-06-29T02:04:57Z" } + }, + "stream_descriptor": { "name": "stargazers" } } }, - "workflows": { - "airbytehq/integration-test": { - "updated_at": "2121-12-31T23:59:59Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2121-12-31T23:59:59Z" } + }, + "stream_descriptor": { "name": "workflows" } } }, - "workflow_runs": { - "airbytehq/integration-test": { - "updated_at": "2121-12-31T23:59:59Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2121-12-31T23:59:59Z" } + }, + "stream_descriptor": { "name": "workflow_runs" } } }, - "workflow_jobs": { - "airbytehq/integration-test": { - "completed_at": "2121-12-31T23:59:59Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "completed_at": "2121-12-31T23:59:59Z" } + }, + "stream_descriptor": { "name": "workflow_jobs" } } } -} +] diff --git a/airbyte-integrations/connectors/source-github/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-github/integration_tests/sample_state.json index 30d7036b88498..80561c1deefda 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-github/integration_tests/sample_state.json @@ -1,77 +1,137 @@ -{ - "commit_comments": { - "airbytehq/integration-test": { - "updated_at": "2021-04-30T20:36:17Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2021-04-30T20:36:17Z" } + }, + "stream_descriptor": { "name": "commit_comments" } } }, - "projects": { - "airbytehq/integration-test": { - "updated_at": "2021-06-28T17:24:51Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2021-06-28T17:24:51Z" } + }, + "stream_descriptor": { "name": "projects" } } }, - "stargazers": { - "airbytehq/integration-test": { - "starred_at": "2021-06-29T02:04:57Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "starred_at": "2021-06-29T02:04:57Z" } + }, + "stream_descriptor": { "name": "stargazers" } } }, - "events": { - "airbytehq/integration-test": { - "created_at": "2021-06-29T03:44:45Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "created_at": "2021-06-29T03:44:45Z" } + }, + "stream_descriptor": { "name": "events" } } }, - "issue_events": { - "airbytehq/integration-test": { - "created_at": "2021-06-29T01:49:42Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "created_at": "2021-06-29T01:49:42Z" } + }, + "stream_descriptor": { "name": "issue_events" } } }, - "releases": { - "airbytehq/integration-test": { - "created_at": "2021-06-23T23:57:07Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "created_at": "2021-06-23T23:57:07Z" } + }, + "stream_descriptor": { "name": "releases" } } }, - "pull_request_stats": { - "airbytehq/integration-test": { - "updated_at": "2021-08-30T12:01:15Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2021-08-30T12:01:15Z" } + }, + "stream_descriptor": { "name": "pull_request_stats" } } }, - "pull_requests": { - "airbytehq/integration-test": { - "updated_at": "2021-06-28T23:36:35Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2021-06-28T23:36:35Z" } + }, + "stream_descriptor": { "name": "pull_requests" } } }, - "issue_milestones": { - "airbytehq/integration-test": { - "updated_at": "2021-06-25T22:28:33Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2021-06-25T22:28:33Z" } + }, + "stream_descriptor": { "name": "issue_milestones" } } }, - "issues": { - "airbytehq/integration-test": { - "updated_at": "2021-06-30T11:32:49Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2021-06-30T11:32:49Z" } + }, + "stream_descriptor": { "name": "issues" } } }, - "comments": { - "airbytehq/integration-test": { - "updated_at": "2021-06-30T10:22:10Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2021-06-30T10:22:10Z" } + }, + "stream_descriptor": { "name": "comments" } } }, - "commits": { - "airbytehq/integration-test": { - "created_at": "2021-06-30T10:04:41Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "created_at": "2021-06-30T10:04:41Z" } + }, + "stream_descriptor": { "name": "commits" } } }, - "reviews": { - "airbytehq/integration-test": { - "updated_at": "2021-08-30T12:01:15Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "updated_at": "2021-08-30T12:01:15Z" } + }, + "stream_descriptor": { "name": "reviews" } } }, - "workflow_runs": { - "airbytehq/integration-test": { - "completed_at": "2021-08-30T12:01:15Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "completed_at": "2021-08-30T12:01:15Z" } + }, + "stream_descriptor": { "name": "workflow_runs" } } }, - "workflow_jobs": { - "airbytehq/integration-test": { - "completed_at": "2021-08-30T12:01:15Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "airbytehq/integration-test": { "completed_at": "2021-08-30T12:01:15Z" } + }, + "stream_descriptor": { "name": "workflow_jobs" } } } -} +] diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py index fba2e03189445..db18092f14c7a 100644 --- a/airbyte-integrations/connectors/source-github/source_github/source.py +++ b/airbyte-integrations/connectors/source-github/source_github/source.py @@ -225,7 +225,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: team_members_stream, Users(**organization_args), Workflows(**repository_args_with_start_date), - WorkflowRuns(**repository_args_with_start_date), + workflow_runs_stream, WorkflowJobs(parent=workflow_runs_stream, **repository_args_with_start_date), TeamMemberships(parent=team_members_stream, **repository_args), ] diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index 17c42df169ec1..7c495444a067f 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -1353,18 +1353,26 @@ def __init__(self, parent: WorkflowRuns, **kwargs): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"repos/{stream_slice['repository']}/actions/runs/{stream_slice['run_id']}/jobs" - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( - sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state - ) + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + parent_stream_state = None + if stream_state is not None: + parent_stream_state = {repository: {self.parent.cursor_field: v[self.cursor_field]} for repository, v in stream_state.items()} + parent_stream_slices = self.parent.stream_slices(sync_mode=sync_mode, cursor_field=cursor_field, stream_state=parent_stream_state) for stream_slice in parent_stream_slices: parent_records = self.parent.read_records( - sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=parent_stream_state ) for record in parent_records: - yield {"repository": record["repository"]["full_name"], "run_id": record["id"]} + stream_slice["run_id"] = record["id"] + yield from super().read_records( + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) def parse_response( self, @@ -1373,14 +1381,16 @@ def parse_response( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> Iterable[Mapping]: - for record in response.json().get("jobs"): # GitHub puts records in an array. - yield self.transform(record=record, stream_slice=stream_slice) + for record in response.json()["jobs"]: + if record.get(self.cursor_field): + yield self.transform(record=record, stream_slice=stream_slice) - def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]: - record = super().transform(record=record, stream_slice=stream_slice) - record["run_id"] = stream_slice["run_id"] - record["repository"] = stream_slice["repository"] - return record + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + params["filter"] = "all" + return params class TeamMembers(GithubStream): diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py index b4d5cafd1a542..c72497fc0acc0 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py @@ -1077,426 +1077,105 @@ def test_stream_workflow_runs_read_incremental(monkeypatch): @responses.activate -def test_stream_workflow_jobs_read_incremental(): +def test_stream_workflow_jobs_read(): repository_args = { "repositories": ["org/repo"], "page_size_for_large_streams": 100, } - repository_args_with_start_date = {**repository_args, "start_date": "2022-09-01T00:00:00Z"} + repository_args_with_start_date = {**repository_args, "start_date": "2022-09-02T09:05:00Z"} workflow_runs_stream = WorkflowRuns(**repository_args_with_start_date) stream = WorkflowJobs(workflow_runs_stream, **repository_args_with_start_date) - data = [ + workflow_runs = [ { "id": 1, - "completed_at": "2022-09-02T09:11:02Z", - "run_id": 1, - "steps": [ - {"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1}, - {"name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", "status": "completed", "conclusion": "success", "number": 2}, - ], + "created_at": "2022-09-02T09:00:00Z", + "updated_at": "2022-09-02T09:10:02Z", + "repository": {"full_name": "org/repo"}, }, { "id": 2, - "completed_at": "2022-09-02T10:11:02Z", - "run_id": 1, - "steps": [ - {"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1}, - {"name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", "status": "completed", "conclusion": "success", "number": 2}, - ], - }, - { - "id": 3, - "completed_at": "2022-09-02T09:11:02Z", - "run_id": 2, - "steps": [ - {"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1}, - {"name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", "status": "completed", "conclusion": "success", "number": 2}, - ], - }, - { - "id": 4, - "completed_at": "2022-09-02T10:11:02Z", - "run_id": 2, - "steps": [ - {"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1}, - {"name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", "status": "completed", "conclusion": "success", "number": 2}, - ], + "created_at": "2022-09-02T09:06:00Z", + "updated_at": "2022-09-02T09:08:00Z", + "repository": {"full_name": "org/repo"}, }, ] - responses.add( - "GET", - "https://api.github.com/repos/org/repo/actions/runs", - json={ - "total_count": 2, - "workflow_runs": [ - { - "id": 1, - "created_at": "2022-09-02T09:10:02Z", - "updated_at": "2022-09-02T09:10:02Z", - "repository": {"full_name": "org/repo"}, - }, - { - "id": 2, - "created_at": "2022-09-02T09:10:04Z", - "updated_at": "2022-09-02T09:10:04Z", - "repository": {"full_name": "org/repo"}, - }, - ], - }, - ) + workflow_jobs_1 = [ + {"id": 1, "completed_at": "2022-09-02T09:02:00Z", "run_id": 1}, + {"id": 4, "completed_at": "2022-09-02T09:10:00Z", "run_id": 1}, + {"id": 5, "completed_at": None, "run_id": 1}, + ] - responses.add("GET", "https://api.github.com/repos/org/repo/actions/runs/1/jobs", json={"jobs": data[0:2]}) + workflow_jobs_2 = [ + {"id": 2, "completed_at": "2022-09-02T09:07:00Z", "run_id": 2}, + {"id": 3, "completed_at": "2022-09-02T09:08:00Z", "run_id": 2}, + ] responses.add( "GET", - "https://api.github.com/repos/org/repo/actions/runs/2/jobs", - json={"jobs": data[2:4]}, + "https://api.github.com/repos/org/repo/actions/runs", + json={"total_count": len(workflow_runs), "workflow_runs": workflow_runs}, ) + responses.add("GET", "https://api.github.com/repos/org/repo/actions/runs/1/jobs", json={"jobs": workflow_jobs_1}) + responses.add("GET", "https://api.github.com/repos/org/repo/actions/runs/2/jobs", json={"jobs": workflow_jobs_2}) state = {} records = read_incremental(stream, state) - - assert state == {"org/repo": {"completed_at": "2022-09-02T10:11:02Z"}} + assert state == {"org/repo": {"completed_at": "2022-09-02T09:10:00Z"}} assert records == [ - { - "id": 1, - "completed_at": "2022-09-02T09:11:02Z", - "run_id": 1, - "steps": [ - {"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1}, - {"name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", "status": "completed", "conclusion": "success", "number": 2}, - ], - "repository": "org/repo", - }, - { - "id": 2, - "completed_at": "2022-09-02T10:11:02Z", - "run_id": 1, - "steps": [ - {"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1}, - {"name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", "status": "completed", "conclusion": "success", "number": 2}, - ], - "repository": "org/repo", - }, - { - "id": 3, - "completed_at": "2022-09-02T09:11:02Z", - "run_id": 2, - "steps": [ - {"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1}, - {"name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", "status": "completed", "conclusion": "success", "number": 2}, - ], - "repository": "org/repo", - }, - { - "id": 4, - "completed_at": "2022-09-02T10:11:02Z", - "run_id": 2, - "steps": [ - {"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1}, - {"name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", "status": "completed", "conclusion": "success", "number": 2}, - ], - "repository": "org/repo", - }, + {"completed_at": "2022-09-02T09:10:00Z", "id": 4, "repository": "org/repo", "run_id": 1}, + {"completed_at": "2022-09-02T09:07:00Z", "id": 2, "repository": "org/repo", "run_id": 2}, + {"completed_at": "2022-09-02T09:08:00Z", "id": 3, "repository": "org/repo", "run_id": 2}, ] assert len(responses.calls) == 3 - data.insert( - 0, + workflow_jobs_1[2]["completed_at"] = "2022-09-02T09:12:00Z" + workflow_runs[0]["updated_at"] = "2022-09-02T09:12:01Z" + workflow_runs.append( { - "id": 5, - "completed_at": "2022-09-03T01:00:00Z", - "run_id": 2, - "steps": [ - {"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1}, - {"name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", "status": "completed", "conclusion": "success", "number": 2}, - ], - }, - ) - - data[2]["completed_at"] = "2022-09-04T01:00:00Z" # data with ID 2 - - responses.add("GET", "https://api.github.com/repos/org/repo/actions/runs/1/jobs", json={"jobs": data[0:1]}) - - responses.add("GET", "https://api.github.com/repos/org/repo/actions/runs/1/jobs", json={"jobs": data[1:2]}) - - responses.add( - "GET", - "https://api.github.com/repos/org/repo/actions/runs/2/jobs", - json={"jobs": data[2:3]}, + "id": 3, + "created_at": "2022-09-02T09:14:00Z", + "updated_at": "2022-09-02T09:15:00Z", + "repository": {"full_name": "org/repo"}, + } ) + workflow_jobs_3 = [ + {"id": 6, "completed_at": "2022-09-02T09:15:00Z", "run_id": 3}, + {"id": 7, "completed_at": None, "run_id": 3}, + ] responses.add( "GET", - "https://api.github.com/repos/org/repo/actions/runs/2/jobs", - json={"jobs": data[3:4]}, + "https://api.github.com/repos/org/repo/actions/runs", + json={"total_count": len(workflow_runs), "workflow_runs": workflow_runs}, ) + responses.add("GET", "https://api.github.com/repos/org/repo/actions/runs/1/jobs", json={"jobs": workflow_jobs_1}) + responses.add("GET", "https://api.github.com/repos/org/repo/actions/runs/2/jobs", json={"jobs": workflow_jobs_2}) + responses.add("GET", "https://api.github.com/repos/org/repo/actions/runs/3/jobs", json={"jobs": workflow_jobs_3}) responses.calls.reset() records = read_incremental(stream, state) - assert state == {"org/repo": {"completed_at": "2022-09-04T01:00:00Z"}} + assert state == {"org/repo": {"completed_at": "2022-09-02T09:15:00Z"}} assert records == [ - { - "id": 5, - "completed_at": "2022-09-03T01:00:00Z", - "run_id": 1, - "steps": [ - {"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1}, - {"name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", "status": "completed", "conclusion": "success", "number": 2}, - ], - "repository": "org/repo", - }, - { - "id": 2, - "completed_at": "2022-09-04T01:00:00Z", - "run_id": 2, - "steps": [ - {"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1}, - {"name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", "status": "completed", "conclusion": "success", "number": 2}, - ], - "repository": "org/repo", - }, + {"completed_at": "2022-09-02T09:12:00Z", "id": 5, "repository": "org/repo", "run_id": 1}, + {"completed_at": "2022-09-02T09:15:00Z", "id": 6, "repository": "org/repo", "run_id": 3}, ] - -@responses.activate -def test_stream_workflow_jobs_full_refresh(): - - repository_args = { - "repositories": ["org/repo"], - "page_size_for_large_streams": 100, - } - repository_args_with_start_date = {**repository_args, "start_date": "2022-09-01T00:00:00Z"} - - workflow_runs_stream = WorkflowRuns(**repository_args_with_start_date) - stream = WorkflowJobs(workflow_runs_stream, **repository_args) - - responses.add( - "GET", - "https://api.github.com/repos/org/repo/actions/runs", - json={ - "total_count": 2, - "workflow_runs": [ - { - "id": 1, - "created_at": "2022-09-02T09:10:02Z", - "updated_at": "2022-09-02T09:10:02Z", - "repository": {"full_name": "org/repo"}, - }, - { - "id": 2, - "created_at": "2022-09-02T09:10:04Z", - "updated_at": "2022-09-02T09:10:04Z", - "repository": {"full_name": "org/repo"}, - }, - ], - }, - ) - - responses.add( - "GET", - "https://api.github.com/repos/org/repo/actions/runs/1/jobs", - json={ - "jobs": [ - { - "id": 1, - "completed_at": "2022-09-02T09:11:02Z", - "run_id": 1, - "steps": [ - { - "name": "Set up job", - "status": "completed", - "conclusion": "success", - "number": 1, - "completed_at": "2022-09-02T09:01:00.000-00:00", - }, - { - "name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", - "status": "completed", - "conclusion": "success", - "number": 2, - "completed_at": "2022-09-02T09:02:00.000-00:00", - }, - ], - }, - { - "id": 2, - "completed_at": "2022-09-02T10:11:02Z", - "run_id": 1, - "steps": [ - { - "name": "Set up job", - "status": "completed", - "conclusion": "success", - "number": 1, - "completed_at": "2022-09-02T10:01:00.000-00:00", - }, - { - "name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", - "status": "completed", - "conclusion": "success", - "number": 2, - "completed_at": "2022-09-02T10:02:00.000-00:00", - }, - ], - }, - ] - }, - ) - - responses.add( - "GET", - "https://api.github.com/repos/org/repo/actions/runs/2/jobs", - json={ - "jobs": [ - { - "id": 3, - "completed_at": "2022-09-02T09:11:02Z", - "run_id": 2, - "steps": [ - { - "name": "Set up job", - "status": "completed", - "conclusion": "success", - "number": 1, - "completed_at": "2022-09-02T09:01:00.000-00:00", - }, - { - "name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", - "status": "completed", - "conclusion": "success", - "number": 2, - "completed_at": "2022-09-02T09:02:00.000-00:00", - }, - ], - }, - { - "id": 4, - "completed_at": "2022-09-02T10:11:02Z", - "run_id": 2, - "steps": [ - { - "name": "Set up job", - "status": "completed", - "conclusion": "success", - "number": 1, - "completed_at": "2022-09-02T10:01:00.000-00:00", - }, - { - "name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", - "status": "completed", - "conclusion": "success", - "number": 2, - "completed_at": "2022-09-02T10:02:00.000-00:00", - }, - ], - }, - ] - }, - ) - records = list(read_full_refresh(stream)) - assert records == [ - { - "id": 1, - "completed_at": "2022-09-02T09:11:02Z", - "run_id": 1, - "steps": [ - { - "name": "Set up job", - "status": "completed", - "conclusion": "success", - "number": 1, - "completed_at": "2022-09-02T09:01:00.000-00:00", - }, - { - "name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", - "status": "completed", - "conclusion": "success", - "number": 2, - "completed_at": "2022-09-02T09:02:00.000-00:00", - }, - ], - "repository": "org/repo", - }, - { - "id": 2, - "completed_at": "2022-09-02T10:11:02Z", - "run_id": 1, - "steps": [ - { - "name": "Set up job", - "status": "completed", - "conclusion": "success", - "number": 1, - "completed_at": "2022-09-02T10:01:00.000-00:00", - }, - { - "name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", - "status": "completed", - "conclusion": "success", - "number": 2, - "completed_at": "2022-09-02T10:02:00.000-00:00", - }, - ], - "repository": "org/repo", - }, - { - "id": 3, - "completed_at": "2022-09-02T09:11:02Z", - "run_id": 2, - "steps": [ - { - "name": "Set up job", - "status": "completed", - "conclusion": "success", - "number": 1, - "completed_at": "2022-09-02T09:01:00.000-00:00", - }, - { - "name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", - "status": "completed", - "conclusion": "success", - "number": 2, - "completed_at": "2022-09-02T09:02:00.000-00:00", - }, - ], - "repository": "org/repo", - }, - { - "id": 4, - "completed_at": "2022-09-02T10:11:02Z", - "run_id": 2, - "steps": [ - { - "name": "Set up job", - "status": "completed", - "conclusion": "success", - "number": 1, - "completed_at": "2022-09-02T10:01:00.000-00:00", - }, - { - "name": "Pull ghcr.io/rtcamp/action-slack-notify:v2.2.0", - "status": "completed", - "conclusion": "success", - "number": 2, - "completed_at": "2022-09-02T10:02:00.000-00:00", - }, - ], - "repository": "org/repo", - }, + {"id": 4, "completed_at": "2022-09-02T09:10:00Z", "run_id": 1, "repository": "org/repo"}, + {"id": 5, "completed_at": "2022-09-02T09:12:00Z", "run_id": 1, "repository": "org/repo"}, + {"id": 2, "completed_at": "2022-09-02T09:07:00Z", "run_id": 2, "repository": "org/repo"}, + {"id": 3, "completed_at": "2022-09-02T09:08:00Z", "run_id": 2, "repository": "org/repo"}, + {"id": 6, "completed_at": "2022-09-02T09:15:00Z", "run_id": 3, "repository": "org/repo"}, ] - assert len(responses.calls) == 3 - @responses.activate def test_stream_pull_request_comment_reactions_read(): diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile index 50b797af4f233..ccb8d79694f34 100644 --- a/airbyte-integrations/connectors/source-google-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-google-ads/Dockerfile @@ -13,5 +13,5 @@ COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-google-ads diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py index a5e9b8f655aff..c510eb402160a 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py @@ -34,10 +34,6 @@ def configured_catalog(): def test_incremental_sync(config, configured_catalog): - today = pendulum.now().date() - start_date = today.subtract(months=3) - config["start_date"] = start_date.to_date_string() - config.pop("end_date", "") google_ads_client = SourceGoogleAds() records = list(google_ads_client.read(logging.getLogger("airbyte"), config, ConfiguredAirbyteCatalog.parse_obj(configured_catalog))) latest_state = None @@ -51,7 +47,7 @@ def test_incremental_sync(config, configured_catalog): continue cursor_value = message.record.data["segments.date"] assert cursor_value <= latest_state - assert cursor_value >= start_date.subtract(days=GAP_DAYS).to_date_string() + assert cursor_value >= pendulum.parse(config["start_date"]).subtract(days=GAP_DAYS).to_date_string() # next sync records = list( diff --git a/airbyte-integrations/connectors/source-google-ads/setup.py b/airbyte-integrations/connectors/source-google-ads/setup.py index 59237952ce387..5d271ee09994e 100644 --- a/airbyte-integrations/connectors/source-google-ads/setup.py +++ b/airbyte-integrations/connectors/source-google-ads/setup.py @@ -7,7 +7,7 @@ # pin protobuf==3.20.0 as other versions may cause problems on different architectures # (see https://github.com/airbytehq/airbyte/issues/13580) -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "google-ads==17.0.0", "protobuf==3.20.0", "pendulum"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "google-ads==17.0.0", "protobuf==3.20.0", "pendulum"] TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock", "freezegun", "requests-mock"] diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile index 1f225969a8abd..4ce0459699b37 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile +++ b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.25 +LABEL io.airbyte.version=0.1.26 LABEL io.airbyte.name=airbyte/source-google-analytics-v4 diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/abnormal_state.json index 0cb746bf6ba36..6886688610f5e 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/abnormal_state.json @@ -1,14 +1,86 @@ -{ - "website_overview": { "ga_date": "2050-05-01" }, - "traffic_sources": { "ga_date": "2050-05-01" }, - "pages": { "ga_date": "2050-05-01" }, - "locations": { "ga_date": "2050-05-01" }, - "monthly_active_users": { "ga_date": "2050-05-01" }, - "four_weekly_active_users": { "ga_date": "2050-05-01" }, - "two_weekly_active_users": { "ga_date": "2050-05-01" }, - "weekly_active_users": { "ga_date": "2050-05-01" }, - "daily_active_users": { "ga_date": "2050-05-01" }, - "devices": { "ga_date": "2050-05-01" }, - "users_per_day": { "ga_date": "2050-05-01" }, - "new_users_per_day": { "ga_date": "2050-05-01" } -} +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "website_overview" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "traffic_sources" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "pages" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "locations" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "monthly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "four_weekly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "two_weekly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "weekly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "daily_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "devices" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "users_per_day" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2050-05-01" }, + "stream_descriptor": { "name": "new_users_per_day" } + } + } +] diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/sample_state.json index d636881bc2a22..0c8625660e070 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/sample_state.json @@ -1,14 +1,86 @@ -{ - "website_overview": { "ga_date": "2021-02-11" }, - "traffic_sources": { "ga_date": "2021-02-11" }, - "pages": { "ga_date": "2021-02-11" }, - "locations": { "ga_date": "2021-02-11" }, - "monthly_active_users": { "ga_date": "2021-02-11" }, - "four_weekly_active_users": { "ga_date": "2021-02-11" }, - "two_weekly_active_users": { "ga_date": "2021-02-11" }, - "weekly_active_users": { "ga_date": "2021-02-11" }, - "daily_active_users": { "ga_date": "2021-02-11" }, - "devices": { "ga_date": "2021-02-11" }, - "users_per_day": { "ga_date": "2021-02-11" }, - "new_users_per_day": { "ga_date": "2021-02-11" } -} +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "website_overview" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "traffic_sources" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "pages" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "locations" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "monthly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "four_weekly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "two_weekly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "weekly_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "daily_active_users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "devices" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "users_per_day" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "ga_date": "2021-02-11" }, + "stream_descriptor": { "name": "new_users_per_day" } + } + } +] diff --git a/airbyte-integrations/connectors/source-google-search-console/Dockerfile b/airbyte-integrations/connectors/source-google-search-console/Dockerfile index 01f5c226f56a1..f8d9e0e92c15f 100755 --- a/airbyte-integrations/connectors/source-google-search-console/Dockerfile +++ b/airbyte-integrations/connectors/source-google-search-console/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.15 +LABEL io.airbyte.version=0.1.16 LABEL io.airbyte.name=airbyte/source-google-search-console diff --git a/airbyte-integrations/connectors/source-google-search-console/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-google-search-console/integration_tests/abnormal_state.json index 94c564562d151..00a2163baf264 100755 --- a/airbyte-integrations/connectors/source-google-search-console/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-google-search-console/integration_tests/abnormal_state.json @@ -1,114 +1,100 @@ -{ - "search_analytics_by_date": { - "https://airbyte.io/": { - "web": { - "date": "2023-08-28" - }, - "news": { - "date": "2023-08-28" - }, - "image": { - "date": "2023-08-28" - }, - "video": { - "date": "2023-08-28" - } +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "https://airbyte.io/": { + "web": { "date": "2023-08-28" }, + "news": { "date": "2023-08-28" }, + "image": { "date": "2023-08-28" }, + "video": { "date": "2023-08-28" } + } + }, + "stream_descriptor": { "name": "search_analytics_by_date" } } }, - "search_analytics_by_country": { - "https://airbyte.io/": { - "web": { - "date": "2023-08-28" - }, - "news": { - "date": "2023-08-28" - }, - "image": { - "date": "2023-08-28" - }, - "video": { - "date": "2023-08-28" - } + { + "type": "STREAM", + "stream": { + "stream_state": { + "https://airbyte.io/": { + "web": { "date": "2023-08-28" }, + "news": { "date": "2023-08-28" }, + "image": { "date": "2023-08-28" }, + "video": { "date": "2023-08-28" } + } + }, + "stream_descriptor": { "name": "search_analytics_by_country" } } }, - "search_analytics_by_device": { - "https://airbyte.io/": { - "web": { - "date": "2023-08-28" - }, - "news": { - "date": "2023-08-28" - }, - "image": { - "date": "2023-08-28" - }, - "video": { - "date": "2023-08-28" - } + { + "type": "STREAM", + "stream": { + "stream_state": { + "https://airbyte.io/": { + "web": { "date": "2023-08-28" }, + "news": { "date": "2023-08-28" }, + "image": { "date": "2023-08-28" }, + "video": { "date": "2023-08-28" } + } + }, + "stream_descriptor": { "name": "search_analytics_by_device" } } }, - "search_analytics_by_page": { - "https://airbyte.io/": { - "web": { - "date": "2023-08-28" - }, - "news": { - "date": "2023-08-28" - }, - "image": { - "date": "2023-08-28" - }, - "video": { - "date": "2023-08-28" - } + { + "type": "STREAM", + "stream": { + "stream_state": { + "https://airbyte.io/": { + "web": { "date": "2023-08-28" }, + "news": { "date": "2023-08-28" }, + "image": { "date": "2023-08-28" }, + "video": { "date": "2023-08-28" } + } + }, + "stream_descriptor": { "name": "search_analytics_by_page" } } }, - "search_analytics_by_query": { - "https://airbyte.io/": { - "web": { - "date": "2023-08-28" - }, - "news": { - "date": "2023-08-28" - }, - "image": { - "date": "2023-08-28" - }, - "video": { - "date": "2023-08-28" - } + { + "type": "STREAM", + "stream": { + "stream_state": { + "https://airbyte.io/": { + "web": { "date": "2023-08-28" }, + "news": { "date": "2023-08-28" }, + "image": { "date": "2023-08-28" }, + "video": { "date": "2023-08-28" } + } + }, + "stream_descriptor": { "name": "search_analytics_by_query" } } }, - "search_analytics_all_fields": { - "https://airbyte.io/": { - "web": { - "date": "2023-08-28" - }, - "news": { - "date": "2023-08-28" - }, - "image": { - "date": "2023-08-28" - }, - "video": { - "date": "2023-08-28" - } + { + "type": "STREAM", + "stream": { + "stream_state": { + "https://airbyte.io/": { + "web": { "date": "2023-08-28" }, + "news": { "date": "2023-08-28" }, + "image": { "date": "2023-08-28" }, + "video": { "date": "2023-08-28" } + } + }, + "stream_descriptor": { "name": "search_analytics_all_fields" } } }, - "custom_dimensions": { - "https://airbyte.io/": { - "web": { - "date": "2023-08-28" - }, - "news": { - "date": "2023-08-28" - }, - "image": { - "date": "2023-08-28" - }, - "video": { - "date": "2023-08-28" - } + { + "type": "STREAM", + "stream": { + "stream_state": { + "https://airbyte.io/": { + "web": { "date": "2023-08-28" }, + "news": { "date": "2023-08-28" }, + "image": { "date": "2023-08-28" }, + "video": { "date": "2023-08-28" } + } + }, + "stream_descriptor": { "name": "custom_dimensions" } } } -} +] diff --git a/airbyte-integrations/connectors/source-google-sheets/Dockerfile b/airbyte-integrations/connectors/source-google-sheets/Dockerfile index 27819f8a245d9..9e3cf439c68ac 100644 --- a/airbyte-integrations/connectors/source-google-sheets/Dockerfile +++ b/airbyte-integrations/connectors/source-google-sheets/Dockerfile @@ -34,5 +34,5 @@ COPY google_sheets_source ./google_sheets_source ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.17 +LABEL io.airbyte.version=0.2.19 LABEL io.airbyte.name=airbyte/source-google-sheets diff --git a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py index 5ea77cee62a08..8588720eeba3c 100644 --- a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py +++ b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py @@ -5,13 +5,14 @@ import json import socket -from typing import Dict, Generator +from typing import Any, Generator, List, MutableMapping, Union from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models.airbyte_protocol import ( AirbyteCatalog, AirbyteConnectionStatus, AirbyteMessage, + AirbyteStateMessage, ConfiguredAirbyteCatalog, Status, Type, @@ -125,7 +126,11 @@ def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: raise Exception(f"Could not run discovery: {reason}") def read( - self, logger: AirbyteLogger, config: json, catalog: ConfiguredAirbyteCatalog, state: Dict[str, any] + self, + logger: AirbyteLogger, + config: json, + catalog: ConfiguredAirbyteCatalog, + state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None, ) -> Generator[AirbyteMessage, None, None]: client = GoogleSheetsClient(self.get_credentials(config)) diff --git a/airbyte-integrations/connectors/source-google-sheets/setup.py b/airbyte-integrations/connectors/source-google-sheets/setup.py index 05a8321684092..6a24a19567213 100644 --- a/airbyte-integrations/connectors/source-google-sheets/setup.py +++ b/airbyte-integrations/connectors/source-google-sheets/setup.py @@ -12,7 +12,7 @@ "google-auth-httplib2", "google-api-python-client", "PyYAML==5.4", - "pydantic==1.6.2", + "pydantic~=1.9.2", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-greenhouse/Dockerfile b/airbyte-integrations/connectors/source-greenhouse/Dockerfile index 66f56636848b9..f0c9562367595 100644 --- a/airbyte-integrations/connectors/source-greenhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-greenhouse/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.10 +LABEL io.airbyte.version=0.2.11 LABEL io.airbyte.name=airbyte/source-greenhouse diff --git a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml index bd26bb598f731..bb4e50f9ad088 100644 --- a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml @@ -5,8 +5,6 @@ tests: spec: - spec_path: "source_greenhouse/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/config_invalid.json" @@ -19,8 +17,16 @@ tests: configured_catalog_path: "integration_tests/configured_catalog.json" expect_records: path: "integration_tests/expected_records.txt" - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_users_only.json" + - config_path: "secrets/config_users_only.json" + # test we do not fail when encounter 403 error + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [ + "applications", "applications_demographics_answers", "applications_interviews", "candidates", "close_reasons", + "custom_fields", "degrees", "demographics_answers", "demographics_answer_options", "questions", + "demographics_answers_answer_options", "demographics_question_sets", "demographics_question_sets_questions", + "departments", "jobs", "jobs_openings", "interviews", "job_posts", "job_stages", "jobs_stages", "offers", + "rejection_reasons", "scorecards", "sources", "demographics_questions" + ] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt index c5ae337060fdf..105b06b32ab80 100644 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt @@ -1,530 +1,536 @@ -{"stream": "applications", "data": {"status": "active", "source": {"public_name": "HRMARKET", "id": 4000067003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "John Lafleur", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:24:37.049Z", "jobs": [], "job_post_id": null, "id": 19214950003, "current_stage": null, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130511003, "attachments": [], "applied_at": "2020-11-24T23:24:37.023Z", "answers": []}, "emitted_at": 1662402660037} -{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Jobs page on your website", "id": 4000177003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "John Lafleur", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:25:13.804Z", "jobs": [], "job_post_id": null, "id": 19214993003, "current_stage": null, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130554003, "attachments": [], "applied_at": "2020-11-24T23:25:13.781Z", "answers": []}, "emitted_at": 1662402660042} -{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Internal Applicant", "id": 4000142003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2020-11-24T23:28:19.779Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 19215172003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130732003, "attachments": [], "applied_at": "2020-11-24T23:28:19.712Z", "answers": []}, "emitted_at": 1662402660043} -{"stream": "applications", "data": {"status": "rejected", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": {"type": {"name": "We rejected them", "id": 4000000003}, "name": "Other (add notes below)", "id": 4000004003}, "rejection_details": {}, "rejected_at": "2021-09-29T16:38:03.637Z", "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-09-29T16:38:03.660Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 44933447003, "current_stage": {"name": "Phone Interview", "id": 5245805003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 40513954003, "attachments": [], "applied_at": "2021-09-29T16:37:27.589Z", "answers": []}, "emitted_at": 1662402660044} -{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-10-10T16:22:13.708Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 44937562003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "Greenhouse Admin", "last_name": "Admin", "id": 4218085003, "first_name": "Greenhouse", "employee_id": null}, "candidate_id": 40517966003, "attachments": [], "applied_at": "2021-09-29T17:20:36.063Z", "answers": []}, "emitted_at": 1662402660045} -{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-11-03T19:56:07.402Z", "jobs": [{"name": "Test job 3", "id": 4466310003}], "job_post_id": 4797691003, "id": 47459993003, "current_stage": {"name": "Application Review", "id": 7332462003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 42921157003, "attachments": [], "applied_at": "2021-11-03T19:51:14.644Z", "answers": [{"question": "Website", "answer": null}, {"question": "LinkedIn Profile", "answer": null}]}, "emitted_at": 1662402660046} -{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Bubblesort", "id": 4000032003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-11-22T08:41:55.713Z", "jobs": [{"name": "Copy of Test Job 2", "id": 4446240003}], "job_post_id": null, "id": 48693310003, "current_stage": {"name": "Application Review", "id": 7179760003}, "credited_to": {"name": "emily.brooks+airbyte_integration@greenhouse.io", "last_name": null, "id": 4218087003, "first_name": null, "employee_id": null}, "candidate_id": 44081361003, "attachments": [], "applied_at": "2021-11-22T08:41:55.640Z", "answers": []}, "emitted_at": 1662402660047} -{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:24:37.050Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Test", "last_activity": "2020-11-24T23:24:37.049Z", "is_private": false, "id": 17130511003, "first_name": "Test", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:24:37.018Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "HRMARKET", "id": 4000067003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "John Lafleur", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:24:37.049Z", "jobs": [], "job_post_id": null, "id": 19214950003, "current_stage": null, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130511003, "attachments": [], "applied_at": "2020-11-24T23:24:37.023Z", "answers": []}], "application_ids": [19214950003], "addresses": []}, "emitted_at": 1662403631906} -{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:25:13.806Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Test2", "last_activity": "2020-11-24T23:25:13.804Z", "is_private": false, "id": 17130554003, "first_name": "Test2", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:25:13.777Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Jobs page on your website", "id": 4000177003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "John Lafleur", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:25:13.804Z", "jobs": [], "job_post_id": null, "id": 19214993003, "current_stage": null, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130554003, "attachments": [], "applied_at": "2020-11-24T23:25:13.781Z", "answers": []}], "application_ids": [19214993003], "addresses": []}, "emitted_at": 1662403631910} -{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:28:19.781Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Lastname", "last_activity": "2020-11-24T23:28:19.779Z", "is_private": false, "id": 17130732003, "first_name": "Name", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:28:19.710Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Internal Applicant", "id": 4000142003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2020-11-24T23:28:19.779Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 19215172003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130732003, "attachments": [], "applied_at": "2020-11-24T23:28:19.712Z", "answers": []}], "application_ids": [19215172003], "addresses": []}, "emitted_at": 1662403631912} -{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2021-09-29T16:38:03.672Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "User", "last_activity": "2021-09-29T16:38:03.660Z", "is_private": false, "id": 40513954003, "first_name": "Test", "employments": [], "email_addresses": [], "educations": [], "created_at": "2021-09-29T16:37:27.585Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "rejected", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": {"type": {"name": "We rejected them", "id": 4000000003}, "name": "Other (add notes below)", "id": 4000004003}, "rejection_details": {}, "rejected_at": "2021-09-29T16:38:03.637Z", "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-09-29T16:38:03.660Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 44933447003, "current_stage": {"name": "Phone Interview", "id": 5245805003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 40513954003, "attachments": [], "applied_at": "2021-09-29T16:37:27.589Z", "answers": []}], "application_ids": [44933447003], "addresses": []}, "emitted_at": 1662403631914} -{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2021-10-10T16:22:13.718Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Scheduled Interview", "last_activity": "2021-10-10T16:22:13.708Z", "is_private": false, "id": 40517966003, "first_name": "Test", "employments": [], "email_addresses": [{"value": "vadym.hevlich@zazmic.com", "type": "personal"}], "educations": [], "created_at": "2021-09-29T17:20:36.038Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-10-10T16:22:13.708Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 44937562003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "Greenhouse Admin", "last_name": "Admin", "id": 4218085003, "first_name": "Greenhouse", "employee_id": null}, "candidate_id": 40517966003, "attachments": [], "applied_at": "2021-09-29T17:20:36.063Z", "answers": []}], "application_ids": [44937562003], "addresses": []}, "emitted_at": 1662403631916} -{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2021-11-03T19:56:07.423Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Candidate", "last_activity": "2021-11-03T19:56:07.402Z", "is_private": false, "id": 42921157003, "first_name": "Test", "employments": [], "email_addresses": [{"value": "vadym.hevlich@zazmic.com", "type": "work"}], "educations": [], "created_at": "2021-11-03T19:51:14.639Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-11-03T19:56:07.402Z", "jobs": [{"name": "Test job 3", "id": 4466310003}], "job_post_id": 4797691003, "id": 47459993003, "current_stage": {"name": "Application Review", "id": 7332462003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 42921157003, "attachments": [], "applied_at": "2021-11-03T19:51:14.644Z", "answers": [{"question": "Website", "answer": null}, {"question": "LinkedIn Profile", "answer": null}]}], "application_ids": [47459993003], "addresses": []}, "emitted_at": 1662403631917} -{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2021-11-22T08:41:55.716Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Cherniaev", "last_activity": "2021-11-22T08:41:55.713Z", "is_private": false, "id": 44081361003, "first_name": "Yurii", "employments": [], "email_addresses": [], "educations": [], "created_at": "2021-11-22T08:41:55.634Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Bubblesort", "id": 4000032003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-11-22T08:41:55.713Z", "jobs": [{"name": "Copy of Test Job 2", "id": 4446240003}], "job_post_id": null, "id": 48693310003, "current_stage": {"name": "Application Review", "id": 7179760003}, "credited_to": {"name": "emily.brooks+airbyte_integration@greenhouse.io", "last_name": null, "id": 4218087003, "first_name": null, "employee_id": null}, "candidate_id": 44081361003, "attachments": [], "applied_at": "2021-11-22T08:41:55.640Z", "answers": []}], "application_ids": [48693310003], "addresses": []}, "emitted_at": 1662403631919} -{"stream":"close_reasons","data":{"id":4010635003,"name":"Not Filling"},"emitted_at":1660156523420} -{"stream":"close_reasons","data":{"id":4010634003,"name":"On Hold"},"emitted_at":1660156523422} -{"stream":"close_reasons","data":{"id":4010633003,"name":"Hire - New Headcount"},"emitted_at":1660156523422} -{"stream":"close_reasons","data":{"id":4010632003,"name":"Hire - Backfill"},"emitted_at":1660156523422} -{"stream":"degrees","data":{"id":10848287003,"name":"High School","priority":0,"external_id":null},"emitted_at":1660156523707} -{"stream":"degrees","data":{"id":10848288003,"name":"Associate's Degree","priority":1,"external_id":null},"emitted_at":1660156523708} -{"stream":"degrees","data":{"id":10848289003,"name":"Bachelor's Degree","priority":2,"external_id":null},"emitted_at":1660156523709} -{"stream":"degrees","data":{"id":10848290003,"name":"Master's Degree","priority":3,"external_id":null},"emitted_at":1660156523709} -{"stream":"degrees","data":{"id":10848291003,"name":"Master of Business Administration (M.B.A.)","priority":4,"external_id":null},"emitted_at":1660156523709} -{"stream":"degrees","data":{"id":10848292003,"name":"Juris Doctor (J.D.)","priority":5,"external_id":null},"emitted_at":1660156523709} -{"stream":"degrees","data":{"id":10848293003,"name":"Doctor of Medicine (M.D.)","priority":6,"external_id":null},"emitted_at":1660156523709} -{"stream":"degrees","data":{"id":10848294003,"name":"Doctor of Philosophy (Ph.D.)","priority":7,"external_id":null},"emitted_at":1660156523710} -{"stream":"degrees","data":{"id":10848295003,"name":"Engineer's Degree","priority":8,"external_id":null},"emitted_at":1660156523710} -{"stream":"degrees","data":{"id":10848296003,"name":"Other","priority":9,"external_id":null},"emitted_at":1660156523710} -{"stream":"departments","data":{"id":4028123003,"name":"test dep 2","parent_id":null,"parent_department_external_id":null,"child_ids":[],"child_department_external_ids":[],"external_id":null},"emitted_at":1660156524114} -{"stream":"departments","data":{"id":4028122003,"name":"Test dep1","parent_id":null,"parent_department_external_id":null,"child_ids":[],"child_department_external_ids":[],"external_id":null},"emitted_at":1660156524116} -{"stream":"job_posts","data":{"id":4252332003,"active":true,"live":false,"first_published_at":null,"title":"Test job","location":{"id":4219721003,"name":"test","office_id":null,"job_post_location_type":{"id":4000000003,"name":"Free Text"}},"internal":false,"external":true,"job_id":4177046003,"content":"

Test description

","internal_content":null,"updated_at":"2021-04-02T17:38:54.835Z","created_at":"2020-11-24T23:29:24.315Z","demographic_question_set_id":null,"questions":[{"required":true,"private":false,"label":"First Name","name":"first_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Last Name","name":"last_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Email","name":"email","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Phone","name":"phone","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Resume","name":"resume","type":"attachment","values":[],"description":null},{"required":false,"private":false,"label":"Cover Letter","name":"cover_letter","type":"attachment","values":[],"description":null},{"required":null,"private":false,"label":"LinkedIn Profile","name":"question_5125927003","type":"short_text","values":[],"description":null},{"required":null,"private":false,"label":"Website","name":"question_5125928003","type":"short_text","values":[],"description":null}]},"emitted_at":1660156524473} -{"stream":"job_posts","data":{"id":4751597003,"active":true,"live":false,"first_published_at":null,"title":"Test Job 2","location":{"id":4700649003,"name":"US","office_id":null,"job_post_location_type":{"id":4000000003,"name":"Free Text"}},"internal":false,"external":true,"job_id":4177048003,"content":"

Job post content

","internal_content":null,"updated_at":"2021-10-07T18:46:59.032Z","created_at":"2021-10-07T18:46:58.846Z","demographic_question_set_id":null,"questions":[{"required":true,"private":false,"label":"First Name","name":"first_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Last Name","name":"last_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Email","name":"email","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Phone","name":"phone","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Resume","name":"resume","type":"attachment","values":[],"description":null},{"required":false,"private":false,"label":"Cover Letter","name":"cover_letter","type":"attachment","values":[],"description":null},{"required":null,"private":false,"label":"LinkedIn Profile","name":"question_7911674003","type":"short_text","values":[],"description":null},{"required":null,"private":false,"label":"Website","name":"question_7911675003","type":"short_text","values":[],"description":null}]},"emitted_at":1660156524477} -{"stream":"job_posts","data":{"id":4752433003,"active":true,"live":false,"first_published_at":null,"title":"Test Job 2","location":{"id":4701484003,"name":"US","office_id":null,"job_post_location_type":{"id":4000000003,"name":"Free Text"}},"internal":false,"external":true,"job_id":4446240003,"content":"

Job post content

","internal_content":null,"updated_at":"2021-10-08T08:19:42.720Z","created_at":"2021-10-08T08:19:42.720Z","demographic_question_set_id":null,"questions":[{"required":true,"private":false,"label":"First Name","name":"first_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Last Name","name":"last_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Email","name":"email","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Phone","name":"phone","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Resume","name":"resume","type":"attachment","values":[],"description":null},{"required":false,"private":false,"label":"Cover Letter","name":"cover_letter","type":"attachment","values":[],"description":null},{"required":null,"private":false,"label":"LinkedIn Profile","name":"question_7918434003","type":"short_text","values":[],"description":null},{"required":null,"private":false,"label":"Website","name":"question_7918435003","type":"short_text","values":[],"description":null}]},"emitted_at":1660156524478} -{"stream":"job_posts","data":{"id":4797691003,"active":true,"live":false,"first_published_at":null,"title":"Test job 3","location":{"id":4746722003,"name":"US","office_id":null,"job_post_location_type":{"id":4000000003,"name":"Free Text"}},"internal":false,"external":true,"job_id":4466310003,"content":"

job description

","internal_content":null,"updated_at":"2021-11-03T19:48:29.808Z","created_at":"2021-11-03T19:48:29.629Z","demographic_question_set_id":4000198003,"questions":[{"required":true,"private":false,"label":"First Name","name":"first_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Last Name","name":"last_name","type":"short_text","values":[],"description":null},{"required":true,"private":false,"label":"Email","name":"email","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Phone","name":"phone","type":"short_text","values":[],"description":null},{"required":false,"private":false,"label":"Resume","name":"resume","type":"attachment","values":[],"description":null},{"required":false,"private":false,"label":"Cover Letter","name":"cover_letter","type":"attachment","values":[],"description":null},{"required":null,"private":false,"label":"LinkedIn Profile","name":"question_8215275003","type":"short_text","values":[],"description":null},{"required":null,"private":false,"label":"Website","name":"question_8215276003","type":"short_text","values":[],"description":null}]},"emitted_at":1660156524479} -{"stream":"jobs","data":{"id":4177046003,"name":"Test job","requisition_id":"3","notes":null,"confidential":false,"is_template":true,"copied_from_id":null,"status":"open","created_at":"2020-11-24T23:27:11.699Z","opened_at":"2020-11-24T23:27:11.878Z","closed_at":null,"updated_at":"2021-04-21T17:48:24.779Z","departments":[{"id":4028122003,"name":"Test dep1","parent_id":null,"parent_department_external_id":null,"child_ids":[],"child_department_external_ids":[],"external_id":null}],"offices":[{"id":4019854003,"name":"Test office","location":{"name":null},"primary_contact_user_id":4218086003,"parent_id":null,"parent_office_external_id":null,"child_ids":[],"child_office_external_ids":[],"external_id":null}],"hiring_team":{"hiring_managers":[],"recruiters":[],"coordinators":[],"sourcers":[]},"openings":[{"id":4320015003,"opening_id":"3-1","status":"open","opened_at":"2020-11-24T23:27:11.723Z","closed_at":null,"application_id":null,"close_reason":null}],"custom_fields":{"employment_type":null},"keyed_custom_fields":{"employment_type":{"name":"Employment Type","type":"single_select","value":null}}},"emitted_at":1660156524869} -{"stream":"jobs","data":{"id":4177048003,"name":"Test Job 2","requisition_id":"4","notes":null,"confidential":false,"is_template":false,"copied_from_id":null,"status":"open","created_at":"2020-11-24T23:27:45.634Z","opened_at":"2020-11-24T23:27:45.878Z","closed_at":null,"updated_at":"2021-10-07T18:46:58.982Z","departments":[{"id":4028123003,"name":"test dep 2","parent_id":null,"parent_department_external_id":null,"child_ids":[],"child_department_external_ids":[],"external_id":null}],"offices":[{"id":4019854003,"name":"Test office","location":{"name":null},"primary_contact_user_id":4218086003,"parent_id":null,"parent_office_external_id":null,"child_ids":[],"child_office_external_ids":[],"external_id":null}],"hiring_team":{"hiring_managers":[],"recruiters":[],"coordinators":[],"sourcers":[]},"openings":[{"id":4320018003,"opening_id":"4-1","status":"open","opened_at":"2020-11-24T23:27:45.665Z","closed_at":null,"application_id":null,"close_reason":null}],"custom_fields":{"employment_type":null},"keyed_custom_fields":{"employment_type":{"name":"Employment Type","type":"single_select","value":null}}},"emitted_at":1660156524875} -{"stream":"jobs","data":{"id":4446240003,"name":"Copy of Test Job 2","requisition_id":"5","notes":null,"confidential":false,"is_template":false,"copied_from_id":4177048003,"status":"open","created_at":"2021-10-08T08:19:42.383Z","opened_at":"2021-10-08T08:19:42.818Z","closed_at":null,"updated_at":"2021-10-08T08:19:42.821Z","departments":[{"id":4028123003,"name":"test dep 2","parent_id":null,"parent_department_external_id":null,"child_ids":[],"child_department_external_ids":[],"external_id":null}],"offices":[{"id":4019854003,"name":"Test office","location":{"name":null},"primary_contact_user_id":4218086003,"parent_id":null,"parent_office_external_id":null,"child_ids":[],"child_office_external_ids":[],"external_id":null}],"hiring_team":{"hiring_managers":[],"recruiters":[],"coordinators":[],"sourcers":[]},"openings":[{"id":4928188003,"opening_id":"5-2","status":"open","opened_at":"2021-10-10T16:39:24.949Z","closed_at":null,"application_id":null,"close_reason":null},{"id":4928187003,"opening_id":"5-2","status":"open","opened_at":"2021-10-10T16:39:08.365Z","closed_at":null,"application_id":null,"close_reason":null},{"id":4928186003,"opening_id":"5-1","status":"open","opened_at":"2021-10-10T16:38:57.407Z","closed_at":null,"application_id":null,"close_reason":null},{"id":4926182003,"opening_id":"5-1","status":"open","opened_at":"2021-10-08T08:19:42.457Z","closed_at":null,"application_id":null,"close_reason":null},{"id":4926183003,"opening_id":"5-2","status":"open","opened_at":"2021-10-08T08:19:42.457Z","closed_at":null,"application_id":null,"close_reason":null}],"custom_fields":{"employment_type":"Full-time"},"keyed_custom_fields":{"employment_type":{"name":"Employment Type","type":"single_select","value":"Full-time"}}},"emitted_at":1660156524875} -{"stream":"jobs","data":{"id":4466310003,"name":"Test job 3","requisition_id":"6","notes":null,"confidential":false,"is_template":false,"copied_from_id":null,"status":"open","created_at":"2021-11-03T19:46:51.107Z","opened_at":"2021-11-03T19:46:51.347Z","closed_at":null,"updated_at":"2021-11-03T19:48:29.760Z","departments":[{"id":4028122003,"name":"Test dep1","parent_id":null,"parent_department_external_id":null,"child_ids":[],"child_department_external_ids":[],"external_id":null}],"offices":[{"id":4019854003,"name":"Test office","location":{"name":null},"primary_contact_user_id":4218086003,"parent_id":null,"parent_office_external_id":null,"child_ids":[],"child_office_external_ids":[],"external_id":null}],"hiring_team":{"hiring_managers":[],"recruiters":[],"coordinators":[],"sourcers":[]},"openings":[{"id":4970166003,"opening_id":"6-1","status":"open","opened_at":"2021-11-30T01:00:00.000Z","closed_at":null,"application_id":null,"close_reason":null}],"custom_fields":{"employment_type":"Full-time"},"keyed_custom_fields":{"employment_type":{"name":"Employment Type","type":"single_select","value":"Full-time"}}},"emitted_at":1660156524876} -{"stream":"offers","data":{"id":4154100003,"version":1,"application_id":19215333003,"created_at":"2020-11-24T23:32:25.760Z","updated_at":"2020-11-24T23:32:25.772Z","sent_at":null,"resolved_at":null,"starts_at":"2020-12-04","status":"unresolved","job_id":4177048003,"candidate_id":17130848003,"opening":{"id":4320018003,"opening_id":"4-1","status":"open","opened_at":"2020-11-24T23:27:45.665Z","closed_at":null,"application_id":null,"close_reason":null},"custom_fields":{"employment_type":"Contract"},"keyed_custom_fields":{"employment_type":{"name":"Employment Type","type":"single_select","value":"Contract"}}},"emitted_at":1660156525177} -{"stream":"scorecards","data":{"id":5253031003,"updated_at":"2020-11-24T23:33:10.440Z","created_at":"2020-11-24T23:33:10.440Z","interview":"Application Review","interview_step":{"id":5628634003,"name":"Application Review"},"candidate_id":17130848003,"application_id":19215333003,"interviewed_at":"2020-11-25T01:00:00.000Z","submitted_by":{"id":4218086003,"first_name":"John","last_name":"Lafleur","name":"John Lafleur","employee_id":null},"interviewer":{"id":4218086003,"first_name":"John","last_name":"Lafleur","name":"John Lafleur","employee_id":null},"submitted_at":"2020-11-24T23:33:10.440Z","overall_recommendation":"no_decision","attributes":[{"name":"Willing to do required travel","type":"Details","note":null,"rating":"no_decision"},{"name":"Three to five years of experience","type":"Qualifications","note":null,"rating":"no_decision"},{"name":"Personable","type":"Personality Traits","note":null,"rating":"no_decision"},{"name":"Passionate","type":"Personality Traits","note":null,"rating":"no_decision"},{"name":"Organizational Skills","type":"Skills","note":null,"rating":"no_decision"},{"name":"Manage competing priorities","type":"Skills","note":null,"rating":"no_decision"},{"name":"Fits our salary range","type":"Details","note":null,"rating":"no_decision"},{"name":"Empathetic","type":"Personality Traits","note":null,"rating":"no_decision"},{"name":"Currently based locally","type":"Details","note":null,"rating":"no_decision"},{"name":"Communication","type":"Skills","note":null,"rating":"no_decision"}],"ratings":{"definitely_not":[],"no":[],"mixed":[],"yes":[],"strong_yes":[]},"questions":[{"id":null,"question":"Key Take-Aways","answer":""},{"id":null,"question":"Private Notes","answer":""}]},"emitted_at":1660156525508} -{"stream":"scorecards","data":{"id":9664505003,"updated_at":"2021-09-29T17:23:11.468Z","created_at":"2021-09-29T17:23:11.468Z","interview":"Preliminary Screening Call","interview_step":{"id":5628615003,"name":"Preliminary Screening Call"},"candidate_id":40517966003,"application_id":44937562003,"interviewed_at":"2021-09-29T01:00:00.000Z","submitted_by":{"id":4218086003,"first_name":"John","last_name":"Lafleur","name":"John Lafleur","employee_id":null},"interviewer":{"id":4218086003,"first_name":"John","last_name":"Lafleur","name":"John Lafleur","employee_id":null},"submitted_at":"2021-09-29T17:23:11.468Z","overall_recommendation":"no_decision","attributes":[{"name":"Willing to do required travel","type":"Details","note":null,"rating":"yes"},{"name":"Three to five years of experience","type":"Qualifications","note":null,"rating":"mixed"},{"name":"Personable","type":"Personality Traits","note":null,"rating":"yes"},{"name":"Passionate","type":"Personality Traits","note":null,"rating":"mixed"},{"name":"Organizational Skills","type":"Skills","note":null,"rating":"yes"},{"name":"Manage competing priorities","type":"Skills","note":null,"rating":"yes"},{"name":"Fits our salary range","type":"Details","note":null,"rating":"yes"},{"name":"Empathetic","type":"Personality Traits","note":null,"rating":"strong_yes"},{"name":"Currently based locally","type":"Details","note":null,"rating":"mixed"},{"name":"Communication","type":"Skills","note":null,"rating":"no"}],"ratings":{"definitely_not":[],"no":["Communication"],"mixed":["Three to five years of experience","Passionate","Currently based locally"],"yes":["Willing to do required travel","Personable","Organizational Skills","Manage competing priorities","Fits our salary range"],"strong_yes":["Empathetic"]},"questions":[{"id":null,"question":"Key Take-Aways","answer":"test"},{"id":null,"question":"Private Notes","answer":""}]},"emitted_at":1660156525511} -{"stream":"users","data":{"id":4218085003,"name":"Greenhouse Admin","first_name":"Greenhouse","last_name":"Admin","primary_email_address":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","updated_at":"2020-11-18T14:09:08.401Z","created_at":"2020-11-18T14:09:08.401Z","disabled":false,"site_admin":true,"emails":["scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com"],"employee_id":null,"linked_candidate_ids":[]},"emitted_at":1660156525823} -{"stream":"users","data":{"id":4218086003,"name":"John Lafleur","first_name":"John","last_name":"Lafleur","primary_email_address":"integration-test@airbyte.io","updated_at":"2022-08-21T20:10:15.420Z","created_at":"2020-11-18T14:09:08.481Z","disabled":false,"site_admin":true,"emails":["integration-test@airbyte.io"],"employee_id":null,"linked_candidate_ids":[]},"emitted_at":1660156525826} -{"stream":"users","data":{"id":4218087003,"name":"emily.brooks+airbyte_integration@greenhouse.io","first_name":null,"last_name":null,"primary_email_address":"emily.brooks+airbyte_integration@greenhouse.io","updated_at":"2020-11-18T14:09:08.991Z","created_at":"2020-11-18T14:09:08.809Z","disabled":false,"site_admin":true,"emails":["emily.brooks+airbyte_integration@greenhouse.io"],"employee_id":null,"linked_candidate_ids":[]},"emitted_at":1660156525826} -{"stream":"users","data":{"id":4460715003,"name":"Vadym Ratniuk","first_name":"Vadym","last_name":"Ratniuk","primary_email_address":"vadym.ratniuk@globallogic.com","updated_at":"2021-09-18T10:09:16.846Z","created_at":"2021-09-14T14:03:01.050Z","disabled":false,"site_admin":false,"emails":["vadym.ratniuk@globallogic.com"],"employee_id":null,"linked_candidate_ids":[]},"emitted_at":1660156525826} -{"stream":"users","data":{"id":4481107003,"name":"Vadym Hevlich","first_name":"Vadym","last_name":"Hevlich","primary_email_address":"vadym.hevlich@zazmic.com","updated_at":"2021-10-10T17:49:28.058Z","created_at":"2021-10-10T17:48:41.978Z","disabled":false,"site_admin":true,"emails":["vadym.hevlich@zazmic.com"],"employee_id":null,"linked_candidate_ids":[]},"emitted_at":1660156525827} -{"stream":"custom_fields","data":{"id":4680899003,"name":"Degree","active":true,"field_type":"candidate","priority":1,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"degree","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10848287003,"name":"High School","priority":0,"external_id":null},{"id":10848288003,"name":"Associate's Degree","priority":1,"external_id":null},{"id":10848289003,"name":"Bachelor's Degree","priority":2,"external_id":null},{"id":10848290003,"name":"Master's Degree","priority":3,"external_id":null},{"id":10848291003,"name":"Master of Business Administration (M.B.A.)","priority":4,"external_id":null},{"id":10848292003,"name":"Juris Doctor (J.D.)","priority":5,"external_id":null},{"id":10848293003,"name":"Doctor of Medicine (M.D.)","priority":6,"external_id":null},{"id":10848294003,"name":"Doctor of Philosophy (Ph.D.)","priority":7,"external_id":null},{"id":10848295003,"name":"Engineer's Degree","priority":8,"external_id":null},{"id":10848296003,"name":"Other","priority":9,"external_id":null}]},"emitted_at":1660156526606} -{"stream":"custom_fields","data":{"id":4680900003,"name":"Discipline","active":true,"field_type":"candidate","priority":2,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"discipline","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10848297003,"name":"Accounting","priority":0,"external_id":null},{"id":10848298003,"name":"African Studies","priority":1,"external_id":null},{"id":10848299003,"name":"Agriculture","priority":2,"external_id":null},{"id":10848300003,"name":"Anthropology","priority":3,"external_id":null},{"id":10848301003,"name":"Applied Health Services","priority":4,"external_id":null},{"id":10848302003,"name":"Architecture","priority":5,"external_id":null},{"id":10848303003,"name":"Art","priority":6,"external_id":null},{"id":10848304003,"name":"Asian Studies","priority":7,"external_id":null},{"id":10848305003,"name":"Biology","priority":8,"external_id":null},{"id":10848306003,"name":"Business","priority":9,"external_id":null},{"id":10848307003,"name":"Business Administration","priority":10,"external_id":null},{"id":10848308003,"name":"Chemistry","priority":11,"external_id":null},{"id":10848309003,"name":"Classical Languages","priority":12,"external_id":null},{"id":10848310003,"name":"Communications & Film","priority":13,"external_id":null},{"id":10848311003,"name":"Computer Science","priority":14,"external_id":null},{"id":10848312003,"name":"Dentistry","priority":15,"external_id":null},{"id":10848313003,"name":"Developing Nations","priority":16,"external_id":null},{"id":10848314003,"name":"Discipline Unknown","priority":17,"external_id":null},{"id":10848315003,"name":"Earth Sciences","priority":18,"external_id":null},{"id":10848316003,"name":"Economics","priority":19,"external_id":null},{"id":10848317003,"name":"Education","priority":20,"external_id":null},{"id":10848318003,"name":"Electronics","priority":21,"external_id":null},{"id":10848319003,"name":"Engineering","priority":22,"external_id":null},{"id":10848320003,"name":"English Studies","priority":23,"external_id":null},{"id":10848321003,"name":"Environmental Studies","priority":24,"external_id":null},{"id":10848322003,"name":"European Studies","priority":25,"external_id":null},{"id":10848323003,"name":"Fashion","priority":26,"external_id":null},{"id":10848324003,"name":"Finance","priority":27,"external_id":null},{"id":10848325003,"name":"Fine Arts","priority":28,"external_id":null},{"id":10848326003,"name":"General Studies","priority":29,"external_id":null},{"id":10848327003,"name":"Health Services","priority":30,"external_id":null},{"id":10848328003,"name":"History","priority":31,"external_id":null},{"id":10848329003,"name":"Human Resources Management","priority":32,"external_id":null},{"id":10848330003,"name":"Humanities","priority":33,"external_id":null},{"id":10848331003,"name":"Industrial Arts & Carpentry","priority":34,"external_id":null},{"id":10848332003,"name":"Information Systems","priority":35,"external_id":null},{"id":10848333003,"name":"International Relations","priority":36,"external_id":null},{"id":10848334003,"name":"Journalism","priority":37,"external_id":null},{"id":10848335003,"name":"Languages","priority":38,"external_id":null},{"id":10848336003,"name":"Latin American Studies","priority":39,"external_id":null},{"id":10848337003,"name":"Law","priority":40,"external_id":null},{"id":10848338003,"name":"Linguistics","priority":41,"external_id":null},{"id":10848339003,"name":"Manufacturing & Mechanics","priority":42,"external_id":null},{"id":10848340003,"name":"Mathematics","priority":43,"external_id":null},{"id":10848341003,"name":"Medicine","priority":44,"external_id":null},{"id":10848342003,"name":"Middle Eastern Studies","priority":45,"external_id":null},{"id":10848343003,"name":"Naval Science","priority":46,"external_id":null},{"id":10848344003,"name":"North American Studies","priority":47,"external_id":null},{"id":10848345003,"name":"Nuclear Technics","priority":48,"external_id":null},{"id":10848346003,"name":"Operations Research & Strategy","priority":49,"external_id":null},{"id":10848347003,"name":"Organizational Theory","priority":50,"external_id":null},{"id":10848348003,"name":"Philosophy","priority":51,"external_id":null},{"id":10848349003,"name":"Physical Education","priority":52,"external_id":null},{"id":10848350003,"name":"Physical Sciences","priority":53,"external_id":null},{"id":10848351003,"name":"Physics","priority":54,"external_id":null},{"id":10848352003,"name":"Political Science","priority":55,"external_id":null},{"id":10848353003,"name":"Psychology","priority":56,"external_id":null},{"id":10848354003,"name":"Public Policy","priority":57,"external_id":null},{"id":10848355003,"name":"Public Service","priority":58,"external_id":null},{"id":10848356003,"name":"Religious Studies","priority":59,"external_id":null},{"id":10848357003,"name":"Russian & Soviet Studies","priority":60,"external_id":null},{"id":10848358003,"name":"Scandinavian Studies","priority":61,"external_id":null},{"id":10848359003,"name":"Science","priority":62,"external_id":null},{"id":10848360003,"name":"Slavic Studies","priority":63,"external_id":null},{"id":10848361003,"name":"Social Science","priority":64,"external_id":null},{"id":10848362003,"name":"Social Sciences","priority":65,"external_id":null},{"id":10848363003,"name":"Sociology","priority":66,"external_id":null},{"id":10848364003,"name":"Speech","priority":67,"external_id":null},{"id":10848365003,"name":"Statistics & Decision Theory","priority":68,"external_id":null},{"id":10848366003,"name":"Urban Studies","priority":69,"external_id":null},{"id":10848367003,"name":"Veterinary Medicine","priority":70,"external_id":null},{"id":10848368003,"name":"Other","priority":71,"external_id":null}]},"emitted_at":1660156526607} -{"stream": "custom_fields", "data": {"id": 4680901003, "name": "Employment Type", "active": true, "field_type": "job", "priority": 0, "value_type": "single_select", "private": false, "required": false, "require_approval": true, "trigger_new_version": false, "name_key": "employment_type", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845796003, "name": "Full-time", "priority": 0, "external_id": null}, {"id": 10845797003, "name": "Part-time", "priority": 1, "external_id": null}, {"id": 10845798003, "name": "Intern", "priority": 2, "external_id": null}, {"id": 10845799003, "name": "Contract", "priority": 3, "external_id": null}, {"id": 10845800003, "name": "Temporary", "priority": 4, "external_id": null}]}, "emitted_at": 1662401663417} -{"stream": "custom_fields", "data": {"id": 4680902003, "name": "Start Date", "active": true, "field_type": "offer", "priority": 0, "value_type": "date", "private": true, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "start_date", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663418} -{"stream": "custom_fields", "data": {"id": 4680903003, "name": "Employment Type", "active": true, "field_type": "offer", "priority": 1, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": true, "name_key": "employment_type", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845801003, "name": "Full-time", "priority": 0, "external_id": null}, {"id": 10845802003, "name": "Part-time", "priority": 1, "external_id": null}, {"id": 10845803003, "name": "Intern", "priority": 2, "external_id": null}, {"id": 10845804003, "name": "Contract", "priority": 3, "external_id": null}, {"id": 10845805003, "name": "Temporary", "priority": 4, "external_id": null}]}, "emitted_at": 1662401663418} -{"stream": "custom_fields", "data": {"id": 4680904003, "name": "Offer Documents", "active": true, "field_type": "offer", "priority": 2, "value_type": "short_text", "private": true, "required": false, "require_approval": false, "trigger_new_version": true, "name_key": "offer_documents", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663418} -{"stream": "custom_fields", "data": {"id": 4680905003, "name": "Relationship", "active": true, "field_type": "referral_question", "priority": 0, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "relationship", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845806003, "name": "Coworker", "priority": 0, "external_id": null}, {"id": 10845807003, "name": "School", "priority": 1, "external_id": null}, {"id": 10845808003, "name": "Manager", "priority": 2, "external_id": null}, {"id": 10845809003, "name": "Reported", "priority": 3, "external_id": null}, {"id": 10845810003, "name": "Friend", "priority": 4, "external_id": null}, {"id": 10845811003, "name": "Do not know", "priority": 5, "external_id": null}]}, "emitted_at": 1662401663418} -{"stream": "custom_fields", "data": {"id": 4680906003, "name": "Work History", "active": true, "field_type": "referral_question", "priority": 1, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "work_history", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845812003, "name": "0-1", "priority": 0, "external_id": null}, {"id": 10845813003, "name": "2-5", "priority": 1, "external_id": null}, {"id": 10845814003, "name": "5+", "priority": 2, "external_id": null}]}, "emitted_at": 1662401663418} -{"stream": "custom_fields", "data": {"id": 4680907003, "name": "Rating", "active": true, "field_type": "referral_question", "priority": 2, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "rating", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845815003, "name": "Superstar", "priority": 0, "external_id": null}, {"id": 10845816003, "name": "Top 5%", "priority": 1, "external_id": null}, {"id": 10845817003, "name": "Top 10%", "priority": 2, "external_id": null}, {"id": 10845818003, "name": "Top 25%", "priority": 3, "external_id": null}, {"id": 10845819003, "name": "Top 50%", "priority": 4, "external_id": null}]}, "emitted_at": 1662401663419} -{"stream": "custom_fields", "data": {"id": 4680908003, "name": "When we reach out", "active": true, "field_type": "referral_question", "priority": 3, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "when_we_reach_out", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845820003, "name": "You may mention me", "priority": 0, "external_id": null}, {"id": 10845821003, "name": "I wish to remain anonymous", "priority": 1, "external_id": null}]}, "emitted_at": 1662401663419} -{"stream": "custom_fields", "data": {"id": 4680909003, "name": "They know they're being referred", "active": true, "field_type": "referral_question", "priority": 4, "value_type": "yes_no", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "they_know_they_re_being_referred", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663419} -{"stream": "custom_fields", "data": {"id": 4680910003, "name": "Referral Notes", "active": true, "field_type": "referral_question", "priority": 5, "value_type": "long_text", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "referral_notes", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663419} -{"stream": "custom_fields", "data": {"id": 7431124003, "name": "Test User", "active": true, "field_type": "agency_question", "priority": 0, "value_type": "yes_no", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "test_user", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663419} -{"stream": "custom_fields", "data": {"id": 7431125003, "name": "Test User", "active": true, "field_type": "agency_question", "priority": 1, "value_type": "short_text", "private": false, "required": true, "require_approval": false, "trigger_new_version": false, "name_key": "test_user_agency_question_1633884465.559642", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663419} -{"stream": "custom_fields", "data": {"id": 7431126003, "name": "Test User", "active": true, "field_type": "referral_question", "priority": 6, "value_type": "yes_no", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "test_user", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663419} -{"stream":"demographics_question_sets","data":{"title":"Test Question Set 1","id":4000197003,"description":"

Test Question Set 1 description

","active":true},"emitted_at":1660156526996} -{"stream":"demographics_question_sets","data":{"title":"Test Question Set 2","id":4000198003,"description":"

Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.

","active":true},"emitted_at":1660156526998} -{"stream":"demographics_question_sets","data":{"title":"U.S. Standard Demographic Questions","id":4002702003,"description":"We invite applicants to share their demographic background. If you choose to complete this survey, your responses may be used to identify areas of improvement in our hiring process.","active":true},"emitted_at":1660156526998} -{"stream":"demographics_questions","data":{"translations":[{"name":"q1","language":"en"}],"required":false,"name":"q1","id":4000714003,"demographic_question_set_id":4000197003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156527371} -{"stream":"demographics_questions","data":{"translations":[{"name":"q2","language":"en"}],"required":false,"name":"q2","id":4000715003,"demographic_question_set_id":4000197003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156527375} -{"stream":"demographics_questions","data":{"translations":[{"name":"question1","language":"en"}],"required":false,"name":"question1","id":4000716003,"demographic_question_set_id":4000198003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156527375} -{"stream":"demographics_questions","data":{"translations":[{"name":"question2","language":"en"}],"required":true,"name":"question2","id":4000717003,"demographic_question_set_id":4000198003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156527389} -{"stream":"demographics_questions","data":{"translations":[{"name":"Are you a veteran or active member of the United States Armed Forces?","language":"en"}],"required":false,"name":"Are you a veteran or active member of the United States Armed Forces?","id":4015594003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156527389} -{"stream":"demographics_questions","data":{"translations":[{"name":"Do you have a disability or chronic condition (physical, visual, auditory, cognitive, mental, emotional, or other) that substantially limits one or more of your major life activities, including mobility, communication (seeing, hearing, speaking), and learning?","language":"en"}],"required":false,"name":"Do you have a disability or chronic condition (physical, visual, auditory, cognitive, mental, emotional, or other) that substantially limits one or more of your major life activities, including mobility, communication (seeing, hearing, speaking), and learning?","id":4015596003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156527389} -{"stream":"demographics_questions","data":{"translations":[{"name":"Do you identify as transgender?","language":"en"}],"required":false,"name":"Do you identify as transgender?","id":4015598003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156527389} -{"stream":"demographics_questions","data":{"translations":[{"name":"How would you describe your sexual orientation? (mark all that apply)","language":"en"}],"required":false,"name":"How would you describe your sexual orientation? (mark all that apply)","id":4015599003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156527389} -{"stream":"demographics_questions","data":{"translations":[{"name":"How would you describe your racial/ethnic background? (mark all that apply)","language":"en"}],"required":false,"name":"How would you describe your racial/ethnic background? (mark all that apply)","id":4015601003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156527389} -{"stream":"demographics_questions","data":{"translations":[{"name":"How would you describe your gender identity? (mark all that apply)","language":"en"}],"required":false,"name":"How would you describe your gender identity? (mark all that apply)","id":4015603003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156527390} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"a1","language":"en"}],"name":"a1","id":4004258003,"free_form":false,"demographic_question_id":4000714003,"active":true},"emitted_at":1660156527772} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"a2","language":"en"}],"name":"a2","id":4004259003,"free_form":false,"demographic_question_id":4000715003,"active":true},"emitted_at":1660156527775} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"a3","language":"en"}],"name":"a3","id":4004260003,"free_form":false,"demographic_question_id":4000715003,"active":true},"emitted_at":1660156527775} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"s1","language":"en"}],"name":"s1","id":4004261003,"free_form":true,"demographic_question_id":4000715003,"active":true},"emitted_at":1660156527775} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"answer1","language":"en"}],"name":"answer1","id":4004262003,"free_form":false,"demographic_question_id":4000716003,"active":true},"emitted_at":1660156527775} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"answer-self1","language":"en"}],"name":"answer-self1","id":4004263003,"free_form":true,"demographic_question_id":4000716003,"active":true},"emitted_at":1660156527775} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"answer2","language":"en"}],"name":"answer2","id":4004264003,"free_form":false,"demographic_question_id":4000716003,"active":true},"emitted_at":1660156527775} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"answer1","language":"en"}],"name":"answer1","id":4004265003,"free_form":false,"demographic_question_id":4000717003,"active":true},"emitted_at":1660156527776} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4004266003,"free_form":false,"demographic_question_id":4000717003,"active":true},"emitted_at":1660156527776} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"answer2","language":"en"}],"name":"answer2","id":4004267003,"free_form":false,"demographic_question_id":4000717003,"active":true},"emitted_at":1660156527776} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093930003,"free_form":false,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156527776} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093931003,"free_form":true,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156527776} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"No, I am not a veteran or active member","language":"en"}],"name":"No, I am not a veteran or active member","id":4093932003,"free_form":false,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156527776} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Yes, I am a veteran or active member","language":"en"}],"name":"Yes, I am a veteran or active member","id":4093934003,"free_form":false,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156527777} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093937003,"free_form":false,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156527777} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093939003,"free_form":true,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156527777} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"No","language":"en"}],"name":"No","id":4093940003,"free_form":false,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156527777} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Yes","language":"en"}],"name":"Yes","id":4093941003,"free_form":false,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156527777} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093944003,"free_form":false,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156527777} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093946003,"free_form":true,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156527778} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"No","language":"en"}],"name":"No","id":4093948003,"free_form":false,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156527778} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Yes","language":"en"}],"name":"Yes","id":4093950003,"free_form":false,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156527779} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093953003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527779} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093955003,"free_form":true,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527779} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Queer","language":"en"}],"name":"Queer","id":4093956003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527781} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Lesbian","language":"en"}],"name":"Lesbian","id":4093957003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527781} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Heterosexual","language":"en"}],"name":"Heterosexual","id":4093959003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527782} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Gay","language":"en"}],"name":"Gay","id":4093961003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527783} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Bisexual and/or pansexual","language":"en"}],"name":"Bisexual and/or pansexual","id":4093963003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527784} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Asexual","language":"en"}],"name":"Asexual","id":4093965003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156527784} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093971003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527785} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093973003,"free_form":true,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527785} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"White or European","language":"en"}],"name":"White or European","id":4093975003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527785} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Southeast Asian","language":"en"}],"name":"Southeast Asian","id":4093976003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527786} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"South Asian","language":"en"}],"name":"South Asian","id":4093977003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527786} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Native Hawaiian or Pacific Islander","language":"en"}],"name":"Native Hawaiian or Pacific Islander","id":4093979003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527786} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Middle Eastern or North African","language":"en"}],"name":"Middle Eastern or North African","id":4093981003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527787} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Indigenous, American Indian or Alaska Native","language":"en"}],"name":"Indigenous, American Indian or Alaska Native","id":4093983003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527787} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Hispanic, Latinx or of Spanish Origin","language":"en"}],"name":"Hispanic, Latinx or of Spanish Origin","id":4093985003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527788} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"East Asian","language":"en"}],"name":"East Asian","id":4093986003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527789} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Black or of African descent","language":"en"}],"name":"Black or of African descent","id":4093988003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156527789} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093989003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156527789} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093990003,"free_form":true,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156527790} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Woman","language":"en"}],"name":"Woman","id":4093991003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156527790} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Non-binary","language":"en"}],"name":"Non-binary","id":4093993003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156527790} -{"stream":"demographics_answer_options","data":{"translations":[{"name":"Man","language":"en"}],"name":"Man","id":4093995003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156527790} -{"stream":"demographics_answers","data":{"updated_at":"2021-11-03T19:56:07.248Z","id":9308815003,"free_form_text":null,"demographic_question_id":4000716003,"demographic_answer_option_id":4004262003,"created_at":"2021-11-03T19:56:07.248Z","application_id":47459993003},"emitted_at":1660156528190} -{"stream":"demographics_answers","data":{"updated_at":"2021-11-03T19:56:07.252Z","id":9308816003,"free_form_text":"custom answer","demographic_question_id":4000716003,"demographic_answer_option_id":4004263003,"created_at":"2021-11-03T19:56:07.252Z","application_id":47459993003},"emitted_at":1660156528193} -{"stream":"demographics_answers","data":{"updated_at":"2021-11-03T19:56:07.259Z","id":9308817003,"free_form_text":null,"demographic_question_id":4000717003,"demographic_answer_option_id":4004266003,"created_at":"2021-11-03T19:56:07.259Z","application_id":47459993003},"emitted_at":1660156528193} -{"stream":"applications_demographics_answers","data":{"updated_at":"2021-11-03T19:56:07.248Z","id":9308815003,"free_form_text":null,"demographic_question_id":4000716003,"demographic_answer_option_id":4004262003,"created_at":"2021-11-03T19:56:07.248Z","application_id":47459993003},"emitted_at":1660156529467} -{"stream":"applications_demographics_answers","data":{"updated_at":"2021-11-03T19:56:07.252Z","id":9308816003,"free_form_text":"custom answer","demographic_question_id":4000716003,"demographic_answer_option_id":4004263003,"created_at":"2021-11-03T19:56:07.252Z","application_id":47459993003},"emitted_at":1660156529468} -{"stream":"applications_demographics_answers","data":{"updated_at":"2021-11-03T19:56:07.259Z","id":9308817003,"free_form_text":null,"demographic_question_id":4000717003,"demographic_answer_option_id":4004266003,"created_at":"2021-11-03T19:56:07.259Z","application_id":47459993003},"emitted_at":1660156529469} -{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"q1","language":"en"}],"required":false,"name":"q1","id":4000714003,"demographic_question_set_id":4000197003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156531201} -{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"q2","language":"en"}],"required":false,"name":"q2","id":4000715003,"demographic_question_set_id":4000197003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156531205} -{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"question1","language":"en"}],"required":false,"name":"question1","id":4000716003,"demographic_question_set_id":4000198003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156531420} -{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"question2","language":"en"}],"required":true,"name":"question2","id":4000717003,"demographic_question_set_id":4000198003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156531422} -{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"Are you a veteran or active member of the United States Armed Forces?","language":"en"}],"required":false,"name":"Are you a veteran or active member of the United States Armed Forces?","id":4015594003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156531636} -{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"Do you have a disability or chronic condition (physical, visual, auditory, cognitive, mental, emotional, or other) that substantially limits one or more of your major life activities, including mobility, communication (seeing, hearing, speaking), and learning?","language":"en"}],"required":false,"name":"Do you have a disability or chronic condition (physical, visual, auditory, cognitive, mental, emotional, or other) that substantially limits one or more of your major life activities, including mobility, communication (seeing, hearing, speaking), and learning?","id":4015596003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156531636} -{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"Do you identify as transgender?","language":"en"}],"required":false,"name":"Do you identify as transgender?","id":4015598003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1660156531636} -{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"How would you describe your sexual orientation? (mark all that apply)","language":"en"}],"required":false,"name":"How would you describe your sexual orientation? (mark all that apply)","id":4015599003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156531637} -{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"How would you describe your racial/ethnic background? (mark all that apply)","language":"en"}],"required":false,"name":"How would you describe your racial/ethnic background? (mark all that apply)","id":4015601003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156531637} -{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"How would you describe your gender identity? (mark all that apply)","language":"en"}],"required":false,"name":"How would you describe your gender identity? (mark all that apply)","id":4015603003,"demographic_question_set_id":4002702003,"answer_type":"multi_value_multi_select","active":true},"emitted_at":1660156531637} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"a1","language":"en"}],"name":"a1","id":4004258003,"free_form":false,"demographic_question_id":4000714003,"active":true},"emitted_at":1660156532379} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"a2","language":"en"}],"name":"a2","id":4004259003,"free_form":false,"demographic_question_id":4000715003,"active":true},"emitted_at":1660156532578} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"a3","language":"en"}],"name":"a3","id":4004260003,"free_form":false,"demographic_question_id":4000715003,"active":true},"emitted_at":1660156532579} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"s1","language":"en"}],"name":"s1","id":4004261003,"free_form":true,"demographic_question_id":4000715003,"active":true},"emitted_at":1660156532579} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"answer1","language":"en"}],"name":"answer1","id":4004262003,"free_form":false,"demographic_question_id":4000716003,"active":true},"emitted_at":1660156532783} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"answer-self1","language":"en"}],"name":"answer-self1","id":4004263003,"free_form":true,"demographic_question_id":4000716003,"active":true},"emitted_at":1660156532783} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"answer2","language":"en"}],"name":"answer2","id":4004264003,"free_form":false,"demographic_question_id":4000716003,"active":true},"emitted_at":1660156532783} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"answer1","language":"en"}],"name":"answer1","id":4004265003,"free_form":false,"demographic_question_id":4000717003,"active":true},"emitted_at":1660156532977} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4004266003,"free_form":false,"demographic_question_id":4000717003,"active":true},"emitted_at":1660156532977} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"answer2","language":"en"}],"name":"answer2","id":4004267003,"free_form":false,"demographic_question_id":4000717003,"active":true},"emitted_at":1660156532977} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093930003,"free_form":false,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156533171} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093931003,"free_form":true,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156533171} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"No, I am not a veteran or active member","language":"en"}],"name":"No, I am not a veteran or active member","id":4093932003,"free_form":false,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156533172} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Yes, I am a veteran or active member","language":"en"}],"name":"Yes, I am a veteran or active member","id":4093934003,"free_form":false,"demographic_question_id":4015594003,"active":true},"emitted_at":1660156533172} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093937003,"free_form":false,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156533376} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093939003,"free_form":true,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156533378} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"No","language":"en"}],"name":"No","id":4093940003,"free_form":false,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156533379} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Yes","language":"en"}],"name":"Yes","id":4093941003,"free_form":false,"demographic_question_id":4015596003,"active":true},"emitted_at":1660156533379} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093944003,"free_form":false,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156533581} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093946003,"free_form":true,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156533582} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"No","language":"en"}],"name":"No","id":4093948003,"free_form":false,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156533582} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Yes","language":"en"}],"name":"Yes","id":4093950003,"free_form":false,"demographic_question_id":4015598003,"active":true},"emitted_at":1660156533582} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093953003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533793} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093955003,"free_form":true,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533793} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Queer","language":"en"}],"name":"Queer","id":4093956003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533793} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Lesbian","language":"en"}],"name":"Lesbian","id":4093957003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533794} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Heterosexual","language":"en"}],"name":"Heterosexual","id":4093959003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533794} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Gay","language":"en"}],"name":"Gay","id":4093961003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533794} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Bisexual and/or pansexual","language":"en"}],"name":"Bisexual and/or pansexual","id":4093963003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533794} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Asexual","language":"en"}],"name":"Asexual","id":4093965003,"free_form":false,"demographic_question_id":4015599003,"active":true},"emitted_at":1660156533795} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093971003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534000} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093973003,"free_form":true,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534000} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"White or European","language":"en"}],"name":"White or European","id":4093975003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534000} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Southeast Asian","language":"en"}],"name":"Southeast Asian","id":4093976003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534001} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"South Asian","language":"en"}],"name":"South Asian","id":4093977003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534001} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Native Hawaiian or Pacific Islander","language":"en"}],"name":"Native Hawaiian or Pacific Islander","id":4093979003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534002} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Middle Eastern or North African","language":"en"}],"name":"Middle Eastern or North African","id":4093981003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534006} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Indigenous, American Indian or Alaska Native","language":"en"}],"name":"Indigenous, American Indian or Alaska Native","id":4093983003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534007} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Hispanic, Latinx or of Spanish Origin","language":"en"}],"name":"Hispanic, Latinx or of Spanish Origin","id":4093985003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534007} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"East Asian","language":"en"}],"name":"East Asian","id":4093986003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534008} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Black or of African descent","language":"en"}],"name":"Black or of African descent","id":4093988003,"free_form":false,"demographic_question_id":4015601003,"active":true},"emitted_at":1660156534008} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I don't wish to answer","language":"en"}],"name":"I don't wish to answer","id":4093989003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156534199} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"I prefer to self-describe","language":"en"}],"name":"I prefer to self-describe","id":4093990003,"free_form":true,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156534199} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Woman","language":"en"}],"name":"Woman","id":4093991003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156534200} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Non-binary","language":"en"}],"name":"Non-binary","id":4093993003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156534200} -{"stream":"demographics_answers_answer_options","data":{"translations":[{"name":"Man","language":"en"}],"name":"Man","id":4093995003,"free_form":false,"demographic_question_id":4015603003,"active":true},"emitted_at":1660156534201} -{"stream":"interviews","data":{"id":40387397003,"application_id":44937562003,"external_event_id":"123456789","start":{"date_time":"2021-12-12T13:15:00.000Z"},"end":{"date_time":"2021-12-12T14:15:00.000Z"},"location":null,"video_conferencing_url":null,"status":"awaiting_feedback","created_at":"2021-10-10T16:21:44.107Z","updated_at":"2021-12-12T15:15:02.894Z","interview":{"id":5628615003,"name":"Preliminary Screening Call"},"organizer":{"id":4218085003,"first_name":"Greenhouse","last_name":"Admin","name":"Greenhouse Admin","employee_id":null},"interviewers":[{"id":4218085003,"employee_id":null,"name":"Greenhouse Admin","email":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","response_status":"accepted","scorecard_id":null}]},"emitted_at":1660156535168} -{"stream":"interviews","data":{"id":40387426003,"application_id":44937562003,"external_event_id":"12345678","start":{"date_time":"2021-12-13T13:15:00.000Z"},"end":{"date_time":"2021-12-13T14:15:00.000Z"},"location":null,"video_conferencing_url":null,"status":"awaiting_feedback","created_at":"2021-10-10T16:22:04.561Z","updated_at":"2021-12-13T15:15:13.252Z","interview":{"id":5628615003,"name":"Preliminary Screening Call"},"organizer":{"id":4218085003,"first_name":"Greenhouse","last_name":"Admin","name":"Greenhouse Admin","employee_id":null},"interviewers":[{"id":4218085003,"employee_id":null,"name":"Greenhouse Admin","email":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","response_status":"accepted","scorecard_id":null}]},"emitted_at":1660156535172} -{"stream":"interviews","data":{"id":40387431003,"application_id":44937562003,"external_event_id":"1234567","start":{"date_time":"2021-12-14T13:15:00.000Z"},"end":{"date_time":"2021-12-14T14:15:00.000Z"},"location":null,"video_conferencing_url":null,"status":"awaiting_feedback","created_at":"2021-10-10T16:22:13.681Z","updated_at":"2021-12-14T15:15:12.118Z","interview":{"id":5628615003,"name":"Preliminary Screening Call"},"organizer":{"id":4218085003,"first_name":"Greenhouse","last_name":"Admin","name":"Greenhouse Admin","employee_id":null},"interviewers":[{"id":4218085003,"employee_id":null,"name":"Greenhouse Admin","email":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","response_status":"accepted","scorecard_id":null}]},"emitted_at":1660156535173} -{"stream":"applications_interviews","data":{"id":40387397003,"application_id":44937562003,"external_event_id":"123456789","start":{"date_time":"2021-12-12T13:15:00.000Z"},"end":{"date_time":"2021-12-12T14:15:00.000Z"},"location":null,"video_conferencing_url":null,"status":"awaiting_feedback","created_at":"2021-10-10T16:21:44.107Z","updated_at":"2021-12-12T15:15:02.894Z","interview":{"id":5628615003,"name":"Preliminary Screening Call"},"organizer":{"id":4218085003,"first_name":"Greenhouse","last_name":"Admin","name":"Greenhouse Admin","employee_id":null},"interviewers":[{"id":4218085003,"employee_id":null,"name":"Greenhouse Admin","email":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","response_status":"accepted","scorecard_id":null}]},"emitted_at":1660156536478} -{"stream":"applications_interviews","data":{"id":40387426003,"application_id":44937562003,"external_event_id":"12345678","start":{"date_time":"2021-12-13T13:15:00.000Z"},"end":{"date_time":"2021-12-13T14:15:00.000Z"},"location":null,"video_conferencing_url":null,"status":"awaiting_feedback","created_at":"2021-10-10T16:22:04.561Z","updated_at":"2021-12-13T15:15:13.252Z","interview":{"id":5628615003,"name":"Preliminary Screening Call"},"organizer":{"id":4218085003,"first_name":"Greenhouse","last_name":"Admin","name":"Greenhouse Admin","employee_id":null},"interviewers":[{"id":4218085003,"employee_id":null,"name":"Greenhouse Admin","email":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","response_status":"accepted","scorecard_id":null}]},"emitted_at":1660156536483} -{"stream":"applications_interviews","data":{"id":40387431003,"application_id":44937562003,"external_event_id":"1234567","start":{"date_time":"2021-12-14T13:15:00.000Z"},"end":{"date_time":"2021-12-14T14:15:00.000Z"},"location":null,"video_conferencing_url":null,"status":"awaiting_feedback","created_at":"2021-10-10T16:22:13.681Z","updated_at":"2021-12-14T15:15:12.118Z","interview":{"id":5628615003,"name":"Preliminary Screening Call"},"organizer":{"id":4218085003,"first_name":"Greenhouse","last_name":"Admin","name":"Greenhouse Admin","employee_id":null},"interviewers":[{"id":4218085003,"employee_id":null,"name":"Greenhouse Admin","email":"scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com","response_status":"accepted","scorecard_id":null}]},"emitted_at":1660156536483} -{"stream":"sources","data":{"id":4000000003,"name":"Recurse","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537034} -{"stream":"sources","data":{"id":4000001003,"name":"cliquify","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537037} -{"stream":"sources","data":{"id":4000002003,"name":"ContactOut","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537038} -{"stream":"sources","data":{"id":4000003003,"name":"Crosschq","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537038} -{"stream":"sources","data":{"id":4000004003,"name":"Talentpair","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537038} -{"stream":"sources","data":{"id":4000005003,"name":"Sompani Talent Pools","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537039} -{"stream":"sources","data":{"id":4000006003,"name":"ScoutFor","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537039} -{"stream":"sources","data":{"id":4000007003,"name":"Gem","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537039} -{"stream":"sources","data":{"id":4000008003,"name":"Findem","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537039} -{"stream":"sources","data":{"id":4000009003,"name":"goldi staging","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537039} -{"stream":"sources","data":{"id":4000010003,"name":"MoBerries","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537039} -{"stream":"sources","data":{"id":4000011003,"name":"Onramp","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537039} -{"stream":"sources","data":{"id":4000012003,"name":"Knowledge Officer","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} -{"stream":"sources","data":{"id":4000013003,"name":"Sourceress","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} -{"stream":"sources","data":{"id":4000014003,"name":"Resume Library","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} -{"stream":"sources","data":{"id":4000015003,"name":"Command E","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} -{"stream":"sources","data":{"id":4000016003,"name":"Attract","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} -{"stream":"sources","data":{"id":4000017003,"name":"WePow","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} -{"stream":"sources","data":{"id":4000018003,"name":"Planted","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537040} -{"stream":"sources","data":{"id":4000019003,"name":"Birch Local","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537041} -{"stream":"sources","data":{"id":4000020003,"name":"Birch","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537041} -{"stream":"sources","data":{"id":4000021003,"name":"Consider","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537041} -{"stream":"sources","data":{"id":4000022003,"name":"Eightfold","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537041} -{"stream":"sources","data":{"id":4000023003,"name":"Google (Job Search)","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537041} -{"stream":"sources","data":{"id":4000024003,"name":"Hundred5","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537041} -{"stream":"sources","data":{"id":4000025003,"name":"Work4 Labs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537041} -{"stream":"sources","data":{"id":4000026003,"name":"Nudj","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} -{"stream":"sources","data":{"id":4000027003,"name":"Handshake","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} -{"stream":"sources","data":{"id":4000028003,"name":"goldi","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} -{"stream":"sources","data":{"id":4000029003,"name":"Honeypot.io","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} -{"stream":"sources","data":{"id":4000030003,"name":"Joonko","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} -{"stream":"sources","data":{"id":4000031003,"name":"Untapped","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} -{"stream":"sources","data":{"id":4000032003,"name":"Bubblesort","type":{"id":4000007003,"name":"Agencies"}},"emitted_at":1660156537042} -{"stream":"sources","data":{"id":4000033003,"name":"Fetcher","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537042} -{"stream":"sources","data":{"id":4000034003,"name":"WorksHub","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} -{"stream":"sources","data":{"id":4000035003,"name":"CareerBuilder Quick Apply","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} -{"stream":"sources","data":{"id":4000036003,"name":"BountyJobs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} -{"stream":"sources","data":{"id":4000037003,"name":"SmartDreamers","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} -{"stream":"sources","data":{"id":4000038003,"name":"Gloat","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} -{"stream":"sources","data":{"id":4000039003,"name":"Selected","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} -{"stream":"sources","data":{"id":4000040003,"name":"SeekOut","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537043} -{"stream":"sources","data":{"id":4000041003,"name":"Jobmailer","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537044} -{"stream":"sources","data":{"id":4000042003,"name":"MindMatch","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537044} -{"stream":"sources","data":{"id":4000043003,"name":"Hackajob","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537044} -{"stream":"sources","data":{"id":4000044003,"name":"Snap.hr","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537044} -{"stream":"sources","data":{"id":4000045003,"name":"JamieAI","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537044} -{"stream":"sources","data":{"id":4000046003,"name":"Visage","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537044} -{"stream":"sources","data":{"id":4000047003,"name":"XING ReferralManager","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537045} -{"stream":"sources","data":{"id":4000048003,"name":"WorkShape.io","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537045} -{"stream":"sources","data":{"id":4000049003,"name":"Workey","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537045} -{"stream":"sources","data":{"id":4000050003,"name":"Uncommon","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537045} -{"stream":"sources","data":{"id":4000051003,"name":"Talentful","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537046} -{"stream":"sources","data":{"id":4000052003,"name":"TalentBin® by Monster","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537046} -{"stream":"sources","data":{"id":4000053003,"name":"Riviera Partners","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537047} -{"stream":"sources","data":{"id":4000054003,"name":"SingleSprout","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537047} -{"stream":"sources","data":{"id":4000055003,"name":"ScoutSavvy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537047} -{"stream":"sources","data":{"id":4000056003,"name":"RippleMatch","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537048} -{"stream":"sources","data":{"id":4000057003,"name":"Project: “Odin”","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537048} -{"stream":"sources","data":{"id":4000058003,"name":"PowerToFly","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537048} -{"stream":"sources","data":{"id":4000059003,"name":"OneWire","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537049} -{"stream":"sources","data":{"id":4000060003,"name":"OfferZen","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537049} -{"stream":"sources","data":{"id":4000061003,"name":"Netin","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537049} -{"stream":"sources","data":{"id":4000062003,"name":"Meritocracy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537049} -{"stream":"sources","data":{"id":4000063003,"name":"Jopwell","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537049} -{"stream":"sources","data":{"id":4000064003,"name":"Jobjet","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537050} -{"stream":"sources","data":{"id":4000065003,"name":"Interviewing.io","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537050} -{"stream":"sources","data":{"id":4000066003,"name":"Interseller","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537050} -{"stream":"sources","data":{"id":4000067003,"name":"HRMARKET","type":{"id":4000007003,"name":"Agencies"}},"emitted_at":1660156537050} -{"stream":"sources","data":{"id":4000068003,"name":"hireEZ","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537050} -{"stream":"sources","data":{"id":4000069003,"name":"HeyJobs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537050} -{"stream":"sources","data":{"id":4000070003,"name":"Hachi","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537050} -{"stream":"sources","data":{"id":4000071003,"name":"getTalent","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} -{"stream":"sources","data":{"id":4000072003,"name":"Functional Works","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} -{"stream":"sources","data":{"id":4000073003,"name":"Firstbird","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} -{"stream":"sources","data":{"id":4000074003,"name":"Crowded","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} -{"stream":"sources","data":{"id":4000075003,"name":"CrediBLL","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} -{"stream":"sources","data":{"id":4000076003,"name":"Celential.ai","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} -{"stream":"sources","data":{"id":4000077003,"name":"AmazingHiring","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537051} -{"stream":"sources","data":{"id":4000078003,"name":"Teamable","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537052} -{"stream":"sources","data":{"id":4000079003,"name":"HubSpot Marketing","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537052} -{"stream":"sources","data":{"id":4000080003,"name":"VolkScience","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537052} -{"stream":"sources","data":{"id":4000081003,"name":"LeapMind","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537052} -{"stream":"sources","data":{"id":4000082003,"name":"Woo","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537052} -{"stream":"sources","data":{"id":4000083003,"name":"ReferralMob","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537053} -{"stream":"sources","data":{"id":4000084003,"name":"Maildrop","type":{"id":4000004003,"name":"Other"}},"emitted_at":1660156537053} -{"stream":"sources","data":{"id":4000085003,"name":"Thumbtack Technology","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} -{"stream":"sources","data":{"id":4000086003,"name":"Wendy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} -{"stream":"sources","data":{"id":4000087003,"name":"Stella","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} -{"stream":"sources","data":{"id":4000088003,"name":"Resource","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} -{"stream":"sources","data":{"id":4000089003,"name":"Talentseer","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} -{"stream":"sources","data":{"id":4000090003,"name":"Door of Clubs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} -{"stream":"sources","data":{"id":4000091003,"name":"untapt","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537054} -{"stream":"sources","data":{"id":4000092003,"name":"vsource","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537055} -{"stream":"sources","data":{"id":4000093003,"name":"Ideal","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537055} -{"stream":"sources","data":{"id":4000094003,"name":"Indeed Prime","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537055} -{"stream":"sources","data":{"id":4000095003,"name":"Predikt","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537055} -{"stream":"sources","data":{"id":4000096003,"name":"Beamery","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537055} -{"stream":"sources","data":{"id":4000097003,"name":"RippleHire","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537056} -{"stream":"sources","data":{"id":4000098003,"name":"LinkedIn (Prospecting)","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537056} -{"stream":"sources","data":{"id":4000099003,"name":"LinkedIn (Ad Posting)","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537056} -{"stream":"sources","data":{"id":4000100003,"name":"FirstJob","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537213} -{"stream":"sources","data":{"id":4000101003,"name":"Bsharp","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537214} -{"stream":"sources","data":{"id":4000102003,"name":"Landing.jobs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537214} -{"stream":"sources","data":{"id":4000103003,"name":"Vettery","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537214} -{"stream":"sources","data":{"id":4000104003,"name":"RAKUNA Recruit","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537215} -{"stream":"sources","data":{"id":4000105003,"name":"E-SS portal","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537215} -{"stream":"sources","data":{"id":4000106003,"name":"Recsolu","type":{"id":4000001003,"name":"In person event"}},"emitted_at":1660156537215} -{"stream":"sources","data":{"id":4000107003,"name":"SocialReferral [DEV]","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537215} -{"stream":"sources","data":{"id":4000108003,"name":"WayUp","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537216} -{"stream":"sources","data":{"id":4000109003,"name":"SocialReferral","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537216} -{"stream":"sources","data":{"id":4000110003,"name":"HumanPredictions","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537216} -{"stream":"sources","data":{"id":4000111003,"name":"Gogohire","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537217} -{"stream":"sources","data":{"id":4000112003,"name":"TalentIQ","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537217} -{"stream":"sources","data":{"id":4000113003,"name":"DoWeKnowThisGuy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537217} -{"stream":"sources","data":{"id":4000114003,"name":"The Muse","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537217} -{"stream":"sources","data":{"id":4000115003,"name":"HackerRank","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537217} -{"stream":"sources","data":{"id":4000116003,"name":"Whitetruffle","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537217} -{"stream":"sources","data":{"id":4000117003,"name":"LinkedIn Limited Listing","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537218} -{"stream":"sources","data":{"id":4000118003,"name":"SpringRole","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537218} -{"stream":"sources","data":{"id":4000119003,"name":"StrongIntro","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537218} -{"stream":"sources","data":{"id":4000120003,"name":"CodeFights","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537218} -{"stream":"sources","data":{"id":4000121003,"name":"Citadel","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537218} -{"stream":"sources","data":{"id":4000122003,"name":"Savvy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537219} -{"stream":"sources","data":{"id":4000123003,"name":"SmashFly","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537219} -{"stream":"sources","data":{"id":4000124003,"name":"Stack","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537219} -{"stream":"sources","data":{"id":4000125003,"name":"Network Monkey","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537219} -{"stream":"sources","data":{"id":4000126003,"name":"Triplebyte","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537219} -{"stream":"sources","data":{"id":4000127003,"name":"HireArt","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537219} -{"stream":"sources","data":{"id":4000128003,"name":"Hirecanvas","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537220} -{"stream":"sources","data":{"id":4000129003,"name":"CloserIQ","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537220} -{"stream":"sources","data":{"id":4000130003,"name":"Codeity","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537220} -{"stream":"sources","data":{"id":4000131003,"name":"ZipRecruiter","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537220} -{"stream":"sources","data":{"id":4000132003,"name":"Drafted","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537220} -{"stream":"sources","data":{"id":4000133003,"name":"ROIKOI","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537220} -{"stream":"sources","data":{"id":4000134003,"name":"DoWeKnowThisGuy Staging","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} -{"stream":"sources","data":{"id":4000135003,"name":"AngelList","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} -{"stream":"sources","data":{"id":4000136003,"name":"Archively","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} -{"stream":"sources","data":{"id":4000137003,"name":"Hired","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} -{"stream":"sources","data":{"id":4000138003,"name":"EmployeeReferrals.com","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} -{"stream":"sources","data":{"id":4000139003,"name":"Aevy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} -{"stream":"sources","data":{"id":4000140003,"name":"Connectifier","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} -{"stream":"sources","data":{"id":4000141003,"name":"Simppler","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537221} -{"stream":"sources","data":{"id":4000142003,"name":"Internal Applicant","type":{"id":4000006003,"name":"Company marketing"}},"emitted_at":1660156537222} -{"stream":"sources","data":{"id":4000143003,"name":"Clinch","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537222} -{"stream":"sources","data":{"id":4000144003,"name":"SwoopTalent","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537222} -{"stream":"sources","data":{"id":4000145003,"name":"Pymetrics","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537222} -{"stream":"sources","data":{"id":4000146003,"name":"YBorder","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537222} -{"stream":"sources","data":{"id":4000147003,"name":"Hirable","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537222} -{"stream":"sources","data":{"id":4000148003,"name":"RecruitiFi","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537222} -{"stream":"sources","data":{"id":4000149003,"name":"RolePoint","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537223} -{"stream":"sources","data":{"id":4000150003,"name":"Entelo","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537223} -{"stream":"sources","data":{"id":4000151003,"name":"Piazza","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537223} -{"stream":"sources","data":{"id":4000152003,"name":"Setter","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537223} -{"stream":"sources","data":{"id":4000153003,"name":"Other","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537223} -{"stream":"sources","data":{"id":4000154003,"name":"Coroflot","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537224} -{"stream":"sources","data":{"id":4000155003,"name":"Startuply","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537224} -{"stream":"sources","data":{"id":4000156003,"name":"Behance","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537225} -{"stream":"sources","data":{"id":4000157003,"name":"Dribbble","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537225} -{"stream":"sources","data":{"id":4000158003,"name":"Glassdoor","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537225} -{"stream":"sources","data":{"id":4000159003,"name":"Careers2.0 by StackOverflow","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537225} -{"stream":"sources","data":{"id":4000160003,"name":"LinkedIn (Social Media)","type":{"id":4000005003,"name":"Social media"}},"emitted_at":1660156537225} -{"stream":"sources","data":{"id":4000161003,"name":"Referral","type":{"id":4000002003,"name":"Referral"}},"emitted_at":1660156537226} -{"stream":"sources","data":{"id":4000162003,"name":"Beyond.com","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537226} -{"stream":"sources","data":{"id":4000163003,"name":"CareerBuilder","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537227} -{"stream":"sources","data":{"id":4000164003,"name":"Monster","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537227} -{"stream":"sources","data":{"id":4000165003,"name":"CareerBuilder","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537227} -{"stream":"sources","data":{"id":4000166003,"name":"Monster","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537228} -{"stream":"sources","data":{"id":4000167003,"name":"craigslist","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537228} -{"stream":"sources","data":{"id":4000168003,"name":"Dice","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537228} -{"stream":"sources","data":{"id":4000169003,"name":"GitHub Jobs","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537228} -{"stream":"sources","data":{"id":4000170003,"name":"Dribbble","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537228} -{"stream":"sources","data":{"id":4000171003,"name":"Social media presence","type":{"id":4000006003,"name":"Company marketing"}},"emitted_at":1660156537229} -{"stream":"sources","data":{"id":4000172003,"name":"Customer newsletter","type":{"id":4000006003,"name":"Company marketing"}},"emitted_at":1660156537229} -{"stream":"sources","data":{"id":4000173003,"name":"Use BountyJobs","type":{"id":4000007003,"name":"Agencies"}},"emitted_at":1660156537229} -{"stream":"sources","data":{"id":4000174003,"name":"Google","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537229} -{"stream":"sources","data":{"id":4000175003,"name":"Indeed","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537229} -{"stream":"sources","data":{"id":4000176003,"name":"Meetups","type":{"id":4000001003,"name":"In person event"}},"emitted_at":1660156537229} -{"stream":"sources","data":{"id":4000177003,"name":"Jobs page on your website","type":{"id":4000006003,"name":"Company marketing"}},"emitted_at":1660156537230} -{"stream":"sources","data":{"id":4000178003,"name":"Twitter","type":{"id":4000005003,"name":"Social media"}},"emitted_at":1660156537230} -{"stream":"sources","data":{"id":4000179003,"name":"Facebook","type":{"id":4000005003,"name":"Social media"}},"emitted_at":1660156537230} -{"stream":"sources","data":{"id":4000180003,"name":"SimplyHired","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537230} -{"stream":"sources","data":{"id":4000181003,"name":"Indeed","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537230} -{"stream":"sources","data":{"id":4000182003,"name":"Job fairs/Conferences/Trade shows","type":{"id":4000001003,"name":"In person event"}},"emitted_at":1660156537230} -{"stream":"sources","data":{"id":4000183003,"name":"Campus recruiting","type":{"id":4000001003,"name":"In person event"}},"emitted_at":1660156537231} -{"stream":"sources","data":{"id":4000184003,"name":"Uncubed","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537231} -{"stream":"sources","data":{"id":4000185003,"name":"Ladders","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537231} -{"stream":"sources","data":{"id":4000186003,"name":"Splash","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537231} -{"stream":"sources","data":{"id":4000187003,"name":"Recruiter.AI","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537231} -{"stream":"sources","data":{"id":4000188003,"name":"Underdog.io","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537231} -{"stream":"sources","data":{"id":4000189003,"name":"UpScored","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} -{"stream":"sources","data":{"id":4000190003,"name":"LinkMatch","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} -{"stream":"sources","data":{"id":4000191003,"name":"Jobbatical","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} -{"stream":"sources","data":{"id":4000192003,"name":"Upsider","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} -{"stream":"sources","data":{"id":4000195003,"name":"zealpath","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} -{"stream":"sources","data":{"id":4000198003,"name":"AppDirect Connector","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} -{"stream":"sources","data":{"id":4000213003,"name":"Helm","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} -{"stream":"sources","data":{"id":4000226003,"name":"Betts","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} -{"stream":"sources","data":{"id":4000228003,"name":"Circular","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537232} -{"stream":"sources","data":{"id":4000241003,"name":"Tempo","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537233} -{"stream":"sources","data":{"id":4000264003,"name":"Woo Auto-sourcer","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537233} -{"stream":"sources","data":{"id":4000344003,"name":"TopFunnel","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537379} -{"stream":"sources","data":{"id":4000538003,"name":"Greenhouse Test","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537379} -{"stream":"sources","data":{"id":4000619003,"name":"Wepow Staging","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537379} -{"stream":"sources","data":{"id":4001253003,"name":"Showcase Jobs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537379} -{"stream":"sources","data":{"id":4001321003,"name":"Showcase QA","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537379} -{"stream":"sources","data":{"id":4001322003,"name":"Showcase Demo","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537379} -{"stream":"sources","data":{"id":4001506003,"name":"Indeed - Sponsored","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537380} -{"stream":"sources","data":{"id":4001507003,"name":"Indeed - Targeted Ad","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537380} -{"stream":"sources","data":{"id":4002280003,"name":"Dash by Dashworks","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537380} -{"stream":"sources","data":{"id":4002742003,"name":"Aleph","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537380} -{"stream":"sources","data":{"id":4002891003,"name":"Xing","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537380} -{"stream":"sources","data":{"id":4003381003,"name":"Talroo","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537380} -{"stream":"sources","data":{"id":4004993003,"name":"include.ai","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537380} -{"stream":"sources","data":{"id":4005607003,"name":"AppDirect","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537381} -{"stream":"sources","data":{"id":4006015003,"name":"Otta","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537381} -{"stream":"sources","data":{"id":4006251003,"name":"Revelo","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537381} -{"stream":"sources","data":{"id":4006550003,"name":"Rainmakers","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537381} -{"stream":"sources","data":{"id":4007366003,"name":"Relode","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537381} -{"stream":"sources","data":{"id":4007690003,"name":"JOIN","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537381} -{"stream":"sources","data":{"id":4008420003,"name":"ERIN","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537381} -{"stream":"sources","data":{"id":4008511003,"name":"VentureBeat Careers, powered by Jobbio","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537382} -{"stream":"sources","data":{"id":4008617003,"name":"Monster Organic","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537382} -{"stream":"sources","data":{"id":4009581003,"name":"Sourcediv dev","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537382} -{"stream":"sources","data":{"id":4009582003,"name":"Sourcediv","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537382} -{"stream":"sources","data":{"id":4009906003,"name":"PandoLogic","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537382} -{"stream":"sources","data":{"id":4010052003,"name":"JOBfindah","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537382} -{"stream":"sources","data":{"id":4010066003,"name":"Cord","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} -{"stream":"sources","data":{"id":4010238003,"name":"purpose.jobs","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} -{"stream":"sources","data":{"id":4010715003,"name":"Scout Hires","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} -{"stream":"sources","data":{"id":4011228003,"name":"CBREX","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} -{"stream":"sources","data":{"id":4012144003,"name":"Talent By Blind (dev)","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} -{"stream":"sources","data":{"id":4012145003,"name":"Talent By Blind (test)","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} -{"stream":"sources","data":{"id":4012387003,"name":"RecruitBot","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537383} -{"stream":"sources","data":{"id":4012474003,"name":"RepVue","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537384} -{"stream":"sources","data":{"id":4012953003,"name":"Jobplanner","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537384} -{"stream":"sources","data":{"id":4013544003,"name":"Test agency","type":{"id":4000007003,"name":"Agencies"}},"emitted_at":1660156537384} -{"stream":"sources","data":{"id":4013712003,"name":"Jobstep","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537384} -{"stream":"sources","data":{"id":4014208003,"name":"Relyance","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537384} -{"stream":"sources","data":{"id":4014433003,"name":"Intrro","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537384} -{"stream":"sources","data":{"id":4014636003,"name":"DataFrenzy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} -{"stream":"sources","data":{"id":4015128003,"name":"Talent By Blind","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} -{"stream":"sources","data":{"id":4015567003,"name":"Real Links","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} -{"stream":"sources","data":{"id":4016305003,"name":"Scouted","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} -{"stream":"sources","data":{"id":4016428003,"name":"Talentry","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} -{"stream":"sources","data":{"id":4016532003,"name":"Careerjet","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} -{"stream":"sources","data":{"id":4018722003,"name":"ProvenBase","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537385} -{"stream":"sources","data":{"id":4018841003,"name":"Homi","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537386} -{"stream":"sources","data":{"id":4019367003,"name":"10x10","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537386} -{"stream":"sources","data":{"id":4019617003,"name":"Secret Tel Aviv","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537387} -{"stream":"sources","data":{"id":4020294003,"name":"BuiltIn","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537387} -{"stream":"sources","data":{"id":4020736003,"name":"Tobu","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537388} -{"stream":"sources","data":{"id":4020760003,"name":"Velents","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537388} -{"stream":"sources","data":{"id":4021421003,"name":"Upward","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537388} -{"stream":"sources","data":{"id":4022468003,"name":"ZoomInfo for Recruiters","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537388} -{"stream":"sources","data":{"id":4023661003,"name":"Prentus","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537388} -{"stream":"sources","data":{"id":4024133003,"name":"Arbeitnow","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537388} -{"stream":"sources","data":{"id":4024421003,"name":"GuidedCompass","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537388} -{"stream":"sources","data":{"id":4024601003,"name":"Elpha","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537389} -{"stream":"sources","data":{"id":4025746003,"name":"Indeed Hiring Platform","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537389} -{"stream":"sources","data":{"id":4025759003,"name":"Elpha Dev","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537389} -{"stream":"sources","data":{"id":4026726003,"name":"Careerpuck","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537389} -{"stream":"sources","data":{"id":4027676003,"name":"Appcast","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537389} -{"stream":"sources","data":{"id":4027972003,"name":"SV Academy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537389} -{"stream":"sources","data":{"id":4028644003,"name":"Tolstoy","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537389} -{"stream":"sources","data":{"id":4028649003,"name":"TolstoyDev","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537389} -{"stream":"sources","data":{"id":4028668003,"name":"OutScout","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537390} -{"stream":"sources","data":{"id":4028669003,"name":"OutScoutDev","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537390} -{"stream":"sources","data":{"id":4028843003,"name":"signNow","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537390} -{"stream":"sources","data":{"id":4029781003,"name":"TitanHouse","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537390} -{"stream":"sources","data":{"id":4030665003,"name":"Reprograma","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537390} -{"stream":"sources","data":{"id":4031825003,"name":"Trinsly","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537390} -{"stream":"sources","data":{"id":4032805003,"name":"Brazen","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} -{"stream":"sources","data":{"id":4034611003,"name":"Wednesday Talent","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} -{"stream":"sources","data":{"id":4034771003,"name":"HRMarket 2","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} -{"stream":"sources","data":{"id":4035349003,"name":"Phenom People","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} -{"stream":"sources","data":{"id":4036053003,"name":"TalentMarketplace","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} -{"stream":"sources","data":{"id":4036865003,"name":"Spleadly","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} -{"stream":"sources","data":{"id":4037066003,"name":"PathMatch","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537391} -{"stream":"sources","data":{"id":4037068003,"name":"Bevov","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537392} -{"stream":"sources","data":{"id":4037490003,"name":"LinkedIn","type":{"id":4000000003,"name":"Third-party boards"}},"emitted_at":1660156537392} -{"stream":"sources","data":{"id":4038632003,"name":"Reflr","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537392} -{"stream":"sources","data":{"id":4039580003,"name":"HeroHunt","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537392} -{"stream":"sources","data":{"id":4040831003,"name":"Inclusively","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537392} -{"stream":"sources","data":{"id":4040946003,"name":"Us in Technology","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537392} -{"stream":"sources","data":{"id":4041406003,"name":"Retorio","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537393} -{"stream":"sources","data":{"id":4041695003,"name":"Waldo Labs Relay","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537393} -{"stream":"sources","data":{"id":4041720003,"name":"Supercharge","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537393} -{"stream":"sources","data":{"id":4041898003,"name":"matched.io","type":{"id":4000003003,"name":"Prospecting"}},"emitted_at":1660156537394} -{"stream":"rejection_reasons","data":{"id":4014678003,"name":"reason1","type":{"id":4000000003,"name":"We rejected them"}},"emitted_at":1660156537678} -{"stream":"rejection_reasons","data":{"id":4014679003,"name":"reason2","type":{"id":4000001003,"name":"They rejected us"}},"emitted_at":1660156537682} -{"stream":"rejection_reasons","data":{"id":4014680003,"name":"reason3","type":{"id":4000002003,"name":"None specified"}},"emitted_at":1660156537682} -{"stream":"jobs_openings","data":{"id":4320015003,"opening_id":"3-1","status":"open","opened_at":"2020-11-24T23:27:11.723Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539008} -{"stream":"jobs_openings","data":{"id":4320018003,"opening_id":"4-1","status":"open","opened_at":"2020-11-24T23:27:45.665Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539258} -{"stream":"jobs_openings","data":{"id":4926182003,"opening_id":"5-1","status":"open","opened_at":"2021-10-08T08:19:42.457Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539555} -{"stream":"jobs_openings","data":{"id":4928186003,"opening_id":"5-1","status":"open","opened_at":"2021-10-10T16:38:57.407Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539555} -{"stream":"jobs_openings","data":{"id":4928187003,"opening_id":"5-2","status":"open","opened_at":"2021-10-10T16:39:08.365Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539555} -{"stream":"jobs_openings","data":{"id":4928188003,"opening_id":"5-2","status":"open","opened_at":"2021-10-10T16:39:24.949Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539555} -{"stream":"jobs_openings","data":{"id":4926183003,"opening_id":"5-2","status":"open","opened_at":"2021-10-08T08:19:42.457Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539555} -{"stream":"jobs_openings","data":{"id":4928186003,"opening_id":"5-1","status":"open","opened_at":"2021-10-10T16:38:57.407Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539692} -{"stream":"jobs_openings","data":{"id":4928187003,"opening_id":"5-2","status":"open","opened_at":"2021-10-10T16:39:08.365Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539693} -{"stream":"jobs_openings","data":{"id":4928188003,"opening_id":"5-2","status":"open","opened_at":"2021-10-10T16:39:24.949Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156539693} -{"stream":"jobs_openings","data":{"id":4970166003,"opening_id":"6-1","status":"open","opened_at":"2021-11-30T01:00:00.000Z","closed_at":null,"application_id":null,"close_reason":null},"emitted_at":1660156545037} -{"stream":"job_stages","data":{"id":5245803003,"name":"Application Review","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":0,"interviews":[{"id":5628614003,"name":"Application Review","schedulable":false,"interview_kit":{"id":5628609003,"content":null,"questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156545600} -{"stream":"job_stages","data":{"id":5245804003,"name":"Preliminary Phone Screen","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":1,"interviews":[{"id":5628615003,"name":"Preliminary Screening Call","schedulable":true,"interview_kit":{"id":5628610003,"content":null,"questions":[]},"estimated_minutes":20,"default_interviewer_users":[]}]},"emitted_at":1660156545605} -{"stream":"job_stages","data":{"id":5245805003,"name":"Phone Interview","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":2,"interviews":[{"id":5628616003,"name":"Behavioral Phone Interview","schedulable":true,"interview_kit":{"id":5628611003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156545605} -{"stream":"job_stages","data":{"id":5245806003,"name":"Face to Face","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":3,"interviews":[{"id":5628617003,"name":"Cultural Add Interview","schedulable":true,"interview_kit":{"id":5628612003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628618003,"name":"Peer Panel Interview","schedulable":true,"interview_kit":{"id":5628613003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628619003,"name":"Case Study","schedulable":true,"interview_kit":{"id":5628614003,"content":null,"questions":[]},"estimated_minutes":45,"default_interviewer_users":[]},{"id":5628620003,"name":"Executive Interview","schedulable":true,"interview_kit":{"id":5628615003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156545606} -{"stream":"job_stages","data":{"id":5245807003,"name":"Reference Check","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":4,"interviews":[{"id":5628621003,"name":"Former Manager","schedulable":false,"interview_kit":{"id":5628616003,"content":null,"questions":[]},"estimated_minutes":15,"default_interviewer_users":[]}]},"emitted_at":1660156545606} -{"stream":"job_stages","data":{"id":5245808003,"name":"Offer","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":5,"interviews":[]},"emitted_at":1660156545607} -{"stream":"job_stages","data":{"id":5245818003,"name":"Application Review","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":0,"interviews":[{"id":5628634003,"name":"Application Review","schedulable":false,"interview_kit":{"id":5628629003,"content":null,"questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156545607} -{"stream":"job_stages","data":{"id":5245819003,"name":"Preliminary Phone Screen","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":1,"interviews":[{"id":5628635003,"name":"Preliminary Screening Call","schedulable":true,"interview_kit":{"id":5628630003,"content":null,"questions":[]},"estimated_minutes":20,"default_interviewer_users":[]}]},"emitted_at":1660156545608} -{"stream":"job_stages","data":{"id":5245820003,"name":"Phone Interview","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":2,"interviews":[{"id":5628636003,"name":"Behavioral Phone Interview","schedulable":true,"interview_kit":{"id":5628631003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156545608} -{"stream":"job_stages","data":{"id":5245821003,"name":"Face to Face","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":3,"interviews":[{"id":5628637003,"name":"Cultural Add Interview","schedulable":true,"interview_kit":{"id":5628632003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628638003,"name":"Peer Panel Interview","schedulable":true,"interview_kit":{"id":5628633003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628639003,"name":"Case Study","schedulable":true,"interview_kit":{"id":5628634003,"content":null,"questions":[]},"estimated_minutes":45,"default_interviewer_users":[]},{"id":5628640003,"name":"Executive Interview","schedulable":true,"interview_kit":{"id":5628635003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156545609} -{"stream":"job_stages","data":{"id":5245822003,"name":"Reference Check","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":4,"interviews":[{"id":5628641003,"name":"Former Manager","schedulable":false,"interview_kit":{"id":5628636003,"content":null,"questions":[]},"estimated_minutes":15,"default_interviewer_users":[]}]},"emitted_at":1660156545609} -{"stream":"job_stages","data":{"id":5245823003,"name":"Offer","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":5,"interviews":[]},"emitted_at":1660156545610} -{"stream":"job_stages","data":{"id":7179760003,"name":"Application Review","created_at":"2021-10-08T08:19:42.584Z","updated_at":"2021-10-08T08:19:42.584Z","job_id":4446240003,"priority":0,"interviews":[{"id":8010560003,"name":"Application Review","schedulable":false,"interview_kit":{"id":8010544003,"content":"

\r\n
Triage the inbox
\r\n
    \r\n
  • Quickly knock out any applications that are spam, duplicates, or clearly not worth your time
    • \r\n
  • Pass along qualified applicants to the next stage for assessment
    • \r\n
  • Reject any applicants who lack the necessary experience or required skills, or whose applications have poor grammar, spelling, or formatting 
    • \r\n
\r\n

","questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156545610} -{"stream":"job_stages","data":{"id":7179761003,"name":"Preliminary Phone Screen","created_at":"2021-10-08T08:19:42.606Z","updated_at":"2021-10-08T08:19:42.606Z","job_id":4446240003,"priority":1,"interviews":[{"id":8010561003,"name":"Preliminary Screening Call","schedulable":true,"interview_kit":{"id":8010545003,"content":"
Purpose
\r\n
    \r\n
  • High-level screening call to make sure the candidate meets the basic requirements
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Explain the company and the job to the candidate: is it what he/she is looking for?
    • \r\n
  • Walk through his/her resume – are there any red flags?
    • \r\n
  • Find out what the candidate is looking for in his/her ideal role. Write this down so you can refer to this later!
    • \r\n
  • Get the candidate's desired salary range: is it in line with the compensation package?
    • \r\n
  • Tell the candidate what to expect next in your process
    • \r\n
\r\n

","questions":[]},"estimated_minutes":20,"default_interviewer_users":[]}]},"emitted_at":1660156545610} -{"stream":"job_stages","data":{"id":7179762003,"name":"Phone Interview","created_at":"2021-10-08T08:19:42.620Z","updated_at":"2021-10-08T08:19:42.620Z","job_id":4446240003,"priority":2,"interviews":[{"id":8010562003,"name":"Behavioral Phone Interview","schedulable":true,"interview_kit":{"id":8010546003,"content":"
Purpose
\r\n
    \r\n
  • Dig in deeper with a set of behavioral questions that target your desired skill set
    • \r\n
  • Specifically look for the following attributes: [Insert desired attributes here]
    • \r\n
  • Determine whether the candidate should be brought in for a face-to-face interview
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Have the candidate give you concrete examples of times when they demonstrated strengths in your desired skill set
    • \r\n
  • Ask a few open-ended “why” questions
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156545611} -{"stream":"job_stages","data":{"id":7179763003,"name":"Face to Face","created_at":"2021-10-08T08:19:42.632Z","updated_at":"2021-10-08T08:19:42.632Z","job_id":4446240003,"priority":3,"interviews":[{"id":8010563003,"name":"Cultural Add Interview","schedulable":true,"interview_kit":{"id":8010547003,"content":"
Purpose
\r\n
    \r\n
  • Determine whether or not the candidate would be a strong addition to the organization
    • \r\n
  • Do they live by your company values? [Insert Company Values Here]
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Ask the candidate for specific examples of times when they demonstrated your company values, e.g., “Tell me about a time when you took ownership of a project from start to finish”
    • \r\n
  • Ask about their motivations in their work - what excites them, what worries them, how do they work best, etc.
    • \r\n
  • Let the candidate know what they should expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":8010564003,"name":"Peer Panel Interview","schedulable":true,"interview_kit":{"id":8010548003,"content":"
    \r\n
  • Ask the candidate behavioral questions that target the skills and personality traits you're looking for
    • \r\n
  • Specifically dig in on the following skills: [Insert skills and traits here]
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Have the candidate give you concrete examples of times when they demonstrated strengths in your desired skill set
    • \r\n
  • Ask a few open-ended “why” questions
    • \r\n
  • Is this someone who you want to work with and would add value to the team?
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":8010565003,"name":"Case Study","schedulable":true,"interview_kit":{"id":8010549003,"content":"

[FIll in your case study problem here]

\r\n

 

\r\n
    \r\n
  • Prepare a case study or problem for the candidate that mimics a real-world situation that he/she would face in the role
    • \r\n
  • Does the candidate approach the problem analytically and logically?
    • \r\n
  • Ask why he/she made certain decisions. Do they demonstrate a desired way of thinking?
    • \r\n
  • Ask follow-up questions to test the candidate further. (e.g., “What would you do this happened? How would you handle the following objection by a superior?”)
    • \r\n
  • Let the candidate know what they should expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":45,"default_interviewer_users":[]},{"id":8010566003,"name":"Executive Interview","schedulable":true,"interview_kit":{"id":8010550003,"content":"
    \r\n
  • Unstructured conversation between Executive or Hiring Manager and the candidate
    • \r\n
  • Find out what motivates the candidate. What would they be most excited about if offered the position? What would they be most nervous about?
    • \r\n
  • Sell the candidate on the vision of the company and the quality of the team
    • \r\n
  • Answer any questions the candidate may have
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156545611} -{"stream":"job_stages","data":{"id":7179764003,"name":"Reference Check","created_at":"2021-10-08T08:19:42.665Z","updated_at":"2021-10-08T08:19:42.665Z","job_id":4446240003,"priority":4,"interviews":[{"id":8010567003,"name":"Former Manager","schedulable":false,"interview_kit":{"id":8010551003,"content":"
    \r\n
  • Have a quick chat with a former boss and confirm what the candidate said. Did the candidate accurately represent his/her responsibilities in their previous job?
    • \r\n
  • What does the boss think the candidate's biggest strengths are? What would the boss be most concerned about if hiring the candidate again?
    • \r\n
  • Ask about any red flags or questions you might have about the candidate
    • \r\n
","questions":[]},"estimated_minutes":15,"default_interviewer_users":[]}]},"emitted_at":1660156545611} -{"stream":"job_stages","data":{"id":7179765003,"name":"Offer","created_at":"2021-10-08T08:19:42.679Z","updated_at":"2021-10-08T08:19:42.679Z","job_id":4446240003,"priority":5,"interviews":[]},"emitted_at":1660156545611} -{"stream":"job_stages","data":{"id":7332462003,"name":"Application Review","created_at":"2021-11-03T19:46:51.185Z","updated_at":"2021-11-03T19:46:51.185Z","job_id":4466310003,"priority":0,"interviews":[{"id":8202995003,"name":"Application Review","schedulable":false,"interview_kit":{"id":8202979003,"content":null,"questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156545612} -{"stream":"job_stages","data":{"id":7332463003,"name":"Offer","created_at":"2021-11-03T19:46:51.185Z","updated_at":"2021-11-03T19:46:51.185Z","job_id":4466310003,"priority":1,"interviews":[]},"emitted_at":1660156545612} -{"stream":"jobs_stages","data":{"id":5245803003,"name":"Application Review","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":0,"interviews":[{"id":5628614003,"name":"Application Review","schedulable":false,"interview_kit":{"id":5628609003,"content":null,"questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156546562} -{"stream":"jobs_stages","data":{"id":5245804003,"name":"Preliminary Phone Screen","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":1,"interviews":[{"id":5628615003,"name":"Preliminary Screening Call","schedulable":true,"interview_kit":{"id":5628610003,"content":null,"questions":[]},"estimated_minutes":20,"default_interviewer_users":[]}]},"emitted_at":1660156546566} -{"stream":"jobs_stages","data":{"id":5245805003,"name":"Phone Interview","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":2,"interviews":[{"id":5628616003,"name":"Behavioral Phone Interview","schedulable":true,"interview_kit":{"id":5628611003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156546566} -{"stream":"jobs_stages","data":{"id":5245806003,"name":"Face to Face","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":3,"interviews":[{"id":5628617003,"name":"Cultural Add Interview","schedulable":true,"interview_kit":{"id":5628612003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628618003,"name":"Peer Panel Interview","schedulable":true,"interview_kit":{"id":5628613003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628619003,"name":"Case Study","schedulable":true,"interview_kit":{"id":5628614003,"content":null,"questions":[]},"estimated_minutes":45,"default_interviewer_users":[]},{"id":5628620003,"name":"Executive Interview","schedulable":true,"interview_kit":{"id":5628615003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156546566} -{"stream":"jobs_stages","data":{"id":5245807003,"name":"Reference Check","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":4,"interviews":[{"id":5628621003,"name":"Former Manager","schedulable":false,"interview_kit":{"id":5628616003,"content":null,"questions":[]},"estimated_minutes":15,"default_interviewer_users":[]}]},"emitted_at":1660156546567} -{"stream":"jobs_stages","data":{"id":5245808003,"name":"Offer","created_at":"2020-11-24T23:27:11.756Z","updated_at":"2020-11-24T23:27:11.756Z","job_id":4177046003,"priority":5,"interviews":[]},"emitted_at":1660156546567} -{"stream":"jobs_stages","data":{"id":5245818003,"name":"Application Review","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":0,"interviews":[{"id":5628634003,"name":"Application Review","schedulable":false,"interview_kit":{"id":5628629003,"content":null,"questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156546749} -{"stream":"jobs_stages","data":{"id":5245819003,"name":"Preliminary Phone Screen","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":1,"interviews":[{"id":5628635003,"name":"Preliminary Screening Call","schedulable":true,"interview_kit":{"id":5628630003,"content":null,"questions":[]},"estimated_minutes":20,"default_interviewer_users":[]}]},"emitted_at":1660156546750} -{"stream":"jobs_stages","data":{"id":5245820003,"name":"Phone Interview","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":2,"interviews":[{"id":5628636003,"name":"Behavioral Phone Interview","schedulable":true,"interview_kit":{"id":5628631003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156546750} -{"stream":"jobs_stages","data":{"id":5245821003,"name":"Face to Face","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":3,"interviews":[{"id":5628637003,"name":"Cultural Add Interview","schedulable":true,"interview_kit":{"id":5628632003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628638003,"name":"Peer Panel Interview","schedulable":true,"interview_kit":{"id":5628633003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":5628639003,"name":"Case Study","schedulable":true,"interview_kit":{"id":5628634003,"content":null,"questions":[]},"estimated_minutes":45,"default_interviewer_users":[]},{"id":5628640003,"name":"Executive Interview","schedulable":true,"interview_kit":{"id":5628635003,"content":null,"questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156546750} -{"stream":"jobs_stages","data":{"id":5245822003,"name":"Reference Check","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":4,"interviews":[{"id":5628641003,"name":"Former Manager","schedulable":false,"interview_kit":{"id":5628636003,"content":null,"questions":[]},"estimated_minutes":15,"default_interviewer_users":[]}]},"emitted_at":1660156546751} -{"stream":"jobs_stages","data":{"id":5245823003,"name":"Offer","created_at":"2020-11-24T23:27:45.704Z","updated_at":"2020-11-24T23:27:45.704Z","job_id":4177048003,"priority":5,"interviews":[]},"emitted_at":1660156546751} -{"stream":"jobs_stages","data":{"id":7179760003,"name":"Application Review","created_at":"2021-10-08T08:19:42.584Z","updated_at":"2021-10-08T08:19:42.584Z","job_id":4446240003,"priority":0,"interviews":[{"id":8010560003,"name":"Application Review","schedulable":false,"interview_kit":{"id":8010544003,"content":"

\r\n
Triage the inbox
\r\n
    \r\n
  • Quickly knock out any applications that are spam, duplicates, or clearly not worth your time
    • \r\n
  • Pass along qualified applicants to the next stage for assessment
    • \r\n
  • Reject any applicants who lack the necessary experience or required skills, or whose applications have poor grammar, spelling, or formatting 
    • \r\n
\r\n

","questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156546992} -{"stream":"jobs_stages","data":{"id":7179761003,"name":"Preliminary Phone Screen","created_at":"2021-10-08T08:19:42.606Z","updated_at":"2021-10-08T08:19:42.606Z","job_id":4446240003,"priority":1,"interviews":[{"id":8010561003,"name":"Preliminary Screening Call","schedulable":true,"interview_kit":{"id":8010545003,"content":"
Purpose
\r\n
    \r\n
  • High-level screening call to make sure the candidate meets the basic requirements
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Explain the company and the job to the candidate: is it what he/she is looking for?
    • \r\n
  • Walk through his/her resume – are there any red flags?
    • \r\n
  • Find out what the candidate is looking for in his/her ideal role. Write this down so you can refer to this later!
    • \r\n
  • Get the candidate's desired salary range: is it in line with the compensation package?
    • \r\n
  • Tell the candidate what to expect next in your process
    • \r\n
\r\n

","questions":[]},"estimated_minutes":20,"default_interviewer_users":[]}]},"emitted_at":1660156546994} -{"stream":"jobs_stages","data":{"id":7179762003,"name":"Phone Interview","created_at":"2021-10-08T08:19:42.620Z","updated_at":"2021-10-08T08:19:42.620Z","job_id":4446240003,"priority":2,"interviews":[{"id":8010562003,"name":"Behavioral Phone Interview","schedulable":true,"interview_kit":{"id":8010546003,"content":"
Purpose
\r\n
    \r\n
  • Dig in deeper with a set of behavioral questions that target your desired skill set
    • \r\n
  • Specifically look for the following attributes: [Insert desired attributes here]
    • \r\n
  • Determine whether the candidate should be brought in for a face-to-face interview
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Have the candidate give you concrete examples of times when they demonstrated strengths in your desired skill set
    • \r\n
  • Ask a few open-ended “why” questions
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156546995} -{"stream":"jobs_stages","data":{"id":7179763003,"name":"Face to Face","created_at":"2021-10-08T08:19:42.632Z","updated_at":"2021-10-08T08:19:42.632Z","job_id":4446240003,"priority":3,"interviews":[{"id":8010563003,"name":"Cultural Add Interview","schedulable":true,"interview_kit":{"id":8010547003,"content":"
Purpose
\r\n
    \r\n
  • Determine whether or not the candidate would be a strong addition to the organization
    • \r\n
  • Do they live by your company values? [Insert Company Values Here]
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Ask the candidate for specific examples of times when they demonstrated your company values, e.g., “Tell me about a time when you took ownership of a project from start to finish”
    • \r\n
  • Ask about their motivations in their work - what excites them, what worries them, how do they work best, etc.
    • \r\n
  • Let the candidate know what they should expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":8010564003,"name":"Peer Panel Interview","schedulable":true,"interview_kit":{"id":8010548003,"content":"
    \r\n
  • Ask the candidate behavioral questions that target the skills and personality traits you're looking for
    • \r\n
  • Specifically dig in on the following skills: [Insert skills and traits here]
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Have the candidate give you concrete examples of times when they demonstrated strengths in your desired skill set
    • \r\n
  • Ask a few open-ended “why” questions
    • \r\n
  • Is this someone who you want to work with and would add value to the team?
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]},{"id":8010565003,"name":"Case Study","schedulable":true,"interview_kit":{"id":8010549003,"content":"

[FIll in your case study problem here]

\r\n

 

\r\n
    \r\n
  • Prepare a case study or problem for the candidate that mimics a real-world situation that he/she would face in the role
    • \r\n
  • Does the candidate approach the problem analytically and logically?
    • \r\n
  • Ask why he/she made certain decisions. Do they demonstrate a desired way of thinking?
    • \r\n
  • Ask follow-up questions to test the candidate further. (e.g., “What would you do this happened? How would you handle the following objection by a superior?”)
    • \r\n
  • Let the candidate know what they should expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":45,"default_interviewer_users":[]},{"id":8010566003,"name":"Executive Interview","schedulable":true,"interview_kit":{"id":8010550003,"content":"
    \r\n
  • Unstructured conversation between Executive or Hiring Manager and the candidate
    • \r\n
  • Find out what motivates the candidate. What would they be most excited about if offered the position? What would they be most nervous about?
    • \r\n
  • Sell the candidate on the vision of the company and the quality of the team
    • \r\n
  • Answer any questions the candidate may have
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
","questions":[]},"estimated_minutes":30,"default_interviewer_users":[]}]},"emitted_at":1660156546995} -{"stream":"jobs_stages","data":{"id":7179764003,"name":"Reference Check","created_at":"2021-10-08T08:19:42.665Z","updated_at":"2021-10-08T08:19:42.665Z","job_id":4446240003,"priority":4,"interviews":[{"id":8010567003,"name":"Former Manager","schedulable":false,"interview_kit":{"id":8010551003,"content":"
    \r\n
  • Have a quick chat with a former boss and confirm what the candidate said. Did the candidate accurately represent his/her responsibilities in their previous job?
    • \r\n
  • What does the boss think the candidate's biggest strengths are? What would the boss be most concerned about if hiring the candidate again?
    • \r\n
  • Ask about any red flags or questions you might have about the candidate
    • \r\n
","questions":[]},"estimated_minutes":15,"default_interviewer_users":[]}]},"emitted_at":1660156546996} -{"stream":"jobs_stages","data":{"id":7179765003,"name":"Offer","created_at":"2021-10-08T08:19:42.679Z","updated_at":"2021-10-08T08:19:42.679Z","job_id":4446240003,"priority":5,"interviews":[]},"emitted_at":1660156546996} -{"stream":"jobs_stages","data":{"id":7332462003,"name":"Application Review","created_at":"2021-11-03T19:46:51.185Z","updated_at":"2021-11-03T19:46:51.185Z","job_id":4466310003,"priority":0,"interviews":[{"id":8202995003,"name":"Application Review","schedulable":false,"interview_kit":{"id":8202979003,"content":null,"questions":[]},"estimated_minutes":1,"default_interviewer_users":[]}]},"emitted_at":1660156547151} -{"stream":"jobs_stages","data":{"id":7332463003,"name":"Offer","created_at":"2021-11-03T19:46:51.185Z","updated_at":"2021-11-03T19:46:51.185Z","job_id":4466310003,"priority":1,"interviews":[]},"emitted_at":1660156547152} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "HRMARKET", "id": 4000067003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "John Lafleur", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:24:37.049Z", "jobs": [], "job_post_id": null, "id": 19214950003, "current_stage": null, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130511003, "attachments": [], "applied_at": "2020-11-24T23:24:37.023Z", "answers": []}, "emitted_at": 1664285613728} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Jobs page on your website", "id": 4000177003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "John Lafleur", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:25:13.804Z", "jobs": [], "job_post_id": null, "id": 19214993003, "current_stage": null, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130554003, "attachments": [], "applied_at": "2020-11-24T23:25:13.781Z", "answers": []}, "emitted_at": 1664285613729} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Internal Applicant", "id": 4000142003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2020-11-24T23:28:19.779Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 19215172003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130732003, "attachments": [], "applied_at": "2020-11-24T23:28:19.712Z", "answers": []}, "emitted_at": 1664285613729} +{"stream": "applications", "data": {"status": "rejected", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": {"type": {"name": "We rejected them", "id": 4000000003}, "name": "Other (add notes below)", "id": 4000004003}, "rejection_details": {}, "rejected_at": "2021-09-29T16:38:03.637Z", "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-09-29T16:38:03.660Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 44933447003, "current_stage": {"name": "Phone Interview", "id": 5245805003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 40513954003, "attachments": [], "applied_at": "2021-09-29T16:37:27.589Z", "answers": []}, "emitted_at": 1664285613730} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-10-10T16:22:13.708Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 44937562003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "Greenhouse Admin", "last_name": "Admin", "id": 4218085003, "first_name": "Greenhouse", "employee_id": null}, "candidate_id": 40517966003, "attachments": [], "applied_at": "2021-09-29T17:20:36.063Z", "answers": []}, "emitted_at": 1664285613730} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-11-03T19:56:07.402Z", "jobs": [{"name": "Test job 3", "id": 4466310003}], "job_post_id": 4797691003, "id": 47459993003, "current_stage": {"name": "Application Review", "id": 7332462003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 42921157003, "attachments": [], "applied_at": "2021-11-03T19:51:14.644Z", "answers": [{"question": "Website", "answer": null}, {"question": "LinkedIn Profile", "answer": null}]}, "emitted_at": 1664285613730} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Bubblesort", "id": 4000032003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-11-22T08:41:55.713Z", "jobs": [{"name": "Copy of Test Job 2", "id": 4446240003}], "job_post_id": null, "id": 48693310003, "current_stage": {"name": "Application Review", "id": 7179760003}, "credited_to": {"name": "emily.brooks+airbyte_integration@greenhouse.io", "last_name": null, "id": 4218087003, "first_name": null, "employee_id": null}, "candidate_id": 44081361003, "attachments": [], "applied_at": "2021-11-22T08:41:55.640Z", "answers": []}, "emitted_at": 1664285613731} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:24:37.050Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Test", "last_activity": "2020-11-24T23:24:37.049Z", "is_private": false, "id": 17130511003, "first_name": "Test", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:24:37.018Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "HRMARKET", "id": 4000067003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "John Lafleur", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:24:37.049Z", "jobs": [], "job_post_id": null, "id": 19214950003, "current_stage": null, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130511003, "attachments": [], "applied_at": "2020-11-24T23:24:37.023Z", "answers": []}], "application_ids": [19214950003], "addresses": []}, "emitted_at": 1664285614231} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:25:13.806Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Test2", "last_activity": "2020-11-24T23:25:13.804Z", "is_private": false, "id": 17130554003, "first_name": "Test2", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:25:13.777Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Jobs page on your website", "id": 4000177003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "John Lafleur", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:25:13.804Z", "jobs": [], "job_post_id": null, "id": 19214993003, "current_stage": null, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130554003, "attachments": [], "applied_at": "2020-11-24T23:25:13.781Z", "answers": []}], "application_ids": [19214993003], "addresses": []}, "emitted_at": 1664285614233} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:28:19.781Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Lastname", "last_activity": "2020-11-24T23:28:19.779Z", "is_private": false, "id": 17130732003, "first_name": "Name", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:28:19.710Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Internal Applicant", "id": 4000142003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2020-11-24T23:28:19.779Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 19215172003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130732003, "attachments": [], "applied_at": "2020-11-24T23:28:19.712Z", "answers": []}], "application_ids": [19215172003], "addresses": []}, "emitted_at": 1664285614233} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2021-09-29T16:38:03.672Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "User", "last_activity": "2021-09-29T16:38:03.660Z", "is_private": false, "id": 40513954003, "first_name": "Test", "employments": [], "email_addresses": [], "educations": [], "created_at": "2021-09-29T16:37:27.585Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "rejected", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": {"type": {"name": "We rejected them", "id": 4000000003}, "name": "Other (add notes below)", "id": 4000004003}, "rejection_details": {}, "rejected_at": "2021-09-29T16:38:03.637Z", "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-09-29T16:38:03.660Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 44933447003, "current_stage": {"name": "Phone Interview", "id": 5245805003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 40513954003, "attachments": [], "applied_at": "2021-09-29T16:37:27.589Z", "answers": []}], "application_ids": [44933447003], "addresses": []}, "emitted_at": 1664285614233} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2021-10-10T16:22:13.718Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Scheduled Interview", "last_activity": "2021-10-10T16:22:13.708Z", "is_private": false, "id": 40517966003, "first_name": "Test", "employments": [], "email_addresses": [{"value": "vadym.hevlich@zazmic.com", "type": "personal"}], "educations": [], "created_at": "2021-09-29T17:20:36.038Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-10-10T16:22:13.708Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 44937562003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "Greenhouse Admin", "last_name": "Admin", "id": 4218085003, "first_name": "Greenhouse", "employee_id": null}, "candidate_id": 40517966003, "attachments": [], "applied_at": "2021-09-29T17:20:36.063Z", "answers": []}], "application_ids": [44937562003], "addresses": []}, "emitted_at": 1664285614234} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2021-11-03T19:56:07.423Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Candidate", "last_activity": "2021-11-03T19:56:07.402Z", "is_private": false, "id": 42921157003, "first_name": "Test", "employments": [], "email_addresses": [{"value": "vadym.hevlich@zazmic.com", "type": "work"}], "educations": [], "created_at": "2021-11-03T19:51:14.639Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-11-03T19:56:07.402Z", "jobs": [{"name": "Test job 3", "id": 4466310003}], "job_post_id": 4797691003, "id": 47459993003, "current_stage": {"name": "Application Review", "id": 7332462003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 42921157003, "attachments": [], "applied_at": "2021-11-03T19:51:14.644Z", "answers": [{"question": "Website", "answer": null}, {"question": "LinkedIn Profile", "answer": null}]}], "application_ids": [47459993003], "addresses": []}, "emitted_at": 1664285614234} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2021-11-22T08:41:55.716Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Cherniaev", "last_activity": "2021-11-22T08:41:55.713Z", "is_private": false, "id": 44081361003, "first_name": "Yurii", "employments": [], "email_addresses": [], "educations": [], "created_at": "2021-11-22T08:41:55.634Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Bubblesort", "id": 4000032003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-11-22T08:41:55.713Z", "jobs": [{"name": "Copy of Test Job 2", "id": 4446240003}], "job_post_id": null, "id": 48693310003, "current_stage": {"name": "Application Review", "id": 7179760003}, "credited_to": {"name": "emily.brooks+airbyte_integration@greenhouse.io", "last_name": null, "id": 4218087003, "first_name": null, "employee_id": null}, "candidate_id": 44081361003, "attachments": [], "applied_at": "2021-11-22T08:41:55.640Z", "answers": []}], "application_ids": [48693310003], "addresses": []}, "emitted_at": 1664285614234} +{"stream": "close_reasons", "data": {"id": 4010635003, "name": "Not Filling"}, "emitted_at": 1664285614694} +{"stream": "close_reasons", "data": {"id": 4010634003, "name": "On Hold"}, "emitted_at": 1664285614695} +{"stream": "close_reasons", "data": {"id": 4010633003, "name": "Hire - New Headcount"}, "emitted_at": 1664285614695} +{"stream": "close_reasons", "data": {"id": 4010632003, "name": "Hire - Backfill"}, "emitted_at": 1664285614695} +{"stream": "degrees", "data": {"id": 10848287003, "name": "High School", "priority": 0, "external_id": null}, "emitted_at": 1664285615360} +{"stream": "degrees", "data": {"id": 10848288003, "name": "Associate's Degree", "priority": 1, "external_id": null}, "emitted_at": 1664285615361} +{"stream": "degrees", "data": {"id": 10848289003, "name": "Bachelor's Degree", "priority": 2, "external_id": null}, "emitted_at": 1664285615361} +{"stream": "degrees", "data": {"id": 10848290003, "name": "Master's Degree", "priority": 3, "external_id": null}, "emitted_at": 1664285615361} +{"stream": "degrees", "data": {"id": 10848291003, "name": "Master of Business Administration (M.B.A.)", "priority": 4, "external_id": null}, "emitted_at": 1664285615361} +{"stream": "degrees", "data": {"id": 10848292003, "name": "Juris Doctor (J.D.)", "priority": 5, "external_id": null}, "emitted_at": 1664285615361} +{"stream": "degrees", "data": {"id": 10848293003, "name": "Doctor of Medicine (M.D.)", "priority": 6, "external_id": null}, "emitted_at": 1664285615362} +{"stream": "degrees", "data": {"id": 10848294003, "name": "Doctor of Philosophy (Ph.D.)", "priority": 7, "external_id": null}, "emitted_at": 1664285615362} +{"stream": "degrees", "data": {"id": 10848295003, "name": "Engineer's Degree", "priority": 8, "external_id": null}, "emitted_at": 1664285615362} +{"stream": "degrees", "data": {"id": 10848296003, "name": "Other", "priority": 9, "external_id": null}, "emitted_at": 1664285615362} +{"stream": "departments", "data": {"id": 4028123003, "name": "test dep 2", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}, "emitted_at": 1664285616280} +{"stream": "departments", "data": {"id": 4028122003, "name": "Test dep1", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}, "emitted_at": 1664285616281} +{"stream": "job_posts", "data": {"id": 4252332003, "active": true, "live": false, "first_published_at": null, "title": "Test job", "location": {"id": 4219721003, "name": "test", "office_id": null, "job_post_location_type": {"id": 4000000003, "name": "Free Text"}}, "internal": false, "external": true, "job_id": 4177046003, "content": "

Test description

", "internal_content": null, "updated_at": "2021-04-02T17:38:54.835Z", "created_at": "2020-11-24T23:29:24.315Z", "demographic_question_set_id": null, "questions": [{"required": true, "private": false, "label": "First Name", "name": "first_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Last Name", "name": "last_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Email", "name": "email", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Phone", "name": "phone", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Resume", "name": "resume", "type": "attachment", "values": [], "description": null}, {"required": false, "private": false, "label": "Cover Letter", "name": "cover_letter", "type": "attachment", "values": [], "description": null}, {"required": null, "private": false, "label": "LinkedIn Profile", "name": "question_5125927003", "type": "short_text", "values": [], "description": null}, {"required": null, "private": false, "label": "Website", "name": "question_5125928003", "type": "short_text", "values": [], "description": null}]}, "emitted_at": 1664285617008} +{"stream": "job_posts", "data": {"id": 4751597003, "active": true, "live": false, "first_published_at": null, "title": "Test Job 2", "location": {"id": 4700649003, "name": "US", "office_id": null, "job_post_location_type": {"id": 4000000003, "name": "Free Text"}}, "internal": false, "external": true, "job_id": 4177048003, "content": "

Job post content

", "internal_content": null, "updated_at": "2021-10-07T18:46:59.032Z", "created_at": "2021-10-07T18:46:58.846Z", "demographic_question_set_id": null, "questions": [{"required": true, "private": false, "label": "First Name", "name": "first_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Last Name", "name": "last_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Email", "name": "email", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Phone", "name": "phone", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Resume", "name": "resume", "type": "attachment", "values": [], "description": null}, {"required": false, "private": false, "label": "Cover Letter", "name": "cover_letter", "type": "attachment", "values": [], "description": null}, {"required": null, "private": false, "label": "LinkedIn Profile", "name": "question_7911674003", "type": "short_text", "values": [], "description": null}, {"required": null, "private": false, "label": "Website", "name": "question_7911675003", "type": "short_text", "values": [], "description": null}]}, "emitted_at": 1664285617009} +{"stream": "job_posts", "data": {"id": 4752433003, "active": true, "live": false, "first_published_at": null, "title": "Test Job 2", "location": {"id": 4701484003, "name": "US", "office_id": null, "job_post_location_type": {"id": 4000000003, "name": "Free Text"}}, "internal": false, "external": true, "job_id": 4446240003, "content": "

Job post content

", "internal_content": null, "updated_at": "2021-10-08T08:19:42.720Z", "created_at": "2021-10-08T08:19:42.720Z", "demographic_question_set_id": null, "questions": [{"required": true, "private": false, "label": "First Name", "name": "first_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Last Name", "name": "last_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Email", "name": "email", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Phone", "name": "phone", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Resume", "name": "resume", "type": "attachment", "values": [], "description": null}, {"required": false, "private": false, "label": "Cover Letter", "name": "cover_letter", "type": "attachment", "values": [], "description": null}, {"required": null, "private": false, "label": "LinkedIn Profile", "name": "question_7918434003", "type": "short_text", "values": [], "description": null}, {"required": null, "private": false, "label": "Website", "name": "question_7918435003", "type": "short_text", "values": [], "description": null}]}, "emitted_at": 1664285617009} +{"stream": "job_posts", "data": {"id": 4797691003, "active": true, "live": false, "first_published_at": null, "title": "Test job 3", "location": {"id": 4746722003, "name": "US", "office_id": null, "job_post_location_type": {"id": 4000000003, "name": "Free Text"}}, "internal": false, "external": true, "job_id": 4466310003, "content": "

job description

", "internal_content": null, "updated_at": "2021-11-03T19:48:29.808Z", "created_at": "2021-11-03T19:48:29.629Z", "demographic_question_set_id": 4000198003, "questions": [{"required": true, "private": false, "label": "First Name", "name": "first_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Last Name", "name": "last_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Email", "name": "email", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Phone", "name": "phone", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Resume", "name": "resume", "type": "attachment", "values": [], "description": null}, {"required": false, "private": false, "label": "Cover Letter", "name": "cover_letter", "type": "attachment", "values": [], "description": null}, {"required": null, "private": false, "label": "LinkedIn Profile", "name": "question_8215275003", "type": "short_text", "values": [], "description": null}, {"required": null, "private": false, "label": "Website", "name": "question_8215276003", "type": "short_text", "values": [], "description": null}]}, "emitted_at": 1664285617010} +{"stream": "jobs", "data": {"id": 4177046003, "name": "Test job", "requisition_id": "3", "notes": null, "confidential": false, "is_template": true, "copied_from_id": null, "status": "open", "created_at": "2020-11-24T23:27:11.699Z", "opened_at": "2020-11-24T23:27:11.878Z", "closed_at": null, "updated_at": "2021-04-21T17:48:24.779Z", "departments": [{"id": 4028122003, "name": "Test dep1", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}], "offices": [{"id": 4019854003, "name": "Test office", "location": {"name": null}, "primary_contact_user_id": 4218086003, "parent_id": null, "parent_office_external_id": null, "child_ids": [], "child_office_external_ids": [], "external_id": null}], "hiring_team": {"hiring_managers": [], "recruiters": [], "coordinators": [], "sourcers": []}, "openings": [{"id": 4320015003, "opening_id": "3-1", "status": "open", "opened_at": "2020-11-24T23:27:11.723Z", "closed_at": null, "application_id": null, "close_reason": null}], "custom_fields": {"employment_type": null}, "keyed_custom_fields": {"employment_type": {"name": "Employment Type", "type": "single_select", "value": null}}}, "emitted_at": 1664285617812} +{"stream": "jobs", "data": {"id": 4177048003, "name": "Test Job 2", "requisition_id": "4", "notes": null, "confidential": false, "is_template": false, "copied_from_id": null, "status": "open", "created_at": "2020-11-24T23:27:45.634Z", "opened_at": "2020-11-24T23:27:45.878Z", "closed_at": null, "updated_at": "2021-10-07T18:46:58.982Z", "departments": [{"id": 4028123003, "name": "test dep 2", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}], "offices": [{"id": 4019854003, "name": "Test office", "location": {"name": null}, "primary_contact_user_id": 4218086003, "parent_id": null, "parent_office_external_id": null, "child_ids": [], "child_office_external_ids": [], "external_id": null}], "hiring_team": {"hiring_managers": [], "recruiters": [], "coordinators": [], "sourcers": []}, "openings": [{"id": 4320018003, "opening_id": "4-1", "status": "open", "opened_at": "2020-11-24T23:27:45.665Z", "closed_at": null, "application_id": null, "close_reason": null}], "custom_fields": {"employment_type": null}, "keyed_custom_fields": {"employment_type": {"name": "Employment Type", "type": "single_select", "value": null}}}, "emitted_at": 1664285617813} +{"stream": "jobs", "data": {"id": 4446240003, "name": "Copy of Test Job 2", "requisition_id": "5", "notes": null, "confidential": false, "is_template": false, "copied_from_id": 4177048003, "status": "open", "created_at": "2021-10-08T08:19:42.383Z", "opened_at": "2021-10-08T08:19:42.818Z", "closed_at": null, "updated_at": "2021-10-08T08:19:42.821Z", "departments": [{"id": 4028123003, "name": "test dep 2", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}], "offices": [{"id": 4019854003, "name": "Test office", "location": {"name": null}, "primary_contact_user_id": 4218086003, "parent_id": null, "parent_office_external_id": null, "child_ids": [], "child_office_external_ids": [], "external_id": null}], "hiring_team": {"hiring_managers": [], "recruiters": [], "coordinators": [], "sourcers": []}, "openings": [{"id": 4928188003, "opening_id": "5-2", "status": "open", "opened_at": "2021-10-10T16:39:24.949Z", "closed_at": null, "application_id": null, "close_reason": null}, {"id": 4928187003, "opening_id": "5-2", "status": "open", "opened_at": "2021-10-10T16:39:08.365Z", "closed_at": null, "application_id": null, "close_reason": null}, {"id": 4928186003, "opening_id": "5-1", "status": "open", "opened_at": "2021-10-10T16:38:57.407Z", "closed_at": null, "application_id": null, "close_reason": null}, {"id": 4926182003, "opening_id": "5-1", "status": "open", "opened_at": "2021-10-08T08:19:42.457Z", "closed_at": null, "application_id": null, "close_reason": null}, {"id": 4926183003, "opening_id": "5-2", "status": "open", "opened_at": "2021-10-08T08:19:42.457Z", "closed_at": null, "application_id": null, "close_reason": null}], "custom_fields": {"employment_type": "Full-time"}, "keyed_custom_fields": {"employment_type": {"name": "Employment Type", "type": "single_select", "value": "Full-time"}}}, "emitted_at": 1664285617813} +{"stream": "jobs", "data": {"id": 4466310003, "name": "Test job 3", "requisition_id": "6", "notes": null, "confidential": false, "is_template": false, "copied_from_id": null, "status": "open", "created_at": "2021-11-03T19:46:51.107Z", "opened_at": "2021-11-03T19:46:51.347Z", "closed_at": null, "updated_at": "2021-11-03T19:48:29.760Z", "departments": [{"id": 4028122003, "name": "Test dep1", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}], "offices": [{"id": 4019854003, "name": "Test office", "location": {"name": null}, "primary_contact_user_id": 4218086003, "parent_id": null, "parent_office_external_id": null, "child_ids": [], "child_office_external_ids": [], "external_id": null}], "hiring_team": {"hiring_managers": [], "recruiters": [], "coordinators": [], "sourcers": []}, "openings": [{"id": 4970166003, "opening_id": "6-1", "status": "open", "opened_at": "2021-11-30T01:00:00.000Z", "closed_at": null, "application_id": null, "close_reason": null}], "custom_fields": {"employment_type": "Full-time"}, "keyed_custom_fields": {"employment_type": {"name": "Employment Type", "type": "single_select", "value": "Full-time"}}}, "emitted_at": 1664285617814} +{"stream": "offers", "data": {"id": 4154100003, "version": 1, "application_id": 19215333003, "created_at": "2020-11-24T23:32:25.760Z", "updated_at": "2020-11-24T23:32:25.772Z", "sent_at": null, "resolved_at": null, "starts_at": "2020-12-04", "status": "unresolved", "job_id": 4177048003, "candidate_id": 17130848003, "opening": {"id": 4320018003, "opening_id": "4-1", "status": "open", "opened_at": "2020-11-24T23:27:45.665Z", "closed_at": null, "application_id": null, "close_reason": null}, "custom_fields": {"employment_type": "Contract"}, "keyed_custom_fields": {"employment_type": {"name": "Employment Type", "type": "single_select", "value": "Contract"}}}, "emitted_at": 1664285618339} +{"stream": "scorecards", "data": {"id": 5253031003, "updated_at": "2020-11-24T23:33:10.440Z", "created_at": "2020-11-24T23:33:10.440Z", "interview": "Application Review", "interview_step": {"id": 5628634003, "name": "Application Review"}, "candidate_id": 17130848003, "application_id": 19215333003, "interviewed_at": "2020-11-25T01:00:00.000Z", "submitted_by": {"id": 4218086003, "first_name": "John", "last_name": "Lafleur", "name": "John Lafleur", "employee_id": null}, "interviewer": {"id": 4218086003, "first_name": "John", "last_name": "Lafleur", "name": "John Lafleur", "employee_id": null}, "submitted_at": "2020-11-24T23:33:10.440Z", "overall_recommendation": "no_decision", "attributes": [{"name": "Willing to do required travel", "type": "Details", "note": null, "rating": "no_decision"}, {"name": "Three to five years of experience", "type": "Qualifications", "note": null, "rating": "no_decision"}, {"name": "Personable", "type": "Personality Traits", "note": null, "rating": "no_decision"}, {"name": "Passionate", "type": "Personality Traits", "note": null, "rating": "no_decision"}, {"name": "Organizational Skills", "type": "Skills", "note": null, "rating": "no_decision"}, {"name": "Manage competing priorities", "type": "Skills", "note": null, "rating": "no_decision"}, {"name": "Fits our salary range", "type": "Details", "note": null, "rating": "no_decision"}, {"name": "Empathetic", "type": "Personality Traits", "note": null, "rating": "no_decision"}, {"name": "Currently based locally", "type": "Details", "note": null, "rating": "no_decision"}, {"name": "Communication", "type": "Skills", "note": null, "rating": "no_decision"}], "ratings": {"definitely_not": [], "no": [], "mixed": [], "yes": [], "strong_yes": []}, "questions": [{"id": null, "question": "Key Take-Aways", "answer": ""}, {"id": null, "question": "Private Notes", "answer": ""}]}, "emitted_at": 1664285619010} +{"stream": "scorecards", "data": {"id": 9664505003, "updated_at": "2021-09-29T17:23:11.468Z", "created_at": "2021-09-29T17:23:11.468Z", "interview": "Preliminary Screening Call", "interview_step": {"id": 5628615003, "name": "Preliminary Screening Call"}, "candidate_id": 40517966003, "application_id": 44937562003, "interviewed_at": "2021-09-29T01:00:00.000Z", "submitted_by": {"id": 4218086003, "first_name": "John", "last_name": "Lafleur", "name": "John Lafleur", "employee_id": null}, "interviewer": {"id": 4218086003, "first_name": "John", "last_name": "Lafleur", "name": "John Lafleur", "employee_id": null}, "submitted_at": "2021-09-29T17:23:11.468Z", "overall_recommendation": "no_decision", "attributes": [{"name": "Willing to do required travel", "type": "Details", "note": null, "rating": "yes"}, {"name": "Three to five years of experience", "type": "Qualifications", "note": null, "rating": "mixed"}, {"name": "Personable", "type": "Personality Traits", "note": null, "rating": "yes"}, {"name": "Passionate", "type": "Personality Traits", "note": null, "rating": "mixed"}, {"name": "Organizational Skills", "type": "Skills", "note": null, "rating": "yes"}, {"name": "Manage competing priorities", "type": "Skills", "note": null, "rating": "yes"}, {"name": "Fits our salary range", "type": "Details", "note": null, "rating": "yes"}, {"name": "Empathetic", "type": "Personality Traits", "note": null, "rating": "strong_yes"}, {"name": "Currently based locally", "type": "Details", "note": null, "rating": "mixed"}, {"name": "Communication", "type": "Skills", "note": null, "rating": "no"}], "ratings": {"definitely_not": [], "no": ["Communication"], "mixed": ["Three to five years of experience", "Passionate", "Currently based locally"], "yes": ["Willing to do required travel", "Personable", "Organizational Skills", "Manage competing priorities", "Fits our salary range"], "strong_yes": ["Empathetic"]}, "questions": [{"id": null, "question": "Key Take-Aways", "answer": "test"}, {"id": null, "question": "Private Notes", "answer": ""}]}, "emitted_at": 1664285619011} +{"stream": "users", "data": {"id": 4218085003, "name": "Greenhouse Admin", "first_name": "Greenhouse", "last_name": "Admin", "primary_email_address": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "updated_at": "2020-11-18T14:09:08.401Z", "created_at": "2020-11-18T14:09:08.401Z", "disabled": false, "site_admin": true, "emails": ["scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com"], "employee_id": null, "linked_candidate_ids": []}, "emitted_at": 1664285619554} +{"stream": "users", "data": {"id": 4218086003, "name": "John Lafleur", "first_name": "John", "last_name": "Lafleur", "primary_email_address": "integration-test@airbyte.io", "updated_at": "2022-09-27T13:00:12.685Z", "created_at": "2020-11-18T14:09:08.481Z", "disabled": false, "site_admin": true, "emails": ["integration-test@airbyte.io"], "employee_id": null, "linked_candidate_ids": []}, "emitted_at": 1664285619555} +{"stream": "users", "data": {"id": 4218087003, "name": "emily.brooks+airbyte_integration@greenhouse.io", "first_name": null, "last_name": null, "primary_email_address": "emily.brooks+airbyte_integration@greenhouse.io", "updated_at": "2020-11-18T14:09:08.991Z", "created_at": "2020-11-18T14:09:08.809Z", "disabled": false, "site_admin": true, "emails": ["emily.brooks+airbyte_integration@greenhouse.io"], "employee_id": null, "linked_candidate_ids": []}, "emitted_at": 1664285619555} +{"stream": "users", "data": {"id": 4460715003, "name": "Vadym Ratniuk", "first_name": "Vadym", "last_name": "Ratniuk", "primary_email_address": "vadym.ratniuk@globallogic.com", "updated_at": "2021-09-18T10:09:16.846Z", "created_at": "2021-09-14T14:03:01.050Z", "disabled": false, "site_admin": false, "emails": ["vadym.ratniuk@globallogic.com"], "employee_id": null, "linked_candidate_ids": []}, "emitted_at": 1664285619556} +{"stream": "users", "data": {"id": 4481107003, "name": "Vadym Hevlich", "first_name": "Vadym", "last_name": "Hevlich", "primary_email_address": "vadym.hevlich@zazmic.com", "updated_at": "2021-10-10T17:49:28.058Z", "created_at": "2021-10-10T17:48:41.978Z", "disabled": false, "site_admin": true, "emails": ["vadym.hevlich@zazmic.com"], "employee_id": null, "linked_candidate_ids": []}, "emitted_at": 1664285619556} +{"stream": "custom_fields", "data": {"id": 4680898003, "name": "School Name", "active": true, "field_type": "candidate", "priority": 0, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "school_name", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845822003, "name": "Abraham Baldwin Agricultural College", "priority": 0, "external_id": null}, {"id": 10845823003, "name": "Academy of Art University", "priority": 1, "external_id": null}, {"id": 10845824003, "name": "Acadia University", "priority": 2, "external_id": null}, {"id": 10845825003, "name": "Adams State University", "priority": 3, "external_id": null}, {"id": 10845826003, "name": "Adelphi University", "priority": 4, "external_id": null}, {"id": 10845827003, "name": "Adrian College", "priority": 5, "external_id": null}, {"id": 10845828003, "name": "Adventist University of Health Sciences", "priority": 6, "external_id": null}, {"id": 10845829003, "name": "Agnes Scott College", "priority": 7, "external_id": null}, {"id": 10845830003, "name": "AIB College of Business", "priority": 8, "external_id": null}, {"id": 10845831003, "name": "Alaska Pacific University", "priority": 9, "external_id": null}, {"id": 10845832003, "name": "Albany College of Pharmacy and Health Sciences", "priority": 10, "external_id": null}, {"id": 10845833003, "name": "Albany State University", "priority": 11, "external_id": null}, {"id": 10845834003, "name": "Albertus Magnus College", "priority": 12, "external_id": null}, {"id": 10845835003, "name": "Albion College", "priority": 13, "external_id": null}, {"id": 10845836003, "name": "Albright College", "priority": 14, "external_id": null}, {"id": 10845837003, "name": "Alderson Broaddus University", "priority": 15, "external_id": null}, {"id": 10845838003, "name": "Alfred University", "priority": 16, "external_id": null}, {"id": 10845839003, "name": "Alice Lloyd College", "priority": 17, "external_id": null}, {"id": 10845840003, "name": "Allegheny College", "priority": 18, "external_id": null}, {"id": 10845841003, "name": "Allen College", "priority": 19, "external_id": null}, {"id": 10845842003, "name": "Allen University", "priority": 20, "external_id": null}, {"id": 10845843003, "name": "Alliant International University", "priority": 21, "external_id": null}, {"id": 10845844003, "name": "Alma College", "priority": 22, "external_id": null}, {"id": 10845845003, "name": "Alvernia University", "priority": 23, "external_id": null}, {"id": 10845846003, "name": "Alverno College", "priority": 24, "external_id": null}, {"id": 10845847003, "name": "Amberton University", "priority": 25, "external_id": null}, {"id": 10845848003, "name": "American Academy of Art", "priority": 26, "external_id": null}, {"id": 10845849003, "name": "American Indian College of the Assemblies of God", "priority": 27, "external_id": null}, {"id": 10845850003, "name": "American InterContinental University", "priority": 28, "external_id": null}, {"id": 10845851003, "name": "American International College", "priority": 29, "external_id": null}, {"id": 10845852003, "name": "American Jewish University", "priority": 30, "external_id": null}, {"id": 10845853003, "name": "American Public University System", "priority": 31, "external_id": null}, {"id": 10845854003, "name": "American University", "priority": 32, "external_id": null}, {"id": 10845855003, "name": "American University in Bulgaria", "priority": 33, "external_id": null}, {"id": 10845856003, "name": "American University in Cairo", "priority": 34, "external_id": null}, {"id": 10845857003, "name": "American University of Beirut", "priority": 35, "external_id": null}, {"id": 10845858003, "name": "American University of Paris", "priority": 36, "external_id": null}, {"id": 10845859003, "name": "American University of Puerto Rico", "priority": 37, "external_id": null}, {"id": 10845860003, "name": "Amherst College", "priority": 38, "external_id": null}, {"id": 10845861003, "name": "Amridge University", "priority": 39, "external_id": null}, {"id": 10845862003, "name": "Anderson University", "priority": 40, "external_id": null}, {"id": 10845863003, "name": "Andrews University", "priority": 41, "external_id": null}, {"id": 10845864003, "name": "Angelo State University", "priority": 42, "external_id": null}, {"id": 10845865003, "name": "Anna Maria College", "priority": 43, "external_id": null}, {"id": 10845866003, "name": "Antioch University", "priority": 44, "external_id": null}, {"id": 10845867003, "name": "Appalachian Bible College", "priority": 45, "external_id": null}, {"id": 10845868003, "name": "Aquinas College", "priority": 46, "external_id": null}, {"id": 10845869003, "name": "Arcadia University", "priority": 47, "external_id": null}, {"id": 10845870003, "name": "Argosy University", "priority": 48, "external_id": null}, {"id": 10845871003, "name": "Arizona Christian University", "priority": 49, "external_id": null}, {"id": 10845872003, "name": "Arizona State University - West", "priority": 50, "external_id": null}, {"id": 10845873003, "name": "Arkansas Baptist College", "priority": 51, "external_id": null}, {"id": 10845874003, "name": "Arkansas Tech University", "priority": 52, "external_id": null}, {"id": 10845875003, "name": "Armstrong Atlantic State University", "priority": 53, "external_id": null}, {"id": 10845876003, "name": "Art Academy of Cincinnati", "priority": 54, "external_id": null}, {"id": 10845877003, "name": "Art Center College of Design", "priority": 55, "external_id": null}, {"id": 10845878003, "name": "Art Institute of Atlanta", "priority": 56, "external_id": null}, {"id": 10845879003, "name": "Art Institute of Colorado", "priority": 57, "external_id": null}, {"id": 10845880003, "name": "Art Institute of Houston", "priority": 58, "external_id": null}, {"id": 10845881003, "name": "Art Institute of Pittsburgh", "priority": 59, "external_id": null}, {"id": 10845882003, "name": "Art Institute of Portland", "priority": 60, "external_id": null}, {"id": 10845883003, "name": "Art Institute of Seattle", "priority": 61, "external_id": null}, {"id": 10845884003, "name": "Asbury University", "priority": 62, "external_id": null}, {"id": 10845885003, "name": "Ashford University", "priority": 63, "external_id": null}, {"id": 10845886003, "name": "Ashland University", "priority": 64, "external_id": null}, {"id": 10845887003, "name": "Assumption College", "priority": 65, "external_id": null}, {"id": 10845888003, "name": "Athens State University", "priority": 66, "external_id": null}, {"id": 10845889003, "name": "Auburn University - Montgomery", "priority": 67, "external_id": null}, {"id": 10845890003, "name": "Augsburg College", "priority": 68, "external_id": null}, {"id": 10845891003, "name": "Augustana College", "priority": 69, "external_id": null}, {"id": 10845892003, "name": "Aurora University", "priority": 70, "external_id": null}, {"id": 10845893003, "name": "Austin College", "priority": 71, "external_id": null}, {"id": 10845894003, "name": "Alcorn State University", "priority": 72, "external_id": null}, {"id": 10845895003, "name": "Ave Maria University", "priority": 73, "external_id": null}, {"id": 10845896003, "name": "Averett University", "priority": 74, "external_id": null}, {"id": 10845897003, "name": "Avila University", "priority": 75, "external_id": null}, {"id": 10845898003, "name": "Azusa Pacific University", "priority": 76, "external_id": null}, {"id": 10845899003, "name": "Babson College", "priority": 77, "external_id": null}, {"id": 10845900003, "name": "Bacone College", "priority": 78, "external_id": null}, {"id": 10845901003, "name": "Baker College of Flint", "priority": 79, "external_id": null}, {"id": 10845902003, "name": "Baker University", "priority": 80, "external_id": null}, {"id": 10845903003, "name": "Baldwin Wallace University", "priority": 81, "external_id": null}, {"id": 10845904003, "name": "Christian Brothers University", "priority": 82, "external_id": null}, {"id": 10845905003, "name": "Abilene Christian University", "priority": 83, "external_id": null}, {"id": 10845906003, "name": "Arizona State University", "priority": 84, "external_id": null}, {"id": 10845907003, "name": "Auburn University", "priority": 85, "external_id": null}, {"id": 10845908003, "name": "Alabama A&M University", "priority": 86, "external_id": null}, {"id": 10845909003, "name": "Alabama State University", "priority": 87, "external_id": null}, {"id": 10845910003, "name": "Arkansas State University", "priority": 88, "external_id": null}, {"id": 10845911003, "name": "Baptist Bible College", "priority": 89, "external_id": null}, {"id": 10845912003, "name": "Baptist Bible College and Seminary", "priority": 90, "external_id": null}, {"id": 10845913003, "name": "Baptist College of Florida", "priority": 91, "external_id": null}, {"id": 10845914003, "name": "Baptist Memorial College of Health Sciences", "priority": 92, "external_id": null}, {"id": 10845915003, "name": "Baptist Missionary Association Theological Seminary", "priority": 93, "external_id": null}, {"id": 10845916003, "name": "Bard College", "priority": 94, "external_id": null}, {"id": 10845917003, "name": "Bard College at Simon's Rock", "priority": 95, "external_id": null}, {"id": 10845918003, "name": "Barnard College", "priority": 96, "external_id": null}, {"id": 10845919003, "name": "Barry University", "priority": 97, "external_id": null}, {"id": 10845920003, "name": "Barton College", "priority": 98, "external_id": null}, {"id": 10845921003, "name": "Bastyr University", "priority": 99, "external_id": null}, {"id": 10845922003, "name": "Bates College", "priority": 100, "external_id": null}, {"id": 10845923003, "name": "Bauder College", "priority": 101, "external_id": null}, {"id": 10845924003, "name": "Bay Path College", "priority": 102, "external_id": null}, {"id": 10845925003, "name": "Bay State College", "priority": 103, "external_id": null}, {"id": 10845926003, "name": "Bayamon Central University", "priority": 104, "external_id": null}, {"id": 10845927003, "name": "Beacon College", "priority": 105, "external_id": null}, {"id": 10845928003, "name": "Becker College", "priority": 106, "external_id": null}, {"id": 10845929003, "name": "Belhaven University", "priority": 107, "external_id": null}, {"id": 10845930003, "name": "Bellarmine University", "priority": 108, "external_id": null}, {"id": 10845931003, "name": "Bellevue College", "priority": 109, "external_id": null}, {"id": 10845932003, "name": "Bellevue University", "priority": 110, "external_id": null}, {"id": 10845933003, "name": "Bellin College", "priority": 111, "external_id": null}, {"id": 10845934003, "name": "Belmont Abbey College", "priority": 112, "external_id": null}, {"id": 10845935003, "name": "Belmont University", "priority": 113, "external_id": null}, {"id": 10845936003, "name": "Beloit College", "priority": 114, "external_id": null}, {"id": 10845937003, "name": "Bemidji State University", "priority": 115, "external_id": null}, {"id": 10845938003, "name": "Benedict College", "priority": 116, "external_id": null}, {"id": 10845939003, "name": "Benedictine College", "priority": 117, "external_id": null}, {"id": 10845940003, "name": "Benedictine University", "priority": 118, "external_id": null}, {"id": 10845941003, "name": "Benjamin Franklin Institute of Technology", "priority": 119, "external_id": null}, {"id": 10845942003, "name": "Bennett College", "priority": 120, "external_id": null}, {"id": 10845943003, "name": "Bennington College", "priority": 121, "external_id": null}, {"id": 10845944003, "name": "Bentley University", "priority": 122, "external_id": null}, {"id": 10845945003, "name": "Berea College", "priority": 123, "external_id": null}, {"id": 10845946003, "name": "Berkeley College", "priority": 124, "external_id": null}, {"id": 10845947003, "name": "Berklee College of Music", "priority": 125, "external_id": null}, {"id": 10845948003, "name": "Berry College", "priority": 126, "external_id": null}, {"id": 10845949003, "name": "Bethany College", "priority": 127, "external_id": null}, {"id": 10845950003, "name": "Bethany Lutheran College", "priority": 128, "external_id": null}, {"id": 10845951003, "name": "Bethel College", "priority": 129, "external_id": null}, {"id": 10845952003, "name": "Bethel University", "priority": 130, "external_id": null}, {"id": 10845953003, "name": "BI Norwegian Business School", "priority": 131, "external_id": null}, {"id": 10845954003, "name": "Binghamton University - SUNY", "priority": 132, "external_id": null}, {"id": 10845955003, "name": "Biola University", "priority": 133, "external_id": null}, {"id": 10845956003, "name": "Birmingham-Southern College", "priority": 134, "external_id": null}, {"id": 10845957003, "name": "Bismarck State College", "priority": 135, "external_id": null}, {"id": 10845958003, "name": "Black Hills State University", "priority": 136, "external_id": null}, {"id": 10845959003, "name": "Blackburn College", "priority": 137, "external_id": null}, {"id": 10845960003, "name": "Blessing-Rieman College of Nursing", "priority": 138, "external_id": null}, {"id": 10845961003, "name": "Bloomfield College", "priority": 139, "external_id": null}, {"id": 10845962003, "name": "Bloomsburg University of Pennsylvania", "priority": 140, "external_id": null}, {"id": 10845963003, "name": "Blue Mountain College", "priority": 141, "external_id": null}, {"id": 10845964003, "name": "Bluefield College", "priority": 142, "external_id": null}, {"id": 10845965003, "name": "Bluefield State College", "priority": 143, "external_id": null}, {"id": 10845966003, "name": "Bluffton University", "priority": 144, "external_id": null}, {"id": 10845967003, "name": "Boricua College", "priority": 145, "external_id": null}, {"id": 10845968003, "name": "Boston Architectural College", "priority": 146, "external_id": null}, {"id": 10845969003, "name": "Boston Conservatory", "priority": 147, "external_id": null}, {"id": 10845970003, "name": "Boston University", "priority": 148, "external_id": null}, {"id": 10845971003, "name": "Bowdoin College", "priority": 149, "external_id": null}, {"id": 10845972003, "name": "Bowie State University", "priority": 150, "external_id": null}, {"id": 10845973003, "name": "Bradley University", "priority": 151, "external_id": null}, {"id": 10845974003, "name": "Brandeis University", "priority": 152, "external_id": null}, {"id": 10845975003, "name": "Brandman University", "priority": 153, "external_id": null}, {"id": 10845976003, "name": "Brazosport College", "priority": 154, "external_id": null}, {"id": 10845977003, "name": "Brenau University", "priority": 155, "external_id": null}, {"id": 10845978003, "name": "Brescia University", "priority": 156, "external_id": null}, {"id": 10845979003, "name": "Brevard College", "priority": 157, "external_id": null}, {"id": 10845980003, "name": "Brewton-Parker College", "priority": 158, "external_id": null}, {"id": 10845981003, "name": "Briar Cliff University", "priority": 159, "external_id": null}, {"id": 10845982003, "name": "Briarcliffe College", "priority": 160, "external_id": null}, {"id": 10845983003, "name": "Bridgewater College", "priority": 161, "external_id": null}, {"id": 10845984003, "name": "Bridgewater State University", "priority": 162, "external_id": null}, {"id": 10845985003, "name": "Brigham Young University - Hawaii", "priority": 163, "external_id": null}, {"id": 10845986003, "name": "Brigham Young University - Idaho", "priority": 164, "external_id": null}, {"id": 10845987003, "name": "Brock University", "priority": 165, "external_id": null}, {"id": 10845988003, "name": "Bryan College", "priority": 166, "external_id": null}, {"id": 10845989003, "name": "Bryn Athyn College of the New Church", "priority": 167, "external_id": null}, {"id": 10845990003, "name": "Bryn Mawr College", "priority": 168, "external_id": null}, {"id": 10845991003, "name": "Boston College", "priority": 169, "external_id": null}, {"id": 10845992003, "name": "Buena Vista University", "priority": 170, "external_id": null}, {"id": 10845993003, "name": "Burlington College", "priority": 171, "external_id": null}, {"id": 10845994003, "name": "Bowling Green State University", "priority": 172, "external_id": null}, {"id": 10845995003, "name": "Brown University", "priority": 173, "external_id": null}, {"id": 10845996003, "name": "Appalachian State University", "priority": 174, "external_id": null}, {"id": 10845997003, "name": "Brigham Young University - Provo", "priority": 175, "external_id": null}, {"id": 10845998003, "name": "Boise State University", "priority": 176, "external_id": null}, {"id": 10845999003, "name": "Bethune-Cookman University", "priority": 177, "external_id": null}, {"id": 10846000003, "name": "Bryant University", "priority": 178, "external_id": null}, {"id": 10846001003, "name": "Cabarrus College of Health Sciences", "priority": 179, "external_id": null}, {"id": 10846002003, "name": "Cabrini College", "priority": 180, "external_id": null}, {"id": 10846003003, "name": "Cairn University", "priority": 181, "external_id": null}, {"id": 10846004003, "name": "Caldwell College", "priority": 182, "external_id": null}, {"id": 10846005003, "name": "California Baptist University", "priority": 183, "external_id": null}, {"id": 10846006003, "name": "California College of the Arts", "priority": 184, "external_id": null}, {"id": 10846007003, "name": "California Institute of Integral Studies", "priority": 185, "external_id": null}, {"id": 10846008003, "name": "California Institute of Technology", "priority": 186, "external_id": null}, {"id": 10846009003, "name": "California Institute of the Arts", "priority": 187, "external_id": null}, {"id": 10846010003, "name": "California Lutheran University", "priority": 188, "external_id": null}, {"id": 10846011003, "name": "California Maritime Academy", "priority": 189, "external_id": null}, {"id": 10846012003, "name": "California State Polytechnic University - Pomona", "priority": 190, "external_id": null}, {"id": 10846013003, "name": "California State University - Bakersfield", "priority": 191, "external_id": null}, {"id": 10846014003, "name": "California State University - Channel Islands", "priority": 192, "external_id": null}, {"id": 10846015003, "name": "California State University - Chico", "priority": 193, "external_id": null}, {"id": 10846016003, "name": "California State University - Dominguez Hills", "priority": 194, "external_id": null}, {"id": 10846017003, "name": "California State University - East Bay", "priority": 195, "external_id": null}, {"id": 10846018003, "name": "California State University - Fullerton", "priority": 196, "external_id": null}, {"id": 10846019003, "name": "California State University - Los Angeles", "priority": 197, "external_id": null}, {"id": 10846020003, "name": "California State University - Monterey Bay", "priority": 198, "external_id": null}, {"id": 10846021003, "name": "California State University - Northridge", "priority": 199, "external_id": null}, {"id": 10846022003, "name": "California State University - San Bernardino", "priority": 200, "external_id": null}, {"id": 10846023003, "name": "California State University - San Marcos", "priority": 201, "external_id": null}, {"id": 10846024003, "name": "California State University - Stanislaus", "priority": 202, "external_id": null}, {"id": 10846025003, "name": "California University of Pennsylvania", "priority": 203, "external_id": null}, {"id": 10846026003, "name": "Calumet College of St. Joseph", "priority": 204, "external_id": null}, {"id": 10846027003, "name": "Calvary Bible College and Theological Seminary", "priority": 205, "external_id": null}, {"id": 10846028003, "name": "Calvin College", "priority": 206, "external_id": null}, {"id": 10846029003, "name": "Cambridge College", "priority": 207, "external_id": null}, {"id": 10846030003, "name": "Cameron University", "priority": 208, "external_id": null}, {"id": 10846031003, "name": "Campbellsville University", "priority": 209, "external_id": null}, {"id": 10846032003, "name": "Canisius College", "priority": 210, "external_id": null}, {"id": 10846033003, "name": "Capella University", "priority": 211, "external_id": null}, {"id": 10846034003, "name": "Capital University", "priority": 212, "external_id": null}, {"id": 10846035003, "name": "Capitol College", "priority": 213, "external_id": null}, {"id": 10846036003, "name": "Cardinal Stritch University", "priority": 214, "external_id": null}, {"id": 10846037003, "name": "Caribbean University", "priority": 215, "external_id": null}, {"id": 10846038003, "name": "Carleton College", "priority": 216, "external_id": null}, {"id": 10846039003, "name": "Carleton University", "priority": 217, "external_id": null}, {"id": 10846040003, "name": "Carlos Albizu University", "priority": 218, "external_id": null}, {"id": 10846041003, "name": "Carlow University", "priority": 219, "external_id": null}, {"id": 10846042003, "name": "Carnegie Mellon University", "priority": 220, "external_id": null}, {"id": 10846043003, "name": "Carroll College", "priority": 221, "external_id": null}, {"id": 10846044003, "name": "Carroll University", "priority": 222, "external_id": null}, {"id": 10846045003, "name": "Carson-Newman University", "priority": 223, "external_id": null}, {"id": 10846046003, "name": "Carthage College", "priority": 224, "external_id": null}, {"id": 10846047003, "name": "Case Western Reserve University", "priority": 225, "external_id": null}, {"id": 10846048003, "name": "Castleton State College", "priority": 226, "external_id": null}, {"id": 10846049003, "name": "Catawba College", "priority": 227, "external_id": null}, {"id": 10846050003, "name": "Cazenovia College", "priority": 228, "external_id": null}, {"id": 10846051003, "name": "Cedar Crest College", "priority": 229, "external_id": null}, {"id": 10846052003, "name": "Cedarville University", "priority": 230, "external_id": null}, {"id": 10846053003, "name": "Centenary College", "priority": 231, "external_id": null}, {"id": 10846054003, "name": "Centenary College of Louisiana", "priority": 232, "external_id": null}, {"id": 10846055003, "name": "Central Baptist College", "priority": 233, "external_id": null}, {"id": 10846056003, "name": "Central Bible College", "priority": 234, "external_id": null}, {"id": 10846057003, "name": "Central Christian College", "priority": 235, "external_id": null}, {"id": 10846058003, "name": "Central College", "priority": 236, "external_id": null}, {"id": 10846059003, "name": "Central Methodist University", "priority": 237, "external_id": null}, {"id": 10846060003, "name": "Central Penn College", "priority": 238, "external_id": null}, {"id": 10846061003, "name": "Central State University", "priority": 239, "external_id": null}, {"id": 10846062003, "name": "Central Washington University", "priority": 240, "external_id": null}, {"id": 10846063003, "name": "Centre College", "priority": 241, "external_id": null}, {"id": 10846064003, "name": "Chadron State College", "priority": 242, "external_id": null}, {"id": 10846065003, "name": "Chamberlain College of Nursing", "priority": 243, "external_id": null}, {"id": 10846066003, "name": "Chaminade University of Honolulu", "priority": 244, "external_id": null}, {"id": 10846067003, "name": "Champlain College", "priority": 245, "external_id": null}, {"id": 10846068003, "name": "Chancellor University", "priority": 246, "external_id": null}, {"id": 10846069003, "name": "Chapman University", "priority": 247, "external_id": null}, {"id": 10846070003, "name": "Charles R. Drew University of Medicine and Science", "priority": 248, "external_id": null}, {"id": 10846071003, "name": "Charter Oak State College", "priority": 249, "external_id": null}, {"id": 10846072003, "name": "Chatham University", "priority": 250, "external_id": null}, {"id": 10846073003, "name": "Chestnut Hill College", "priority": 251, "external_id": null}, {"id": 10846074003, "name": "Cheyney University of Pennsylvania", "priority": 252, "external_id": null}, {"id": 10846075003, "name": "Chicago State University", "priority": 253, "external_id": null}, {"id": 10846076003, "name": "Chipola College", "priority": 254, "external_id": null}, {"id": 10846077003, "name": "Chowan University", "priority": 255, "external_id": null}, {"id": 10846078003, "name": "Christendom College", "priority": 256, "external_id": null}, {"id": 10846079003, "name": "Baylor University", "priority": 257, "external_id": null}, {"id": 10846080003, "name": "Central Connecticut State University", "priority": 258, "external_id": null}, {"id": 10846081003, "name": "Central Michigan University", "priority": 259, "external_id": null}, {"id": 10846082003, "name": "Charleston Southern University", "priority": 260, "external_id": null}, {"id": 10846083003, "name": "California State University - Sacramento", "priority": 261, "external_id": null}, {"id": 10846084003, "name": "California State University - Fresno", "priority": 262, "external_id": null}, {"id": 10846085003, "name": "Campbell University", "priority": 263, "external_id": null}, {"id": 10846086003, "name": "Christopher Newport University", "priority": 264, "external_id": null}, {"id": 10846087003, "name": "Cincinnati Christian University", "priority": 265, "external_id": null}, {"id": 10846088003, "name": "Cincinnati College of Mortuary Science", "priority": 266, "external_id": null}, {"id": 10846089003, "name": "City University of Seattle", "priority": 267, "external_id": null}, {"id": 10846090003, "name": "Claflin University", "priority": 268, "external_id": null}, {"id": 10846091003, "name": "Claremont McKenna College", "priority": 269, "external_id": null}, {"id": 10846092003, "name": "Clarion University of Pennsylvania", "priority": 270, "external_id": null}, {"id": 10846093003, "name": "Clark Atlanta University", "priority": 271, "external_id": null}, {"id": 10846094003, "name": "Clark University", "priority": 272, "external_id": null}, {"id": 10846095003, "name": "Clarke University", "priority": 273, "external_id": null}, {"id": 10846096003, "name": "Clarkson College", "priority": 274, "external_id": null}, {"id": 10846097003, "name": "Clarkson University", "priority": 275, "external_id": null}, {"id": 10846098003, "name": "Clayton State University", "priority": 276, "external_id": null}, {"id": 10846099003, "name": "Clear Creek Baptist Bible College", "priority": 277, "external_id": null}, {"id": 10846100003, "name": "Clearwater Christian College", "priority": 278, "external_id": null}, {"id": 10846101003, "name": "Cleary University", "priority": 279, "external_id": null}, {"id": 10846102003, "name": "College of William and Mary", "priority": 280, "external_id": null}, {"id": 10846103003, "name": "Cleveland Chiropractic College", "priority": 281, "external_id": null}, {"id": 10846104003, "name": "Cleveland Institute of Art", "priority": 282, "external_id": null}, {"id": 10846105003, "name": "Cleveland Institute of Music", "priority": 283, "external_id": null}, {"id": 10846106003, "name": "Cleveland State University", "priority": 284, "external_id": null}, {"id": 10846107003, "name": "Coe College", "priority": 285, "external_id": null}, {"id": 10846108003, "name": "Cogswell Polytechnical College", "priority": 286, "external_id": null}, {"id": 10846109003, "name": "Coker College", "priority": 287, "external_id": null}, {"id": 10846110003, "name": "Colby College", "priority": 288, "external_id": null}, {"id": 10846111003, "name": "Colby-Sawyer College", "priority": 289, "external_id": null}, {"id": 10846112003, "name": "College at Brockport - SUNY", "priority": 290, "external_id": null}, {"id": 10846113003, "name": "College for Creative Studies", "priority": 291, "external_id": null}, {"id": 10846114003, "name": "College of Charleston", "priority": 292, "external_id": null}, {"id": 10846115003, "name": "College of Idaho", "priority": 293, "external_id": null}, {"id": 10846116003, "name": "College of Mount St. Joseph", "priority": 294, "external_id": null}, {"id": 10846117003, "name": "College of Mount St. Vincent", "priority": 295, "external_id": null}, {"id": 10846118003, "name": "College of New Jersey", "priority": 296, "external_id": null}, {"id": 10846119003, "name": "College of New Rochelle", "priority": 297, "external_id": null}, {"id": 10846120003, "name": "College of Our Lady of the Elms", "priority": 298, "external_id": null}, {"id": 10846121003, "name": "College of Saints John Fisher & Thomas More", "priority": 299, "external_id": null}, {"id": 10846122003, "name": "College of Southern Nevada", "priority": 300, "external_id": null}, {"id": 10846123003, "name": "College of St. Benedict", "priority": 301, "external_id": null}, {"id": 10846124003, "name": "College of St. Elizabeth", "priority": 302, "external_id": null}, {"id": 10846125003, "name": "College of St. Joseph", "priority": 303, "external_id": null}, {"id": 10846126003, "name": "College of St. Mary", "priority": 304, "external_id": null}, {"id": 10846127003, "name": "College of St. Rose", "priority": 305, "external_id": null}, {"id": 10846128003, "name": "College of St. Scholastica", "priority": 306, "external_id": null}, {"id": 10846129003, "name": "College of the Atlantic", "priority": 307, "external_id": null}, {"id": 10846130003, "name": "College of the Holy Cross", "priority": 308, "external_id": null}, {"id": 10846131003, "name": "College of the Ozarks", "priority": 309, "external_id": null}, {"id": 10846132003, "name": "College of Wooster", "priority": 310, "external_id": null}, {"id": 10846133003, "name": "Colorado Christian University", "priority": 311, "external_id": null}, {"id": 10846134003, "name": "Colorado College", "priority": 312, "external_id": null}, {"id": 10846135003, "name": "Colorado Mesa University", "priority": 313, "external_id": null}, {"id": 10846136003, "name": "Colorado School of Mines", "priority": 314, "external_id": null}, {"id": 10846137003, "name": "Colorado State University - Pueblo", "priority": 315, "external_id": null}, {"id": 10846138003, "name": "Colorado Technical University", "priority": 316, "external_id": null}, {"id": 10846139003, "name": "Columbia College", "priority": 317, "external_id": null}, {"id": 10846140003, "name": "Columbia College Chicago", "priority": 318, "external_id": null}, {"id": 10846141003, "name": "Columbia College of Nursing", "priority": 319, "external_id": null}, {"id": 10846142003, "name": "Columbia International University", "priority": 320, "external_id": null}, {"id": 10846143003, "name": "Columbus College of Art and Design", "priority": 321, "external_id": null}, {"id": 10846144003, "name": "Columbus State University", "priority": 322, "external_id": null}, {"id": 10846145003, "name": "Conception Seminary College", "priority": 323, "external_id": null}, {"id": 10846146003, "name": "Concord University", "priority": 324, "external_id": null}, {"id": 10846147003, "name": "Concordia College", "priority": 325, "external_id": null}, {"id": 10846148003, "name": "Concordia College - Moorhead", "priority": 326, "external_id": null}, {"id": 10846149003, "name": "Concordia University", "priority": 327, "external_id": null}, {"id": 10846150003, "name": "Concordia University Chicago", "priority": 328, "external_id": null}, {"id": 10846151003, "name": "Concordia University Texas", "priority": 329, "external_id": null}, {"id": 10846152003, "name": "Concordia University Wisconsin", "priority": 330, "external_id": null}, {"id": 10846153003, "name": "Concordia University - St. Paul", "priority": 331, "external_id": null}, {"id": 10846154003, "name": "Connecticut College", "priority": 332, "external_id": null}, {"id": 10846155003, "name": "Converse College", "priority": 333, "external_id": null}, {"id": 10846156003, "name": "Cooper Union", "priority": 334, "external_id": null}, {"id": 10846157003, "name": "Coppin State University", "priority": 335, "external_id": null}, {"id": 10846158003, "name": "Corban University", "priority": 336, "external_id": null}, {"id": 10846159003, "name": "Corcoran College of Art and Design", "priority": 337, "external_id": null}, {"id": 10846160003, "name": "Cornell College", "priority": 338, "external_id": null}, {"id": 10846161003, "name": "Cornerstone University", "priority": 339, "external_id": null}, {"id": 10846162003, "name": "Cornish College of the Arts", "priority": 340, "external_id": null}, {"id": 10846163003, "name": "Covenant College", "priority": 341, "external_id": null}, {"id": 10846164003, "name": "Cox College", "priority": 342, "external_id": null}, {"id": 10846165003, "name": "Creighton University", "priority": 343, "external_id": null}, {"id": 10846166003, "name": "Criswell College", "priority": 344, "external_id": null}, {"id": 10846167003, "name": "Crown College", "priority": 345, "external_id": null}, {"id": 10846168003, "name": "Culinary Institute of America", "priority": 346, "external_id": null}, {"id": 10846169003, "name": "Culver-Stockton College", "priority": 347, "external_id": null}, {"id": 10846170003, "name": "Cumberland University", "priority": 348, "external_id": null}, {"id": 10846171003, "name": "Columbia University", "priority": 349, "external_id": null}, {"id": 10846172003, "name": "Cornell University", "priority": 350, "external_id": null}, {"id": 10846173003, "name": "Colorado State University", "priority": 351, "external_id": null}, {"id": 10846174003, "name": "University of Virginia", "priority": 352, "external_id": null}, {"id": 10846175003, "name": "Colgate University", "priority": 353, "external_id": null}, {"id": 10846176003, "name": "CUNY - Baruch College", "priority": 354, "external_id": null}, {"id": 10846177003, "name": "CUNY - Brooklyn College", "priority": 355, "external_id": null}, {"id": 10846178003, "name": "CUNY - City College", "priority": 356, "external_id": null}, {"id": 10846179003, "name": "CUNY - College of Staten Island", "priority": 357, "external_id": null}, {"id": 10846180003, "name": "CUNY - Hunter College", "priority": 358, "external_id": null}, {"id": 10846181003, "name": "CUNY - John Jay College of Criminal Justice", "priority": 359, "external_id": null}, {"id": 10846182003, "name": "CUNY - Lehman College", "priority": 360, "external_id": null}, {"id": 10846183003, "name": "CUNY - Medgar Evers College", "priority": 361, "external_id": null}, {"id": 10846184003, "name": "CUNY - New York City College of Technology", "priority": 362, "external_id": null}, {"id": 10846185003, "name": "CUNY - Queens College", "priority": 363, "external_id": null}, {"id": 10846186003, "name": "CUNY - York College", "priority": 364, "external_id": null}, {"id": 10846187003, "name": "Curry College", "priority": 365, "external_id": null}, {"id": 10846188003, "name": "Curtis Institute of Music", "priority": 366, "external_id": null}, {"id": 10846189003, "name": "D'Youville College", "priority": 367, "external_id": null}, {"id": 10846190003, "name": "Daemen College", "priority": 368, "external_id": null}, {"id": 10846191003, "name": "Dakota State University", "priority": 369, "external_id": null}, {"id": 10846192003, "name": "Dakota Wesleyan University", "priority": 370, "external_id": null}, {"id": 10846193003, "name": "Dalhousie University", "priority": 371, "external_id": null}, {"id": 10846194003, "name": "Dallas Baptist University", "priority": 372, "external_id": null}, {"id": 10846195003, "name": "Dallas Christian College", "priority": 373, "external_id": null}, {"id": 10846196003, "name": "Dalton State College", "priority": 374, "external_id": null}, {"id": 10846197003, "name": "Daniel Webster College", "priority": 375, "external_id": null}, {"id": 10846198003, "name": "Davenport University", "priority": 376, "external_id": null}, {"id": 10846199003, "name": "Davis and Elkins College", "priority": 377, "external_id": null}, {"id": 10846200003, "name": "Davis College", "priority": 378, "external_id": null}, {"id": 10846201003, "name": "Daytona State College", "priority": 379, "external_id": null}, {"id": 10846202003, "name": "Dean College", "priority": 380, "external_id": null}, {"id": 10846203003, "name": "Defiance College", "priority": 381, "external_id": null}, {"id": 10846204003, "name": "Delaware Valley College", "priority": 382, "external_id": null}, {"id": 10846205003, "name": "Delta State University", "priority": 383, "external_id": null}, {"id": 10846206003, "name": "Denison University", "priority": 384, "external_id": null}, {"id": 10846207003, "name": "DePaul University", "priority": 385, "external_id": null}, {"id": 10846208003, "name": "DePauw University", "priority": 386, "external_id": null}, {"id": 10846209003, "name": "DEREE - The American College of Greece", "priority": 387, "external_id": null}, {"id": 10846210003, "name": "DeSales University", "priority": 388, "external_id": null}, {"id": 10846211003, "name": "DeVry University", "priority": 389, "external_id": null}, {"id": 10846212003, "name": "Dickinson College", "priority": 390, "external_id": null}, {"id": 10846213003, "name": "Dickinson State University", "priority": 391, "external_id": null}, {"id": 10846214003, "name": "Dillard University", "priority": 392, "external_id": null}, {"id": 10846215003, "name": "Divine Word College", "priority": 393, "external_id": null}, {"id": 10846216003, "name": "Dixie State College of Utah", "priority": 394, "external_id": null}, {"id": 10846217003, "name": "Doane College", "priority": 395, "external_id": null}, {"id": 10846218003, "name": "Dominican College", "priority": 396, "external_id": null}, {"id": 10846219003, "name": "Dominican University", "priority": 397, "external_id": null}, {"id": 10846220003, "name": "Dominican University of California", "priority": 398, "external_id": null}, {"id": 10846221003, "name": "Donnelly College", "priority": 399, "external_id": null}, {"id": 10846222003, "name": "Dordt College", "priority": 400, "external_id": null}, {"id": 10846223003, "name": "Dowling College", "priority": 401, "external_id": null}, {"id": 10846224003, "name": "Drew University", "priority": 402, "external_id": null}, {"id": 10846225003, "name": "Drexel University", "priority": 403, "external_id": null}, {"id": 10846226003, "name": "Drury University", "priority": 404, "external_id": null}, {"id": 10846227003, "name": "Dunwoody College of Technology", "priority": 405, "external_id": null}, {"id": 10846228003, "name": "Earlham College", "priority": 406, "external_id": null}, {"id": 10846229003, "name": "Drake University", "priority": 407, "external_id": null}, {"id": 10846230003, "name": "East Central University", "priority": 408, "external_id": null}, {"id": 10846231003, "name": "East Stroudsburg University of Pennsylvania", "priority": 409, "external_id": null}, {"id": 10846232003, "name": "East Tennessee State University", "priority": 410, "external_id": null}, {"id": 10846233003, "name": "East Texas Baptist University", "priority": 411, "external_id": null}, {"id": 10846234003, "name": "East-West University", "priority": 412, "external_id": null}, {"id": 10846235003, "name": "Eastern Connecticut State University", "priority": 413, "external_id": null}, {"id": 10846236003, "name": "Eastern Mennonite University", "priority": 414, "external_id": null}, {"id": 10846237003, "name": "Eastern Nazarene College", "priority": 415, "external_id": null}, {"id": 10846238003, "name": "Eastern New Mexico University", "priority": 416, "external_id": null}, {"id": 10846239003, "name": "Eastern Oregon University", "priority": 417, "external_id": null}, {"id": 10846240003, "name": "Eastern University", "priority": 418, "external_id": null}, {"id": 10846241003, "name": "Eckerd College", "priority": 419, "external_id": null}, {"id": 10846242003, "name": "ECPI University", "priority": 420, "external_id": null}, {"id": 10846243003, "name": "Edgewood College", "priority": 421, "external_id": null}, {"id": 10846244003, "name": "Edinboro University of Pennsylvania", "priority": 422, "external_id": null}, {"id": 10846245003, "name": "Edison State College", "priority": 423, "external_id": null}, {"id": 10846246003, "name": "Edward Waters College", "priority": 424, "external_id": null}, {"id": 10846247003, "name": "Elizabeth City State University", "priority": 425, "external_id": null}, {"id": 10846248003, "name": "Elizabethtown College", "priority": 426, "external_id": null}, {"id": 10846249003, "name": "Elmhurst College", "priority": 427, "external_id": null}, {"id": 10846250003, "name": "Elmira College", "priority": 428, "external_id": null}, {"id": 10846251003, "name": "Embry-Riddle Aeronautical University", "priority": 429, "external_id": null}, {"id": 10846252003, "name": "Embry-Riddle Aeronautical University - Prescott", "priority": 430, "external_id": null}, {"id": 10846253003, "name": "Emerson College", "priority": 431, "external_id": null}, {"id": 10846254003, "name": "Duquesne University", "priority": 432, "external_id": null}, {"id": 10846255003, "name": "Eastern Washington University", "priority": 433, "external_id": null}, {"id": 10846256003, "name": "Eastern Illinois University", "priority": 434, "external_id": null}, {"id": 10846257003, "name": "Eastern Kentucky University", "priority": 435, "external_id": null}, {"id": 10846258003, "name": "Eastern Michigan University", "priority": 436, "external_id": null}, {"id": 10846259003, "name": "Elon University", "priority": 437, "external_id": null}, {"id": 10846260003, "name": "Delaware State University", "priority": 438, "external_id": null}, {"id": 10846261003, "name": "Duke University", "priority": 439, "external_id": null}, {"id": 10846262003, "name": "California Polytechnic State University - San Luis Obispo", "priority": 440, "external_id": null}, {"id": 10846263003, "name": "Emmanuel College", "priority": 441, "external_id": null}, {"id": 10846264003, "name": "Emmaus Bible College", "priority": 442, "external_id": null}, {"id": 10846265003, "name": "Emory and Henry College", "priority": 443, "external_id": null}, {"id": 10846266003, "name": "Emory University", "priority": 444, "external_id": null}, {"id": 10846267003, "name": "Emporia State University", "priority": 445, "external_id": null}, {"id": 10846268003, "name": "Endicott College", "priority": 446, "external_id": null}, {"id": 10846269003, "name": "Erskine College", "priority": 447, "external_id": null}, {"id": 10846270003, "name": "Escuela de Artes Plasticas de Puerto Rico", "priority": 448, "external_id": null}, {"id": 10846271003, "name": "Eureka College", "priority": 449, "external_id": null}, {"id": 10846272003, "name": "Evangel University", "priority": 450, "external_id": null}, {"id": 10846273003, "name": "Everest College - Phoenix", "priority": 451, "external_id": null}, {"id": 10846274003, "name": "Everglades University", "priority": 452, "external_id": null}, {"id": 10846275003, "name": "Evergreen State College", "priority": 453, "external_id": null}, {"id": 10846276003, "name": "Excelsior College", "priority": 454, "external_id": null}, {"id": 10846277003, "name": "Fairfield University", "priority": 455, "external_id": null}, {"id": 10846278003, "name": "Fairleigh Dickinson University", "priority": 456, "external_id": null}, {"id": 10846279003, "name": "Fairmont State University", "priority": 457, "external_id": null}, {"id": 10846280003, "name": "Faith Baptist Bible College and Theological Seminary", "priority": 458, "external_id": null}, {"id": 10846281003, "name": "Farmingdale State College - SUNY", "priority": 459, "external_id": null}, {"id": 10846282003, "name": "Fashion Institute of Technology", "priority": 460, "external_id": null}, {"id": 10846283003, "name": "Faulkner University", "priority": 461, "external_id": null}, {"id": 10846284003, "name": "Fayetteville State University", "priority": 462, "external_id": null}, {"id": 10846285003, "name": "Felician College", "priority": 463, "external_id": null}, {"id": 10846286003, "name": "Ferris State University", "priority": 464, "external_id": null}, {"id": 10846287003, "name": "Ferrum College", "priority": 465, "external_id": null}, {"id": 10846288003, "name": "Finlandia University", "priority": 466, "external_id": null}, {"id": 10846289003, "name": "Fisher College", "priority": 467, "external_id": null}, {"id": 10846290003, "name": "Fisk University", "priority": 468, "external_id": null}, {"id": 10846291003, "name": "Fitchburg State University", "priority": 469, "external_id": null}, {"id": 10846292003, "name": "Five Towns College", "priority": 470, "external_id": null}, {"id": 10846293003, "name": "Flagler College", "priority": 471, "external_id": null}, {"id": 10846294003, "name": "Florida Christian College", "priority": 472, "external_id": null}, {"id": 10846295003, "name": "Florida College", "priority": 473, "external_id": null}, {"id": 10846296003, "name": "Florida Gulf Coast University", "priority": 474, "external_id": null}, {"id": 10846297003, "name": "Florida Institute of Technology", "priority": 475, "external_id": null}, {"id": 10846298003, "name": "Florida Memorial University", "priority": 476, "external_id": null}, {"id": 10846299003, "name": "Florida Southern College", "priority": 477, "external_id": null}, {"id": 10846300003, "name": "Florida State College - Jacksonville", "priority": 478, "external_id": null}, {"id": 10846301003, "name": "Fontbonne University", "priority": 479, "external_id": null}, {"id": 10846302003, "name": "Fort Hays State University", "priority": 480, "external_id": null}, {"id": 10846303003, "name": "Fort Lewis College", "priority": 481, "external_id": null}, {"id": 10846304003, "name": "Fort Valley State University", "priority": 482, "external_id": null}, {"id": 10846305003, "name": "Framingham State University", "priority": 483, "external_id": null}, {"id": 10846306003, "name": "Francis Marion University", "priority": 484, "external_id": null}, {"id": 10846307003, "name": "Franciscan University of Steubenville", "priority": 485, "external_id": null}, {"id": 10846308003, "name": "Frank Lloyd Wright School of Architecture", "priority": 486, "external_id": null}, {"id": 10846309003, "name": "Franklin and Marshall College", "priority": 487, "external_id": null}, {"id": 10846310003, "name": "Franklin College", "priority": 488, "external_id": null}, {"id": 10846311003, "name": "Franklin College Switzerland", "priority": 489, "external_id": null}, {"id": 10846312003, "name": "Franklin Pierce University", "priority": 490, "external_id": null}, {"id": 10846313003, "name": "Franklin University", "priority": 491, "external_id": null}, {"id": 10846314003, "name": "Franklin W. Olin College of Engineering", "priority": 492, "external_id": null}, {"id": 10846315003, "name": "Freed-Hardeman University", "priority": 493, "external_id": null}, {"id": 10846316003, "name": "Fresno Pacific University", "priority": 494, "external_id": null}, {"id": 10846317003, "name": "Friends University", "priority": 495, "external_id": null}, {"id": 10846318003, "name": "Frostburg State University", "priority": 496, "external_id": null}, {"id": 10846319003, "name": "Gallaudet University", "priority": 497, "external_id": null}, {"id": 10846320003, "name": "Gannon University", "priority": 498, "external_id": null}, {"id": 10846321003, "name": "Geneva College", "priority": 499, "external_id": null}, {"id": 10846322003, "name": "George Fox University", "priority": 500, "external_id": null}, {"id": 10846323003, "name": "George Mason University", "priority": 501, "external_id": null}, {"id": 10846324003, "name": "George Washington University", "priority": 502, "external_id": null}, {"id": 10846325003, "name": "Georgetown College", "priority": 503, "external_id": null}, {"id": 10846326003, "name": "Georgia College & State University", "priority": 504, "external_id": null}, {"id": 10846327003, "name": "Georgia Gwinnett College", "priority": 505, "external_id": null}, {"id": 10846328003, "name": "Georgia Regents University", "priority": 506, "external_id": null}, {"id": 10846329003, "name": "Georgia Southwestern State University", "priority": 507, "external_id": null}, {"id": 10846330003, "name": "Georgian Court University", "priority": 508, "external_id": null}, {"id": 10846331003, "name": "Gettysburg College", "priority": 509, "external_id": null}, {"id": 10846332003, "name": "Glenville State College", "priority": 510, "external_id": null}, {"id": 10846333003, "name": "God's Bible School and College", "priority": 511, "external_id": null}, {"id": 10846334003, "name": "Goddard College", "priority": 512, "external_id": null}, {"id": 10846335003, "name": "Golden Gate University", "priority": 513, "external_id": null}, {"id": 10846336003, "name": "Goldey-Beacom College", "priority": 514, "external_id": null}, {"id": 10846337003, "name": "Goldfarb School of Nursing at Barnes-Jewish College", "priority": 515, "external_id": null}, {"id": 10846338003, "name": "Gonzaga University", "priority": 516, "external_id": null}, {"id": 10846339003, "name": "Gordon College", "priority": 517, "external_id": null}, {"id": 10846340003, "name": "Fordham University", "priority": 518, "external_id": null}, {"id": 10846341003, "name": "Georgia Institute of Technology", "priority": 519, "external_id": null}, {"id": 10846342003, "name": "Gardner-Webb University", "priority": 520, "external_id": null}, {"id": 10846343003, "name": "Georgia Southern University", "priority": 521, "external_id": null}, {"id": 10846344003, "name": "Georgia State University", "priority": 522, "external_id": null}, {"id": 10846345003, "name": "Florida State University", "priority": 523, "external_id": null}, {"id": 10846346003, "name": "Dartmouth College", "priority": 524, "external_id": null}, {"id": 10846347003, "name": "Florida International University", "priority": 525, "external_id": null}, {"id": 10846348003, "name": "Georgetown University", "priority": 526, "external_id": null}, {"id": 10846349003, "name": "Furman University", "priority": 527, "external_id": null}, {"id": 10846350003, "name": "Gordon State College", "priority": 528, "external_id": null}, {"id": 10846351003, "name": "Goshen College", "priority": 529, "external_id": null}, {"id": 10846352003, "name": "Goucher College", "priority": 530, "external_id": null}, {"id": 10846353003, "name": "Governors State University", "priority": 531, "external_id": null}, {"id": 10846354003, "name": "Grace Bible College", "priority": 532, "external_id": null}, {"id": 10846355003, "name": "Grace College and Seminary", "priority": 533, "external_id": null}, {"id": 10846356003, "name": "Grace University", "priority": 534, "external_id": null}, {"id": 10846357003, "name": "Graceland University", "priority": 535, "external_id": null}, {"id": 10846358003, "name": "Grand Canyon University", "priority": 536, "external_id": null}, {"id": 10846359003, "name": "Grand Valley State University", "priority": 537, "external_id": null}, {"id": 10846360003, "name": "Grand View University", "priority": 538, "external_id": null}, {"id": 10846361003, "name": "Granite State College", "priority": 539, "external_id": null}, {"id": 10846362003, "name": "Gratz College", "priority": 540, "external_id": null}, {"id": 10846363003, "name": "Great Basin College", "priority": 541, "external_id": null}, {"id": 10846364003, "name": "Great Lakes Christian College", "priority": 542, "external_id": null}, {"id": 10846365003, "name": "Green Mountain College", "priority": 543, "external_id": null}, {"id": 10846366003, "name": "Greensboro College", "priority": 544, "external_id": null}, {"id": 10846367003, "name": "Greenville College", "priority": 545, "external_id": null}, {"id": 10846368003, "name": "Grinnell College", "priority": 546, "external_id": null}, {"id": 10846369003, "name": "Grove City College", "priority": 547, "external_id": null}, {"id": 10846370003, "name": "Guilford College", "priority": 548, "external_id": null}, {"id": 10846371003, "name": "Gustavus Adolphus College", "priority": 549, "external_id": null}, {"id": 10846372003, "name": "Gwynedd-Mercy College", "priority": 550, "external_id": null}, {"id": 10846373003, "name": "Hamilton College", "priority": 551, "external_id": null}, {"id": 10846374003, "name": "Hamline University", "priority": 552, "external_id": null}, {"id": 10846375003, "name": "Hampden-Sydney College", "priority": 553, "external_id": null}, {"id": 10846376003, "name": "Hampshire College", "priority": 554, "external_id": null}, {"id": 10846377003, "name": "Hannibal-LaGrange University", "priority": 555, "external_id": null}, {"id": 10846378003, "name": "Hanover College", "priority": 556, "external_id": null}, {"id": 10846379003, "name": "Hardin-Simmons University", "priority": 557, "external_id": null}, {"id": 10846380003, "name": "Harding University", "priority": 558, "external_id": null}, {"id": 10846381003, "name": "Harrington College of Design", "priority": 559, "external_id": null}, {"id": 10846382003, "name": "Harris-Stowe State University", "priority": 560, "external_id": null}, {"id": 10846383003, "name": "Harrisburg University of Science and Technology", "priority": 561, "external_id": null}, {"id": 10846384003, "name": "Hartwick College", "priority": 562, "external_id": null}, {"id": 10846385003, "name": "Harvey Mudd College", "priority": 563, "external_id": null}, {"id": 10846386003, "name": "Haskell Indian Nations University", "priority": 564, "external_id": null}, {"id": 10846387003, "name": "Hastings College", "priority": 565, "external_id": null}, {"id": 10846388003, "name": "Haverford College", "priority": 566, "external_id": null}, {"id": 10846389003, "name": "Hawaii Pacific University", "priority": 567, "external_id": null}, {"id": 10846390003, "name": "Hebrew Theological College", "priority": 568, "external_id": null}, {"id": 10846391003, "name": "Heidelberg University", "priority": 569, "external_id": null}, {"id": 10846392003, "name": "Hellenic College", "priority": 570, "external_id": null}, {"id": 10846393003, "name": "Henderson State University", "priority": 571, "external_id": null}, {"id": 10846394003, "name": "Hendrix College", "priority": 572, "external_id": null}, {"id": 10846395003, "name": "Heritage University", "priority": 573, "external_id": null}, {"id": 10846396003, "name": "Herzing University", "priority": 574, "external_id": null}, {"id": 10846397003, "name": "Hesser College", "priority": 575, "external_id": null}, {"id": 10846398003, "name": "High Point University", "priority": 576, "external_id": null}, {"id": 10846399003, "name": "Hilbert College", "priority": 577, "external_id": null}, {"id": 10846400003, "name": "Hillsdale College", "priority": 578, "external_id": null}, {"id": 10846401003, "name": "Hiram College", "priority": 579, "external_id": null}, {"id": 10846402003, "name": "Hobart and William Smith Colleges", "priority": 580, "external_id": null}, {"id": 10846403003, "name": "Hodges University", "priority": 581, "external_id": null}, {"id": 10846404003, "name": "Hofstra University", "priority": 582, "external_id": null}, {"id": 10846405003, "name": "Hollins University", "priority": 583, "external_id": null}, {"id": 10846406003, "name": "Holy Apostles College and Seminary", "priority": 584, "external_id": null}, {"id": 10846407003, "name": "Indiana State University", "priority": 585, "external_id": null}, {"id": 10846408003, "name": "Holy Family University", "priority": 586, "external_id": null}, {"id": 10846409003, "name": "Holy Names University", "priority": 587, "external_id": null}, {"id": 10846410003, "name": "Hood College", "priority": 588, "external_id": null}, {"id": 10846411003, "name": "Hope College", "priority": 589, "external_id": null}, {"id": 10846412003, "name": "Hope International University", "priority": 590, "external_id": null}, {"id": 10846413003, "name": "Houghton College", "priority": 591, "external_id": null}, {"id": 10846414003, "name": "Howard Payne University", "priority": 592, "external_id": null}, {"id": 10846415003, "name": "Hult International Business School", "priority": 593, "external_id": null}, {"id": 10846416003, "name": "Humboldt State University", "priority": 594, "external_id": null}, {"id": 10846417003, "name": "Humphreys College", "priority": 595, "external_id": null}, {"id": 10846418003, "name": "Huntingdon College", "priority": 596, "external_id": null}, {"id": 10846419003, "name": "Huntington University", "priority": 597, "external_id": null}, {"id": 10846420003, "name": "Husson University", "priority": 598, "external_id": null}, {"id": 10846421003, "name": "Huston-Tillotson University", "priority": 599, "external_id": null}, {"id": 10846422003, "name": "Illinois College", "priority": 600, "external_id": null}, {"id": 10846423003, "name": "Illinois Institute of Art at Chicago", "priority": 601, "external_id": null}, {"id": 10846424003, "name": "Illinois Institute of Technology", "priority": 602, "external_id": null}, {"id": 10846425003, "name": "Illinois Wesleyan University", "priority": 603, "external_id": null}, {"id": 10846426003, "name": "Immaculata University", "priority": 604, "external_id": null}, {"id": 10846427003, "name": "Indian River State College", "priority": 605, "external_id": null}, {"id": 10846428003, "name": "Indiana Institute of Technology", "priority": 606, "external_id": null}, {"id": 10846429003, "name": "Indiana University East", "priority": 607, "external_id": null}, {"id": 10846430003, "name": "Indiana University Northwest", "priority": 608, "external_id": null}, {"id": 10846431003, "name": "Indiana University of Pennsylvania", "priority": 609, "external_id": null}, {"id": 10846432003, "name": "Indiana University Southeast", "priority": 610, "external_id": null}, {"id": 10846433003, "name": "Illinois State University", "priority": 611, "external_id": null}, {"id": 10846434003, "name": "Indiana University - Bloomington", "priority": 612, "external_id": null}, {"id": 10846435003, "name": "Davidson College", "priority": 613, "external_id": null}, {"id": 10846436003, "name": "Idaho State University", "priority": 614, "external_id": null}, {"id": 10846437003, "name": "Harvard University", "priority": 615, "external_id": null}, {"id": 10846438003, "name": "Howard University", "priority": 616, "external_id": null}, {"id": 10846439003, "name": "Houston Baptist University", "priority": 617, "external_id": null}, {"id": 10846440003, "name": "Indiana University - Kokomo", "priority": 618, "external_id": null}, {"id": 10846441003, "name": "Indiana University - South Bend", "priority": 619, "external_id": null}, {"id": 10846442003, "name": "Indiana University-Purdue University - Fort Wayne", "priority": 620, "external_id": null}, {"id": 10846443003, "name": "Indiana University-Purdue University - Indianapolis", "priority": 621, "external_id": null}, {"id": 10846444003, "name": "Indiana Wesleyan University", "priority": 622, "external_id": null}, {"id": 10846445003, "name": "Institute of American Indian and Alaska Native Culture and Arts Development", "priority": 623, "external_id": null}, {"id": 10846446003, "name": "Inter American University of Puerto Rico - Aguadilla", "priority": 624, "external_id": null}, {"id": 10846447003, "name": "Inter American University of Puerto Rico - Arecibo", "priority": 625, "external_id": null}, {"id": 10846448003, "name": "Inter American University of Puerto Rico - Barranquitas", "priority": 626, "external_id": null}, {"id": 10846449003, "name": "Inter American University of Puerto Rico - Bayamon", "priority": 627, "external_id": null}, {"id": 10846450003, "name": "Inter American University of Puerto Rico - Fajardo", "priority": 628, "external_id": null}, {"id": 10846451003, "name": "Inter American University of Puerto Rico - Guayama", "priority": 629, "external_id": null}, {"id": 10846452003, "name": "Inter American University of Puerto Rico - Metropolitan Campus", "priority": 630, "external_id": null}, {"id": 10846453003, "name": "Inter American University of Puerto Rico - Ponce", "priority": 631, "external_id": null}, {"id": 10846454003, "name": "Inter American University of Puerto Rico - San German", "priority": 632, "external_id": null}, {"id": 10846455003, "name": "International College of the Cayman Islands", "priority": 633, "external_id": null}, {"id": 10846456003, "name": "Iona College", "priority": 634, "external_id": null}, {"id": 10846457003, "name": "Iowa Wesleyan College", "priority": 635, "external_id": null}, {"id": 10846458003, "name": "Ithaca College", "priority": 636, "external_id": null}, {"id": 10846459003, "name": "Jarvis Christian College", "priority": 637, "external_id": null}, {"id": 10846460003, "name": "Jewish Theological Seminary of America", "priority": 638, "external_id": null}, {"id": 10846461003, "name": "John Brown University", "priority": 639, "external_id": null}, {"id": 10846462003, "name": "John Carroll University", "priority": 640, "external_id": null}, {"id": 10846463003, "name": "John F. Kennedy University", "priority": 641, "external_id": null}, {"id": 10846464003, "name": "Johns Hopkins University", "priority": 642, "external_id": null}, {"id": 10846465003, "name": "Johnson & Wales University", "priority": 643, "external_id": null}, {"id": 10846466003, "name": "Johnson C. Smith University", "priority": 644, "external_id": null}, {"id": 10846467003, "name": "Johnson State College", "priority": 645, "external_id": null}, {"id": 10846468003, "name": "Johnson University", "priority": 646, "external_id": null}, {"id": 10846469003, "name": "Jones International University", "priority": 647, "external_id": null}, {"id": 10846470003, "name": "Judson College", "priority": 648, "external_id": null}, {"id": 10846471003, "name": "Judson University", "priority": 649, "external_id": null}, {"id": 10846472003, "name": "Juilliard School", "priority": 650, "external_id": null}, {"id": 10846473003, "name": "Juniata College", "priority": 651, "external_id": null}, {"id": 10846474003, "name": "Kalamazoo College", "priority": 652, "external_id": null}, {"id": 10846475003, "name": "Kansas City Art Institute", "priority": 653, "external_id": null}, {"id": 10846476003, "name": "Kansas Wesleyan University", "priority": 654, "external_id": null}, {"id": 10846477003, "name": "Kaplan University", "priority": 655, "external_id": null}, {"id": 10846478003, "name": "Kean University", "priority": 656, "external_id": null}, {"id": 10846479003, "name": "Keene State College", "priority": 657, "external_id": null}, {"id": 10846480003, "name": "Keiser University", "priority": 658, "external_id": null}, {"id": 10846481003, "name": "Kendall College", "priority": 659, "external_id": null}, {"id": 10846482003, "name": "Kennesaw State University", "priority": 660, "external_id": null}, {"id": 10846483003, "name": "Kentucky Christian University", "priority": 661, "external_id": null}, {"id": 10846484003, "name": "Kentucky State University", "priority": 662, "external_id": null}, {"id": 10846485003, "name": "Kentucky Wesleyan College", "priority": 663, "external_id": null}, {"id": 10846486003, "name": "Kenyon College", "priority": 664, "external_id": null}, {"id": 10846487003, "name": "Kettering College", "priority": 665, "external_id": null}, {"id": 10846488003, "name": "Kettering University", "priority": 666, "external_id": null}, {"id": 10846489003, "name": "Keuka College", "priority": 667, "external_id": null}, {"id": 10846490003, "name": "Keystone College", "priority": 668, "external_id": null}, {"id": 10846491003, "name": "King University", "priority": 669, "external_id": null}, {"id": 10846492003, "name": "King's College", "priority": 670, "external_id": null}, {"id": 10846493003, "name": "Knox College", "priority": 671, "external_id": null}, {"id": 10846494003, "name": "Kutztown University of Pennsylvania", "priority": 672, "external_id": null}, {"id": 10846495003, "name": "Kuyper College", "priority": 673, "external_id": null}, {"id": 10846496003, "name": "La Roche College", "priority": 674, "external_id": null}, {"id": 10846497003, "name": "La Salle University", "priority": 675, "external_id": null}, {"id": 10846498003, "name": "La Sierra University", "priority": 676, "external_id": null}, {"id": 10846499003, "name": "LaGrange College", "priority": 677, "external_id": null}, {"id": 10846500003, "name": "Laguna College of Art and Design", "priority": 678, "external_id": null}, {"id": 10846501003, "name": "Lake Erie College", "priority": 679, "external_id": null}, {"id": 10846502003, "name": "Lake Forest College", "priority": 680, "external_id": null}, {"id": 10846503003, "name": "Lake Superior State University", "priority": 681, "external_id": null}, {"id": 10846504003, "name": "Lakeland College", "priority": 682, "external_id": null}, {"id": 10846505003, "name": "Lakeview College of Nursing", "priority": 683, "external_id": null}, {"id": 10846506003, "name": "Lancaster Bible College", "priority": 684, "external_id": null}, {"id": 10846507003, "name": "Lander University", "priority": 685, "external_id": null}, {"id": 10846508003, "name": "Lane College", "priority": 686, "external_id": null}, {"id": 10846509003, "name": "Langston University", "priority": 687, "external_id": null}, {"id": 10846510003, "name": "Lasell College", "priority": 688, "external_id": null}, {"id": 10846511003, "name": "Lawrence Technological University", "priority": 689, "external_id": null}, {"id": 10846512003, "name": "Lawrence University", "priority": 690, "external_id": null}, {"id": 10846513003, "name": "Le Moyne College", "priority": 691, "external_id": null}, {"id": 10846514003, "name": "Lebanon Valley College", "priority": 692, "external_id": null}, {"id": 10846515003, "name": "Lee University", "priority": 693, "external_id": null}, {"id": 10846516003, "name": "Lees-McRae College", "priority": 694, "external_id": null}, {"id": 10846517003, "name": "Kansas State University", "priority": 695, "external_id": null}, {"id": 10846518003, "name": "James Madison University", "priority": 696, "external_id": null}, {"id": 10846519003, "name": "Lafayette College", "priority": 697, "external_id": null}, {"id": 10846520003, "name": "Jacksonville University", "priority": 698, "external_id": null}, {"id": 10846521003, "name": "Kent State University", "priority": 699, "external_id": null}, {"id": 10846522003, "name": "Lamar University", "priority": 700, "external_id": null}, {"id": 10846523003, "name": "Jackson State University", "priority": 701, "external_id": null}, {"id": 10846524003, "name": "Lehigh University", "priority": 702, "external_id": null}, {"id": 10846525003, "name": "Jacksonville State University", "priority": 703, "external_id": null}, {"id": 10846526003, "name": "LeMoyne-Owen College", "priority": 704, "external_id": null}, {"id": 10846527003, "name": "Lenoir-Rhyne University", "priority": 705, "external_id": null}, {"id": 10846528003, "name": "Lesley University", "priority": 706, "external_id": null}, {"id": 10846529003, "name": "LeTourneau University", "priority": 707, "external_id": null}, {"id": 10846530003, "name": "Lewis & Clark College", "priority": 708, "external_id": null}, {"id": 10846531003, "name": "Lewis University", "priority": 709, "external_id": null}, {"id": 10846532003, "name": "Lewis-Clark State College", "priority": 710, "external_id": null}, {"id": 10846533003, "name": "Lexington College", "priority": 711, "external_id": null}, {"id": 10846534003, "name": "Life Pacific College", "priority": 712, "external_id": null}, {"id": 10846535003, "name": "Life University", "priority": 713, "external_id": null}, {"id": 10846536003, "name": "LIM College", "priority": 714, "external_id": null}, {"id": 10846537003, "name": "Limestone College", "priority": 715, "external_id": null}, {"id": 10846538003, "name": "Lincoln Christian University", "priority": 716, "external_id": null}, {"id": 10846539003, "name": "Lincoln College", "priority": 717, "external_id": null}, {"id": 10846540003, "name": "Lincoln Memorial University", "priority": 718, "external_id": null}, {"id": 10846541003, "name": "Lincoln University", "priority": 719, "external_id": null}, {"id": 10846542003, "name": "Lindenwood University", "priority": 720, "external_id": null}, {"id": 10846543003, "name": "Lindsey Wilson College", "priority": 721, "external_id": null}, {"id": 10846544003, "name": "Linfield College", "priority": 722, "external_id": null}, {"id": 10846545003, "name": "Lipscomb University", "priority": 723, "external_id": null}, {"id": 10846546003, "name": "LIU Post", "priority": 724, "external_id": null}, {"id": 10846547003, "name": "Livingstone College", "priority": 725, "external_id": null}, {"id": 10846548003, "name": "Lock Haven University of Pennsylvania", "priority": 726, "external_id": null}, {"id": 10846549003, "name": "Loma Linda University", "priority": 727, "external_id": null}, {"id": 10846550003, "name": "Longwood University", "priority": 728, "external_id": null}, {"id": 10846551003, "name": "Loras College", "priority": 729, "external_id": null}, {"id": 10846552003, "name": "Louisiana College", "priority": 730, "external_id": null}, {"id": 10846553003, "name": "Louisiana State University Health Sciences Center", "priority": 731, "external_id": null}, {"id": 10846554003, "name": "Louisiana State University - Alexandria", "priority": 732, "external_id": null}, {"id": 10846555003, "name": "Louisiana State University - Shreveport", "priority": 733, "external_id": null}, {"id": 10846556003, "name": "Lourdes University", "priority": 734, "external_id": null}, {"id": 10846557003, "name": "Loyola Marymount University", "priority": 735, "external_id": null}, {"id": 10846558003, "name": "Loyola University Chicago", "priority": 736, "external_id": null}, {"id": 10846559003, "name": "Loyola University Maryland", "priority": 737, "external_id": null}, {"id": 10846560003, "name": "Loyola University New Orleans", "priority": 738, "external_id": null}, {"id": 10846561003, "name": "Lubbock Christian University", "priority": 739, "external_id": null}, {"id": 10846562003, "name": "Luther College", "priority": 740, "external_id": null}, {"id": 10846563003, "name": "Lycoming College", "priority": 741, "external_id": null}, {"id": 10846564003, "name": "Lyme Academy College of Fine Arts", "priority": 742, "external_id": null}, {"id": 10846565003, "name": "Lynchburg College", "priority": 743, "external_id": null}, {"id": 10846566003, "name": "Lyndon State College", "priority": 744, "external_id": null}, {"id": 10846567003, "name": "Lynn University", "priority": 745, "external_id": null}, {"id": 10846568003, "name": "Lyon College", "priority": 746, "external_id": null}, {"id": 10846569003, "name": "Macalester College", "priority": 747, "external_id": null}, {"id": 10846570003, "name": "MacMurray College", "priority": 748, "external_id": null}, {"id": 10846571003, "name": "Madonna University", "priority": 749, "external_id": null}, {"id": 10846572003, "name": "Maharishi University of Management", "priority": 750, "external_id": null}, {"id": 10846573003, "name": "Maine College of Art", "priority": 751, "external_id": null}, {"id": 10846574003, "name": "Maine Maritime Academy", "priority": 752, "external_id": null}, {"id": 10846575003, "name": "Malone University", "priority": 753, "external_id": null}, {"id": 10846576003, "name": "Manchester University", "priority": 754, "external_id": null}, {"id": 10846577003, "name": "Manhattan Christian College", "priority": 755, "external_id": null}, {"id": 10846578003, "name": "Manhattan College", "priority": 756, "external_id": null}, {"id": 10846579003, "name": "Manhattan School of Music", "priority": 757, "external_id": null}, {"id": 10846580003, "name": "Manhattanville College", "priority": 758, "external_id": null}, {"id": 10846581003, "name": "Mansfield University of Pennsylvania", "priority": 759, "external_id": null}, {"id": 10846582003, "name": "Maranatha Baptist Bible College", "priority": 760, "external_id": null}, {"id": 10846583003, "name": "Marian University", "priority": 761, "external_id": null}, {"id": 10846584003, "name": "Marietta College", "priority": 762, "external_id": null}, {"id": 10846585003, "name": "Marlboro College", "priority": 763, "external_id": null}, {"id": 10846586003, "name": "Marquette University", "priority": 764, "external_id": null}, {"id": 10846587003, "name": "Mars Hill University", "priority": 765, "external_id": null}, {"id": 10846588003, "name": "Martin Luther College", "priority": 766, "external_id": null}, {"id": 10846589003, "name": "Martin Methodist College", "priority": 767, "external_id": null}, {"id": 10846590003, "name": "Martin University", "priority": 768, "external_id": null}, {"id": 10846591003, "name": "Mary Baldwin College", "priority": 769, "external_id": null}, {"id": 10846592003, "name": "Marygrove College", "priority": 770, "external_id": null}, {"id": 10846593003, "name": "Maryland Institute College of Art", "priority": 771, "external_id": null}, {"id": 10846594003, "name": "Marylhurst University", "priority": 772, "external_id": null}, {"id": 10846595003, "name": "Marymount Manhattan College", "priority": 773, "external_id": null}, {"id": 10846596003, "name": "Marymount University", "priority": 774, "external_id": null}, {"id": 10846597003, "name": "Maryville College", "priority": 775, "external_id": null}, {"id": 10846598003, "name": "Maryville University of St. Louis", "priority": 776, "external_id": null}, {"id": 10846599003, "name": "Marywood University", "priority": 777, "external_id": null}, {"id": 10846600003, "name": "Massachusetts College of Art and Design", "priority": 778, "external_id": null}, {"id": 10846601003, "name": "Massachusetts College of Liberal Arts", "priority": 779, "external_id": null}, {"id": 10846602003, "name": "Massachusetts College of Pharmacy and Health Sciences", "priority": 780, "external_id": null}, {"id": 10846603003, "name": "Massachusetts Institute of Technology", "priority": 781, "external_id": null}, {"id": 10846604003, "name": "Massachusetts Maritime Academy", "priority": 782, "external_id": null}, {"id": 10846605003, "name": "Master's College and Seminary", "priority": 783, "external_id": null}, {"id": 10846606003, "name": "Mayville State University", "priority": 784, "external_id": null}, {"id": 10846607003, "name": "McDaniel College", "priority": 785, "external_id": null}, {"id": 10846608003, "name": "McGill University", "priority": 786, "external_id": null}, {"id": 10846609003, "name": "McKendree University", "priority": 787, "external_id": null}, {"id": 10846610003, "name": "McMurry University", "priority": 788, "external_id": null}, {"id": 10846611003, "name": "McPherson College", "priority": 789, "external_id": null}, {"id": 10846612003, "name": "Medaille College", "priority": 790, "external_id": null}, {"id": 10846613003, "name": "Marist College", "priority": 791, "external_id": null}, {"id": 10846614003, "name": "McNeese State University", "priority": 792, "external_id": null}, {"id": 10846615003, "name": "Louisiana Tech University", "priority": 793, "external_id": null}, {"id": 10846616003, "name": "Marshall University", "priority": 794, "external_id": null}, {"id": 10846617003, "name": "Medical University of South Carolina", "priority": 795, "external_id": null}, {"id": 10846618003, "name": "Memorial University of Newfoundland", "priority": 796, "external_id": null}, {"id": 10846619003, "name": "Memphis College of Art", "priority": 797, "external_id": null}, {"id": 10846620003, "name": "Menlo College", "priority": 798, "external_id": null}, {"id": 10846621003, "name": "Mercy College", "priority": 799, "external_id": null}, {"id": 10846622003, "name": "Mercy College of Health Sciences", "priority": 800, "external_id": null}, {"id": 10846623003, "name": "Mercy College of Ohio", "priority": 801, "external_id": null}, {"id": 10846624003, "name": "Mercyhurst University", "priority": 802, "external_id": null}, {"id": 10846625003, "name": "Meredith College", "priority": 803, "external_id": null}, {"id": 10846626003, "name": "Merrimack College", "priority": 804, "external_id": null}, {"id": 10846627003, "name": "Messiah College", "priority": 805, "external_id": null}, {"id": 10846628003, "name": "Methodist University", "priority": 806, "external_id": null}, {"id": 10846629003, "name": "Metropolitan College of New York", "priority": 807, "external_id": null}, {"id": 10846630003, "name": "Metropolitan State University", "priority": 808, "external_id": null}, {"id": 10846631003, "name": "Metropolitan State University of Denver", "priority": 809, "external_id": null}, {"id": 10846632003, "name": "Miami Dade College", "priority": 810, "external_id": null}, {"id": 10846633003, "name": "Miami International University of Art & Design", "priority": 811, "external_id": null}, {"id": 10846634003, "name": "Michigan Technological University", "priority": 812, "external_id": null}, {"id": 10846635003, "name": "Mid-America Christian University", "priority": 813, "external_id": null}, {"id": 10846636003, "name": "Mid-Atlantic Christian University", "priority": 814, "external_id": null}, {"id": 10846637003, "name": "Mid-Continent University", "priority": 815, "external_id": null}, {"id": 10846638003, "name": "MidAmerica Nazarene University", "priority": 816, "external_id": null}, {"id": 10846639003, "name": "Middle Georgia State College", "priority": 817, "external_id": null}, {"id": 10846640003, "name": "Middlebury College", "priority": 818, "external_id": null}, {"id": 10846641003, "name": "Midland College", "priority": 819, "external_id": null}, {"id": 10846642003, "name": "Midland University", "priority": 820, "external_id": null}, {"id": 10846643003, "name": "Midstate College", "priority": 821, "external_id": null}, {"id": 10846644003, "name": "Midway College", "priority": 822, "external_id": null}, {"id": 10846645003, "name": "Midwestern State University", "priority": 823, "external_id": null}, {"id": 10846646003, "name": "Miles College", "priority": 824, "external_id": null}, {"id": 10846647003, "name": "Millersville University of Pennsylvania", "priority": 825, "external_id": null}, {"id": 10846648003, "name": "Milligan College", "priority": 826, "external_id": null}, {"id": 10846649003, "name": "Millikin University", "priority": 827, "external_id": null}, {"id": 10846650003, "name": "Mills College", "priority": 828, "external_id": null}, {"id": 10846651003, "name": "Millsaps College", "priority": 829, "external_id": null}, {"id": 10846652003, "name": "Milwaukee Institute of Art and Design", "priority": 830, "external_id": null}, {"id": 10846653003, "name": "Milwaukee School of Engineering", "priority": 831, "external_id": null}, {"id": 10846654003, "name": "Minneapolis College of Art and Design", "priority": 832, "external_id": null}, {"id": 10846655003, "name": "Minnesota State University - Mankato", "priority": 833, "external_id": null}, {"id": 10846656003, "name": "Minnesota State University - Moorhead", "priority": 834, "external_id": null}, {"id": 10846657003, "name": "Minot State University", "priority": 835, "external_id": null}, {"id": 10846658003, "name": "Misericordia University", "priority": 836, "external_id": null}, {"id": 10846659003, "name": "Mississippi College", "priority": 837, "external_id": null}, {"id": 10846660003, "name": "Mississippi University for Women", "priority": 838, "external_id": null}, {"id": 10846661003, "name": "Missouri Baptist University", "priority": 839, "external_id": null}, {"id": 10846662003, "name": "Missouri Southern State University", "priority": 840, "external_id": null}, {"id": 10846663003, "name": "Missouri University of Science & Technology", "priority": 841, "external_id": null}, {"id": 10846664003, "name": "Missouri Valley College", "priority": 842, "external_id": null}, {"id": 10846665003, "name": "Missouri Western State University", "priority": 843, "external_id": null}, {"id": 10846666003, "name": "Mitchell College", "priority": 844, "external_id": null}, {"id": 10846667003, "name": "Molloy College", "priority": 845, "external_id": null}, {"id": 10846668003, "name": "Monmouth College", "priority": 846, "external_id": null}, {"id": 10846669003, "name": "Monroe College", "priority": 847, "external_id": null}, {"id": 10846670003, "name": "Montana State University - Billings", "priority": 848, "external_id": null}, {"id": 10846671003, "name": "Montana State University - Northern", "priority": 849, "external_id": null}, {"id": 10846672003, "name": "Montana Tech of the University of Montana", "priority": 850, "external_id": null}, {"id": 10846673003, "name": "Montclair State University", "priority": 851, "external_id": null}, {"id": 10846674003, "name": "Monterrey Institute of Technology and Higher Education - Monterrey", "priority": 852, "external_id": null}, {"id": 10846675003, "name": "Montreat College", "priority": 853, "external_id": null}, {"id": 10846676003, "name": "Montserrat College of Art", "priority": 854, "external_id": null}, {"id": 10846677003, "name": "Moody Bible Institute", "priority": 855, "external_id": null}, {"id": 10846678003, "name": "Moore College of Art & Design", "priority": 856, "external_id": null}, {"id": 10846679003, "name": "Moravian College", "priority": 857, "external_id": null}, {"id": 10846680003, "name": "Morehouse College", "priority": 858, "external_id": null}, {"id": 10846681003, "name": "Morningside College", "priority": 859, "external_id": null}, {"id": 10846682003, "name": "Morris College", "priority": 860, "external_id": null}, {"id": 10846683003, "name": "Morrisville State College", "priority": 861, "external_id": null}, {"id": 10846684003, "name": "Mount Aloysius College", "priority": 862, "external_id": null}, {"id": 10846685003, "name": "Mount Angel Seminary", "priority": 863, "external_id": null}, {"id": 10846686003, "name": "Mount Carmel College of Nursing", "priority": 864, "external_id": null}, {"id": 10846687003, "name": "Mount Holyoke College", "priority": 865, "external_id": null}, {"id": 10846688003, "name": "Mount Ida College", "priority": 866, "external_id": null}, {"id": 10846689003, "name": "Mount Marty College", "priority": 867, "external_id": null}, {"id": 10846690003, "name": "Mount Mary University", "priority": 868, "external_id": null}, {"id": 10846691003, "name": "Mount Mercy University", "priority": 869, "external_id": null}, {"id": 10846692003, "name": "Mount Olive College", "priority": 870, "external_id": null}, {"id": 10846693003, "name": "Mississippi State University", "priority": 871, "external_id": null}, {"id": 10846694003, "name": "Montana State University", "priority": 872, "external_id": null}, {"id": 10846695003, "name": "Mississippi Valley State University", "priority": 873, "external_id": null}, {"id": 10846696003, "name": "Monmouth University", "priority": 874, "external_id": null}, {"id": 10846697003, "name": "Morehead State University", "priority": 875, "external_id": null}, {"id": 10846698003, "name": "Miami University - Oxford", "priority": 876, "external_id": null}, {"id": 10846699003, "name": "Morgan State University", "priority": 877, "external_id": null}, {"id": 10846700003, "name": "Missouri State University", "priority": 878, "external_id": null}, {"id": 10846701003, "name": "Michigan State University", "priority": 879, "external_id": null}, {"id": 10846702003, "name": "Mount St. Mary College", "priority": 880, "external_id": null}, {"id": 10846703003, "name": "Mount St. Mary's College", "priority": 881, "external_id": null}, {"id": 10846704003, "name": "Mount St. Mary's University", "priority": 882, "external_id": null}, {"id": 10846705003, "name": "Mount Vernon Nazarene University", "priority": 883, "external_id": null}, {"id": 10846706003, "name": "Muhlenberg College", "priority": 884, "external_id": null}, {"id": 10846707003, "name": "Multnomah University", "priority": 885, "external_id": null}, {"id": 10846708003, "name": "Muskingum University", "priority": 886, "external_id": null}, {"id": 10846709003, "name": "Naropa University", "priority": 887, "external_id": null}, {"id": 10846710003, "name": "National American University", "priority": 888, "external_id": null}, {"id": 10846711003, "name": "National Graduate School of Quality Management", "priority": 889, "external_id": null}, {"id": 10846712003, "name": "National Hispanic University", "priority": 890, "external_id": null}, {"id": 10846713003, "name": "National Labor College", "priority": 891, "external_id": null}, {"id": 10846714003, "name": "National University", "priority": 892, "external_id": null}, {"id": 10846715003, "name": "National-Louis University", "priority": 893, "external_id": null}, {"id": 10846716003, "name": "Nazarene Bible College", "priority": 894, "external_id": null}, {"id": 10846717003, "name": "Nazareth College", "priority": 895, "external_id": null}, {"id": 10846718003, "name": "Nebraska Methodist College", "priority": 896, "external_id": null}, {"id": 10846719003, "name": "Nebraska Wesleyan University", "priority": 897, "external_id": null}, {"id": 10846720003, "name": "Neumann University", "priority": 898, "external_id": null}, {"id": 10846721003, "name": "Nevada State College", "priority": 899, "external_id": null}, {"id": 10846722003, "name": "New College of Florida", "priority": 900, "external_id": null}, {"id": 10846723003, "name": "New England College", "priority": 901, "external_id": null}, {"id": 10846724003, "name": "New England Conservatory of Music", "priority": 902, "external_id": null}, {"id": 10846725003, "name": "New England Institute of Art", "priority": 903, "external_id": null}, {"id": 10846726003, "name": "New England Institute of Technology", "priority": 904, "external_id": null}, {"id": 10846727003, "name": "New Jersey City University", "priority": 905, "external_id": null}, {"id": 10846728003, "name": "New Jersey Institute of Technology", "priority": 906, "external_id": null}, {"id": 10846729003, "name": "New Mexico Highlands University", "priority": 907, "external_id": null}, {"id": 10846730003, "name": "New Mexico Institute of Mining and Technology", "priority": 908, "external_id": null}, {"id": 10846731003, "name": "New Orleans Baptist Theological Seminary", "priority": 909, "external_id": null}, {"id": 10846732003, "name": "New School", "priority": 910, "external_id": null}, {"id": 10846733003, "name": "New York Institute of Technology", "priority": 911, "external_id": null}, {"id": 10846734003, "name": "New York University", "priority": 912, "external_id": null}, {"id": 10846735003, "name": "Newberry College", "priority": 913, "external_id": null}, {"id": 10846736003, "name": "Newbury College", "priority": 914, "external_id": null}, {"id": 10846737003, "name": "Newman University", "priority": 915, "external_id": null}, {"id": 10846738003, "name": "Niagara University", "priority": 916, "external_id": null}, {"id": 10846739003, "name": "Nichols College", "priority": 917, "external_id": null}, {"id": 10846740003, "name": "North Carolina Wesleyan College", "priority": 918, "external_id": null}, {"id": 10846741003, "name": "North Central College", "priority": 919, "external_id": null}, {"id": 10846742003, "name": "North Central University", "priority": 920, "external_id": null}, {"id": 10846743003, "name": "North Greenville University", "priority": 921, "external_id": null}, {"id": 10846744003, "name": "North Park University", "priority": 922, "external_id": null}, {"id": 10846745003, "name": "Northcentral University", "priority": 923, "external_id": null}, {"id": 10846746003, "name": "Northeastern Illinois University", "priority": 924, "external_id": null}, {"id": 10846747003, "name": "Northeastern State University", "priority": 925, "external_id": null}, {"id": 10846748003, "name": "Northeastern University", "priority": 926, "external_id": null}, {"id": 10846749003, "name": "Northern Kentucky University", "priority": 927, "external_id": null}, {"id": 10846750003, "name": "Northern Michigan University", "priority": 928, "external_id": null}, {"id": 10846751003, "name": "Northern New Mexico College", "priority": 929, "external_id": null}, {"id": 10846752003, "name": "Northern State University", "priority": 930, "external_id": null}, {"id": 10846753003, "name": "Northland College", "priority": 931, "external_id": null}, {"id": 10846754003, "name": "Northwest Christian University", "priority": 932, "external_id": null}, {"id": 10846755003, "name": "Northwest Florida State College", "priority": 933, "external_id": null}, {"id": 10846756003, "name": "Northwest Missouri State University", "priority": 934, "external_id": null}, {"id": 10846757003, "name": "Northwest Nazarene University", "priority": 935, "external_id": null}, {"id": 10846758003, "name": "Northwest University", "priority": 936, "external_id": null}, {"id": 10846759003, "name": "Northwestern College", "priority": 937, "external_id": null}, {"id": 10846760003, "name": "Northwestern Health Sciences University", "priority": 938, "external_id": null}, {"id": 10846761003, "name": "Northwestern Oklahoma State University", "priority": 939, "external_id": null}, {"id": 10846762003, "name": "Northwood University", "priority": 940, "external_id": null}, {"id": 10846763003, "name": "Norwich University", "priority": 941, "external_id": null}, {"id": 10846764003, "name": "Notre Dame College of Ohio", "priority": 942, "external_id": null}, {"id": 10846765003, "name": "Notre Dame de Namur University", "priority": 943, "external_id": null}, {"id": 10846766003, "name": "Notre Dame of Maryland University", "priority": 944, "external_id": null}, {"id": 10846767003, "name": "Nova Scotia College of Art and Design", "priority": 945, "external_id": null}, {"id": 10846768003, "name": "Nova Southeastern University", "priority": 946, "external_id": null}, {"id": 10846769003, "name": "Nyack College", "priority": 947, "external_id": null}, {"id": 10846770003, "name": "Oakland City University", "priority": 948, "external_id": null}, {"id": 10846771003, "name": "Oakland University", "priority": 949, "external_id": null}, {"id": 10846772003, "name": "Oakwood University", "priority": 950, "external_id": null}, {"id": 10846773003, "name": "Oberlin College", "priority": 951, "external_id": null}, {"id": 10846774003, "name": "Occidental College", "priority": 952, "external_id": null}, {"id": 10846775003, "name": "Oglala Lakota College", "priority": 953, "external_id": null}, {"id": 10846776003, "name": "North Carolina A&T State University", "priority": 954, "external_id": null}, {"id": 10846777003, "name": "Northern Illinois University", "priority": 955, "external_id": null}, {"id": 10846778003, "name": "North Dakota State University", "priority": 956, "external_id": null}, {"id": 10846779003, "name": "Nicholls State University", "priority": 957, "external_id": null}, {"id": 10846780003, "name": "North Carolina Central University", "priority": 958, "external_id": null}, {"id": 10846781003, "name": "Norfolk State University", "priority": 959, "external_id": null}, {"id": 10846782003, "name": "Northwestern State University of Louisiana", "priority": 960, "external_id": null}, {"id": 10846783003, "name": "Northern Arizona University", "priority": 961, "external_id": null}, {"id": 10846784003, "name": "North Carolina State University - Raleigh", "priority": 962, "external_id": null}, {"id": 10846785003, "name": "Northwestern University", "priority": 963, "external_id": null}, {"id": 10846786003, "name": "Oglethorpe University", "priority": 964, "external_id": null}, {"id": 10846787003, "name": "Ohio Christian University", "priority": 965, "external_id": null}, {"id": 10846788003, "name": "Ohio Dominican University", "priority": 966, "external_id": null}, {"id": 10846789003, "name": "Ohio Northern University", "priority": 967, "external_id": null}, {"id": 10846790003, "name": "Ohio Valley University", "priority": 968, "external_id": null}, {"id": 10846791003, "name": "Ohio Wesleyan University", "priority": 969, "external_id": null}, {"id": 10846792003, "name": "Oklahoma Baptist University", "priority": 970, "external_id": null}, {"id": 10846793003, "name": "Oklahoma Christian University", "priority": 971, "external_id": null}, {"id": 10846794003, "name": "Oklahoma City University", "priority": 972, "external_id": null}, {"id": 10846795003, "name": "Oklahoma Panhandle State University", "priority": 973, "external_id": null}, {"id": 10846796003, "name": "Oklahoma State University Institute of Technology - Okmulgee", "priority": 974, "external_id": null}, {"id": 10846797003, "name": "Oklahoma State University - Oklahoma City", "priority": 975, "external_id": null}, {"id": 10846798003, "name": "Oklahoma Wesleyan University", "priority": 976, "external_id": null}, {"id": 10846799003, "name": "Olivet College", "priority": 977, "external_id": null}, {"id": 10846800003, "name": "Olivet Nazarene University", "priority": 978, "external_id": null}, {"id": 10846801003, "name": "Olympic College", "priority": 979, "external_id": null}, {"id": 10846802003, "name": "Oral Roberts University", "priority": 980, "external_id": null}, {"id": 10846803003, "name": "Oregon College of Art and Craft", "priority": 981, "external_id": null}, {"id": 10846804003, "name": "Oregon Health and Science University", "priority": 982, "external_id": null}, {"id": 10846805003, "name": "Oregon Institute of Technology", "priority": 983, "external_id": null}, {"id": 10846806003, "name": "Otis College of Art and Design", "priority": 984, "external_id": null}, {"id": 10846807003, "name": "Ottawa University", "priority": 985, "external_id": null}, {"id": 10846808003, "name": "Otterbein University", "priority": 986, "external_id": null}, {"id": 10846809003, "name": "Ouachita Baptist University", "priority": 987, "external_id": null}, {"id": 10846810003, "name": "Our Lady of Holy Cross College", "priority": 988, "external_id": null}, {"id": 10846811003, "name": "Our Lady of the Lake College", "priority": 989, "external_id": null}, {"id": 10846812003, "name": "Our Lady of the Lake University", "priority": 990, "external_id": null}, {"id": 10846813003, "name": "Pace University", "priority": 991, "external_id": null}, {"id": 10846814003, "name": "Pacific Lutheran University", "priority": 992, "external_id": null}, {"id": 10846815003, "name": "Pacific Northwest College of Art", "priority": 993, "external_id": null}, {"id": 10846816003, "name": "Pacific Oaks College", "priority": 994, "external_id": null}, {"id": 10846817003, "name": "Pacific Union College", "priority": 995, "external_id": null}, {"id": 10846818003, "name": "Pacific University", "priority": 996, "external_id": null}, {"id": 10846819003, "name": "Paine College", "priority": 997, "external_id": null}, {"id": 10846820003, "name": "Palm Beach Atlantic University", "priority": 998, "external_id": null}, {"id": 10846821003, "name": "Palmer College of Chiropractic", "priority": 999, "external_id": null}, {"id": 10846822003, "name": "Park University", "priority": 1000, "external_id": null}, {"id": 10846823003, "name": "Parker University", "priority": 1001, "external_id": null}, {"id": 10846824003, "name": "Patten University", "priority": 1002, "external_id": null}, {"id": 10846825003, "name": "Paul Smith's College", "priority": 1003, "external_id": null}, {"id": 10846826003, "name": "Peirce College", "priority": 1004, "external_id": null}, {"id": 10846827003, "name": "Peninsula College", "priority": 1005, "external_id": null}, {"id": 10846828003, "name": "Pennsylvania College of Art and Design", "priority": 1006, "external_id": null}, {"id": 10846829003, "name": "Pennsylvania College of Technology", "priority": 1007, "external_id": null}, {"id": 10846830003, "name": "Pennsylvania State University - Erie, The Behrend College", "priority": 1008, "external_id": null}, {"id": 10846831003, "name": "Pennsylvania State University - Harrisburg", "priority": 1009, "external_id": null}, {"id": 10846832003, "name": "Pepperdine University", "priority": 1010, "external_id": null}, {"id": 10846833003, "name": "Peru State College", "priority": 1011, "external_id": null}, {"id": 10846834003, "name": "Pfeiffer University", "priority": 1012, "external_id": null}, {"id": 10846835003, "name": "Philadelphia University", "priority": 1013, "external_id": null}, {"id": 10846836003, "name": "Philander Smith College", "priority": 1014, "external_id": null}, {"id": 10846837003, "name": "Piedmont College", "priority": 1015, "external_id": null}, {"id": 10846838003, "name": "Pine Manor College", "priority": 1016, "external_id": null}, {"id": 10846839003, "name": "Pittsburg State University", "priority": 1017, "external_id": null}, {"id": 10846840003, "name": "Pitzer College", "priority": 1018, "external_id": null}, {"id": 10846841003, "name": "Plaza College", "priority": 1019, "external_id": null}, {"id": 10846842003, "name": "Plymouth State University", "priority": 1020, "external_id": null}, {"id": 10846843003, "name": "Point Loma Nazarene University", "priority": 1021, "external_id": null}, {"id": 10846844003, "name": "Point Park University", "priority": 1022, "external_id": null}, {"id": 10846845003, "name": "Point University", "priority": 1023, "external_id": null}, {"id": 10846846003, "name": "Polytechnic Institute of New York University", "priority": 1024, "external_id": null}, {"id": 10846847003, "name": "Pomona College", "priority": 1025, "external_id": null}, {"id": 10846848003, "name": "Pontifical Catholic University of Puerto Rico", "priority": 1026, "external_id": null}, {"id": 10846849003, "name": "Pontifical College Josephinum", "priority": 1027, "external_id": null}, {"id": 10846850003, "name": "Post University", "priority": 1028, "external_id": null}, {"id": 10846851003, "name": "Potomac College", "priority": 1029, "external_id": null}, {"id": 10846852003, "name": "Pratt Institute", "priority": 1030, "external_id": null}, {"id": 10846853003, "name": "Prescott College", "priority": 1031, "external_id": null}, {"id": 10846854003, "name": "Presentation College", "priority": 1032, "external_id": null}, {"id": 10846855003, "name": "Principia College", "priority": 1033, "external_id": null}, {"id": 10846856003, "name": "Providence College", "priority": 1034, "external_id": null}, {"id": 10846857003, "name": "Puerto Rico Conservatory of Music", "priority": 1035, "external_id": null}, {"id": 10846858003, "name": "Purchase College - SUNY", "priority": 1036, "external_id": null}, {"id": 10846859003, "name": "Purdue University - Calumet", "priority": 1037, "external_id": null}, {"id": 10846860003, "name": "Purdue University - North Central", "priority": 1038, "external_id": null}, {"id": 10846861003, "name": "Queens University of Charlotte", "priority": 1039, "external_id": null}, {"id": 10846862003, "name": "Oklahoma State University", "priority": 1040, "external_id": null}, {"id": 10846863003, "name": "Oregon State University", "priority": 1041, "external_id": null}, {"id": 10846864003, "name": "Portland State University", "priority": 1042, "external_id": null}, {"id": 10846865003, "name": "Old Dominion University", "priority": 1043, "external_id": null}, {"id": 10846866003, "name": "Prairie View A&M University", "priority": 1044, "external_id": null}, {"id": 10846867003, "name": "Presbyterian College", "priority": 1045, "external_id": null}, {"id": 10846868003, "name": "Purdue University - West Lafayette", "priority": 1046, "external_id": null}, {"id": 10846869003, "name": "Ohio University", "priority": 1047, "external_id": null}, {"id": 10846870003, "name": "Princeton University", "priority": 1048, "external_id": null}, {"id": 10846871003, "name": "Quincy University", "priority": 1049, "external_id": null}, {"id": 10846872003, "name": "Quinnipiac University", "priority": 1050, "external_id": null}, {"id": 10846873003, "name": "Radford University", "priority": 1051, "external_id": null}, {"id": 10846874003, "name": "Ramapo College of New Jersey", "priority": 1052, "external_id": null}, {"id": 10846875003, "name": "Randolph College", "priority": 1053, "external_id": null}, {"id": 10846876003, "name": "Randolph-Macon College", "priority": 1054, "external_id": null}, {"id": 10846877003, "name": "Ranken Technical College", "priority": 1055, "external_id": null}, {"id": 10846878003, "name": "Reed College", "priority": 1056, "external_id": null}, {"id": 10846879003, "name": "Regent University", "priority": 1057, "external_id": null}, {"id": 10846880003, "name": "Regent's American College London", "priority": 1058, "external_id": null}, {"id": 10846881003, "name": "Regis College", "priority": 1059, "external_id": null}, {"id": 10846882003, "name": "Regis University", "priority": 1060, "external_id": null}, {"id": 10846883003, "name": "Reinhardt University", "priority": 1061, "external_id": null}, {"id": 10846884003, "name": "Rensselaer Polytechnic Institute", "priority": 1062, "external_id": null}, {"id": 10846885003, "name": "Research College of Nursing", "priority": 1063, "external_id": null}, {"id": 10846886003, "name": "Resurrection University", "priority": 1064, "external_id": null}, {"id": 10846887003, "name": "Rhode Island College", "priority": 1065, "external_id": null}, {"id": 10846888003, "name": "Rhode Island School of Design", "priority": 1066, "external_id": null}, {"id": 10846889003, "name": "Rhodes College", "priority": 1067, "external_id": null}, {"id": 10846890003, "name": "Richard Stockton College of New Jersey", "priority": 1068, "external_id": null}, {"id": 10846891003, "name": "Richmond - The American International University in London", "priority": 1069, "external_id": null}, {"id": 10846892003, "name": "Rider University", "priority": 1070, "external_id": null}, {"id": 10846893003, "name": "Ringling College of Art and Design", "priority": 1071, "external_id": null}, {"id": 10846894003, "name": "Ripon College", "priority": 1072, "external_id": null}, {"id": 10846895003, "name": "Rivier University", "priority": 1073, "external_id": null}, {"id": 10846896003, "name": "Roanoke College", "priority": 1074, "external_id": null}, {"id": 10846897003, "name": "Robert B. Miller College", "priority": 1075, "external_id": null}, {"id": 10846898003, "name": "Roberts Wesleyan College", "priority": 1076, "external_id": null}, {"id": 10846899003, "name": "Rochester College", "priority": 1077, "external_id": null}, {"id": 10846900003, "name": "Rochester Institute of Technology", "priority": 1078, "external_id": null}, {"id": 10846901003, "name": "Rockford University", "priority": 1079, "external_id": null}, {"id": 10846902003, "name": "Rockhurst University", "priority": 1080, "external_id": null}, {"id": 10846903003, "name": "Rocky Mountain College", "priority": 1081, "external_id": null}, {"id": 10846904003, "name": "Rocky Mountain College of Art and Design", "priority": 1082, "external_id": null}, {"id": 10846905003, "name": "Roger Williams University", "priority": 1083, "external_id": null}, {"id": 10846906003, "name": "Rogers State University", "priority": 1084, "external_id": null}, {"id": 10846907003, "name": "Rollins College", "priority": 1085, "external_id": null}, {"id": 10846908003, "name": "Roosevelt University", "priority": 1086, "external_id": null}, {"id": 10846909003, "name": "Rosalind Franklin University of Medicine and Science", "priority": 1087, "external_id": null}, {"id": 10846910003, "name": "Rose-Hulman Institute of Technology", "priority": 1088, "external_id": null}, {"id": 10846911003, "name": "Rosemont College", "priority": 1089, "external_id": null}, {"id": 10846912003, "name": "Rowan University", "priority": 1090, "external_id": null}, {"id": 10846913003, "name": "Rush University", "priority": 1091, "external_id": null}, {"id": 10846914003, "name": "Rust College", "priority": 1092, "external_id": null}, {"id": 10846915003, "name": "Rutgers, the State University of New Jersey - Camden", "priority": 1093, "external_id": null}, {"id": 10846916003, "name": "Rutgers, the State University of New Jersey - Newark", "priority": 1094, "external_id": null}, {"id": 10846917003, "name": "Ryerson University", "priority": 1095, "external_id": null}, {"id": 10846918003, "name": "Sacred Heart Major Seminary", "priority": 1096, "external_id": null}, {"id": 10846919003, "name": "Saginaw Valley State University", "priority": 1097, "external_id": null}, {"id": 10846920003, "name": "Salem College", "priority": 1098, "external_id": null}, {"id": 10846921003, "name": "Salem International University", "priority": 1099, "external_id": null}, {"id": 10846922003, "name": "Salem State University", "priority": 1100, "external_id": null}, {"id": 10846923003, "name": "Salisbury University", "priority": 1101, "external_id": null}, {"id": 10846924003, "name": "Salish Kootenai College", "priority": 1102, "external_id": null}, {"id": 10846925003, "name": "Salve Regina University", "priority": 1103, "external_id": null}, {"id": 10846926003, "name": "Samuel Merritt University", "priority": 1104, "external_id": null}, {"id": 10846927003, "name": "San Diego Christian College", "priority": 1105, "external_id": null}, {"id": 10846928003, "name": "San Francisco Art Institute", "priority": 1106, "external_id": null}, {"id": 10846929003, "name": "San Francisco Conservatory of Music", "priority": 1107, "external_id": null}, {"id": 10846930003, "name": "San Francisco State University", "priority": 1108, "external_id": null}, {"id": 10846931003, "name": "Sanford College of Nursing", "priority": 1109, "external_id": null}, {"id": 10846932003, "name": "Santa Clara University", "priority": 1110, "external_id": null}, {"id": 10846933003, "name": "Santa Fe University of Art and Design", "priority": 1111, "external_id": null}, {"id": 10846934003, "name": "Sarah Lawrence College", "priority": 1112, "external_id": null}, {"id": 10846935003, "name": "Savannah College of Art and Design", "priority": 1113, "external_id": null}, {"id": 10846936003, "name": "School of the Art Institute of Chicago", "priority": 1114, "external_id": null}, {"id": 10846937003, "name": "School of Visual Arts", "priority": 1115, "external_id": null}, {"id": 10846938003, "name": "Schreiner University", "priority": 1116, "external_id": null}, {"id": 10846939003, "name": "Scripps College", "priority": 1117, "external_id": null}, {"id": 10846940003, "name": "Seattle Pacific University", "priority": 1118, "external_id": null}, {"id": 10846941003, "name": "Seattle University", "priority": 1119, "external_id": null}, {"id": 10846942003, "name": "Seton Hall University", "priority": 1120, "external_id": null}, {"id": 10846943003, "name": "Seton Hill University", "priority": 1121, "external_id": null}, {"id": 10846944003, "name": "Sewanee - University of the South", "priority": 1122, "external_id": null}, {"id": 10846945003, "name": "Shaw University", "priority": 1123, "external_id": null}, {"id": 10846946003, "name": "Shawnee State University", "priority": 1124, "external_id": null}, {"id": 10846947003, "name": "Shenandoah University", "priority": 1125, "external_id": null}, {"id": 10846948003, "name": "Shepherd University", "priority": 1126, "external_id": null}, {"id": 10846949003, "name": "Shimer College", "priority": 1127, "external_id": null}, {"id": 10846950003, "name": "Sacred Heart University", "priority": 1128, "external_id": null}, {"id": 10846951003, "name": "Robert Morris University", "priority": 1129, "external_id": null}, {"id": 10846952003, "name": "Sam Houston State University", "priority": 1130, "external_id": null}, {"id": 10846953003, "name": "Samford University", "priority": 1131, "external_id": null}, {"id": 10846954003, "name": "Savannah State University", "priority": 1132, "external_id": null}, {"id": 10846955003, "name": "San Jose State University", "priority": 1133, "external_id": null}, {"id": 10846956003, "name": "Rutgers, the State University of New Jersey - New Brunswick", "priority": 1134, "external_id": null}, {"id": 10846957003, "name": "San Diego State University", "priority": 1135, "external_id": null}, {"id": 10846958003, "name": "Shippensburg University of Pennsylvania", "priority": 1136, "external_id": null}, {"id": 10846959003, "name": "Shorter University", "priority": 1137, "external_id": null}, {"id": 10846960003, "name": "Siena College", "priority": 1138, "external_id": null}, {"id": 10846961003, "name": "Siena Heights University", "priority": 1139, "external_id": null}, {"id": 10846962003, "name": "Sierra Nevada College", "priority": 1140, "external_id": null}, {"id": 10846963003, "name": "Silver Lake College", "priority": 1141, "external_id": null}, {"id": 10846964003, "name": "Simmons College", "priority": 1142, "external_id": null}, {"id": 10846965003, "name": "Simon Fraser University", "priority": 1143, "external_id": null}, {"id": 10846966003, "name": "Simpson College", "priority": 1144, "external_id": null}, {"id": 10846967003, "name": "Simpson University", "priority": 1145, "external_id": null}, {"id": 10846968003, "name": "Sinte Gleska University", "priority": 1146, "external_id": null}, {"id": 10846969003, "name": "Sitting Bull College", "priority": 1147, "external_id": null}, {"id": 10846970003, "name": "Skidmore College", "priority": 1148, "external_id": null}, {"id": 10846971003, "name": "Slippery Rock University of Pennsylvania", "priority": 1149, "external_id": null}, {"id": 10846972003, "name": "Smith College", "priority": 1150, "external_id": null}, {"id": 10846973003, "name": "Sojourner-Douglass College", "priority": 1151, "external_id": null}, {"id": 10846974003, "name": "Soka University of America", "priority": 1152, "external_id": null}, {"id": 10846975003, "name": "Sonoma State University", "priority": 1153, "external_id": null}, {"id": 10846976003, "name": "South College", "priority": 1154, "external_id": null}, {"id": 10846977003, "name": "South Dakota School of Mines and Technology", "priority": 1155, "external_id": null}, {"id": 10846978003, "name": "South Seattle Community College", "priority": 1156, "external_id": null}, {"id": 10846979003, "name": "South Texas College", "priority": 1157, "external_id": null}, {"id": 10846980003, "name": "South University", "priority": 1158, "external_id": null}, {"id": 10846981003, "name": "Southeastern Oklahoma State University", "priority": 1159, "external_id": null}, {"id": 10846982003, "name": "Southeastern University", "priority": 1160, "external_id": null}, {"id": 10846983003, "name": "Southern Adventist University", "priority": 1161, "external_id": null}, {"id": 10846984003, "name": "Southern Arkansas University", "priority": 1162, "external_id": null}, {"id": 10846985003, "name": "Southern Baptist Theological Seminary", "priority": 1163, "external_id": null}, {"id": 10846986003, "name": "Southern California Institute of Architecture", "priority": 1164, "external_id": null}, {"id": 10846987003, "name": "Southern Connecticut State University", "priority": 1165, "external_id": null}, {"id": 10846988003, "name": "Southern Illinois University - Edwardsville", "priority": 1166, "external_id": null}, {"id": 10846989003, "name": "Southern Nazarene University", "priority": 1167, "external_id": null}, {"id": 10846990003, "name": "Southern New Hampshire University", "priority": 1168, "external_id": null}, {"id": 10846991003, "name": "Southern Oregon University", "priority": 1169, "external_id": null}, {"id": 10846992003, "name": "Southern Polytechnic State University", "priority": 1170, "external_id": null}, {"id": 10846993003, "name": "Southern University - New Orleans", "priority": 1171, "external_id": null}, {"id": 10846994003, "name": "Southern Vermont College", "priority": 1172, "external_id": null}, {"id": 10846995003, "name": "Southern Wesleyan University", "priority": 1173, "external_id": null}, {"id": 10846996003, "name": "Southwest Baptist University", "priority": 1174, "external_id": null}, {"id": 10846997003, "name": "Southwest Minnesota State University", "priority": 1175, "external_id": null}, {"id": 10846998003, "name": "Southwest University of Visual Arts", "priority": 1176, "external_id": null}, {"id": 10846999003, "name": "Southwestern Adventist University", "priority": 1177, "external_id": null}, {"id": 10847000003, "name": "Southwestern Assemblies of God University", "priority": 1178, "external_id": null}, {"id": 10847001003, "name": "Southwestern Christian College", "priority": 1179, "external_id": null}, {"id": 10847002003, "name": "Southwestern Christian University", "priority": 1180, "external_id": null}, {"id": 10847003003, "name": "Southwestern College", "priority": 1181, "external_id": null}, {"id": 10847004003, "name": "Southwestern Oklahoma State University", "priority": 1182, "external_id": null}, {"id": 10847005003, "name": "Southwestern University", "priority": 1183, "external_id": null}, {"id": 10847006003, "name": "Spalding University", "priority": 1184, "external_id": null}, {"id": 10847007003, "name": "Spelman College", "priority": 1185, "external_id": null}, {"id": 10847008003, "name": "Spring Arbor University", "priority": 1186, "external_id": null}, {"id": 10847009003, "name": "Spring Hill College", "priority": 1187, "external_id": null}, {"id": 10847010003, "name": "Springfield College", "priority": 1188, "external_id": null}, {"id": 10847011003, "name": "St. Ambrose University", "priority": 1189, "external_id": null}, {"id": 10847012003, "name": "St. Anselm College", "priority": 1190, "external_id": null}, {"id": 10847013003, "name": "St. Anthony College of Nursing", "priority": 1191, "external_id": null}, {"id": 10847014003, "name": "St. Augustine College", "priority": 1192, "external_id": null}, {"id": 10847015003, "name": "St. Augustine's University", "priority": 1193, "external_id": null}, {"id": 10847016003, "name": "St. Bonaventure University", "priority": 1194, "external_id": null}, {"id": 10847017003, "name": "St. Catharine College", "priority": 1195, "external_id": null}, {"id": 10847018003, "name": "St. Catherine University", "priority": 1196, "external_id": null}, {"id": 10847019003, "name": "St. Charles Borromeo Seminary", "priority": 1197, "external_id": null}, {"id": 10847020003, "name": "St. Cloud State University", "priority": 1198, "external_id": null}, {"id": 10847021003, "name": "St. Edward's University", "priority": 1199, "external_id": null}, {"id": 10847022003, "name": "St. Francis College", "priority": 1200, "external_id": null}, {"id": 10847023003, "name": "St. Francis Medical Center College of Nursing", "priority": 1201, "external_id": null}, {"id": 10847024003, "name": "St. Gregory's University", "priority": 1202, "external_id": null}, {"id": 10847025003, "name": "St. John Fisher College", "priority": 1203, "external_id": null}, {"id": 10847026003, "name": "St. John Vianney College Seminary", "priority": 1204, "external_id": null}, {"id": 10847027003, "name": "St. John's College", "priority": 1205, "external_id": null}, {"id": 10847028003, "name": "St. John's University", "priority": 1206, "external_id": null}, {"id": 10847029003, "name": "St. Joseph Seminary College", "priority": 1207, "external_id": null}, {"id": 10847030003, "name": "St. Joseph's College", "priority": 1208, "external_id": null}, {"id": 10847031003, "name": "St. Joseph's College New York", "priority": 1209, "external_id": null}, {"id": 10847032003, "name": "St. Joseph's University", "priority": 1210, "external_id": null}, {"id": 10847033003, "name": "St. Lawrence University", "priority": 1211, "external_id": null}, {"id": 10847034003, "name": "St. Leo University", "priority": 1212, "external_id": null}, {"id": 10847035003, "name": "Southern University and A&M College", "priority": 1213, "external_id": null}, {"id": 10847036003, "name": "Southern Methodist University", "priority": 1214, "external_id": null}, {"id": 10847037003, "name": "Southeast Missouri State University", "priority": 1215, "external_id": null}, {"id": 10847038003, "name": "Southern Utah University", "priority": 1216, "external_id": null}, {"id": 10847039003, "name": "South Dakota State University", "priority": 1217, "external_id": null}, {"id": 10847040003, "name": "St. Francis University", "priority": 1218, "external_id": null}, {"id": 10847041003, "name": "Southeastern Louisiana University", "priority": 1219, "external_id": null}, {"id": 10847042003, "name": "Southern Illinois University - Carbondale", "priority": 1220, "external_id": null}, {"id": 10847043003, "name": "St. Louis College of Pharmacy", "priority": 1221, "external_id": null}, {"id": 10847044003, "name": "St. Louis University", "priority": 1222, "external_id": null}, {"id": 10847045003, "name": "St. Luke's College of Health Sciences", "priority": 1223, "external_id": null}, {"id": 10847046003, "name": "St. Martin's University", "priority": 1224, "external_id": null}, {"id": 10847047003, "name": "St. Mary's College", "priority": 1225, "external_id": null}, {"id": 10847048003, "name": "St. Mary's College of California", "priority": 1226, "external_id": null}, {"id": 10847049003, "name": "St. Mary's College of Maryland", "priority": 1227, "external_id": null}, {"id": 10847050003, "name": "St. Mary's Seminary and University", "priority": 1228, "external_id": null}, {"id": 10847051003, "name": "St. Mary's University of Minnesota", "priority": 1229, "external_id": null}, {"id": 10847052003, "name": "St. Mary's University of San Antonio", "priority": 1230, "external_id": null}, {"id": 10847053003, "name": "St. Mary-of-the-Woods College", "priority": 1231, "external_id": null}, {"id": 10847054003, "name": "St. Michael's College", "priority": 1232, "external_id": null}, {"id": 10847055003, "name": "St. Norbert College", "priority": 1233, "external_id": null}, {"id": 10847056003, "name": "St. Olaf College", "priority": 1234, "external_id": null}, {"id": 10847057003, "name": "St. Paul's College", "priority": 1235, "external_id": null}, {"id": 10847058003, "name": "St. Peter's University", "priority": 1236, "external_id": null}, {"id": 10847059003, "name": "St. Petersburg College", "priority": 1237, "external_id": null}, {"id": 10847060003, "name": "St. Thomas Aquinas College", "priority": 1238, "external_id": null}, {"id": 10847061003, "name": "St. Thomas University", "priority": 1239, "external_id": null}, {"id": 10847062003, "name": "St. Vincent College", "priority": 1240, "external_id": null}, {"id": 10847063003, "name": "St. Xavier University", "priority": 1241, "external_id": null}, {"id": 10847064003, "name": "Stephens College", "priority": 1242, "external_id": null}, {"id": 10847065003, "name": "Sterling College", "priority": 1243, "external_id": null}, {"id": 10847066003, "name": "Stevens Institute of Technology", "priority": 1244, "external_id": null}, {"id": 10847067003, "name": "Stevenson University", "priority": 1245, "external_id": null}, {"id": 10847068003, "name": "Stillman College", "priority": 1246, "external_id": null}, {"id": 10847069003, "name": "Stonehill College", "priority": 1247, "external_id": null}, {"id": 10847070003, "name": "Strayer University", "priority": 1248, "external_id": null}, {"id": 10847071003, "name": "Suffolk University", "priority": 1249, "external_id": null}, {"id": 10847072003, "name": "Sul Ross State University", "priority": 1250, "external_id": null}, {"id": 10847073003, "name": "Sullivan University", "priority": 1251, "external_id": null}, {"id": 10847074003, "name": "SUNY Buffalo State", "priority": 1252, "external_id": null}, {"id": 10847075003, "name": "SUNY College of Agriculture and Technology - Cobleskill", "priority": 1253, "external_id": null}, {"id": 10847076003, "name": "SUNY College of Environmental Science and Forestry", "priority": 1254, "external_id": null}, {"id": 10847077003, "name": "SUNY College of Technology - Alfred", "priority": 1255, "external_id": null}, {"id": 10847078003, "name": "SUNY College of Technology - Canton", "priority": 1256, "external_id": null}, {"id": 10847079003, "name": "SUNY College of Technology - Delhi", "priority": 1257, "external_id": null}, {"id": 10847080003, "name": "SUNY College - Cortland", "priority": 1258, "external_id": null}, {"id": 10847081003, "name": "SUNY College - Old Westbury", "priority": 1259, "external_id": null}, {"id": 10847082003, "name": "SUNY College - Oneonta", "priority": 1260, "external_id": null}, {"id": 10847083003, "name": "SUNY College - Potsdam", "priority": 1261, "external_id": null}, {"id": 10847084003, "name": "SUNY Downstate Medical Center", "priority": 1262, "external_id": null}, {"id": 10847085003, "name": "SUNY Empire State College", "priority": 1263, "external_id": null}, {"id": 10847086003, "name": "SUNY Institute of Technology - Utica/Rome", "priority": 1264, "external_id": null}, {"id": 10847087003, "name": "SUNY Maritime College", "priority": 1265, "external_id": null}, {"id": 10847088003, "name": "SUNY Upstate Medical University", "priority": 1266, "external_id": null}, {"id": 10847089003, "name": "SUNY - Fredonia", "priority": 1267, "external_id": null}, {"id": 10847090003, "name": "SUNY - Geneseo", "priority": 1268, "external_id": null}, {"id": 10847091003, "name": "SUNY - New Paltz", "priority": 1269, "external_id": null}, {"id": 10847092003, "name": "SUNY - Oswego", "priority": 1270, "external_id": null}, {"id": 10847093003, "name": "SUNY - Plattsburgh", "priority": 1271, "external_id": null}, {"id": 10847094003, "name": "Swarthmore College", "priority": 1272, "external_id": null}, {"id": 10847095003, "name": "Sweet Briar College", "priority": 1273, "external_id": null}, {"id": 10847096003, "name": "Tabor College", "priority": 1274, "external_id": null}, {"id": 10847097003, "name": "Talladega College", "priority": 1275, "external_id": null}, {"id": 10847098003, "name": "Tarleton State University", "priority": 1276, "external_id": null}, {"id": 10847099003, "name": "Taylor University", "priority": 1277, "external_id": null}, {"id": 10847100003, "name": "Tennessee Wesleyan College", "priority": 1278, "external_id": null}, {"id": 10847101003, "name": "Texas A&M International University", "priority": 1279, "external_id": null}, {"id": 10847102003, "name": "Texas A&M University - Commerce", "priority": 1280, "external_id": null}, {"id": 10847103003, "name": "Texas A&M University - Corpus Christi", "priority": 1281, "external_id": null}, {"id": 10847104003, "name": "Texas A&M University - Galveston", "priority": 1282, "external_id": null}, {"id": 10847105003, "name": "Texas A&M University - Kingsville", "priority": 1283, "external_id": null}, {"id": 10847106003, "name": "Texas A&M University - Texarkana", "priority": 1284, "external_id": null}, {"id": 10847107003, "name": "Texas College", "priority": 1285, "external_id": null}, {"id": 10847108003, "name": "Texas Lutheran University", "priority": 1286, "external_id": null}, {"id": 10847109003, "name": "Bucknell University", "priority": 1287, "external_id": null}, {"id": 10847110003, "name": "Butler University", "priority": 1288, "external_id": null}, {"id": 10847111003, "name": "Stephen F. Austin State University", "priority": 1289, "external_id": null}, {"id": 10847112003, "name": "Texas A&M University - College Station", "priority": 1290, "external_id": null}, {"id": 10847113003, "name": "Stanford University", "priority": 1291, "external_id": null}, {"id": 10847114003, "name": "Stetson University", "priority": 1292, "external_id": null}, {"id": 10847115003, "name": "Stony Brook University - SUNY", "priority": 1293, "external_id": null}, {"id": 10847116003, "name": "Syracuse University", "priority": 1294, "external_id": null}, {"id": 10847117003, "name": "Texas Christian University", "priority": 1295, "external_id": null}, {"id": 10847118003, "name": "Temple University", "priority": 1296, "external_id": null}, {"id": 10847119003, "name": "Clemson University", "priority": 1297, "external_id": null}, {"id": 10847120003, "name": "Texas Southern University", "priority": 1298, "external_id": null}, {"id": 10847121003, "name": "Austin Peay State University", "priority": 1299, "external_id": null}, {"id": 10847122003, "name": "Tennessee State University", "priority": 1300, "external_id": null}, {"id": 10847123003, "name": "Ball State University", "priority": 1301, "external_id": null}, {"id": 10847124003, "name": "Texas Tech University Health Sciences Center", "priority": 1302, "external_id": null}, {"id": 10847125003, "name": "Texas Wesleyan University", "priority": 1303, "external_id": null}, {"id": 10847126003, "name": "Texas Woman's University", "priority": 1304, "external_id": null}, {"id": 10847127003, "name": "The Catholic University of America", "priority": 1305, "external_id": null}, {"id": 10847128003, "name": "The Sage Colleges", "priority": 1306, "external_id": null}, {"id": 10847129003, "name": "Thiel College", "priority": 1307, "external_id": null}, {"id": 10847130003, "name": "Thomas Aquinas College", "priority": 1308, "external_id": null}, {"id": 10847131003, "name": "Thomas College", "priority": 1309, "external_id": null}, {"id": 10847132003, "name": "Thomas Edison State College", "priority": 1310, "external_id": null}, {"id": 10847133003, "name": "Thomas Jefferson University", "priority": 1311, "external_id": null}, {"id": 10847134003, "name": "Thomas More College", "priority": 1312, "external_id": null}, {"id": 10847135003, "name": "Thomas More College of Liberal Arts", "priority": 1313, "external_id": null}, {"id": 10847136003, "name": "Thomas University", "priority": 1314, "external_id": null}, {"id": 10847137003, "name": "Tiffin University", "priority": 1315, "external_id": null}, {"id": 10847138003, "name": "Tilburg University", "priority": 1316, "external_id": null}, {"id": 10847139003, "name": "Toccoa Falls College", "priority": 1317, "external_id": null}, {"id": 10847140003, "name": "Tougaloo College", "priority": 1318, "external_id": null}, {"id": 10847141003, "name": "Touro College", "priority": 1319, "external_id": null}, {"id": 10847142003, "name": "Transylvania University", "priority": 1320, "external_id": null}, {"id": 10847143003, "name": "Trent University", "priority": 1321, "external_id": null}, {"id": 10847144003, "name": "Trevecca Nazarene University", "priority": 1322, "external_id": null}, {"id": 10847145003, "name": "Trident University International", "priority": 1323, "external_id": null}, {"id": 10847146003, "name": "Trine University", "priority": 1324, "external_id": null}, {"id": 10847147003, "name": "Trinity Christian College", "priority": 1325, "external_id": null}, {"id": 10847148003, "name": "Trinity College", "priority": 1326, "external_id": null}, {"id": 10847149003, "name": "Trinity College of Nursing & Health Sciences", "priority": 1327, "external_id": null}, {"id": 10847150003, "name": "Trinity International University", "priority": 1328, "external_id": null}, {"id": 10847151003, "name": "Trinity Lutheran College", "priority": 1329, "external_id": null}, {"id": 10847152003, "name": "Trinity University", "priority": 1330, "external_id": null}, {"id": 10847153003, "name": "Trinity Western University", "priority": 1331, "external_id": null}, {"id": 10847154003, "name": "Truett McConnell College", "priority": 1332, "external_id": null}, {"id": 10847155003, "name": "Truman State University", "priority": 1333, "external_id": null}, {"id": 10847156003, "name": "Tufts University", "priority": 1334, "external_id": null}, {"id": 10847157003, "name": "Tusculum College", "priority": 1335, "external_id": null}, {"id": 10847158003, "name": "Tuskegee University", "priority": 1336, "external_id": null}, {"id": 10847159003, "name": "Union College", "priority": 1337, "external_id": null}, {"id": 10847160003, "name": "Union Institute and University", "priority": 1338, "external_id": null}, {"id": 10847161003, "name": "Union University", "priority": 1339, "external_id": null}, {"id": 10847162003, "name": "United States Coast Guard Academy", "priority": 1340, "external_id": null}, {"id": 10847163003, "name": "United States International University - Kenya", "priority": 1341, "external_id": null}, {"id": 10847164003, "name": "United States Merchant Marine Academy", "priority": 1342, "external_id": null}, {"id": 10847165003, "name": "United States Sports Academy", "priority": 1343, "external_id": null}, {"id": 10847166003, "name": "Unity College", "priority": 1344, "external_id": null}, {"id": 10847167003, "name": "Universidad Adventista de las Antillas", "priority": 1345, "external_id": null}, {"id": 10847168003, "name": "Universidad del Este", "priority": 1346, "external_id": null}, {"id": 10847169003, "name": "Universidad del Turabo", "priority": 1347, "external_id": null}, {"id": 10847170003, "name": "Universidad Metropolitana", "priority": 1348, "external_id": null}, {"id": 10847171003, "name": "Universidad Politecnica De Puerto Rico", "priority": 1349, "external_id": null}, {"id": 10847172003, "name": "University of Advancing Technology", "priority": 1350, "external_id": null}, {"id": 10847173003, "name": "University of Alabama - Huntsville", "priority": 1351, "external_id": null}, {"id": 10847174003, "name": "University of Alaska - Anchorage", "priority": 1352, "external_id": null}, {"id": 10847175003, "name": "University of Alaska - Fairbanks", "priority": 1353, "external_id": null}, {"id": 10847176003, "name": "University of Alaska - Southeast", "priority": 1354, "external_id": null}, {"id": 10847177003, "name": "University of Alberta", "priority": 1355, "external_id": null}, {"id": 10847178003, "name": "University of Arkansas for Medical Sciences", "priority": 1356, "external_id": null}, {"id": 10847179003, "name": "University of Arkansas - Fort Smith", "priority": 1357, "external_id": null}, {"id": 10847180003, "name": "University of Arkansas - Little Rock", "priority": 1358, "external_id": null}, {"id": 10847181003, "name": "University of Arkansas - Monticello", "priority": 1359, "external_id": null}, {"id": 10847182003, "name": "University of Baltimore", "priority": 1360, "external_id": null}, {"id": 10847183003, "name": "University of Bridgeport", "priority": 1361, "external_id": null}, {"id": 10847184003, "name": "University of British Columbia", "priority": 1362, "external_id": null}, {"id": 10847185003, "name": "University of Calgary", "priority": 1363, "external_id": null}, {"id": 10847186003, "name": "University of California - Riverside", "priority": 1364, "external_id": null}, {"id": 10847187003, "name": "Holy Cross College", "priority": 1365, "external_id": null}, {"id": 10847188003, "name": "Towson University", "priority": 1366, "external_id": null}, {"id": 10847189003, "name": "United States Military Academy", "priority": 1367, "external_id": null}, {"id": 10847190003, "name": "The Citadel", "priority": 1368, "external_id": null}, {"id": 10847191003, "name": "Troy University", "priority": 1369, "external_id": null}, {"id": 10847192003, "name": "University of California - Davis", "priority": 1370, "external_id": null}, {"id": 10847193003, "name": "Grambling State University", "priority": 1371, "external_id": null}, {"id": 10847194003, "name": "University at Albany - SUNY", "priority": 1372, "external_id": null}, {"id": 10847195003, "name": "University at Buffalo - SUNY", "priority": 1373, "external_id": null}, {"id": 10847196003, "name": "United States Naval Academy", "priority": 1374, "external_id": null}, {"id": 10847197003, "name": "University of Arizona", "priority": 1375, "external_id": null}, {"id": 10847198003, "name": "University of California - Los Angeles", "priority": 1376, "external_id": null}, {"id": 10847199003, "name": "Florida A&M University", "priority": 1377, "external_id": null}, {"id": 10847200003, "name": "Texas State University", "priority": 1378, "external_id": null}, {"id": 10847201003, "name": "University of Alabama - Birmingham", "priority": 1379, "external_id": null}, {"id": 10847202003, "name": "University of California - Santa Cruz", "priority": 1380, "external_id": null}, {"id": 10847203003, "name": "University of Central Missouri", "priority": 1381, "external_id": null}, {"id": 10847204003, "name": "University of Central Oklahoma", "priority": 1382, "external_id": null}, {"id": 10847205003, "name": "University of Charleston", "priority": 1383, "external_id": null}, {"id": 10847206003, "name": "University of Chicago", "priority": 1384, "external_id": null}, {"id": 10847207003, "name": "University of Cincinnati - UC Blue Ash College", "priority": 1385, "external_id": null}, {"id": 10847208003, "name": "University of Colorado - Colorado Springs", "priority": 1386, "external_id": null}, {"id": 10847209003, "name": "University of Colorado - Denver", "priority": 1387, "external_id": null}, {"id": 10847210003, "name": "University of Dallas", "priority": 1388, "external_id": null}, {"id": 10847211003, "name": "University of Denver", "priority": 1389, "external_id": null}, {"id": 10847212003, "name": "University of Detroit Mercy", "priority": 1390, "external_id": null}, {"id": 10847213003, "name": "University of Dubuque", "priority": 1391, "external_id": null}, {"id": 10847214003, "name": "University of Evansville", "priority": 1392, "external_id": null}, {"id": 10847215003, "name": "University of Findlay", "priority": 1393, "external_id": null}, {"id": 10847216003, "name": "University of Great Falls", "priority": 1394, "external_id": null}, {"id": 10847217003, "name": "University of Guam", "priority": 1395, "external_id": null}, {"id": 10847218003, "name": "University of Guelph", "priority": 1396, "external_id": null}, {"id": 10847219003, "name": "University of Hartford", "priority": 1397, "external_id": null}, {"id": 10847220003, "name": "University of Hawaii - Hilo", "priority": 1398, "external_id": null}, {"id": 10847221003, "name": "University of Hawaii - Maui College", "priority": 1399, "external_id": null}, {"id": 10847222003, "name": "University of Hawaii - West Oahu", "priority": 1400, "external_id": null}, {"id": 10847223003, "name": "University of Houston - Clear Lake", "priority": 1401, "external_id": null}, {"id": 10847224003, "name": "University of Houston - Downtown", "priority": 1402, "external_id": null}, {"id": 10847225003, "name": "University of Houston - Victoria", "priority": 1403, "external_id": null}, {"id": 10847226003, "name": "University of Illinois - Chicago", "priority": 1404, "external_id": null}, {"id": 10847227003, "name": "University of Illinois - Springfield", "priority": 1405, "external_id": null}, {"id": 10847228003, "name": "University of Indianapolis", "priority": 1406, "external_id": null}, {"id": 10847229003, "name": "University of Jamestown", "priority": 1407, "external_id": null}, {"id": 10847230003, "name": "University of La Verne", "priority": 1408, "external_id": null}, {"id": 10847231003, "name": "University of Maine - Augusta", "priority": 1409, "external_id": null}, {"id": 10847232003, "name": "University of Maine - Farmington", "priority": 1410, "external_id": null}, {"id": 10847233003, "name": "University of Maine - Fort Kent", "priority": 1411, "external_id": null}, {"id": 10847234003, "name": "University of Maine - Machias", "priority": 1412, "external_id": null}, {"id": 10847235003, "name": "University of Maine - Presque Isle", "priority": 1413, "external_id": null}, {"id": 10847236003, "name": "University of Mary", "priority": 1414, "external_id": null}, {"id": 10847237003, "name": "University of Mary Hardin-Baylor", "priority": 1415, "external_id": null}, {"id": 10847238003, "name": "University of Mary Washington", "priority": 1416, "external_id": null}, {"id": 10847239003, "name": "University of Maryland - Baltimore", "priority": 1417, "external_id": null}, {"id": 10847240003, "name": "University of Maryland - Baltimore County", "priority": 1418, "external_id": null}, {"id": 10847241003, "name": "University of Maryland - Eastern Shore", "priority": 1419, "external_id": null}, {"id": 10847242003, "name": "University of Maryland - University College", "priority": 1420, "external_id": null}, {"id": 10847243003, "name": "University of Massachusetts - Boston", "priority": 1421, "external_id": null}, {"id": 10847244003, "name": "University of Massachusetts - Dartmouth", "priority": 1422, "external_id": null}, {"id": 10847245003, "name": "University of Massachusetts - Lowell", "priority": 1423, "external_id": null}, {"id": 10847246003, "name": "University of Medicine and Dentistry of New Jersey", "priority": 1424, "external_id": null}, {"id": 10847247003, "name": "University of Michigan - Dearborn", "priority": 1425, "external_id": null}, {"id": 10847248003, "name": "University of Michigan - Flint", "priority": 1426, "external_id": null}, {"id": 10847249003, "name": "University of Minnesota - Crookston", "priority": 1427, "external_id": null}, {"id": 10847250003, "name": "University of Minnesota - Duluth", "priority": 1428, "external_id": null}, {"id": 10847251003, "name": "University of Minnesota - Morris", "priority": 1429, "external_id": null}, {"id": 10847252003, "name": "University of Mississippi Medical Center", "priority": 1430, "external_id": null}, {"id": 10847253003, "name": "University of Missouri - Kansas City", "priority": 1431, "external_id": null}, {"id": 10847254003, "name": "University of Missouri - St. Louis", "priority": 1432, "external_id": null}, {"id": 10847255003, "name": "University of Mobile", "priority": 1433, "external_id": null}, {"id": 10847256003, "name": "University of Montana - Western", "priority": 1434, "external_id": null}, {"id": 10847257003, "name": "University of Montevallo", "priority": 1435, "external_id": null}, {"id": 10847258003, "name": "University of Mount Union", "priority": 1436, "external_id": null}, {"id": 10847259003, "name": "University of Nebraska Medical Center", "priority": 1437, "external_id": null}, {"id": 10847260003, "name": "University of Nebraska - Kearney", "priority": 1438, "external_id": null}, {"id": 10847261003, "name": "University of Dayton", "priority": 1439, "external_id": null}, {"id": 10847262003, "name": "University of Delaware", "priority": 1440, "external_id": null}, {"id": 10847263003, "name": "University of Florida", "priority": 1441, "external_id": null}, {"id": 10847264003, "name": "University of Iowa", "priority": 1442, "external_id": null}, {"id": 10847265003, "name": "University of Idaho", "priority": 1443, "external_id": null}, {"id": 10847266003, "name": "University of Kentucky", "priority": 1444, "external_id": null}, {"id": 10847267003, "name": "University of Massachusetts - Amherst", "priority": 1445, "external_id": null}, {"id": 10847268003, "name": "University of Maine", "priority": 1446, "external_id": null}, {"id": 10847269003, "name": "University of Michigan - Ann Arbor", "priority": 1447, "external_id": null}, {"id": 10847270003, "name": "University of Cincinnati", "priority": 1448, "external_id": null}, {"id": 10847271003, "name": "University of Miami", "priority": 1449, "external_id": null}, {"id": 10847272003, "name": "University of Louisiana - Monroe", "priority": 1450, "external_id": null}, {"id": 10847273003, "name": "University of Missouri", "priority": 1451, "external_id": null}, {"id": 10847274003, "name": "University of Mississippi", "priority": 1452, "external_id": null}, {"id": 10847275003, "name": "University of Memphis", "priority": 1453, "external_id": null}, {"id": 10847276003, "name": "University of Houston", "priority": 1454, "external_id": null}, {"id": 10847277003, "name": "University of Colorado - Boulder", "priority": 1455, "external_id": null}, {"id": 10847278003, "name": "University of Nebraska - Omaha", "priority": 1456, "external_id": null}, {"id": 10847279003, "name": "University of New Brunswick", "priority": 1457, "external_id": null}, {"id": 10847280003, "name": "University of New England", "priority": 1458, "external_id": null}, {"id": 10847281003, "name": "University of New Haven", "priority": 1459, "external_id": null}, {"id": 10847282003, "name": "University of New Orleans", "priority": 1460, "external_id": null}, {"id": 10847283003, "name": "University of North Alabama", "priority": 1461, "external_id": null}, {"id": 10847284003, "name": "University of North Carolina School of the Arts", "priority": 1462, "external_id": null}, {"id": 10847285003, "name": "University of North Carolina - Asheville", "priority": 1463, "external_id": null}, {"id": 10847286003, "name": "University of North Carolina - Greensboro", "priority": 1464, "external_id": null}, {"id": 10847287003, "name": "University of North Carolina - Pembroke", "priority": 1465, "external_id": null}, {"id": 10847288003, "name": "University of North Carolina - Wilmington", "priority": 1466, "external_id": null}, {"id": 10847289003, "name": "University of North Florida", "priority": 1467, "external_id": null}, {"id": 10847290003, "name": "University of North Georgia", "priority": 1468, "external_id": null}, {"id": 10847291003, "name": "University of Northwestern Ohio", "priority": 1469, "external_id": null}, {"id": 10847292003, "name": "University of Northwestern - St. Paul", "priority": 1470, "external_id": null}, {"id": 10847293003, "name": "University of Ottawa", "priority": 1471, "external_id": null}, {"id": 10847294003, "name": "University of Phoenix", "priority": 1472, "external_id": null}, {"id": 10847295003, "name": "University of Pikeville", "priority": 1473, "external_id": null}, {"id": 10847296003, "name": "University of Portland", "priority": 1474, "external_id": null}, {"id": 10847297003, "name": "University of Prince Edward Island", "priority": 1475, "external_id": null}, {"id": 10847298003, "name": "University of Puerto Rico - Aguadilla", "priority": 1476, "external_id": null}, {"id": 10847299003, "name": "University of Puerto Rico - Arecibo", "priority": 1477, "external_id": null}, {"id": 10847300003, "name": "University of Puerto Rico - Bayamon", "priority": 1478, "external_id": null}, {"id": 10847301003, "name": "University of Puerto Rico - Cayey", "priority": 1479, "external_id": null}, {"id": 10847302003, "name": "University of Puerto Rico - Humacao", "priority": 1480, "external_id": null}, {"id": 10847303003, "name": "University of Puerto Rico - Mayaguez", "priority": 1481, "external_id": null}, {"id": 10847304003, "name": "University of Puerto Rico - Medical Sciences Campus", "priority": 1482, "external_id": null}, {"id": 10847305003, "name": "University of Puerto Rico - Ponce", "priority": 1483, "external_id": null}, {"id": 10847306003, "name": "University of Puerto Rico - Rio Piedras", "priority": 1484, "external_id": null}, {"id": 10847307003, "name": "University of Puget Sound", "priority": 1485, "external_id": null}, {"id": 10847308003, "name": "University of Redlands", "priority": 1486, "external_id": null}, {"id": 10847309003, "name": "University of Regina", "priority": 1487, "external_id": null}, {"id": 10847310003, "name": "University of Rio Grande", "priority": 1488, "external_id": null}, {"id": 10847311003, "name": "University of Rochester", "priority": 1489, "external_id": null}, {"id": 10847312003, "name": "University of San Francisco", "priority": 1490, "external_id": null}, {"id": 10847313003, "name": "University of Saskatchewan", "priority": 1491, "external_id": null}, {"id": 10847314003, "name": "University of Science and Arts of Oklahoma", "priority": 1492, "external_id": null}, {"id": 10847315003, "name": "University of Scranton", "priority": 1493, "external_id": null}, {"id": 10847316003, "name": "University of Sioux Falls", "priority": 1494, "external_id": null}, {"id": 10847317003, "name": "University of South Carolina - Aiken", "priority": 1495, "external_id": null}, {"id": 10847318003, "name": "University of South Carolina - Beaufort", "priority": 1496, "external_id": null}, {"id": 10847319003, "name": "University of South Carolina - Upstate", "priority": 1497, "external_id": null}, {"id": 10847320003, "name": "University of South Florida - St. Petersburg", "priority": 1498, "external_id": null}, {"id": 10847321003, "name": "University of Southern Indiana", "priority": 1499, "external_id": null}, {"id": 10847322003, "name": "University of Southern Maine", "priority": 1500, "external_id": null}, {"id": 10847323003, "name": "University of St. Francis", "priority": 1501, "external_id": null}, {"id": 10847324003, "name": "University of St. Joseph", "priority": 1502, "external_id": null}, {"id": 10847325003, "name": "University of St. Mary", "priority": 1503, "external_id": null}, {"id": 10847326003, "name": "University of St. Thomas", "priority": 1504, "external_id": null}, {"id": 10847327003, "name": "University of Tampa", "priority": 1505, "external_id": null}, {"id": 10847328003, "name": "University of Texas Health Science Center - Houston", "priority": 1506, "external_id": null}, {"id": 10847329003, "name": "University of Texas Health Science Center - San Antonio", "priority": 1507, "external_id": null}, {"id": 10847330003, "name": "University of Texas Medical Branch - Galveston", "priority": 1508, "external_id": null}, {"id": 10847331003, "name": "University of Texas of the Permian Basin", "priority": 1509, "external_id": null}, {"id": 10847332003, "name": "University of Texas - Arlington", "priority": 1510, "external_id": null}, {"id": 10847333003, "name": "University of Texas - Brownsville", "priority": 1511, "external_id": null}, {"id": 10847334003, "name": "University of Texas - Pan American", "priority": 1512, "external_id": null}, {"id": 10847335003, "name": "University of Oregon", "priority": 1513, "external_id": null}, {"id": 10847336003, "name": "University of New Mexico", "priority": 1514, "external_id": null}, {"id": 10847337003, "name": "University of Pennsylvania", "priority": 1515, "external_id": null}, {"id": 10847338003, "name": "University of North Dakota", "priority": 1516, "external_id": null}, {"id": 10847339003, "name": "University of Nevada - Reno", "priority": 1517, "external_id": null}, {"id": 10847340003, "name": "University of New Hampshire", "priority": 1518, "external_id": null}, {"id": 10847341003, "name": "University of Texas - Austin", "priority": 1519, "external_id": null}, {"id": 10847342003, "name": "University of Southern Mississippi", "priority": 1520, "external_id": null}, {"id": 10847343003, "name": "University of Rhode Island", "priority": 1521, "external_id": null}, {"id": 10847344003, "name": "University of South Dakota", "priority": 1522, "external_id": null}, {"id": 10847345003, "name": "University of Tennessee", "priority": 1523, "external_id": null}, {"id": 10847346003, "name": "University of North Texas", "priority": 1524, "external_id": null}, {"id": 10847347003, "name": "University of North Carolina - Charlotte", "priority": 1525, "external_id": null}, {"id": 10847348003, "name": "University of Texas - San Antonio", "priority": 1526, "external_id": null}, {"id": 10847349003, "name": "University of Notre Dame", "priority": 1527, "external_id": null}, {"id": 10847350003, "name": "University of Southern California", "priority": 1528, "external_id": null}, {"id": 10847351003, "name": "University of Texas - Tyler", "priority": 1529, "external_id": null}, {"id": 10847352003, "name": "University of the Arts", "priority": 1530, "external_id": null}, {"id": 10847353003, "name": "University of the Cumberlands", "priority": 1531, "external_id": null}, {"id": 10847354003, "name": "University of the District of Columbia", "priority": 1532, "external_id": null}, {"id": 10847355003, "name": "University of the Ozarks", "priority": 1533, "external_id": null}, {"id": 10847356003, "name": "University of the Pacific", "priority": 1534, "external_id": null}, {"id": 10847357003, "name": "University of the Sacred Heart", "priority": 1535, "external_id": null}, {"id": 10847358003, "name": "University of the Sciences", "priority": 1536, "external_id": null}, {"id": 10847359003, "name": "University of the Southwest", "priority": 1537, "external_id": null}, {"id": 10847360003, "name": "University of the Virgin Islands", "priority": 1538, "external_id": null}, {"id": 10847361003, "name": "University of the West", "priority": 1539, "external_id": null}, {"id": 10847362003, "name": "University of Toronto", "priority": 1540, "external_id": null}, {"id": 10847363003, "name": "University of Vermont", "priority": 1541, "external_id": null}, {"id": 10847364003, "name": "University of Victoria", "priority": 1542, "external_id": null}, {"id": 10847365003, "name": "University of Virginia - Wise", "priority": 1543, "external_id": null}, {"id": 10847366003, "name": "University of Waterloo", "priority": 1544, "external_id": null}, {"id": 10847367003, "name": "University of West Alabama", "priority": 1545, "external_id": null}, {"id": 10847368003, "name": "University of West Florida", "priority": 1546, "external_id": null}, {"id": 10847369003, "name": "University of West Georgia", "priority": 1547, "external_id": null}, {"id": 10847370003, "name": "University of Windsor", "priority": 1548, "external_id": null}, {"id": 10847371003, "name": "University of Winnipeg", "priority": 1549, "external_id": null}, {"id": 10847372003, "name": "University of Wisconsin - Eau Claire", "priority": 1550, "external_id": null}, {"id": 10847373003, "name": "University of Wisconsin - Green Bay", "priority": 1551, "external_id": null}, {"id": 10847374003, "name": "University of Wisconsin - La Crosse", "priority": 1552, "external_id": null}, {"id": 10847375003, "name": "University of Wisconsin - Milwaukee", "priority": 1553, "external_id": null}, {"id": 10847376003, "name": "University of Wisconsin - Oshkosh", "priority": 1554, "external_id": null}, {"id": 10847377003, "name": "University of Wisconsin - Parkside", "priority": 1555, "external_id": null}, {"id": 10847378003, "name": "University of Wisconsin - Platteville", "priority": 1556, "external_id": null}, {"id": 10847379003, "name": "University of Wisconsin - River Falls", "priority": 1557, "external_id": null}, {"id": 10847380003, "name": "University of Wisconsin - Stevens Point", "priority": 1558, "external_id": null}, {"id": 10847381003, "name": "University of Wisconsin - Stout", "priority": 1559, "external_id": null}, {"id": 10847382003, "name": "University of Wisconsin - Superior", "priority": 1560, "external_id": null}, {"id": 10847383003, "name": "University of Wisconsin - Whitewater", "priority": 1561, "external_id": null}, {"id": 10847384003, "name": "Upper Iowa University", "priority": 1562, "external_id": null}, {"id": 10847385003, "name": "Urbana University", "priority": 1563, "external_id": null}, {"id": 10847386003, "name": "Ursinus College", "priority": 1564, "external_id": null}, {"id": 10847387003, "name": "Ursuline College", "priority": 1565, "external_id": null}, {"id": 10847388003, "name": "Utah Valley University", "priority": 1566, "external_id": null}, {"id": 10847389003, "name": "Utica College", "priority": 1567, "external_id": null}, {"id": 10847390003, "name": "Valdosta State University", "priority": 1568, "external_id": null}, {"id": 10847391003, "name": "Valley City State University", "priority": 1569, "external_id": null}, {"id": 10847392003, "name": "Valley Forge Christian College", "priority": 1570, "external_id": null}, {"id": 10847393003, "name": "VanderCook College of Music", "priority": 1571, "external_id": null}, {"id": 10847394003, "name": "Vanguard University of Southern California", "priority": 1572, "external_id": null}, {"id": 10847395003, "name": "Vassar College", "priority": 1573, "external_id": null}, {"id": 10847396003, "name": "Vaughn College of Aeronautics and Technology", "priority": 1574, "external_id": null}, {"id": 10847397003, "name": "Vermont Technical College", "priority": 1575, "external_id": null}, {"id": 10847398003, "name": "Victory University", "priority": 1576, "external_id": null}, {"id": 10847399003, "name": "Vincennes University", "priority": 1577, "external_id": null}, {"id": 10847400003, "name": "Virginia Commonwealth University", "priority": 1578, "external_id": null}, {"id": 10847401003, "name": "Virginia Intermont College", "priority": 1579, "external_id": null}, {"id": 10847402003, "name": "Virginia State University", "priority": 1580, "external_id": null}, {"id": 10847403003, "name": "Virginia Union University", "priority": 1581, "external_id": null}, {"id": 10847404003, "name": "Virginia Wesleyan College", "priority": 1582, "external_id": null}, {"id": 10847405003, "name": "Viterbo University", "priority": 1583, "external_id": null}, {"id": 10847406003, "name": "Voorhees College", "priority": 1584, "external_id": null}, {"id": 10847407003, "name": "Wabash College", "priority": 1585, "external_id": null}, {"id": 10847408003, "name": "Walden University", "priority": 1586, "external_id": null}, {"id": 10847409003, "name": "Waldorf College", "priority": 1587, "external_id": null}, {"id": 10847410003, "name": "Walla Walla University", "priority": 1588, "external_id": null}, {"id": 10847411003, "name": "Walsh College of Accountancy and Business Administration", "priority": 1589, "external_id": null}, {"id": 10847412003, "name": "Walsh University", "priority": 1590, "external_id": null}, {"id": 10847413003, "name": "Warner Pacific College", "priority": 1591, "external_id": null}, {"id": 10847414003, "name": "Warner University", "priority": 1592, "external_id": null}, {"id": 10847415003, "name": "Warren Wilson College", "priority": 1593, "external_id": null}, {"id": 10847416003, "name": "Wartburg College", "priority": 1594, "external_id": null}, {"id": 10847417003, "name": "Washburn University", "priority": 1595, "external_id": null}, {"id": 10847418003, "name": "Washington Adventist University", "priority": 1596, "external_id": null}, {"id": 10847419003, "name": "Washington and Jefferson College", "priority": 1597, "external_id": null}, {"id": 10847420003, "name": "Washington and Lee University", "priority": 1598, "external_id": null}, {"id": 10847421003, "name": "Washington College", "priority": 1599, "external_id": null}, {"id": 10847422003, "name": "Washington University in St. Louis", "priority": 1600, "external_id": null}, {"id": 10847423003, "name": "Watkins College of Art, Design & Film", "priority": 1601, "external_id": null}, {"id": 10847424003, "name": "Wayland Baptist University", "priority": 1602, "external_id": null}, {"id": 10847425003, "name": "Wayne State College", "priority": 1603, "external_id": null}, {"id": 10847426003, "name": "Wayne State University", "priority": 1604, "external_id": null}, {"id": 10847427003, "name": "Waynesburg University", "priority": 1605, "external_id": null}, {"id": 10847428003, "name": "Valparaiso University", "priority": 1606, "external_id": null}, {"id": 10847429003, "name": "Villanova University", "priority": 1607, "external_id": null}, {"id": 10847430003, "name": "Virginia Tech", "priority": 1608, "external_id": null}, {"id": 10847431003, "name": "Washington State University", "priority": 1609, "external_id": null}, {"id": 10847432003, "name": "University of Toledo", "priority": 1610, "external_id": null}, {"id": 10847433003, "name": "Wagner College", "priority": 1611, "external_id": null}, {"id": 10847434003, "name": "University of Wyoming", "priority": 1612, "external_id": null}, {"id": 10847435003, "name": "University of Wisconsin - Madison", "priority": 1613, "external_id": null}, {"id": 10847436003, "name": "University of Tulsa", "priority": 1614, "external_id": null}, {"id": 10847437003, "name": "Webb Institute", "priority": 1615, "external_id": null}, {"id": 10847438003, "name": "Webber International University", "priority": 1616, "external_id": null}, {"id": 10847439003, "name": "Webster University", "priority": 1617, "external_id": null}, {"id": 10847440003, "name": "Welch College", "priority": 1618, "external_id": null}, {"id": 10847441003, "name": "Wellesley College", "priority": 1619, "external_id": null}, {"id": 10847442003, "name": "Wells College", "priority": 1620, "external_id": null}, {"id": 10847443003, "name": "Wentworth Institute of Technology", "priority": 1621, "external_id": null}, {"id": 10847444003, "name": "Wesley College", "priority": 1622, "external_id": null}, {"id": 10847445003, "name": "Wesleyan College", "priority": 1623, "external_id": null}, {"id": 10847446003, "name": "Wesleyan University", "priority": 1624, "external_id": null}, {"id": 10847447003, "name": "West Chester University of Pennsylvania", "priority": 1625, "external_id": null}, {"id": 10847448003, "name": "West Liberty University", "priority": 1626, "external_id": null}, {"id": 10847449003, "name": "West Texas A&M University", "priority": 1627, "external_id": null}, {"id": 10847450003, "name": "West Virginia State University", "priority": 1628, "external_id": null}, {"id": 10847451003, "name": "West Virginia University Institute of Technology", "priority": 1629, "external_id": null}, {"id": 10847452003, "name": "West Virginia University - Parkersburg", "priority": 1630, "external_id": null}, {"id": 10847453003, "name": "West Virginia Wesleyan College", "priority": 1631, "external_id": null}, {"id": 10847454003, "name": "Western Connecticut State University", "priority": 1632, "external_id": null}, {"id": 10847455003, "name": "Western Governors University", "priority": 1633, "external_id": null}, {"id": 10847456003, "name": "Western International University", "priority": 1634, "external_id": null}, {"id": 10847457003, "name": "Western Nevada College", "priority": 1635, "external_id": null}, {"id": 10847458003, "name": "Western New England University", "priority": 1636, "external_id": null}, {"id": 10847459003, "name": "Western New Mexico University", "priority": 1637, "external_id": null}, {"id": 10847460003, "name": "Western Oregon University", "priority": 1638, "external_id": null}, {"id": 10847461003, "name": "Western State Colorado University", "priority": 1639, "external_id": null}, {"id": 10847462003, "name": "Western University", "priority": 1640, "external_id": null}, {"id": 10847463003, "name": "Western Washington University", "priority": 1641, "external_id": null}, {"id": 10847464003, "name": "Westfield State University", "priority": 1642, "external_id": null}, {"id": 10847465003, "name": "Westminster College", "priority": 1643, "external_id": null}, {"id": 10847466003, "name": "Westmont College", "priority": 1644, "external_id": null}, {"id": 10847467003, "name": "Wheaton College", "priority": 1645, "external_id": null}, {"id": 10847468003, "name": "Wheeling Jesuit University", "priority": 1646, "external_id": null}, {"id": 10847469003, "name": "Wheelock College", "priority": 1647, "external_id": null}, {"id": 10847470003, "name": "Whitman College", "priority": 1648, "external_id": null}, {"id": 10847471003, "name": "Whittier College", "priority": 1649, "external_id": null}, {"id": 10847472003, "name": "Whitworth University", "priority": 1650, "external_id": null}, {"id": 10847473003, "name": "Wichita State University", "priority": 1651, "external_id": null}, {"id": 10847474003, "name": "Widener University", "priority": 1652, "external_id": null}, {"id": 10847475003, "name": "Wilberforce University", "priority": 1653, "external_id": null}, {"id": 10847476003, "name": "Wiley College", "priority": 1654, "external_id": null}, {"id": 10847477003, "name": "Wilkes University", "priority": 1655, "external_id": null}, {"id": 10847478003, "name": "Willamette University", "priority": 1656, "external_id": null}, {"id": 10847479003, "name": "William Carey University", "priority": 1657, "external_id": null}, {"id": 10847480003, "name": "William Jessup University", "priority": 1658, "external_id": null}, {"id": 10847481003, "name": "William Jewell College", "priority": 1659, "external_id": null}, {"id": 10847482003, "name": "William Paterson University of New Jersey", "priority": 1660, "external_id": null}, {"id": 10847483003, "name": "William Peace University", "priority": 1661, "external_id": null}, {"id": 10847484003, "name": "William Penn University", "priority": 1662, "external_id": null}, {"id": 10847485003, "name": "William Woods University", "priority": 1663, "external_id": null}, {"id": 10847486003, "name": "Williams Baptist College", "priority": 1664, "external_id": null}, {"id": 10847487003, "name": "Williams College", "priority": 1665, "external_id": null}, {"id": 10847488003, "name": "Wilmington College", "priority": 1666, "external_id": null}, {"id": 10847489003, "name": "Wilmington University", "priority": 1667, "external_id": null}, {"id": 10847490003, "name": "Wilson College", "priority": 1668, "external_id": null}, {"id": 10847491003, "name": "Wingate University", "priority": 1669, "external_id": null}, {"id": 10847492003, "name": "Winona State University", "priority": 1670, "external_id": null}, {"id": 10847493003, "name": "Winston-Salem State University", "priority": 1671, "external_id": null}, {"id": 10847494003, "name": "Winthrop University", "priority": 1672, "external_id": null}, {"id": 10847495003, "name": "Wisconsin Lutheran College", "priority": 1673, "external_id": null}, {"id": 10847496003, "name": "Wittenberg University", "priority": 1674, "external_id": null}, {"id": 10847497003, "name": "Woodbury University", "priority": 1675, "external_id": null}, {"id": 10847498003, "name": "Worcester Polytechnic Institute", "priority": 1676, "external_id": null}, {"id": 10847499003, "name": "Worcester State University", "priority": 1677, "external_id": null}, {"id": 10847500003, "name": "Wright State University", "priority": 1678, "external_id": null}, {"id": 10847501003, "name": "Xavier University", "priority": 1679, "external_id": null}, {"id": 10847502003, "name": "Xavier University of Louisiana", "priority": 1680, "external_id": null}, {"id": 10847503003, "name": "Yeshiva University", "priority": 1681, "external_id": null}, {"id": 10847504003, "name": "York College", "priority": 1682, "external_id": null}, {"id": 10847505003, "name": "York College of Pennsylvania", "priority": 1683, "external_id": null}, {"id": 10847506003, "name": "York University", "priority": 1684, "external_id": null}, {"id": 10847507003, "name": "University of Cambridge", "priority": 1685, "external_id": null}, {"id": 10847508003, "name": "UCL (University College London)", "priority": 1686, "external_id": null}, {"id": 10847509003, "name": "Imperial College London", "priority": 1687, "external_id": null}, {"id": 10847510003, "name": "University of Oxford", "priority": 1688, "external_id": null}, {"id": 10847511003, "name": "ETH Zurich (Swiss Federal Institute of Technology)", "priority": 1689, "external_id": null}, {"id": 10847512003, "name": "University of Edinburgh", "priority": 1690, "external_id": null}, {"id": 10847513003, "name": "Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne", "priority": 1691, "external_id": null}, {"id": 10847514003, "name": "King's College London (KCL)", "priority": 1692, "external_id": null}, {"id": 10847515003, "name": "National University of Singapore (NUS)", "priority": 1693, "external_id": null}, {"id": 10847516003, "name": "University of Hong Kong", "priority": 1694, "external_id": null}, {"id": 10847517003, "name": "Australian National University", "priority": 1695, "external_id": null}, {"id": 10847518003, "name": "Ecole normale sup\u00e9rieure, Paris", "priority": 1696, "external_id": null}, {"id": 10847519003, "name": "University of Bristol", "priority": 1697, "external_id": null}, {"id": 10847520003, "name": "The University of Melbourne", "priority": 1698, "external_id": null}, {"id": 10847521003, "name": "The University of Tokyo", "priority": 1699, "external_id": null}, {"id": 10847522003, "name": "The University of Manchester", "priority": 1700, "external_id": null}, {"id": 10847523003, "name": "Western Illinois University", "priority": 1701, "external_id": null}, {"id": 10847524003, "name": "Wofford College", "priority": 1702, "external_id": null}, {"id": 10847525003, "name": "Western Carolina University", "priority": 1703, "external_id": null}, {"id": 10847526003, "name": "West Virginia University", "priority": 1704, "external_id": null}, {"id": 10847527003, "name": "Yale University", "priority": 1705, "external_id": null}, {"id": 10847528003, "name": "The Hong Kong University of Science and Technology", "priority": 1706, "external_id": null}, {"id": 10847529003, "name": "Kyoto University", "priority": 1707, "external_id": null}, {"id": 10847530003, "name": "Seoul National University", "priority": 1708, "external_id": null}, {"id": 10847531003, "name": "The University of Sydney", "priority": 1709, "external_id": null}, {"id": 10847532003, "name": "The Chinese University of Hong Kong", "priority": 1710, "external_id": null}, {"id": 10847533003, "name": "Ecole Polytechnique", "priority": 1711, "external_id": null}, {"id": 10847534003, "name": "Nanyang Technological University (NTU)", "priority": 1712, "external_id": null}, {"id": 10847535003, "name": "The University of Queensland", "priority": 1713, "external_id": null}, {"id": 10847536003, "name": "University of Copenhagen", "priority": 1714, "external_id": null}, {"id": 10847537003, "name": "Peking University", "priority": 1715, "external_id": null}, {"id": 10847538003, "name": "Tsinghua University", "priority": 1716, "external_id": null}, {"id": 10847539003, "name": "Ruprecht-Karls-Universit\u00e4t Heidelberg", "priority": 1717, "external_id": null}, {"id": 10847540003, "name": "University of Glasgow", "priority": 1718, "external_id": null}, {"id": 10847541003, "name": "The University of New South Wales", "priority": 1719, "external_id": null}, {"id": 10847542003, "name": "Technische Universit\u00e4t M\u00fcnchen", "priority": 1720, "external_id": null}, {"id": 10847543003, "name": "Osaka University", "priority": 1721, "external_id": null}, {"id": 10847544003, "name": "University of Amsterdam", "priority": 1722, "external_id": null}, {"id": 10847545003, "name": "KAIST - Korea Advanced Institute of Science & Technology", "priority": 1723, "external_id": null}, {"id": 10847546003, "name": "Trinity College Dublin", "priority": 1724, "external_id": null}, {"id": 10847547003, "name": "University of Birmingham", "priority": 1725, "external_id": null}, {"id": 10847548003, "name": "The University of Warwick", "priority": 1726, "external_id": null}, {"id": 10847549003, "name": "Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen", "priority": 1727, "external_id": null}, {"id": 10847550003, "name": "Tokyo Institute of Technology", "priority": 1728, "external_id": null}, {"id": 10847551003, "name": "Lund University", "priority": 1729, "external_id": null}, {"id": 10847552003, "name": "London School of Economics and Political Science (LSE)", "priority": 1730, "external_id": null}, {"id": 10847553003, "name": "Monash University", "priority": 1731, "external_id": null}, {"id": 10847554003, "name": "University of Helsinki", "priority": 1732, "external_id": null}, {"id": 10847555003, "name": "The University of Sheffield", "priority": 1733, "external_id": null}, {"id": 10847556003, "name": "University of Geneva", "priority": 1734, "external_id": null}, {"id": 10847557003, "name": "Leiden University", "priority": 1735, "external_id": null}, {"id": 10847558003, "name": "The University of Nottingham", "priority": 1736, "external_id": null}, {"id": 10847559003, "name": "Tohoku University", "priority": 1737, "external_id": null}, {"id": 10847560003, "name": "KU Leuven", "priority": 1738, "external_id": null}, {"id": 10847561003, "name": "University of Zurich", "priority": 1739, "external_id": null}, {"id": 10847562003, "name": "Uppsala University", "priority": 1740, "external_id": null}, {"id": 10847563003, "name": "Utrecht University", "priority": 1741, "external_id": null}, {"id": 10847564003, "name": "National Taiwan University (NTU)", "priority": 1742, "external_id": null}, {"id": 10847565003, "name": "University of St Andrews", "priority": 1743, "external_id": null}, {"id": 10847566003, "name": "The University of Western Australia", "priority": 1744, "external_id": null}, {"id": 10847567003, "name": "University of Southampton", "priority": 1745, "external_id": null}, {"id": 10847568003, "name": "Fudan University", "priority": 1746, "external_id": null}, {"id": 10847569003, "name": "University of Oslo", "priority": 1747, "external_id": null}, {"id": 10847570003, "name": "Durham University", "priority": 1748, "external_id": null}, {"id": 10847571003, "name": "Aarhus University", "priority": 1749, "external_id": null}, {"id": 10847572003, "name": "Erasmus University Rotterdam", "priority": 1750, "external_id": null}, {"id": 10847573003, "name": "Universit\u00e9 de Montr\u00e9al", "priority": 1751, "external_id": null}, {"id": 10847574003, "name": "The University of Auckland", "priority": 1752, "external_id": null}, {"id": 10847575003, "name": "Delft University of Technology", "priority": 1753, "external_id": null}, {"id": 10847576003, "name": "University of Groningen", "priority": 1754, "external_id": null}, {"id": 10847577003, "name": "University of Leeds", "priority": 1755, "external_id": null}, {"id": 10847578003, "name": "Nagoya University", "priority": 1756, "external_id": null}, {"id": 10847579003, "name": "Universit\u00e4t Freiburg", "priority": 1757, "external_id": null}, {"id": 10847580003, "name": "City University of Hong Kong", "priority": 1758, "external_id": null}, {"id": 10847581003, "name": "The University of Adelaide", "priority": 1759, "external_id": null}, {"id": 10847582003, "name": "Pohang University of Science And Technology (POSTECH)", "priority": 1760, "external_id": null}, {"id": 10847583003, "name": "Freie Universit\u00e4t Berlin", "priority": 1761, "external_id": null}, {"id": 10847584003, "name": "University of Basel", "priority": 1762, "external_id": null}, {"id": 10847585003, "name": "University of Lausanne", "priority": 1763, "external_id": null}, {"id": 10847586003, "name": "Universit\u00e9 Pierre et Marie Curie (UPMC)", "priority": 1764, "external_id": null}, {"id": 10847587003, "name": "Yonsei University", "priority": 1765, "external_id": null}, {"id": 10847588003, "name": "University of York", "priority": 1766, "external_id": null}, {"id": 10847589003, "name": "Queen Mary, University of London (QMUL)", "priority": 1767, "external_id": null}, {"id": 10847590003, "name": "Karlsruhe Institute of Technology (KIT)", "priority": 1768, "external_id": null}, {"id": 10847591003, "name": "KTH, Royal Institute of Technology", "priority": 1769, "external_id": null}, {"id": 10847592003, "name": "Lomonosov Moscow State University", "priority": 1770, "external_id": null}, {"id": 10847593003, "name": "Maastricht University", "priority": 1771, "external_id": null}, {"id": 10847594003, "name": "University of Ghent", "priority": 1772, "external_id": null}, {"id": 10847595003, "name": "Shanghai Jiao Tong University", "priority": 1773, "external_id": null}, {"id": 10847596003, "name": "Humboldt-Universit\u00e4t zu Berlin", "priority": 1774, "external_id": null}, {"id": 10847597003, "name": "Universidade de S\u00e3o Paulo (USP)", "priority": 1775, "external_id": null}, {"id": 10847598003, "name": "Georg-August-Universit\u00e4t G\u00f6ttingen", "priority": 1776, "external_id": null}, {"id": 10847599003, "name": "Newcastle University", "priority": 1777, "external_id": null}, {"id": 10847600003, "name": "University of Liverpool", "priority": 1778, "external_id": null}, {"id": 10847601003, "name": "Kyushu University", "priority": 1779, "external_id": null}, {"id": 10847602003, "name": "Eberhard Karls Universit\u00e4t T\u00fcbingen", "priority": 1780, "external_id": null}, {"id": 10847603003, "name": "Technical University of Denmark", "priority": 1781, "external_id": null}, {"id": 10847604003, "name": "Cardiff University", "priority": 1782, "external_id": null}, {"id": 10847605003, "name": "Universit\u00e9 Catholique de Louvain (UCL)", "priority": 1783, "external_id": null}, {"id": 10847606003, "name": "University College Dublin", "priority": 1784, "external_id": null}, {"id": 10847607003, "name": "McMaster University", "priority": 1785, "external_id": null}, {"id": 10847608003, "name": "Hebrew University of Jerusalem", "priority": 1786, "external_id": null}, {"id": 10847609003, "name": "Radboud University Nijmegen", "priority": 1787, "external_id": null}, {"id": 10847610003, "name": "Hokkaido University", "priority": 1788, "external_id": null}, {"id": 10847611003, "name": "Korea University", "priority": 1789, "external_id": null}, {"id": 10847612003, "name": "University of Cape Town", "priority": 1790, "external_id": null}, {"id": 10847613003, "name": "Rheinisch-Westf\u00e4lische Technische Hochschule Aachen", "priority": 1791, "external_id": null}, {"id": 10847614003, "name": "University of Aberdeen", "priority": 1792, "external_id": null}, {"id": 10847615003, "name": "Wageningen University", "priority": 1793, "external_id": null}, {"id": 10847616003, "name": "University of Bergen", "priority": 1794, "external_id": null}, {"id": 10847617003, "name": "University of Bern", "priority": 1795, "external_id": null}, {"id": 10847618003, "name": "University of Otago", "priority": 1796, "external_id": null}, {"id": 10847619003, "name": "Lancaster University", "priority": 1797, "external_id": null}, {"id": 10847620003, "name": "Eindhoven University of Technology", "priority": 1798, "external_id": null}, {"id": 10847621003, "name": "Ecole Normale Sup\u00e9rieure de Lyon", "priority": 1799, "external_id": null}, {"id": 10847622003, "name": "University of Vienna", "priority": 1800, "external_id": null}, {"id": 10847623003, "name": "The Hong Kong Polytechnic University", "priority": 1801, "external_id": null}, {"id": 10847624003, "name": "Sungkyunkwan University", "priority": 1802, "external_id": null}, {"id": 10847625003, "name": "Rheinische Friedrich-Wilhelms-Universit\u00e4t Bonn", "priority": 1803, "external_id": null}, {"id": 10847626003, "name": "Universidad Nacional Aut\u00f3noma de M\u00e9xico (UNAM)", "priority": 1804, "external_id": null}, {"id": 10847627003, "name": "Zhejiang University", "priority": 1805, "external_id": null}, {"id": 10847628003, "name": "Pontificia Universidad Cat\u00f3lica de Chile", "priority": 1806, "external_id": null}, {"id": 10847629003, "name": "Universiti Malaya (UM)", "priority": 1807, "external_id": null}, {"id": 10847630003, "name": "Universit\u00e9 Libre de Bruxelles (ULB)", "priority": 1808, "external_id": null}, {"id": 10847631003, "name": "University of Exeter", "priority": 1809, "external_id": null}, {"id": 10847632003, "name": "Stockholm University", "priority": 1810, "external_id": null}, {"id": 10847633003, "name": "Queen's University of Belfast", "priority": 1811, "external_id": null}, {"id": 10847634003, "name": "Vrije Universiteit Brussel (VUB)", "priority": 1812, "external_id": null}, {"id": 10847635003, "name": "University of Science and Technology of China", "priority": 1813, "external_id": null}, {"id": 10847636003, "name": "Nanjing University", "priority": 1814, "external_id": null}, {"id": 10847637003, "name": "Universitat Aut\u00f3noma de Barcelona", "priority": 1815, "external_id": null}, {"id": 10847638003, "name": "University of Barcelona", "priority": 1816, "external_id": null}, {"id": 10847639003, "name": "VU University Amsterdam", "priority": 1817, "external_id": null}, {"id": 10847640003, "name": "Technion - Israel Institute of Technology", "priority": 1818, "external_id": null}, {"id": 10847641003, "name": "Technische Universit\u00e4t Berlin", "priority": 1819, "external_id": null}, {"id": 10847642003, "name": "University of Antwerp", "priority": 1820, "external_id": null}, {"id": 10847643003, "name": "Universit\u00e4t Hamburg", "priority": 1821, "external_id": null}, {"id": 10847644003, "name": "University of Bath", "priority": 1822, "external_id": null}, {"id": 10847645003, "name": "University of Bologna", "priority": 1823, "external_id": null}, {"id": 10847646003, "name": "Queen's University, Ontario", "priority": 1824, "external_id": null}, {"id": 10847647003, "name": "Universit\u00e9 Paris-Sud 11", "priority": 1825, "external_id": null}, {"id": 10847648003, "name": "Keio University", "priority": 1826, "external_id": null}, {"id": 10847649003, "name": "University of Sussex", "priority": 1827, "external_id": null}, {"id": 10847650003, "name": "Universidad Aut\u00f3noma de Madrid", "priority": 1828, "external_id": null}, {"id": 10847651003, "name": "Aalto University", "priority": 1829, "external_id": null}, {"id": 10847652003, "name": "Sapienza University of Rome", "priority": 1830, "external_id": null}, {"id": 10847653003, "name": "Tel Aviv University", "priority": 1831, "external_id": null}, {"id": 10847654003, "name": "National Tsing Hua University", "priority": 1832, "external_id": null}, {"id": 10847655003, "name": "Chalmers University of Technology", "priority": 1833, "external_id": null}, {"id": 10847656003, "name": "University of Leicester", "priority": 1834, "external_id": null}, {"id": 10847657003, "name": "Universit\u00e9 Paris Diderot - Paris 7", "priority": 1835, "external_id": null}, {"id": 10847658003, "name": "University of Gothenburg", "priority": 1836, "external_id": null}, {"id": 10847659003, "name": "University of Turku", "priority": 1837, "external_id": null}, {"id": 10847660003, "name": "Universit\u00e4t Frankfurt am Main", "priority": 1838, "external_id": null}, {"id": 10847661003, "name": "Universidad de Buenos Aires", "priority": 1839, "external_id": null}, {"id": 10847662003, "name": "University College Cork", "priority": 1840, "external_id": null}, {"id": 10847663003, "name": "University of Tsukuba", "priority": 1841, "external_id": null}, {"id": 10847664003, "name": "University of Reading", "priority": 1842, "external_id": null}, {"id": 10847665003, "name": "Sciences Po Paris", "priority": 1843, "external_id": null}, {"id": 10847666003, "name": "Universidade Estadual de Campinas", "priority": 1844, "external_id": null}, {"id": 10847667003, "name": "King Fahd University of Petroleum & Minerals", "priority": 1845, "external_id": null}, {"id": 10847668003, "name": "University Complutense Madrid", "priority": 1846, "external_id": null}, {"id": 10847669003, "name": "Universit\u00e9 Paris-Sorbonne (Paris IV)", "priority": 1847, "external_id": null}, {"id": 10847670003, "name": "University of Dundee", "priority": 1848, "external_id": null}, {"id": 10847671003, "name": "Universit\u00e9 Joseph Fourier - Grenoble 1", "priority": 1849, "external_id": null}, {"id": 10847672003, "name": "Waseda University", "priority": 1850, "external_id": null}, {"id": 10847673003, "name": "Indian Institute of Technology Delhi (IITD)", "priority": 1851, "external_id": null}, {"id": 10847674003, "name": "Universidad de Chile", "priority": 1852, "external_id": null}, {"id": 10847675003, "name": "Universit\u00e9 Paris 1 Panth\u00e9on-Sorbonne", "priority": 1853, "external_id": null}, {"id": 10847676003, "name": "Universit\u00e9 de Strasbourg", "priority": 1854, "external_id": null}, {"id": 10847677003, "name": "University of Twente", "priority": 1855, "external_id": null}, {"id": 10847678003, "name": "University of East Anglia (UEA)", "priority": 1856, "external_id": null}, {"id": 10847679003, "name": "National Chiao Tung University", "priority": 1857, "external_id": null}, {"id": 10847680003, "name": "Politecnico di Milano", "priority": 1858, "external_id": null}, {"id": 10847681003, "name": "Charles University", "priority": 1859, "external_id": null}, {"id": 10847682003, "name": "Indian Institute of Technology Bombay (IITB)", "priority": 1860, "external_id": null}, {"id": 10847683003, "name": "University of Milano", "priority": 1861, "external_id": null}, {"id": 10847684003, "name": "Westf\u00e4lische Wilhelms-Universit\u00e4t M\u00fcnster", "priority": 1862, "external_id": null}, {"id": 10847685003, "name": "University of Canterbury", "priority": 1863, "external_id": null}, {"id": 10847686003, "name": "Chulalongkorn University", "priority": 1864, "external_id": null}, {"id": 10847687003, "name": "Saint-Petersburg State University", "priority": 1865, "external_id": null}, {"id": 10847688003, "name": "University of Liege", "priority": 1866, "external_id": null}, {"id": 10847689003, "name": "Universit\u00e4t zu K\u00f6ln", "priority": 1867, "external_id": null}, {"id": 10847690003, "name": "Loughborough University", "priority": 1868, "external_id": null}, {"id": 10847691003, "name": "National Cheng Kung University", "priority": 1869, "external_id": null}, {"id": 10847692003, "name": "Universit\u00e4t Stuttgart", "priority": 1870, "external_id": null}, {"id": 10847693003, "name": "Hanyang University", "priority": 1871, "external_id": null}, {"id": 10847694003, "name": "American University of Beirut (AUB)", "priority": 1872, "external_id": null}, {"id": 10847695003, "name": "Norwegian University of Science And Technology", "priority": 1873, "external_id": null}, {"id": 10847696003, "name": "Beijing Normal University", "priority": 1874, "external_id": null}, {"id": 10847697003, "name": "King Saud University", "priority": 1875, "external_id": null}, {"id": 10847698003, "name": "University of Oulu", "priority": 1876, "external_id": null}, {"id": 10847699003, "name": "Kyung Hee University", "priority": 1877, "external_id": null}, {"id": 10847700003, "name": "University of Strathclyde", "priority": 1878, "external_id": null}, {"id": 10847701003, "name": "Universit\u00e4t Ulm", "priority": 1879, "external_id": null}, {"id": 10847702003, "name": "University of Pisa", "priority": 1880, "external_id": null}, {"id": 10847703003, "name": "Technische Universit\u00e4t Darmstadt", "priority": 1881, "external_id": null}, {"id": 10847704003, "name": "Technische Universit\u00e4t Dresden", "priority": 1882, "external_id": null}, {"id": 10847705003, "name": "Macquarie University", "priority": 1883, "external_id": null}, {"id": 10847706003, "name": "Vienna University of Technology", "priority": 1884, "external_id": null}, {"id": 10847707003, "name": "Royal Holloway University of London", "priority": 1885, "external_id": null}, {"id": 10847708003, "name": "Victoria University of Wellington", "priority": 1886, "external_id": null}, {"id": 10847709003, "name": "University of Padua", "priority": 1887, "external_id": null}, {"id": 10847710003, "name": "Universiti Kebangsaan Malaysia (UKM)", "priority": 1888, "external_id": null}, {"id": 10847711003, "name": "University of Technology, Sydney", "priority": 1889, "external_id": null}, {"id": 10847712003, "name": "Universit\u00e4t Konstanz", "priority": 1890, "external_id": null}, {"id": 10847713003, "name": "Universidad de Los Andes Colombia", "priority": 1891, "external_id": null}, {"id": 10847714003, "name": "Universit\u00e9 Paris Descartes", "priority": 1892, "external_id": null}, {"id": 10847715003, "name": "Tokyo Medical and Dental University", "priority": 1893, "external_id": null}, {"id": 10847716003, "name": "University of Wollongong", "priority": 1894, "external_id": null}, {"id": 10847717003, "name": "Universit\u00e4t Erlangen-N\u00fcrnberg", "priority": 1895, "external_id": null}, {"id": 10847718003, "name": "Queensland University of Technology", "priority": 1896, "external_id": null}, {"id": 10847719003, "name": "Tecnol\u00f3gico de Monterrey (ITESM)", "priority": 1897, "external_id": null}, {"id": 10847720003, "name": "Universit\u00e4t Mannheim", "priority": 1898, "external_id": null}, {"id": 10847721003, "name": "Universitat Pompeu Fabra", "priority": 1899, "external_id": null}, {"id": 10847722003, "name": "Mahidol University", "priority": 1900, "external_id": null}, {"id": 10847723003, "name": "Curtin University", "priority": 1901, "external_id": null}, {"id": 10847724003, "name": "National University of Ireland, Galway", "priority": 1902, "external_id": null}, {"id": 10847725003, "name": "Universidade Federal do Rio de Janeiro", "priority": 1903, "external_id": null}, {"id": 10847726003, "name": "University of Surrey", "priority": 1904, "external_id": null}, {"id": 10847727003, "name": "Hong Kong Baptist University", "priority": 1905, "external_id": null}, {"id": 10847728003, "name": "Ume\u00e5 University", "priority": 1906, "external_id": null}, {"id": 10847729003, "name": "Universit\u00e4t Innsbruck", "priority": 1907, "external_id": null}, {"id": 10847730003, "name": "RMIT University", "priority": 1908, "external_id": null}, {"id": 10847731003, "name": "University of Eastern Finland", "priority": 1909, "external_id": null}, {"id": 10847732003, "name": "Christian-Albrechts-Universit\u00e4t zu Kiel", "priority": 1910, "external_id": null}, {"id": 10847733003, "name": "Indian Institute of Technology Kanpur (IITK)", "priority": 1911, "external_id": null}, {"id": 10847734003, "name": "National Yang Ming University", "priority": 1912, "external_id": null}, {"id": 10847735003, "name": "Johannes Gutenberg Universit\u00e4t Mainz", "priority": 1913, "external_id": null}, {"id": 10847736003, "name": "The University of Newcastle", "priority": 1914, "external_id": null}, {"id": 10847737003, "name": "Al-Farabi Kazakh National University", "priority": 1915, "external_id": null}, {"id": 10847738003, "name": "\u00c9cole des Ponts ParisTech", "priority": 1916, "external_id": null}, {"id": 10847739003, "name": "University of Jyv\u00e4skyl\u00e4", "priority": 1917, "external_id": null}, {"id": 10847740003, "name": "L.N. Gumilyov Eurasian National University", "priority": 1918, "external_id": null}, {"id": 10847741003, "name": "Kobe University", "priority": 1919, "external_id": null}, {"id": 10847742003, "name": "University of Tromso", "priority": 1920, "external_id": null}, {"id": 10847743003, "name": "Hiroshima University", "priority": 1921, "external_id": null}, {"id": 10847744003, "name": "Universit\u00e9 Bordeaux 1, Sciences Technologies", "priority": 1922, "external_id": null}, {"id": 10847745003, "name": "University of Indonesia", "priority": 1923, "external_id": null}, {"id": 10847746003, "name": "Universit\u00e4t Leipzig", "priority": 1924, "external_id": null}, {"id": 10847747003, "name": "University of Southern Denmark", "priority": 1925, "external_id": null}, {"id": 10847748003, "name": "Indian Institute of Technology Madras (IITM)", "priority": 1926, "external_id": null}, {"id": 10847749003, "name": "University of The Witwatersrand", "priority": 1927, "external_id": null}, {"id": 10847750003, "name": "University of Navarra", "priority": 1928, "external_id": null}, {"id": 10847751003, "name": "Universidad Austral - Argentina", "priority": 1929, "external_id": null}, {"id": 10847752003, "name": "Universidad Carlos III de Madrid", "priority": 1930, "external_id": null}, {"id": 10847753003, "name": "Universit\u00e0\u00a1 degli Studi di Roma - Tor Vergata", "priority": 1931, "external_id": null}, {"id": 10847754003, "name": "Pontificia Universidad Cat\u00f3lica Argentina Santa Mar\u00eda de los Buenos Aires", "priority": 1932, "external_id": null}, {"id": 10847755003, "name": "UCA", "priority": 1933, "external_id": null}, {"id": 10847756003, "name": "Julius-Maximilians-Universit\u00e4t W\u00fcrzburg", "priority": 1934, "external_id": null}, {"id": 10847757003, "name": "Universidad Nacional de Colombia", "priority": 1935, "external_id": null}, {"id": 10847758003, "name": "Laval University", "priority": 1936, "external_id": null}, {"id": 10847759003, "name": "Ben Gurion University of The Negev", "priority": 1937, "external_id": null}, {"id": 10847760003, "name": "Link\u00f6ping University", "priority": 1938, "external_id": null}, {"id": 10847761003, "name": "Aalborg University", "priority": 1939, "external_id": null}, {"id": 10847762003, "name": "Bauman Moscow State Technical University", "priority": 1940, "external_id": null}, {"id": 10847763003, "name": "Ecole Normale Sup\u00e9rieure de Cachan", "priority": 1941, "external_id": null}, {"id": 10847764003, "name": "SOAS - School of Oriental and African Studies, University of London", "priority": 1942, "external_id": null}, {"id": 10847765003, "name": "University of Essex", "priority": 1943, "external_id": null}, {"id": 10847766003, "name": "University of Warsaw", "priority": 1944, "external_id": null}, {"id": 10847767003, "name": "Griffith University", "priority": 1945, "external_id": null}, {"id": 10847768003, "name": "University of South Australia", "priority": 1946, "external_id": null}, {"id": 10847769003, "name": "Massey University", "priority": 1947, "external_id": null}, {"id": 10847770003, "name": "University of Porto", "priority": 1948, "external_id": null}, {"id": 10847771003, "name": "Universitat Polit\u00e8cnica de Catalunya", "priority": 1949, "external_id": null}, {"id": 10847772003, "name": "Indian Institute of Technology Kharagpur (IITKGP)", "priority": 1950, "external_id": null}, {"id": 10847773003, "name": "City University London", "priority": 1951, "external_id": null}, {"id": 10847774003, "name": "Dublin City University", "priority": 1952, "external_id": null}, {"id": 10847775003, "name": "Pontificia Universidad Javeriana", "priority": 1953, "external_id": null}, {"id": 10847776003, "name": "James Cook University", "priority": 1954, "external_id": null}, {"id": 10847777003, "name": "Novosibirsk State University", "priority": 1955, "external_id": null}, {"id": 10847778003, "name": "Universidade Nova de Lisboa", "priority": 1956, "external_id": null}, {"id": 10847779003, "name": "Universit\u00e9 Aix-Marseille", "priority": 1957, "external_id": null}, {"id": 10847780003, "name": "Universiti Sains Malaysia (USM)", "priority": 1958, "external_id": null}, {"id": 10847781003, "name": "Universiti Teknologi Malaysia (UTM)", "priority": 1959, "external_id": null}, {"id": 10847782003, "name": "Universit\u00e9 Paris Dauphine", "priority": 1960, "external_id": null}, {"id": 10847783003, "name": "University of Coimbra", "priority": 1961, "external_id": null}, {"id": 10847784003, "name": "Brunel University", "priority": 1962, "external_id": null}, {"id": 10847785003, "name": "King Abdul Aziz University (KAU)", "priority": 1963, "external_id": null}, {"id": 10847786003, "name": "Ewha Womans University", "priority": 1964, "external_id": null}, {"id": 10847787003, "name": "Nankai University", "priority": 1965, "external_id": null}, {"id": 10847788003, "name": "Taipei Medical University", "priority": 1966, "external_id": null}, {"id": 10847789003, "name": "Universit\u00e4t Jena", "priority": 1967, "external_id": null}, {"id": 10847790003, "name": "Ruhr-Universit\u00e4t Bochum", "priority": 1968, "external_id": null}, {"id": 10847791003, "name": "Heriot-Watt University", "priority": 1969, "external_id": null}, {"id": 10847792003, "name": "Politecnico di Torino", "priority": 1970, "external_id": null}, {"id": 10847793003, "name": "Universit\u00e4t Bremen", "priority": 1971, "external_id": null}, {"id": 10847794003, "name": "Xi'an Jiaotong University", "priority": 1972, "external_id": null}, {"id": 10847795003, "name": "Birkbeck College, University of London", "priority": 1973, "external_id": null}, {"id": 10847796003, "name": "Oxford Brookes University", "priority": 1974, "external_id": null}, {"id": 10847797003, "name": "Jagiellonian University", "priority": 1975, "external_id": null}, {"id": 10847798003, "name": "University of Tampere", "priority": 1976, "external_id": null}, {"id": 10847799003, "name": "University of Florence", "priority": 1977, "external_id": null}, {"id": 10847800003, "name": "Deakin University", "priority": 1978, "external_id": null}, {"id": 10847801003, "name": "University of the Philippines", "priority": 1979, "external_id": null}, {"id": 10847802003, "name": "Universitat Polit\u00e8cnica de Val\u00e8ncia", "priority": 1980, "external_id": null}, {"id": 10847803003, "name": "Sun Yat-sen University", "priority": 1981, "external_id": null}, {"id": 10847804003, "name": "Universit\u00e9 Montpellier 2, Sciences et Techniques du Languedoc", "priority": 1982, "external_id": null}, {"id": 10847805003, "name": "Moscow State Institute of International Relations (MGIMO-University)", "priority": 1983, "external_id": null}, {"id": 10847806003, "name": "Stellenbosch University", "priority": 1984, "external_id": null}, {"id": 10847807003, "name": "Polit\u00e9cnica de Madrid", "priority": 1985, "external_id": null}, {"id": 10847808003, "name": "Instituto Tecnol\u00f3gico de Buenos Aires (ITBA)", "priority": 1986, "external_id": null}, {"id": 10847809003, "name": "La Trobe University", "priority": 1987, "external_id": null}, {"id": 10847810003, "name": "Universit\u00e9 Paul Sabatier Toulouse III", "priority": 1988, "external_id": null}, {"id": 10847811003, "name": "Karl-Franzens-Universit\u00e4t Graz", "priority": 1989, "external_id": null}, {"id": 10847812003, "name": "Universit\u00e4t D\u00fcsseldorf", "priority": 1990, "external_id": null}, {"id": 10847813003, "name": "University of Naples - Federico Ii", "priority": 1991, "external_id": null}, {"id": 10847814003, "name": "Aston University", "priority": 1992, "external_id": null}, {"id": 10847815003, "name": "University of Turin", "priority": 1993, "external_id": null}, {"id": 10847816003, "name": "Beihang University (former BUAA)", "priority": 1994, "external_id": null}, {"id": 10847817003, "name": "Indian Institute of Technology Roorkee (IITR)", "priority": 1995, "external_id": null}, {"id": 10847818003, "name": "National Central University", "priority": 1996, "external_id": null}, {"id": 10847819003, "name": "Sogang University", "priority": 1997, "external_id": null}, {"id": 10847820003, "name": "Universit\u00e4t Regensburg", "priority": 1998, "external_id": null}, {"id": 10847821003, "name": "Universit\u00e9 Lille 1, Sciences et Technologie", "priority": 1999, "external_id": null}, {"id": 10847822003, "name": "University of Tasmania", "priority": 2000, "external_id": null}, {"id": 10847823003, "name": "University of Waikato", "priority": 2001, "external_id": null}, {"id": 10847824003, "name": "Wuhan University", "priority": 2002, "external_id": null}, {"id": 10847825003, "name": "National Taiwan University of Science And Technology", "priority": 2003, "external_id": null}, {"id": 10847826003, "name": "Universidade Federal de S\u00e3o Paulo (UNIFESP)", "priority": 2004, "external_id": null}, {"id": 10847827003, "name": "Universit\u00e0 degli Studi di Pavia", "priority": 2005, "external_id": null}, {"id": 10847828003, "name": "Universit\u00e4t Bayreuth", "priority": 2006, "external_id": null}, {"id": 10847829003, "name": "Universit\u00e9 Claude Bernard Lyon 1", "priority": 2007, "external_id": null}, {"id": 10847830003, "name": "Universit\u00e9 du Qu\u00e9bec", "priority": 2008, "external_id": null}, {"id": 10847831003, "name": "Universiti Putra Malaysia (UPM)", "priority": 2009, "external_id": null}, {"id": 10847832003, "name": "University of Kent", "priority": 2010, "external_id": null}, {"id": 10847833003, "name": "University of St Gallen (HSG)", "priority": 2011, "external_id": null}, {"id": 10847834003, "name": "Bond University", "priority": 2012, "external_id": null}, {"id": 10847835003, "name": "United Arab Emirates University", "priority": 2013, "external_id": null}, {"id": 10847836003, "name": "Universidad de San Andr\u00c3\u00a9s", "priority": 2014, "external_id": null}, {"id": 10847837003, "name": "Universidad Nacional de La Plata", "priority": 2015, "external_id": null}, {"id": 10847838003, "name": "Universit\u00e4t des Saarlandes", "priority": 2016, "external_id": null}, {"id": 10847839003, "name": "American University of Sharjah (AUS)", "priority": 2017, "external_id": null}, {"id": 10847840003, "name": "Bilkent University", "priority": 2018, "external_id": null}, {"id": 10847841003, "name": "Flinders University", "priority": 2019, "external_id": null}, {"id": 10847842003, "name": "Hankuk (Korea) University of Foreign Studies", "priority": 2020, "external_id": null}, {"id": 10847843003, "name": "Middle East Technical University", "priority": 2021, "external_id": null}, {"id": 10847844003, "name": "Philipps-Universit\u00e4t Marburg", "priority": 2022, "external_id": null}, {"id": 10847845003, "name": "Swansea University", "priority": 2023, "external_id": null}, {"id": 10847846003, "name": "Tampere University of Technology", "priority": 2024, "external_id": null}, {"id": 10847847003, "name": "Universit\u00e4t Bielefeld", "priority": 2025, "external_id": null}, {"id": 10847848003, "name": "University of Manitoba", "priority": 2026, "external_id": null}, {"id": 10847849003, "name": "Chiba University", "priority": 2027, "external_id": null}, {"id": 10847850003, "name": "Moscow Institute of Physics and Technology State University", "priority": 2028, "external_id": null}, {"id": 10847851003, "name": "Tallinn University of Technology", "priority": 2029, "external_id": null}, {"id": 10847852003, "name": "Taras Shevchenko National University of Kyiv", "priority": 2030, "external_id": null}, {"id": 10847853003, "name": "Tokyo University of Science", "priority": 2031, "external_id": null}, {"id": 10847854003, "name": "University of Salamanca", "priority": 2032, "external_id": null}, {"id": 10847855003, "name": "University of Trento", "priority": 2033, "external_id": null}, {"id": 10847856003, "name": "Universit\u00e9 de Sherbrooke", "priority": 2034, "external_id": null}, {"id": 10847857003, "name": "Universit\u00e9 Panth\u00e9on-Assas (Paris 2)", "priority": 2035, "external_id": null}, {"id": 10847858003, "name": "University of Delhi", "priority": 2036, "external_id": null}, {"id": 10847859003, "name": "Abo Akademi University", "priority": 2037, "external_id": null}, {"id": 10847860003, "name": "Czech Technical University In Prague", "priority": 2038, "external_id": null}, {"id": 10847861003, "name": "Leibniz Universit\u00e4t Hannover", "priority": 2039, "external_id": null}, {"id": 10847862003, "name": "Pusan National University", "priority": 2040, "external_id": null}, {"id": 10847863003, "name": "Shanghai University", "priority": 2041, "external_id": null}, {"id": 10847864003, "name": "St. Petersburg State Politechnical University", "priority": 2042, "external_id": null}, {"id": 10847865003, "name": "Universit\u00e0 Cattolica del Sacro Cuore", "priority": 2043, "external_id": null}, {"id": 10847866003, "name": "University of Genoa", "priority": 2044, "external_id": null}, {"id": 10847867003, "name": "Bandung Institute of Technology (ITB)", "priority": 2045, "external_id": null}, {"id": 10847868003, "name": "Bogazici University", "priority": 2046, "external_id": null}, {"id": 10847869003, "name": "Goldsmiths, University of London", "priority": 2047, "external_id": null}, {"id": 10847870003, "name": "National Sun Yat-sen University", "priority": 2048, "external_id": null}, {"id": 10847871003, "name": "Renmin (People\u2019s) University of China", "priority": 2049, "external_id": null}, {"id": 10847872003, "name": "Universidad de Costa Rica", "priority": 2050, "external_id": null}, {"id": 10847873003, "name": "Universidad de Santiago de Chile - USACH", "priority": 2051, "external_id": null}, {"id": 10847874003, "name": "University of Tartu", "priority": 2052, "external_id": null}, {"id": 10847875003, "name": "Aristotle University of Thessaloniki", "priority": 2053, "external_id": null}, {"id": 10847876003, "name": "Auckland University of Technology", "priority": 2054, "external_id": null}, {"id": 10847877003, "name": "Bangor University", "priority": 2055, "external_id": null}, {"id": 10847878003, "name": "Charles Darwin University", "priority": 2056, "external_id": null}, {"id": 10847879003, "name": "Kingston University, London", "priority": 2057, "external_id": null}, {"id": 10847880003, "name": "Universitat de Valencia", "priority": 2058, "external_id": null}, {"id": 10847881003, "name": "Universit\u00e9 Montpellier 1", "priority": 2059, "external_id": null}, {"id": 10847882003, "name": "University of Pretoria", "priority": 2060, "external_id": null}, {"id": 10847883003, "name": "Lincoln University", "priority": 2061, "external_id": null}, {"id": 10847884003, "name": "National Taiwan Normal University", "priority": 2062, "external_id": null}, {"id": 10847885003, "name": "National University of Sciences And Technology (NUST) Islamabad", "priority": 2063, "external_id": null}, {"id": 10847886003, "name": "Swinburne University of Technology", "priority": 2064, "external_id": null}, {"id": 10847887003, "name": "Tongji University", "priority": 2065, "external_id": null}, {"id": 10847888003, "name": "Universidad de Zaragoza", "priority": 2066, "external_id": null}, {"id": 10847889003, "name": "Universidade Federal de Minas Gerais", "priority": 2067, "external_id": null}, {"id": 10847890003, "name": "Universit\u00e4t Duisburg-Essen", "priority": 2068, "external_id": null}, {"id": 10847891003, "name": "Al-Imam Mohamed Ibn Saud Islamic University", "priority": 2069, "external_id": null}, {"id": 10847892003, "name": "Harbin Institute of Technology", "priority": 2070, "external_id": null}, {"id": 10847893003, "name": "People's Friendship University of Russia", "priority": 2071, "external_id": null}, {"id": 10847894003, "name": "Universidade Estadual PaulistaJ\u00falio de Mesquita Filho' (UNESP)", "priority": 2072, "external_id": null}, {"id": 10847895003, "name": "Universit\u00e9 Nice Sophia-Antipolis", "priority": 2073, "external_id": null}, {"id": 10847896003, "name": "University of Crete", "priority": 2074, "external_id": null}, {"id": 10847897003, "name": "University of Milano-Bicocca", "priority": 2075, "external_id": null}, {"id": 10847898003, "name": "Ateneo de Manila University", "priority": 2076, "external_id": null}, {"id": 10847899003, "name": "Beijing Institute of Technology", "priority": 2077, "external_id": null}, {"id": 10847900003, "name": "Chang Gung University", "priority": 2078, "external_id": null}, {"id": 10847901003, "name": "hung-Ang University", "priority": 2079, "external_id": null}, {"id": 10847902003, "name": "Dublin Institute of Technology", "priority": 2080, "external_id": null}, {"id": 10847903003, "name": "Huazhong University of Science and Technology", "priority": 2081, "external_id": null}, {"id": 10847904003, "name": "International Islamic University Malaysia (IIUM)", "priority": 2082, "external_id": null}, {"id": 10847905003, "name": "Johannes Kepler University Linz", "priority": 2083, "external_id": null}, {"id": 10847906003, "name": "Justus-Liebig-Universit\u00e4t Gie\u00dfen", "priority": 2084, "external_id": null}, {"id": 10847907003, "name": "Kanazawa University", "priority": 2085, "external_id": null}, {"id": 10847908003, "name": "Keele University", "priority": 2086, "external_id": null}, {"id": 10847909003, "name": "Koc University", "priority": 2087, "external_id": null}, {"id": 10847910003, "name": "National and Kapodistrian University of Athens", "priority": 2088, "external_id": null}, {"id": 10847911003, "name": "National Research University \u2013 Higher School of Economics (HSE)", "priority": 2089, "external_id": null}, {"id": 10847912003, "name": "National Technical University of Athens", "priority": 2090, "external_id": null}, {"id": 10847913003, "name": "Okayama University", "priority": 2091, "external_id": null}, {"id": 10847914003, "name": "Sabanci University", "priority": 2092, "external_id": null}, {"id": 10847915003, "name": "Southeast University", "priority": 2093, "external_id": null}, {"id": 10847916003, "name": "Sultan Qaboos University", "priority": 2094, "external_id": null}, {"id": 10847917003, "name": "Technische Universit\u00e4t Braunschweig", "priority": 2095, "external_id": null}, {"id": 10847918003, "name": "Technische Universit\u00e4t Dortmund", "priority": 2096, "external_id": null}, {"id": 10847919003, "name": "The Catholic University of Korea", "priority": 2097, "external_id": null}, {"id": 10847920003, "name": "Tianjin University", "priority": 2098, "external_id": null}, {"id": 10847921003, "name": "Tokyo Metropolitan University", "priority": 2099, "external_id": null}, {"id": 10847922003, "name": "Universidad de Antioquia", "priority": 2100, "external_id": null}, {"id": 10847923003, "name": "University of Granada", "priority": 2101, "external_id": null}, {"id": 10847924003, "name": "Universidad de Palermo", "priority": 2102, "external_id": null}, {"id": 10847925003, "name": "Universidad Nacional de C\u00f3rdoba", "priority": 2103, "external_id": null}, {"id": 10847926003, "name": "Universidade de Santiago de Compostela", "priority": 2104, "external_id": null}, {"id": 10847927003, "name": "Universidade Federal do Rio Grande Do Sul", "priority": 2105, "external_id": null}, {"id": 10847928003, "name": "University of Siena", "priority": 2106, "external_id": null}, {"id": 10847929003, "name": "University of Trieste", "priority": 2107, "external_id": null}, {"id": 10847930003, "name": "Universitas Gadjah Mada", "priority": 2108, "external_id": null}, {"id": 10847931003, "name": "Universit\u00e9 de Lorraine", "priority": 2109, "external_id": null}, {"id": 10847932003, "name": "Universit\u00e9 de Rennes 1", "priority": 2110, "external_id": null}, {"id": 10847933003, "name": "University of Bradford", "priority": 2111, "external_id": null}, {"id": 10847934003, "name": "University of Hull", "priority": 2112, "external_id": null}, {"id": 10847935003, "name": "University of Kwazulu-Natal", "priority": 2113, "external_id": null}, {"id": 10847936003, "name": "University of Limerick", "priority": 2114, "external_id": null}, {"id": 10847937003, "name": "University of Stirling", "priority": 2115, "external_id": null}, {"id": 10847938003, "name": "University of Szeged", "priority": 2116, "external_id": null}, {"id": 10847939003, "name": "Ural Federal University", "priority": 2117, "external_id": null}, {"id": 10847940003, "name": "Xiamen University", "priority": 2118, "external_id": null}, {"id": 10847941003, "name": "Yokohama City University", "priority": 2119, "external_id": null}, {"id": 10847942003, "name": "Aberystwyth University", "priority": 2120, "external_id": null}, {"id": 10847943003, "name": "Belarus State University", "priority": 2121, "external_id": null}, {"id": 10847944003, "name": "Cairo University", "priority": 2122, "external_id": null}, {"id": 10847945003, "name": "Chiang Mai University", "priority": 2123, "external_id": null}, {"id": 10847946003, "name": "Chonbuk National University", "priority": 2124, "external_id": null}, {"id": 10847947003, "name": "E\u00f6tv\u00f6s Lor\u00e1nd University", "priority": 2125, "external_id": null}, {"id": 10847948003, "name": "Inha University", "priority": 2126, "external_id": null}, {"id": 10847949003, "name": "Instituto Polit\u00e9cnico Nacional (IPN)", "priority": 2127, "external_id": null}, {"id": 10847950003, "name": "Istanbul Technical University", "priority": 2128, "external_id": null}, {"id": 10847951003, "name": "Kumamoto University", "priority": 2129, "external_id": null}, {"id": 10847952003, "name": "Kyungpook National University", "priority": 2130, "external_id": null}, {"id": 10847953003, "name": "Lingnan University (Hong Kong)", "priority": 2131, "external_id": null}, {"id": 10847954003, "name": "Masaryk University", "priority": 2132, "external_id": null}, {"id": 10847955003, "name": "Murdoch University", "priority": 2133, "external_id": null}, {"id": 10847956003, "name": "Nagasaki University", "priority": 2134, "external_id": null}, {"id": 10847957003, "name": "National Chung Hsing University", "priority": 2135, "external_id": null}, {"id": 10847958003, "name": "National Taipei University of Technology", "priority": 2136, "external_id": null}, {"id": 10847959003, "name": "National University of Ireland Maynooth", "priority": 2137, "external_id": null}, {"id": 10847960003, "name": "Osaka City University", "priority": 2138, "external_id": null}, {"id": 10847961003, "name": "Pontificia Universidad Cat\u00f3lica del Per\u00fa", "priority": 2139, "external_id": null}, {"id": 10847962003, "name": "Pontificia Universidade Cat\u00f3lica de S\u00e3o Paulo (PUC -SP)", "priority": 2140, "external_id": null}, {"id": 10847963003, "name": "Pontificia Universidade Cat\u00f3lica do Rio de Janeiro (PUC - Rio)", "priority": 2141, "external_id": null}, {"id": 10847964003, "name": "Qatar University", "priority": 2142, "external_id": null}, {"id": 10847965003, "name": "Rhodes University", "priority": 2143, "external_id": null}, {"id": 10847966003, "name": "Tokyo University of Agriculture and Technology", "priority": 2144, "external_id": null}, {"id": 10847967003, "name": "Tomsk Polytechnic University", "priority": 2145, "external_id": null}, {"id": 10847968003, "name": "Tomsk State University", "priority": 2146, "external_id": null}, {"id": 10847969003, "name": "Umm Al-Qura University", "priority": 2147, "external_id": null}, {"id": 10847970003, "name": "Universidad Cat\u00f3lica Andr\u00e9s Bello - UCAB", "priority": 2148, "external_id": null}, {"id": 10847971003, "name": "Universidad Central de Venezuela - UCV", "priority": 2149, "external_id": null}, {"id": 10847972003, "name": "Universidad de Belgrano", "priority": 2150, "external_id": null}, {"id": 10847973003, "name": "Universidad de Concepci\u00f3n", "priority": 2151, "external_id": null}, {"id": 10847974003, "name": "Universidad de Sevilla", "priority": 2152, "external_id": null}, {"id": 10847975003, "name": "Universidade Catolica Portuguesa, Lisboa", "priority": 2153, "external_id": null}, {"id": 10847976003, "name": "Universidade de Brasilia (UnB)", "priority": 2154, "external_id": null}, {"id": 10847977003, "name": "University of Lisbon", "priority": 2155, "external_id": null}, {"id": 10847978003, "name": "University of Ljubljana", "priority": 2156, "external_id": null}, {"id": 10847979003, "name": "University of Seoul", "priority": 2157, "external_id": null}, {"id": 10847980003, "name": "Abu Dhabi University", "priority": 2158, "external_id": null}, {"id": 10847981003, "name": "Ain Shams University", "priority": 2159, "external_id": null}, {"id": 10847982003, "name": "Ajou University", "priority": 2160, "external_id": null}, {"id": 10847983003, "name": "De La Salle University", "priority": 2161, "external_id": null}, {"id": 10847984003, "name": "Dongguk University", "priority": 2162, "external_id": null}, {"id": 10847985003, "name": "Gifu University", "priority": 2163, "external_id": null}, {"id": 10847986003, "name": "Hacettepe University", "priority": 2164, "external_id": null}, {"id": 10847987003, "name": "Indian Institute of Technology Guwahati (IITG)", "priority": 2165, "external_id": null}, {"id": 10847988003, "name": "Jilin University", "priority": 2166, "external_id": null}, {"id": 10847989003, "name": "Kazan Federal University", "priority": 2167, "external_id": null}, {"id": 10847990003, "name": "King Khalid University", "priority": 2168, "external_id": null}, {"id": 10847991003, "name": "Martin-Luther-Universit\u00e4t Halle-Wittenberg", "priority": 2169, "external_id": null}, {"id": 10847992003, "name": "National Chengchi University", "priority": 2170, "external_id": null}, {"id": 10847993003, "name": "National Technical University of UkraineKyiv Polytechnic Institute'", "priority": 2171, "external_id": null}, {"id": 10847994003, "name": "Niigata University", "priority": 2172, "external_id": null}, {"id": 10847995003, "name": "Osaka Prefecture University", "priority": 2173, "external_id": null}, {"id": 10847996003, "name": "Paris Lodron University of Salzburg", "priority": 2174, "external_id": null}, {"id": 10847997003, "name": "Sharif University of Technology", "priority": 2175, "external_id": null}, {"id": 10847998003, "name": "Southern Federal University", "priority": 2176, "external_id": null}, {"id": 10847999003, "name": "Thammasat University", "priority": 2177, "external_id": null}, {"id": 10848000003, "name": "Universidad de Guadalajara (UDG)", "priority": 2178, "external_id": null}, {"id": 10848001003, "name": "Universidad de la Rep\u00fablica (UdelaR)", "priority": 2179, "external_id": null}, {"id": 10848002003, "name": "Universidad Iberoamericana (UIA)", "priority": 2180, "external_id": null}, {"id": 10848003003, "name": "Universidad Torcuato Di Tella", "priority": 2181, "external_id": null}, {"id": 10848004003, "name": "Universidade Federal da Bahia", "priority": 2182, "external_id": null}, {"id": 10848005003, "name": "Universidade Federal de S\u00e3o Carlos", "priority": 2183, "external_id": null}, {"id": 10848006003, "name": "Universidade Federal de Vi\u00e7osa", "priority": 2184, "external_id": null}, {"id": 10848007003, "name": "Perugia University", "priority": 2185, "external_id": null}, {"id": 10848008003, "name": "Universit\u00e9 de Nantes", "priority": 2186, "external_id": null}, {"id": 10848009003, "name": "Universit\u00e9 Saint-Joseph de Beyrouth", "priority": 2187, "external_id": null}, {"id": 10848010003, "name": "University of Canberra", "priority": 2188, "external_id": null}, {"id": 10848011003, "name": "University of Debrecen", "priority": 2189, "external_id": null}, {"id": 10848012003, "name": "University of Johannesburg", "priority": 2190, "external_id": null}, {"id": 10848013003, "name": "University of Mumbai", "priority": 2191, "external_id": null}, {"id": 10848014003, "name": "University of Patras", "priority": 2192, "external_id": null}, {"id": 10848015003, "name": "University of Tehran", "priority": 2193, "external_id": null}, {"id": 10848016003, "name": "University of Ulsan", "priority": 2194, "external_id": null}, {"id": 10848017003, "name": "University of Ulster", "priority": 2195, "external_id": null}, {"id": 10848018003, "name": "University of Zagreb", "priority": 2196, "external_id": null}, {"id": 10848019003, "name": "Vilnius University", "priority": 2197, "external_id": null}, {"id": 10848020003, "name": "Warsaw University of Technology", "priority": 2198, "external_id": null}, {"id": 10848021003, "name": "Al Azhar University", "priority": 2199, "external_id": null}, {"id": 10848022003, "name": "Bar-Ilan University", "priority": 2200, "external_id": null}, {"id": 10848023003, "name": "Brno University of Technology", "priority": 2201, "external_id": null}, {"id": 10848024003, "name": "Chonnam National University", "priority": 2202, "external_id": null}, {"id": 10848025003, "name": "Chungnam National University", "priority": 2203, "external_id": null}, {"id": 10848026003, "name": "Corvinus University of Budapest", "priority": 2204, "external_id": null}, {"id": 10848027003, "name": "Gunma University", "priority": 2205, "external_id": null}, {"id": 10848028003, "name": "Hallym University", "priority": 2206, "external_id": null}, {"id": 10848029003, "name": "Instituto Tecnol\u00f3gico Autonomo de M\u00e9xico (ITAM)", "priority": 2207, "external_id": null}, {"id": 10848030003, "name": "Istanbul University", "priority": 2208, "external_id": null}, {"id": 10848031003, "name": "Jordan University of Science & Technology", "priority": 2209, "external_id": null}, {"id": 10848032003, "name": "Kasetsart University", "priority": 2210, "external_id": null}, {"id": 10848033003, "name": "Kazakh-British Technical University", "priority": 2211, "external_id": null}, {"id": 10848034003, "name": "Khazar University", "priority": 2212, "external_id": null}, {"id": 10848035003, "name": "London Metropolitan University", "priority": 2213, "external_id": null}, {"id": 10848036003, "name": "Middlesex University", "priority": 2214, "external_id": null}, {"id": 10848037003, "name": "Universidad Industrial de Santander", "priority": 2215, "external_id": null}, {"id": 10848038003, "name": "Pontificia Universidad Cat\u00f3lica de Valpara\u00edso", "priority": 2216, "external_id": null}, {"id": 10848039003, "name": "Pontificia Universidade Cat\u00f3lica do Rio Grande do Sul", "priority": 2217, "external_id": null}, {"id": 10848040003, "name": "Qafqaz University", "priority": 2218, "external_id": null}, {"id": 10848041003, "name": "Ritsumeikan University", "priority": 2219, "external_id": null}, {"id": 10848042003, "name": "Shandong University", "priority": 2220, "external_id": null}, {"id": 10848043003, "name": "University of St. Kliment Ohridski", "priority": 2221, "external_id": null}, {"id": 10848044003, "name": "South Kazakhstan State University (SKSU)", "priority": 2222, "external_id": null}, {"id": 10848045003, "name": "Universidad Adolfo Ib\u00e1\u00f1ez", "priority": 2223, "external_id": null}, {"id": 10848046003, "name": "Universidad Aut\u00f3noma del Estado de M\u00e9xico", "priority": 2224, "external_id": null}, {"id": 10848047003, "name": "Universidad Aut\u00f3noma Metropolitana (UAM)", "priority": 2225, "external_id": null}, {"id": 10848048003, "name": "Universidad de Alcal\u00e1", "priority": 2226, "external_id": null}, {"id": 10848049003, "name": "Universidad Nacional Costa Rica", "priority": 2227, "external_id": null}, {"id": 10848050003, "name": "Universidad Nacional de Mar del Plata", "priority": 2228, "external_id": null}, {"id": 10848051003, "name": "Universidad Peruana Cayetano Heredia", "priority": 2229, "external_id": null}, {"id": 10848052003, "name": "Universidad Sim\u00f3n Bol\u00edvar Venezuela", "priority": 2230, "external_id": null}, {"id": 10848053003, "name": "Universidade Federal de Santa Catarina", "priority": 2231, "external_id": null}, {"id": 10848054003, "name": "Universidade Federal do Paran\u00e1 (UFPR)", "priority": 2232, "external_id": null}, {"id": 10848055003, "name": "Universidade Federal Fluminense", "priority": 2233, "external_id": null}, {"id": 10848056003, "name": "University of Modena", "priority": 2234, "external_id": null}, {"id": 10848057003, "name": "Universit\u00e9 Lumi\u00e8re Lyon 2", "priority": 2235, "external_id": null}, {"id": 10848058003, "name": "Universit\u00e9 Toulouse 1, Capitole", "priority": 2236, "external_id": null}, {"id": 10848059003, "name": "University of Economics Prague", "priority": 2237, "external_id": null}, {"id": 10848060003, "name": "University of Hertfordshire", "priority": 2238, "external_id": null}, {"id": 10848061003, "name": "University of Plymouth", "priority": 2239, "external_id": null}, {"id": 10848062003, "name": "University of Salford", "priority": 2240, "external_id": null}, {"id": 10848063003, "name": "University of Science and Technology Beijing", "priority": 2241, "external_id": null}, {"id": 10848064003, "name": "University of Western Sydney", "priority": 2242, "external_id": null}, {"id": 10848065003, "name": "Yamaguchi University", "priority": 2243, "external_id": null}, {"id": 10848066003, "name": "Yokohama National University", "priority": 2244, "external_id": null}, {"id": 10848067003, "name": "Airlangga University", "priority": 2245, "external_id": null}, {"id": 10848068003, "name": "Alexandria University", "priority": 2246, "external_id": null}, {"id": 10848069003, "name": "Alexandru Ioan Cuza University", "priority": 2247, "external_id": null}, {"id": 10848070003, "name": "Alpen-Adria-Universit\u00e4t Klagenfurt", "priority": 2248, "external_id": null}, {"id": 10848071003, "name": "Aoyama Gakuin University", "priority": 2249, "external_id": null}, {"id": 10848072003, "name": "Athens University of Economy And Business", "priority": 2250, "external_id": null}, {"id": 10848073003, "name": "Babes-Bolyai University", "priority": 2251, "external_id": null}, {"id": 10848074003, "name": "Baku State University", "priority": 2252, "external_id": null}, {"id": 10848075003, "name": "Belarusian National Technical University", "priority": 2253, "external_id": null}, {"id": 10848076003, "name": "Benem\u00e9rita Universidad Aut\u00f3noma de Puebla", "priority": 2254, "external_id": null}, {"id": 10848077003, "name": "Bogor Agricultural University", "priority": 2255, "external_id": null}, {"id": 10848078003, "name": "Coventry University", "priority": 2256, "external_id": null}, {"id": 10848079003, "name": "Cukurova University", "priority": 2257, "external_id": null}, {"id": 10848080003, "name": "Diponegoro University", "priority": 2258, "external_id": null}, {"id": 10848081003, "name": "Donetsk National University", "priority": 2259, "external_id": null}, {"id": 10848082003, "name": "Doshisha University", "priority": 2260, "external_id": null}, {"id": 10848083003, "name": "E.A.Buketov Karaganda State University", "priority": 2261, "external_id": null}, {"id": 10848084003, "name": "Far Eastern Federal University", "priority": 2262, "external_id": null}, {"id": 10848085003, "name": "Fu Jen Catholic University", "priority": 2263, "external_id": null}, {"id": 10848086003, "name": "Kagoshima University", "priority": 2264, "external_id": null}, {"id": 10848087003, "name": "Kaunas University of Technology", "priority": 2265, "external_id": null}, {"id": 10848088003, "name": "Kazakh Ablai khan University of International Relations and World Languages", "priority": 2266, "external_id": null}, {"id": 10848089003, "name": "Kazakh National Pedagogical University Abai", "priority": 2267, "external_id": null}, {"id": 10848090003, "name": "Kazakh National Technical University", "priority": 2268, "external_id": null}, {"id": 10848091003, "name": "Khon Kaen University", "priority": 2269, "external_id": null}, {"id": 10848092003, "name": "King Faisal University", "priority": 2270, "external_id": null}, {"id": 10848093003, "name": "King Mongkut''s University of Technology Thonburi", "priority": 2271, "external_id": null}, {"id": 10848094003, "name": "Kuwait University", "priority": 2272, "external_id": null}, {"id": 10848095003, "name": "Lodz University", "priority": 2273, "external_id": null}, {"id": 10848096003, "name": "Manchester Metropolitan University", "priority": 2274, "external_id": null}, {"id": 10848097003, "name": "Lobachevsky State University of Nizhni Novgorod", "priority": 2275, "external_id": null}, {"id": 10848098003, "name": "National Technical UniversityKharkiv Polytechnic Institute'", "priority": 2276, "external_id": null}, {"id": 10848099003, "name": "Nicolaus Copernicus University", "priority": 2277, "external_id": null}, {"id": 10848100003, "name": "Northumbria University at Newcastle", "priority": 2278, "external_id": null}, {"id": 10848101003, "name": "Nottingham Trent University", "priority": 2279, "external_id": null}, {"id": 10848102003, "name": "Ochanomizu University", "priority": 2280, "external_id": null}, {"id": 10848103003, "name": "Plekhanov Russian University of Economics", "priority": 2281, "external_id": null}, {"id": 10848104003, "name": "Pontificia Universidad Catolica del Ecuador", "priority": 2282, "external_id": null}, {"id": 10848105003, "name": "Prince of Songkla University", "priority": 2283, "external_id": null}, {"id": 10848106003, "name": "S.Seifullin Kazakh Agro Technical University", "priority": 2284, "external_id": null}, {"id": 10848107003, "name": "Saitama University", "priority": 2285, "external_id": null}, {"id": 10848108003, "name": "Sepuluh Nopember Institute of Technology", "priority": 2286, "external_id": null}, {"id": 10848109003, "name": "Shinshu University", "priority": 2287, "external_id": null}, {"id": 10848110003, "name": "The Robert Gordon University", "priority": 2288, "external_id": null}, {"id": 10848111003, "name": "Tokai University", "priority": 2289, "external_id": null}, {"id": 10848112003, "name": "Universidad ANAHUAC", "priority": 2290, "external_id": null}, {"id": 10848113003, "name": "Universidad Austral de Chile", "priority": 2291, "external_id": null}, {"id": 10848114003, "name": "University Aut\u00f3noma de Nuevo Le\u00f3n (UANL)", "priority": 2292, "external_id": null}, {"id": 10848115003, "name": "Universidad de la Habana", "priority": 2293, "external_id": null}, {"id": 10848116003, "name": "Universidad de La Sabana", "priority": 2294, "external_id": null}, {"id": 10848117003, "name": "Universidad de las Am\u00e9ricas Puebla (UDLAP)", "priority": 2295, "external_id": null}, {"id": 10848118003, "name": "Universidad de los Andes M\u00e9rida", "priority": 2296, "external_id": null}, {"id": 10848119003, "name": "University of Murcia", "priority": 2297, "external_id": null}, {"id": 10848120003, "name": "Universidad de Puerto Rico", "priority": 2298, "external_id": null}, {"id": 10848121003, "name": "Universidad de San Francisco de Quito", "priority": 2299, "external_id": null}, {"id": 10848122003, "name": "Universidad de Talca", "priority": 2300, "external_id": null}, {"id": 10848123003, "name": "Universidad del Norte", "priority": 2301, "external_id": null}, {"id": 10848124003, "name": "Universidad del Rosario", "priority": 2302, "external_id": null}, {"id": 10848125003, "name": "Universidad del Valle", "priority": 2303, "external_id": null}, {"id": 10848126003, "name": "Universidad Nacional de Cuyo", "priority": 2304, "external_id": null}, {"id": 10848127003, "name": "Universidad Nacional de Rosario", "priority": 2305, "external_id": null}, {"id": 10848128003, "name": "Universidad Nacional de Tucum\u00e1n", "priority": 2306, "external_id": null}, {"id": 10848129003, "name": "Universidad Nacional del Sur", "priority": 2307, "external_id": null}, {"id": 10848130003, "name": "Universidad Nacional Mayor de San Marcos", "priority": 2308, "external_id": null}, {"id": 10848131003, "name": "Universidad T\u00e9cnica Federico Santa Mar\u00eda", "priority": 2309, "external_id": null}, {"id": 10848132003, "name": "Universidad Tecnol\u00f3gica Nacional (UTN)", "priority": 2310, "external_id": null}, {"id": 10848133003, "name": "Universidade do Estado do Rio de Janeiro (UERJ)", "priority": 2311, "external_id": null}, {"id": 10848134003, "name": "Universidade Estadual de Londrina (UEL)", "priority": 2312, "external_id": null}, {"id": 10848135003, "name": "Universidade Federal de Santa Maria", "priority": 2313, "external_id": null}, {"id": 10848136003, "name": "Universidade Federal do Cear\u00e1 (UFC)", "priority": 2314, "external_id": null}, {"id": 10848137003, "name": "Universidade Federal do Pernambuco", "priority": 2315, "external_id": null}, {"id": 10848138003, "name": "Universit\u00e0 Ca'' Foscari Venezia", "priority": 2316, "external_id": null}, {"id": 10848139003, "name": "Catania University", "priority": 2317, "external_id": null}, {"id": 10848140003, "name": "Universit\u00e0 degli Studi Roma Tre", "priority": 2318, "external_id": null}, {"id": 10848141003, "name": "Universit\u00e9 Charles-de-Gaulle Lille 3", "priority": 2319, "external_id": null}, {"id": 10848142003, "name": "Universit\u00e9 de Caen Basse-Normandie", "priority": 2320, "external_id": null}, {"id": 10848143003, "name": "Universit\u00e9 de Cergy-Pontoise", "priority": 2321, "external_id": null}, {"id": 10848144003, "name": "Universit\u00e9 de Poitiers", "priority": 2322, "external_id": null}, {"id": 10848145003, "name": "Universit\u00e9 Jean Moulin Lyon 3", "priority": 2323, "external_id": null}, {"id": 10848146003, "name": "Universit\u00e9 Lille 2 Droit et Sant\u00e9", "priority": 2324, "external_id": null}, {"id": 10848147003, "name": "Universit\u00e9 Paris Ouest Nanterre La D\u00e9fense", "priority": 2325, "external_id": null}, {"id": 10848148003, "name": "Universit\u00e9 Paul-Val\u00e9ry Montpellier 3", "priority": 2326, "external_id": null}, {"id": 10848149003, "name": "Universit\u00e9 Pierre Mend\u00e8s France - Grenoble 2", "priority": 2327, "external_id": null}, {"id": 10848150003, "name": "Universit\u00e9 Stendhal Grenoble 3", "priority": 2328, "external_id": null}, {"id": 10848151003, "name": "Universit\u00e9 Toulouse II, Le Mirail", "priority": 2329, "external_id": null}, {"id": 10848152003, "name": "Universiti Teknologi MARA - UiTM", "priority": 2330, "external_id": null}, {"id": 10848153003, "name": "University of Baghdad", "priority": 2331, "external_id": null}, {"id": 10848154003, "name": "University of Bahrain", "priority": 2332, "external_id": null}, {"id": 10848155003, "name": "University of Bari", "priority": 2333, "external_id": null}, {"id": 10848156003, "name": "University of Belgrade", "priority": 2334, "external_id": null}, {"id": 10848157003, "name": "University of Brawijaya", "priority": 2335, "external_id": null}, {"id": 10848158003, "name": "University of Brescia", "priority": 2336, "external_id": null}, {"id": 10848159003, "name": "University of Bucharest", "priority": 2337, "external_id": null}, {"id": 10848160003, "name": "University of Calcutta", "priority": 2338, "external_id": null}, {"id": 10848161003, "name": "University of Central Lancashire", "priority": 2339, "external_id": null}, {"id": 10848162003, "name": "University of Colombo", "priority": 2340, "external_id": null}, {"id": 10848163003, "name": "University of Dhaka", "priority": 2341, "external_id": null}, {"id": 10848164003, "name": "University of East London", "priority": 2342, "external_id": null}, {"id": 10848165003, "name": "University of Engineering & Technology (UET) Lahore", "priority": 2343, "external_id": null}, {"id": 10848166003, "name": "University of Greenwich", "priority": 2344, "external_id": null}, {"id": 10848167003, "name": "University of Jordan", "priority": 2345, "external_id": null}, {"id": 10848168003, "name": "University of Karachi", "priority": 2346, "external_id": null}, {"id": 10848169003, "name": "University of Lahore", "priority": 2347, "external_id": null}, {"id": 10848170003, "name": "University of Latvia", "priority": 2348, "external_id": null}, {"id": 10848171003, "name": "University of New England", "priority": 2349, "external_id": null}, {"id": 10848172003, "name": "University of Pune", "priority": 2350, "external_id": null}, {"id": 10848173003, "name": "University of Santo Tomas", "priority": 2351, "external_id": null}, {"id": 10848174003, "name": "University of Southern Queensland", "priority": 2352, "external_id": null}, {"id": 10848175003, "name": "University of Wroclaw", "priority": 2353, "external_id": null}, {"id": 10848176003, "name": "Verona University", "priority": 2354, "external_id": null}, {"id": 10848177003, "name": "Victoria University", "priority": 2355, "external_id": null}, {"id": 10848178003, "name": "Vilnius Gediminas Technical University", "priority": 2356, "external_id": null}, {"id": 10848179003, "name": "Voronezh State University", "priority": 2357, "external_id": null}, {"id": 10848180003, "name": "Vytautas Magnus University", "priority": 2358, "external_id": null}, {"id": 10848181003, "name": "West University of Timisoara", "priority": 2359, "external_id": null}, {"id": 10848182003, "name": "University of South Alabama", "priority": 2360, "external_id": null}, {"id": 10848183003, "name": "University of Arkansas", "priority": 2361, "external_id": null}, {"id": 10848184003, "name": "University of California - Berkeley", "priority": 2362, "external_id": null}, {"id": 10848185003, "name": "University of Connecticut", "priority": 2363, "external_id": null}, {"id": 10848186003, "name": "University of South Florida", "priority": 2364, "external_id": null}, {"id": 10848187003, "name": "University of Georgia", "priority": 2365, "external_id": null}, {"id": 10848188003, "name": "University of Hawaii - Manoa", "priority": 2366, "external_id": null}, {"id": 10848189003, "name": "Iowa State University", "priority": 2367, "external_id": null}, {"id": 10848190003, "name": "Murray State University", "priority": 2368, "external_id": null}, {"id": 10848191003, "name": "University of Louisville", "priority": 2369, "external_id": null}, {"id": 10848192003, "name": "Western Kentucky University", "priority": 2370, "external_id": null}, {"id": 10848193003, "name": "Louisiana State University - Baton Rouge", "priority": 2371, "external_id": null}, {"id": 10848194003, "name": "University of Maryland - College Park", "priority": 2372, "external_id": null}, {"id": 10848195003, "name": "University of Minnesota - Twin Cities", "priority": 2373, "external_id": null}, {"id": 10848196003, "name": "University of Montana", "priority": 2374, "external_id": null}, {"id": 10848197003, "name": "East Carolina University", "priority": 2375, "external_id": null}, {"id": 10848198003, "name": "University of North Carolina - Chapel Hill", "priority": 2376, "external_id": null}, {"id": 10848199003, "name": "Wake Forest University", "priority": 2377, "external_id": null}, {"id": 10848200003, "name": "University of Nebraska - Lincoln", "priority": 2378, "external_id": null}, {"id": 10848201003, "name": "New Mexico State University", "priority": 2379, "external_id": null}, {"id": 10848202003, "name": "Ohio State University - Columbus", "priority": 2380, "external_id": null}, {"id": 10848203003, "name": "University of Oklahoma", "priority": 2381, "external_id": null}, {"id": 10848204003, "name": "Pennsylvania State University - University Park", "priority": 2382, "external_id": null}, {"id": 10848205003, "name": "University of Pittsburgh", "priority": 2383, "external_id": null}, {"id": 10848206003, "name": "University of Tennessee - Chattanooga", "priority": 2384, "external_id": null}, {"id": 10848207003, "name": "Vanderbilt University", "priority": 2385, "external_id": null}, {"id": 10848208003, "name": "Rice University", "priority": 2386, "external_id": null}, {"id": 10848209003, "name": "University of Utah", "priority": 2387, "external_id": null}, {"id": 10848210003, "name": "University of Richmond", "priority": 2388, "external_id": null}, {"id": 10848211003, "name": "University of Arkansas - Pine Bluff", "priority": 2389, "external_id": null}, {"id": 10848212003, "name": "University of Central Florida", "priority": 2390, "external_id": null}, {"id": 10848213003, "name": "Florida Atlantic University", "priority": 2391, "external_id": null}, {"id": 10848214003, "name": "Hampton University", "priority": 2392, "external_id": null}, {"id": 10848215003, "name": "Liberty University", "priority": 2393, "external_id": null}, {"id": 10848216003, "name": "Mercer University", "priority": 2394, "external_id": null}, {"id": 10848217003, "name": "Middle Tennessee State University", "priority": 2395, "external_id": null}, {"id": 10848218003, "name": "University of Nevada - Las Vegas", "priority": 2396, "external_id": null}, {"id": 10848219003, "name": "South Carolina State University", "priority": 2397, "external_id": null}, {"id": 10848220003, "name": "University of Tennessee - Martin", "priority": 2398, "external_id": null}, {"id": 10848221003, "name": "Weber State University", "priority": 2399, "external_id": null}, {"id": 10848222003, "name": "Youngstown State University", "priority": 2400, "external_id": null}, {"id": 10848223003, "name": "University of the Incarnate Word", "priority": 2401, "external_id": null}, {"id": 10848224003, "name": "University of Washington", "priority": 2402, "external_id": null}, {"id": 10848225003, "name": "University of Louisiana - Lafayette", "priority": 2403, "external_id": null}, {"id": 10848226003, "name": "Coastal Carolina University", "priority": 2404, "external_id": null}, {"id": 10848227003, "name": "Utah State University", "priority": 2405, "external_id": null}, {"id": 10848228003, "name": "University of Alabama", "priority": 2406, "external_id": null}, {"id": 10848229003, "name": "University of Illinois - Urbana-Champaign", "priority": 2407, "external_id": null}, {"id": 10848230003, "name": "United States Air Force Academy", "priority": 2408, "external_id": null}, {"id": 10848231003, "name": "University of Akron", "priority": 2409, "external_id": null}, {"id": 10848232003, "name": "University of Central Arkansas", "priority": 2410, "external_id": null}, {"id": 10848233003, "name": "University of Kansas", "priority": 2411, "external_id": null}, {"id": 10848234003, "name": "University of Northern Colorado", "priority": 2412, "external_id": null}, {"id": 10848235003, "name": "University of Northern Iowa", "priority": 2413, "external_id": null}, {"id": 10848236003, "name": "University of South Carolina", "priority": 2414, "external_id": null}, {"id": 10848237003, "name": "Tennessee Technological University", "priority": 2415, "external_id": null}, {"id": 10848238003, "name": "University of Texas - El Paso", "priority": 2416, "external_id": null}, {"id": 10848239003, "name": "Texas Tech University", "priority": 2417, "external_id": null}, {"id": 10848240003, "name": "Tulane University", "priority": 2418, "external_id": null}, {"id": 10848241003, "name": "Virginia Military Institute", "priority": 2419, "external_id": null}, {"id": 10848242003, "name": "Western Michigan University", "priority": 2420, "external_id": null}, {"id": 10848243003, "name": "Wilfrid Laurier University", "priority": 2421, "external_id": null}, {"id": 10848244003, "name": "University of San Diego", "priority": 2422, "external_id": null}, {"id": 10848245003, "name": "University of California - San Diego", "priority": 2423, "external_id": null}, {"id": 10848246003, "name": "Brooks Institute of Photography", "priority": 2424, "external_id": null}, {"id": 10848247003, "name": "Acupuncture and Integrative Medicine College - Berkeley", "priority": 2425, "external_id": null}, {"id": 10848248003, "name": "Southern Alberta Institute of Technology", "priority": 2426, "external_id": null}, {"id": 10848249003, "name": "Susquehanna University", "priority": 2427, "external_id": null}, {"id": 10848250003, "name": "University of Texas - Dallas", "priority": 2428, "external_id": null}, {"id": 10848251003, "name": "Thunderbird School of Global Management", "priority": 2429, "external_id": null}, {"id": 10848252003, "name": "Presidio Graduate School", "priority": 2430, "external_id": null}, {"id": 10848253003, "name": "\u00c9cole sup\u00e9rieure de commerce de Dijon", "priority": 2431, "external_id": null}, {"id": 10848254003, "name": "University of California - San Francisco", "priority": 2432, "external_id": null}, {"id": 10848255003, "name": "Hack Reactor", "priority": 2433, "external_id": null}, {"id": 10848256003, "name": "St. Mary''s College of California", "priority": 2434, "external_id": null}, {"id": 10848257003, "name": "New England Law", "priority": 2435, "external_id": null}, {"id": 10848258003, "name": "University of California, Merced", "priority": 2436, "external_id": null}, {"id": 10848259003, "name": "University of California, Hastings College of the Law", "priority": 2437, "external_id": null}, {"id": 10848260003, "name": "V.N. Karazin Kharkiv National University", "priority": 2438, "external_id": null}, {"id": 10848261003, "name": "SIM University (UniSIM)", "priority": 2439, "external_id": null}, {"id": 10848262003, "name": "Singapore Management University (SMU)", "priority": 2440, "external_id": null}, {"id": 10848263003, "name": "Singapore University of Technology and Design (SUTD)", "priority": 2441, "external_id": null}, {"id": 10848264003, "name": "Singapore Institute of Technology (SIT)", "priority": 2442, "external_id": null}, {"id": 10848265003, "name": "Nanyang Polytechnic (NYP)", "priority": 2443, "external_id": null}, {"id": 10848266003, "name": "Ngee Ann Polytechnic (NP)", "priority": 2444, "external_id": null}, {"id": 10848267003, "name": "Republic Polytechnic (RP)", "priority": 2445, "external_id": null}, {"id": 10848268003, "name": "Singapore Polytechnic (SP)", "priority": 2446, "external_id": null}, {"id": 10848269003, "name": "Temasek Polytechnic (TP)", "priority": 2447, "external_id": null}, {"id": 10848270003, "name": "INSEAD", "priority": 2448, "external_id": null}, {"id": 10848271003, "name": "Funda\u00e7\u00e3o Get\u00falio Vargas", "priority": 2449, "external_id": null}, {"id": 10848272003, "name": "Acharya Nagarjuna University", "priority": 2450, "external_id": null}, {"id": 10848273003, "name": "University of California - Santa Barbara", "priority": 2451, "external_id": null}, {"id": 10848274003, "name": "University of California - Irvine", "priority": 2452, "external_id": null}, {"id": 10848275003, "name": "California State University - Long Beach", "priority": 2453, "external_id": null}, {"id": 10848276003, "name": "Robert Morris University Illinois", "priority": 2454, "external_id": null}, {"id": 10848277003, "name": "Harold Washington College - City Colleges of Chicago", "priority": 2455, "external_id": null}, {"id": 10848278003, "name": "Harry S Truman College - City Colleges of Chicago", "priority": 2456, "external_id": null}, {"id": 10848279003, "name": "Kennedy-King College - City Colleges of Chicago", "priority": 2457, "external_id": null}, {"id": 10848280003, "name": "Malcolm X College - City Colleges of Chicago", "priority": 2458, "external_id": null}, {"id": 10848281003, "name": "Olive-Harvey College - City Colleges of Chicago", "priority": 2459, "external_id": null}, {"id": 10848282003, "name": "Richard J Daley College - City Colleges of Chicago", "priority": 2460, "external_id": null}, {"id": 10848283003, "name": "Wilbur Wright College - City Colleges of Chicago", "priority": 2461, "external_id": null}, {"id": 10848284003, "name": "Abertay University", "priority": 2462, "external_id": null}, {"id": 10848285003, "name": "Pontif\u00edcia Universidade Cat\u00f3lica de Minas Gerais", "priority": 2463, "external_id": null}, {"id": 10848286003, "name": "Other", "priority": 2464, "external_id": null}, {"id": 19126655003, "name": "Atlanta College of Arts", "priority": 2465, "external_id": null}]}, "emitted_at": 1664285620787} +{"stream": "custom_fields", "data": {"id": 4680899003, "name": "Degree", "active": true, "field_type": "candidate", "priority": 1, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "degree", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10848287003, "name": "High School", "priority": 0, "external_id": null}, {"id": 10848288003, "name": "Associate's Degree", "priority": 1, "external_id": null}, {"id": 10848289003, "name": "Bachelor's Degree", "priority": 2, "external_id": null}, {"id": 10848290003, "name": "Master's Degree", "priority": 3, "external_id": null}, {"id": 10848291003, "name": "Master of Business Administration (M.B.A.)", "priority": 4, "external_id": null}, {"id": 10848292003, "name": "Juris Doctor (J.D.)", "priority": 5, "external_id": null}, {"id": 10848293003, "name": "Doctor of Medicine (M.D.)", "priority": 6, "external_id": null}, {"id": 10848294003, "name": "Doctor of Philosophy (Ph.D.)", "priority": 7, "external_id": null}, {"id": 10848295003, "name": "Engineer's Degree", "priority": 8, "external_id": null}, {"id": 10848296003, "name": "Other", "priority": 9, "external_id": null}]}, "emitted_at": 1664285620804} +{"stream": "custom_fields", "data": {"id": 4680900003, "name": "Discipline", "active": true, "field_type": "candidate", "priority": 2, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "discipline", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10848297003, "name": "Accounting", "priority": 0, "external_id": null}, {"id": 10848298003, "name": "African Studies", "priority": 1, "external_id": null}, {"id": 10848299003, "name": "Agriculture", "priority": 2, "external_id": null}, {"id": 10848300003, "name": "Anthropology", "priority": 3, "external_id": null}, {"id": 10848301003, "name": "Applied Health Services", "priority": 4, "external_id": null}, {"id": 10848302003, "name": "Architecture", "priority": 5, "external_id": null}, {"id": 10848303003, "name": "Art", "priority": 6, "external_id": null}, {"id": 10848304003, "name": "Asian Studies", "priority": 7, "external_id": null}, {"id": 10848305003, "name": "Biology", "priority": 8, "external_id": null}, {"id": 10848306003, "name": "Business", "priority": 9, "external_id": null}, {"id": 10848307003, "name": "Business Administration", "priority": 10, "external_id": null}, {"id": 10848308003, "name": "Chemistry", "priority": 11, "external_id": null}, {"id": 10848309003, "name": "Classical Languages", "priority": 12, "external_id": null}, {"id": 10848310003, "name": "Communications & Film", "priority": 13, "external_id": null}, {"id": 10848311003, "name": "Computer Science", "priority": 14, "external_id": null}, {"id": 10848312003, "name": "Dentistry", "priority": 15, "external_id": null}, {"id": 10848313003, "name": "Developing Nations", "priority": 16, "external_id": null}, {"id": 10848314003, "name": "Discipline Unknown", "priority": 17, "external_id": null}, {"id": 10848315003, "name": "Earth Sciences", "priority": 18, "external_id": null}, {"id": 10848316003, "name": "Economics", "priority": 19, "external_id": null}, {"id": 10848317003, "name": "Education", "priority": 20, "external_id": null}, {"id": 10848318003, "name": "Electronics", "priority": 21, "external_id": null}, {"id": 10848319003, "name": "Engineering", "priority": 22, "external_id": null}, {"id": 10848320003, "name": "English Studies", "priority": 23, "external_id": null}, {"id": 10848321003, "name": "Environmental Studies", "priority": 24, "external_id": null}, {"id": 10848322003, "name": "European Studies", "priority": 25, "external_id": null}, {"id": 10848323003, "name": "Fashion", "priority": 26, "external_id": null}, {"id": 10848324003, "name": "Finance", "priority": 27, "external_id": null}, {"id": 10848325003, "name": "Fine Arts", "priority": 28, "external_id": null}, {"id": 10848326003, "name": "General Studies", "priority": 29, "external_id": null}, {"id": 10848327003, "name": "Health Services", "priority": 30, "external_id": null}, {"id": 10848328003, "name": "History", "priority": 31, "external_id": null}, {"id": 10848329003, "name": "Human Resources Management", "priority": 32, "external_id": null}, {"id": 10848330003, "name": "Humanities", "priority": 33, "external_id": null}, {"id": 10848331003, "name": "Industrial Arts & Carpentry", "priority": 34, "external_id": null}, {"id": 10848332003, "name": "Information Systems", "priority": 35, "external_id": null}, {"id": 10848333003, "name": "International Relations", "priority": 36, "external_id": null}, {"id": 10848334003, "name": "Journalism", "priority": 37, "external_id": null}, {"id": 10848335003, "name": "Languages", "priority": 38, "external_id": null}, {"id": 10848336003, "name": "Latin American Studies", "priority": 39, "external_id": null}, {"id": 10848337003, "name": "Law", "priority": 40, "external_id": null}, {"id": 10848338003, "name": "Linguistics", "priority": 41, "external_id": null}, {"id": 10848339003, "name": "Manufacturing & Mechanics", "priority": 42, "external_id": null}, {"id": 10848340003, "name": "Mathematics", "priority": 43, "external_id": null}, {"id": 10848341003, "name": "Medicine", "priority": 44, "external_id": null}, {"id": 10848342003, "name": "Middle Eastern Studies", "priority": 45, "external_id": null}, {"id": 10848343003, "name": "Naval Science", "priority": 46, "external_id": null}, {"id": 10848344003, "name": "North American Studies", "priority": 47, "external_id": null}, {"id": 10848345003, "name": "Nuclear Technics", "priority": 48, "external_id": null}, {"id": 10848346003, "name": "Operations Research & Strategy", "priority": 49, "external_id": null}, {"id": 10848347003, "name": "Organizational Theory", "priority": 50, "external_id": null}, {"id": 10848348003, "name": "Philosophy", "priority": 51, "external_id": null}, {"id": 10848349003, "name": "Physical Education", "priority": 52, "external_id": null}, {"id": 10848350003, "name": "Physical Sciences", "priority": 53, "external_id": null}, {"id": 10848351003, "name": "Physics", "priority": 54, "external_id": null}, {"id": 10848352003, "name": "Political Science", "priority": 55, "external_id": null}, {"id": 10848353003, "name": "Psychology", "priority": 56, "external_id": null}, {"id": 10848354003, "name": "Public Policy", "priority": 57, "external_id": null}, {"id": 10848355003, "name": "Public Service", "priority": 58, "external_id": null}, {"id": 10848356003, "name": "Religious Studies", "priority": 59, "external_id": null}, {"id": 10848357003, "name": "Russian & Soviet Studies", "priority": 60, "external_id": null}, {"id": 10848358003, "name": "Scandinavian Studies", "priority": 61, "external_id": null}, {"id": 10848359003, "name": "Science", "priority": 62, "external_id": null}, {"id": 10848360003, "name": "Slavic Studies", "priority": 63, "external_id": null}, {"id": 10848361003, "name": "Social Science", "priority": 64, "external_id": null}, {"id": 10848362003, "name": "Social Sciences", "priority": 65, "external_id": null}, {"id": 10848363003, "name": "Sociology", "priority": 66, "external_id": null}, {"id": 10848364003, "name": "Speech", "priority": 67, "external_id": null}, {"id": 10848365003, "name": "Statistics & Decision Theory", "priority": 68, "external_id": null}, {"id": 10848366003, "name": "Urban Studies", "priority": 69, "external_id": null}, {"id": 10848367003, "name": "Veterinary Medicine", "priority": 70, "external_id": null}, {"id": 10848368003, "name": "Other", "priority": 71, "external_id": null}]}, "emitted_at": 1664285620804} +{"stream": "custom_fields", "data": {"id": 4680901003, "name": "Employment Type", "active": true, "field_type": "job", "priority": 0, "value_type": "single_select", "private": false, "required": false, "require_approval": true, "trigger_new_version": false, "name_key": "employment_type", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845796003, "name": "Full-time", "priority": 0, "external_id": null}, {"id": 10845797003, "name": "Part-time", "priority": 1, "external_id": null}, {"id": 10845798003, "name": "Intern", "priority": 2, "external_id": null}, {"id": 10845799003, "name": "Contract", "priority": 3, "external_id": null}, {"id": 10845800003, "name": "Temporary", "priority": 4, "external_id": null}]}, "emitted_at": 1664285620805} +{"stream": "custom_fields", "data": {"id": 4680902003, "name": "Start Date", "active": true, "field_type": "offer", "priority": 0, "value_type": "date", "private": true, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "start_date", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1664285620805} +{"stream": "custom_fields", "data": {"id": 4680903003, "name": "Employment Type", "active": true, "field_type": "offer", "priority": 1, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": true, "name_key": "employment_type", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845801003, "name": "Full-time", "priority": 0, "external_id": null}, {"id": 10845802003, "name": "Part-time", "priority": 1, "external_id": null}, {"id": 10845803003, "name": "Intern", "priority": 2, "external_id": null}, {"id": 10845804003, "name": "Contract", "priority": 3, "external_id": null}, {"id": 10845805003, "name": "Temporary", "priority": 4, "external_id": null}]}, "emitted_at": 1664285620805} +{"stream": "custom_fields", "data": {"id": 4680904003, "name": "Offer Documents", "active": true, "field_type": "offer", "priority": 2, "value_type": "short_text", "private": true, "required": false, "require_approval": false, "trigger_new_version": true, "name_key": "offer_documents", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1664285620805} +{"stream": "custom_fields", "data": {"id": 4680905003, "name": "Relationship", "active": true, "field_type": "referral_question", "priority": 0, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "relationship", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845806003, "name": "Coworker", "priority": 0, "external_id": null}, {"id": 10845807003, "name": "School", "priority": 1, "external_id": null}, {"id": 10845808003, "name": "Manager", "priority": 2, "external_id": null}, {"id": 10845809003, "name": "Reported", "priority": 3, "external_id": null}, {"id": 10845810003, "name": "Friend", "priority": 4, "external_id": null}, {"id": 10845811003, "name": "Do not know", "priority": 5, "external_id": null}]}, "emitted_at": 1664285620805} +{"stream": "custom_fields", "data": {"id": 4680906003, "name": "Work History", "active": true, "field_type": "referral_question", "priority": 1, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "work_history", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845812003, "name": "0-1", "priority": 0, "external_id": null}, {"id": 10845813003, "name": "2-5", "priority": 1, "external_id": null}, {"id": 10845814003, "name": "5+", "priority": 2, "external_id": null}]}, "emitted_at": 1664285620806} +{"stream": "custom_fields", "data": {"id": 4680907003, "name": "Rating", "active": true, "field_type": "referral_question", "priority": 2, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "rating", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845815003, "name": "Superstar", "priority": 0, "external_id": null}, {"id": 10845816003, "name": "Top 5%", "priority": 1, "external_id": null}, {"id": 10845817003, "name": "Top 10%", "priority": 2, "external_id": null}, {"id": 10845818003, "name": "Top 25%", "priority": 3, "external_id": null}, {"id": 10845819003, "name": "Top 50%", "priority": 4, "external_id": null}]}, "emitted_at": 1664285620806} +{"stream": "custom_fields", "data": {"id": 4680908003, "name": "When we reach out", "active": true, "field_type": "referral_question", "priority": 3, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "when_we_reach_out", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845820003, "name": "You may mention me", "priority": 0, "external_id": null}, {"id": 10845821003, "name": "I wish to remain anonymous", "priority": 1, "external_id": null}]}, "emitted_at": 1664285620806} +{"stream": "custom_fields", "data": {"id": 4680909003, "name": "They know they're being referred", "active": true, "field_type": "referral_question", "priority": 4, "value_type": "yes_no", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "they_know_they_re_being_referred", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1664285620806} +{"stream": "custom_fields", "data": {"id": 4680910003, "name": "Referral Notes", "active": true, "field_type": "referral_question", "priority": 5, "value_type": "long_text", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "referral_notes", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1664285620806} +{"stream": "custom_fields", "data": {"id": 7431124003, "name": "Test User", "active": true, "field_type": "agency_question", "priority": 0, "value_type": "yes_no", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "test_user", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1664285620806} +{"stream": "custom_fields", "data": {"id": 7431125003, "name": "Test User", "active": true, "field_type": "agency_question", "priority": 1, "value_type": "short_text", "private": false, "required": true, "require_approval": false, "trigger_new_version": false, "name_key": "test_user_agency_question_1633884465.559642", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1664285620806} +{"stream": "custom_fields", "data": {"id": 7431126003, "name": "Test User", "active": true, "field_type": "referral_question", "priority": 6, "value_type": "yes_no", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "test_user", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1664285620807} +{"stream": "demographics_question_sets", "data": {"title": "Test Question Set 1", "id": 4000197003, "description": "

Test Question Set 1 description

", "active": true}, "emitted_at": 1664285621607} +{"stream": "demographics_question_sets", "data": {"title": "Test Question Set 2", "id": 4000198003, "description": "

Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.

", "active": true}, "emitted_at": 1664285621610} +{"stream": "demographics_question_sets", "data": {"title": "U.S. Standard Demographic Questions", "id": 4002702003, "description": "We invite applicants to share their demographic background. If you choose to complete this survey, your responses may be used to identify areas of improvement in our hiring process.", "active": true}, "emitted_at": 1664285621610} +{"stream": "demographics_questions", "data": {"translations": [{"name": "q1", "language": "en"}], "required": false, "name": "q1", "id": 4000714003, "demographic_question_set_id": 4000197003, "answer_type": "multi_value_single_select", "active": true}, "emitted_at": 1664285622528} +{"stream": "demographics_questions", "data": {"translations": [{"name": "q2", "language": "en"}], "required": false, "name": "q2", "id": 4000715003, "demographic_question_set_id": 4000197003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1664285622532} +{"stream": "demographics_questions", "data": {"translations": [{"name": "question1", "language": "en"}], "required": false, "name": "question1", "id": 4000716003, "demographic_question_set_id": 4000198003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1664285622532} +{"stream": "demographics_questions", "data": {"translations": [{"name": "question2", "language": "en"}], "required": true, "name": "question2", "id": 4000717003, "demographic_question_set_id": 4000198003, "answer_type": "multi_value_single_select", "active": true}, "emitted_at": 1664285622533} +{"stream": "demographics_questions", "data": {"translations": [{"name": "Are you a veteran or active member of the United States Armed Forces?", "language": "en"}], "required": false, "name": "Are you a veteran or active member of the United States Armed Forces?", "id": 4015594003, "demographic_question_set_id": 4002702003, "answer_type": "multi_value_single_select", "active": true}, "emitted_at": 1664285622533} +{"stream": "demographics_questions", "data": {"translations": [{"name": "Do you have a disability or chronic condition (physical, visual, auditory, cognitive, mental, emotional, or other) that substantially limits one or more of your major life activities, including mobility, communication (seeing, hearing, speaking), and learning?", "language": "en"}], "required": false, "name": "Do you have a disability or chronic condition (physical, visual, auditory, cognitive, mental, emotional, or other) that substantially limits one or more of your major life activities, including mobility, communication (seeing, hearing, speaking), and learning?", "id": 4015596003, "demographic_question_set_id": 4002702003, "answer_type": "multi_value_single_select", "active": true}, "emitted_at": 1664285622534} +{"stream": "demographics_questions", "data": {"translations": [{"name": "Do you identify as transgender?", "language": "en"}], "required": false, "name": "Do you identify as transgender?", "id": 4015598003, "demographic_question_set_id": 4002702003, "answer_type": "multi_value_single_select", "active": true}, "emitted_at": 1664285622535} +{"stream": "demographics_questions", "data": {"translations": [{"name": "How would you describe your sexual orientation? (mark all that apply)", "language": "en"}], "required": false, "name": "How would you describe your sexual orientation? (mark all that apply)", "id": 4015599003, "demographic_question_set_id": 4002702003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1664285622535} +{"stream": "demographics_questions", "data": {"translations": [{"name": "How would you describe your racial/ethnic background? (mark all that apply)", "language": "en"}], "required": false, "name": "How would you describe your racial/ethnic background? (mark all that apply)", "id": 4015601003, "demographic_question_set_id": 4002702003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1664285622536} +{"stream": "demographics_questions", "data": {"translations": [{"name": "How would you describe your gender identity? (mark all that apply)", "language": "en"}], "required": false, "name": "How would you describe your gender identity? (mark all that apply)", "id": 4015603003, "demographic_question_set_id": 4002702003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1664285622536} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "a1", "language": "en"}], "name": "a1", "id": 4004258003, "free_form": false, "demographic_question_id": 4000714003, "active": true}, "emitted_at": 1664285623356} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "a2", "language": "en"}], "name": "a2", "id": 4004259003, "free_form": false, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1664285623359} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "a3", "language": "en"}], "name": "a3", "id": 4004260003, "free_form": false, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1664285623360} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "s1", "language": "en"}], "name": "s1", "id": 4004261003, "free_form": true, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1664285623361} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "answer1", "language": "en"}], "name": "answer1", "id": 4004262003, "free_form": false, "demographic_question_id": 4000716003, "active": true}, "emitted_at": 1664285623361} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "answer-self1", "language": "en"}], "name": "answer-self1", "id": 4004263003, "free_form": true, "demographic_question_id": 4000716003, "active": true}, "emitted_at": 1664285623362} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "answer2", "language": "en"}], "name": "answer2", "id": 4004264003, "free_form": false, "demographic_question_id": 4000716003, "active": true}, "emitted_at": 1664285623362} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "answer1", "language": "en"}], "name": "answer1", "id": 4004265003, "free_form": false, "demographic_question_id": 4000717003, "active": true}, "emitted_at": 1664285623363} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4004266003, "free_form": false, "demographic_question_id": 4000717003, "active": true}, "emitted_at": 1664285623363} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "answer2", "language": "en"}], "name": "answer2", "id": 4004267003, "free_form": false, "demographic_question_id": 4000717003, "active": true}, "emitted_at": 1664285623364} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4093930003, "free_form": false, "demographic_question_id": 4015594003, "active": true}, "emitted_at": 1664285623364} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "I prefer to self-describe", "language": "en"}], "name": "I prefer to self-describe", "id": 4093931003, "free_form": true, "demographic_question_id": 4015594003, "active": true}, "emitted_at": 1664285623364} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "No, I am not a veteran or active member", "language": "en"}], "name": "No, I am not a veteran or active member", "id": 4093932003, "free_form": false, "demographic_question_id": 4015594003, "active": true}, "emitted_at": 1664285623365} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Yes, I am a veteran or active member", "language": "en"}], "name": "Yes, I am a veteran or active member", "id": 4093934003, "free_form": false, "demographic_question_id": 4015594003, "active": true}, "emitted_at": 1664285623366} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4093937003, "free_form": false, "demographic_question_id": 4015596003, "active": true}, "emitted_at": 1664285623366} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "I prefer to self-describe", "language": "en"}], "name": "I prefer to self-describe", "id": 4093939003, "free_form": true, "demographic_question_id": 4015596003, "active": true}, "emitted_at": 1664285623366} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "No", "language": "en"}], "name": "No", "id": 4093940003, "free_form": false, "demographic_question_id": 4015596003, "active": true}, "emitted_at": 1664285623367} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Yes", "language": "en"}], "name": "Yes", "id": 4093941003, "free_form": false, "demographic_question_id": 4015596003, "active": true}, "emitted_at": 1664285623367} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4093944003, "free_form": false, "demographic_question_id": 4015598003, "active": true}, "emitted_at": 1664285623368} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "I prefer to self-describe", "language": "en"}], "name": "I prefer to self-describe", "id": 4093946003, "free_form": true, "demographic_question_id": 4015598003, "active": true}, "emitted_at": 1664285623368} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "No", "language": "en"}], "name": "No", "id": 4093948003, "free_form": false, "demographic_question_id": 4015598003, "active": true}, "emitted_at": 1664285623369} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Yes", "language": "en"}], "name": "Yes", "id": 4093950003, "free_form": false, "demographic_question_id": 4015598003, "active": true}, "emitted_at": 1664285623369} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4093953003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285623370} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "I prefer to self-describe", "language": "en"}], "name": "I prefer to self-describe", "id": 4093955003, "free_form": true, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285623370} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Queer", "language": "en"}], "name": "Queer", "id": 4093956003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285623371} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Lesbian", "language": "en"}], "name": "Lesbian", "id": 4093957003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285623371} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Heterosexual", "language": "en"}], "name": "Heterosexual", "id": 4093959003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285623372} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Gay", "language": "en"}], "name": "Gay", "id": 4093961003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285623372} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Bisexual and/or pansexual", "language": "en"}], "name": "Bisexual and/or pansexual", "id": 4093963003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285623373} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Asexual", "language": "en"}], "name": "Asexual", "id": 4093965003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285623373} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4093971003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285623374} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "I prefer to self-describe", "language": "en"}], "name": "I prefer to self-describe", "id": 4093973003, "free_form": true, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285623374} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "White or European", "language": "en"}], "name": "White or European", "id": 4093975003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285623374} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Southeast Asian", "language": "en"}], "name": "Southeast Asian", "id": 4093976003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285623375} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "South Asian", "language": "en"}], "name": "South Asian", "id": 4093977003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285623375} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Native Hawaiian or Pacific Islander", "language": "en"}], "name": "Native Hawaiian or Pacific Islander", "id": 4093979003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285623375} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Middle Eastern or North African", "language": "en"}], "name": "Middle Eastern or North African", "id": 4093981003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285623376} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Indigenous, American Indian or Alaska Native", "language": "en"}], "name": "Indigenous, American Indian or Alaska Native", "id": 4093983003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285623376} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Hispanic, Latinx or of Spanish Origin", "language": "en"}], "name": "Hispanic, Latinx or of Spanish Origin", "id": 4093985003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285623376} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "East Asian", "language": "en"}], "name": "East Asian", "id": 4093986003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285623377} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Black or of African descent", "language": "en"}], "name": "Black or of African descent", "id": 4093988003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285623377} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4093989003, "free_form": false, "demographic_question_id": 4015603003, "active": true}, "emitted_at": 1664285623377} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "I prefer to self-describe", "language": "en"}], "name": "I prefer to self-describe", "id": 4093990003, "free_form": true, "demographic_question_id": 4015603003, "active": true}, "emitted_at": 1664285623377} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Woman", "language": "en"}], "name": "Woman", "id": 4093991003, "free_form": false, "demographic_question_id": 4015603003, "active": true}, "emitted_at": 1664285623378} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Non-binary", "language": "en"}], "name": "Non-binary", "id": 4093993003, "free_form": false, "demographic_question_id": 4015603003, "active": true}, "emitted_at": 1664285623378} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "Man", "language": "en"}], "name": "Man", "id": 4093995003, "free_form": false, "demographic_question_id": 4015603003, "active": true}, "emitted_at": 1664285623378} +{"stream": "demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.248Z", "id": 9308815003, "free_form_text": null, "demographic_question_id": 4000716003, "demographic_answer_option_id": 4004262003, "created_at": "2021-11-03T19:56:07.248Z", "application_id": 47459993003}, "emitted_at": 1664285624166} +{"stream": "demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.252Z", "id": 9308816003, "free_form_text": "custom answer", "demographic_question_id": 4000716003, "demographic_answer_option_id": 4004263003, "created_at": "2021-11-03T19:56:07.252Z", "application_id": 47459993003}, "emitted_at": 1664285624170} +{"stream": "demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.259Z", "id": 9308817003, "free_form_text": null, "demographic_question_id": 4000717003, "demographic_answer_option_id": 4004266003, "created_at": "2021-11-03T19:56:07.259Z", "application_id": 47459993003}, "emitted_at": 1664285624170} +{"stream": "applications_demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.248Z", "id": 9308815003, "free_form_text": null, "demographic_question_id": 4000716003, "demographic_answer_option_id": 4004262003, "created_at": "2021-11-03T19:56:07.248Z", "application_id": 47459993003}, "emitted_at": 1664285626543} +{"stream": "applications_demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.252Z", "id": 9308816003, "free_form_text": "custom answer", "demographic_question_id": 4000716003, "demographic_answer_option_id": 4004263003, "created_at": "2021-11-03T19:56:07.252Z", "application_id": 47459993003}, "emitted_at": 1664285626546} +{"stream": "applications_demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.259Z", "id": 9308817003, "free_form_text": null, "demographic_question_id": 4000717003, "demographic_answer_option_id": 4004266003, "created_at": "2021-11-03T19:56:07.259Z", "application_id": 47459993003}, "emitted_at": 1664285626547} +{"stream": "demographics_question_sets_questions", "data": {"translations": [{"name": "q1", "language": "en"}], "required": false, "name": "q1", "id": 4000714003, "demographic_question_set_id": 4000197003, "answer_type": "multi_value_single_select", "active": true}, "emitted_at": 1664285627885} +{"stream": "demographics_question_sets_questions", "data": {"translations": [{"name": "q2", "language": "en"}], "required": false, "name": "q2", "id": 4000715003, "demographic_question_set_id": 4000197003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1664285627888} +{"stream": "demographics_question_sets_questions", "data": {"translations": [{"name": "question1", "language": "en"}], "required": false, "name": "question1", "id": 4000716003, "demographic_question_set_id": 4000198003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1664285628366} +{"stream": "demographics_question_sets_questions", "data": {"translations": [{"name": "question2", "language": "en"}], "required": true, "name": "question2", "id": 4000717003, "demographic_question_set_id": 4000198003, "answer_type": "multi_value_single_select", "active": true}, "emitted_at": 1664285628367} +{"stream": "demographics_question_sets_questions", "data": {"translations": [{"name": "Are you a veteran or active member of the United States Armed Forces?", "language": "en"}], "required": false, "name": "Are you a veteran or active member of the United States Armed Forces?", "id": 4015594003, "demographic_question_set_id": 4002702003, "answer_type": "multi_value_single_select", "active": true}, "emitted_at": 1664285628774} +{"stream": "demographics_question_sets_questions", "data": {"translations": [{"name": "Do you have a disability or chronic condition (physical, visual, auditory, cognitive, mental, emotional, or other) that substantially limits one or more of your major life activities, including mobility, communication (seeing, hearing, speaking), and learning?", "language": "en"}], "required": false, "name": "Do you have a disability or chronic condition (physical, visual, auditory, cognitive, mental, emotional, or other) that substantially limits one or more of your major life activities, including mobility, communication (seeing, hearing, speaking), and learning?", "id": 4015596003, "demographic_question_set_id": 4002702003, "answer_type": "multi_value_single_select", "active": true}, "emitted_at": 1664285628774} +{"stream": "demographics_question_sets_questions", "data": {"translations": [{"name": "Do you identify as transgender?", "language": "en"}], "required": false, "name": "Do you identify as transgender?", "id": 4015598003, "demographic_question_set_id": 4002702003, "answer_type": "multi_value_single_select", "active": true}, "emitted_at": 1664285628775} +{"stream": "demographics_question_sets_questions", "data": {"translations": [{"name": "How would you describe your sexual orientation? (mark all that apply)", "language": "en"}], "required": false, "name": "How would you describe your sexual orientation? (mark all that apply)", "id": 4015599003, "demographic_question_set_id": 4002702003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1664285628776} +{"stream": "demographics_question_sets_questions", "data": {"translations": [{"name": "How would you describe your racial/ethnic background? (mark all that apply)", "language": "en"}], "required": false, "name": "How would you describe your racial/ethnic background? (mark all that apply)", "id": 4015601003, "demographic_question_set_id": 4002702003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1664285628776} +{"stream": "demographics_question_sets_questions", "data": {"translations": [{"name": "How would you describe your gender identity? (mark all that apply)", "language": "en"}], "required": false, "name": "How would you describe your gender identity? (mark all that apply)", "id": 4015603003, "demographic_question_set_id": 4002702003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1664285628777} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "a1", "language": "en"}], "name": "a1", "id": 4004258003, "free_form": false, "demographic_question_id": 4000714003, "active": true}, "emitted_at": 1664285630146} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "a2", "language": "en"}], "name": "a2", "id": 4004259003, "free_form": false, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1664285630475} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "a3", "language": "en"}], "name": "a3", "id": 4004260003, "free_form": false, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1664285630476} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "s1", "language": "en"}], "name": "s1", "id": 4004261003, "free_form": true, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1664285630477} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "answer1", "language": "en"}], "name": "answer1", "id": 4004262003, "free_form": false, "demographic_question_id": 4000716003, "active": true}, "emitted_at": 1664285630936} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "answer-self1", "language": "en"}], "name": "answer-self1", "id": 4004263003, "free_form": true, "demographic_question_id": 4000716003, "active": true}, "emitted_at": 1664285630937} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "answer2", "language": "en"}], "name": "answer2", "id": 4004264003, "free_form": false, "demographic_question_id": 4000716003, "active": true}, "emitted_at": 1664285630937} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "answer1", "language": "en"}], "name": "answer1", "id": 4004265003, "free_form": false, "demographic_question_id": 4000717003, "active": true}, "emitted_at": 1664285631274} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4004266003, "free_form": false, "demographic_question_id": 4000717003, "active": true}, "emitted_at": 1664285631274} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "answer2", "language": "en"}], "name": "answer2", "id": 4004267003, "free_form": false, "demographic_question_id": 4000717003, "active": true}, "emitted_at": 1664285631274} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4093930003, "free_form": false, "demographic_question_id": 4015594003, "active": true}, "emitted_at": 1664285631560} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "I prefer to self-describe", "language": "en"}], "name": "I prefer to self-describe", "id": 4093931003, "free_form": true, "demographic_question_id": 4015594003, "active": true}, "emitted_at": 1664285631560} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "No, I am not a veteran or active member", "language": "en"}], "name": "No, I am not a veteran or active member", "id": 4093932003, "free_form": false, "demographic_question_id": 4015594003, "active": true}, "emitted_at": 1664285631560} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Yes, I am a veteran or active member", "language": "en"}], "name": "Yes, I am a veteran or active member", "id": 4093934003, "free_form": false, "demographic_question_id": 4015594003, "active": true}, "emitted_at": 1664285631560} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4093937003, "free_form": false, "demographic_question_id": 4015596003, "active": true}, "emitted_at": 1664285631954} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "I prefer to self-describe", "language": "en"}], "name": "I prefer to self-describe", "id": 4093939003, "free_form": true, "demographic_question_id": 4015596003, "active": true}, "emitted_at": 1664285631954} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "No", "language": "en"}], "name": "No", "id": 4093940003, "free_form": false, "demographic_question_id": 4015596003, "active": true}, "emitted_at": 1664285631955} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Yes", "language": "en"}], "name": "Yes", "id": 4093941003, "free_form": false, "demographic_question_id": 4015596003, "active": true}, "emitted_at": 1664285631955} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4093944003, "free_form": false, "demographic_question_id": 4015598003, "active": true}, "emitted_at": 1664285632355} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "I prefer to self-describe", "language": "en"}], "name": "I prefer to self-describe", "id": 4093946003, "free_form": true, "demographic_question_id": 4015598003, "active": true}, "emitted_at": 1664285632356} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "No", "language": "en"}], "name": "No", "id": 4093948003, "free_form": false, "demographic_question_id": 4015598003, "active": true}, "emitted_at": 1664285632356} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Yes", "language": "en"}], "name": "Yes", "id": 4093950003, "free_form": false, "demographic_question_id": 4015598003, "active": true}, "emitted_at": 1664285632356} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4093953003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285632770} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "I prefer to self-describe", "language": "en"}], "name": "I prefer to self-describe", "id": 4093955003, "free_form": true, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285632771} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Queer", "language": "en"}], "name": "Queer", "id": 4093956003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285632772} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Lesbian", "language": "en"}], "name": "Lesbian", "id": 4093957003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285632772} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Heterosexual", "language": "en"}], "name": "Heterosexual", "id": 4093959003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285632773} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Gay", "language": "en"}], "name": "Gay", "id": 4093961003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285632773} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Bisexual and/or pansexual", "language": "en"}], "name": "Bisexual and/or pansexual", "id": 4093963003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285632774} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Asexual", "language": "en"}], "name": "Asexual", "id": 4093965003, "free_form": false, "demographic_question_id": 4015599003, "active": true}, "emitted_at": 1664285632775} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4093971003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285633134} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "I prefer to self-describe", "language": "en"}], "name": "I prefer to self-describe", "id": 4093973003, "free_form": true, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285633134} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "White or European", "language": "en"}], "name": "White or European", "id": 4093975003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285633135} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Southeast Asian", "language": "en"}], "name": "Southeast Asian", "id": 4093976003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285633135} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "South Asian", "language": "en"}], "name": "South Asian", "id": 4093977003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285633136} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Native Hawaiian or Pacific Islander", "language": "en"}], "name": "Native Hawaiian or Pacific Islander", "id": 4093979003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285633136} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Middle Eastern or North African", "language": "en"}], "name": "Middle Eastern or North African", "id": 4093981003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285633136} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Indigenous, American Indian or Alaska Native", "language": "en"}], "name": "Indigenous, American Indian or Alaska Native", "id": 4093983003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285633137} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Hispanic, Latinx or of Spanish Origin", "language": "en"}], "name": "Hispanic, Latinx or of Spanish Origin", "id": 4093985003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285633137} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "East Asian", "language": "en"}], "name": "East Asian", "id": 4093986003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285633138} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Black or of African descent", "language": "en"}], "name": "Black or of African descent", "id": 4093988003, "free_form": false, "demographic_question_id": 4015601003, "active": true}, "emitted_at": 1664285633138} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "I don't wish to answer", "language": "en"}], "name": "I don't wish to answer", "id": 4093989003, "free_form": false, "demographic_question_id": 4015603003, "active": true}, "emitted_at": 1664285633460} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "I prefer to self-describe", "language": "en"}], "name": "I prefer to self-describe", "id": 4093990003, "free_form": true, "demographic_question_id": 4015603003, "active": true}, "emitted_at": 1664285633460} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Woman", "language": "en"}], "name": "Woman", "id": 4093991003, "free_form": false, "demographic_question_id": 4015603003, "active": true}, "emitted_at": 1664285633461} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Non-binary", "language": "en"}], "name": "Non-binary", "id": 4093993003, "free_form": false, "demographic_question_id": 4015603003, "active": true}, "emitted_at": 1664285633461} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "Man", "language": "en"}], "name": "Man", "id": 4093995003, "free_form": false, "demographic_question_id": 4015603003, "active": true}, "emitted_at": 1664285633462} +{"stream": "interviews", "data": {"id": 40387397003, "application_id": 44937562003, "external_event_id": "123456789", "start": {"date_time": "2021-12-12T13:15:00.000Z"}, "end": {"date_time": "2021-12-12T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:21:44.107Z", "updated_at": "2021-12-12T15:15:02.894Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1664285634418} +{"stream": "interviews", "data": {"id": 40387426003, "application_id": 44937562003, "external_event_id": "12345678", "start": {"date_time": "2021-12-13T13:15:00.000Z"}, "end": {"date_time": "2021-12-13T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:04.561Z", "updated_at": "2021-12-13T15:15:13.252Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1664285634421} +{"stream": "interviews", "data": {"id": 40387431003, "application_id": 44937562003, "external_event_id": "1234567", "start": {"date_time": "2021-12-14T13:15:00.000Z"}, "end": {"date_time": "2021-12-14T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:13.681Z", "updated_at": "2021-12-14T15:15:12.118Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1664285634422} +{"stream": "applications_interviews", "data": {"id": 40387397003, "application_id": 44937562003, "external_event_id": "123456789", "start": {"date_time": "2021-12-12T13:15:00.000Z"}, "end": {"date_time": "2021-12-12T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:21:44.107Z", "updated_at": "2021-12-12T15:15:02.894Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1664285637504} +{"stream": "applications_interviews", "data": {"id": 40387426003, "application_id": 44937562003, "external_event_id": "12345678", "start": {"date_time": "2021-12-13T13:15:00.000Z"}, "end": {"date_time": "2021-12-13T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:04.561Z", "updated_at": "2021-12-13T15:15:13.252Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1664285637508} +{"stream": "applications_interviews", "data": {"id": 40387431003, "application_id": 44937562003, "external_event_id": "1234567", "start": {"date_time": "2021-12-14T13:15:00.000Z"}, "end": {"date_time": "2021-12-14T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:13.681Z", "updated_at": "2021-12-14T15:15:12.118Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1664285637509} +{"stream": "sources", "data": {"id": 4000000003, "name": "Recurse", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638360} +{"stream": "sources", "data": {"id": 4000001003, "name": "cliquify", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638360} +{"stream": "sources", "data": {"id": 4000002003, "name": "ContactOut", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638361} +{"stream": "sources", "data": {"id": 4000003003, "name": "Crosschq", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638361} +{"stream": "sources", "data": {"id": 4000004003, "name": "Talentpair", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638361} +{"stream": "sources", "data": {"id": 4000005003, "name": "Sompani Talent Pools", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638361} +{"stream": "sources", "data": {"id": 4000006003, "name": "ScoutFor", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638361} +{"stream": "sources", "data": {"id": 4000007003, "name": "Gem", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638361} +{"stream": "sources", "data": {"id": 4000008003, "name": "Findem", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638361} +{"stream": "sources", "data": {"id": 4000009003, "name": "goldi staging", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638361} +{"stream": "sources", "data": {"id": 4000010003, "name": "MoBerries", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638361} +{"stream": "sources", "data": {"id": 4000011003, "name": "Onramp", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638361} +{"stream": "sources", "data": {"id": 4000012003, "name": "Knowledge Officer", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638361} +{"stream": "sources", "data": {"id": 4000013003, "name": "Sourceress", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638361} +{"stream": "sources", "data": {"id": 4000014003, "name": "Resume Library", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638362} +{"stream": "sources", "data": {"id": 4000015003, "name": "Command E", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638362} +{"stream": "sources", "data": {"id": 4000016003, "name": "Attract", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638362} +{"stream": "sources", "data": {"id": 4000017003, "name": "WePow", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638362} +{"stream": "sources", "data": {"id": 4000018003, "name": "Planted", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638362} +{"stream": "sources", "data": {"id": 4000019003, "name": "Birch Local", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638362} +{"stream": "sources", "data": {"id": 4000020003, "name": "Birch", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638362} +{"stream": "sources", "data": {"id": 4000021003, "name": "Consider", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638362} +{"stream": "sources", "data": {"id": 4000022003, "name": "Eightfold", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638362} +{"stream": "sources", "data": {"id": 4000023003, "name": "Google (Job Search)", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638362} +{"stream": "sources", "data": {"id": 4000024003, "name": "Hundred5", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638362} +{"stream": "sources", "data": {"id": 4000025003, "name": "Work4 Labs", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638363} +{"stream": "sources", "data": {"id": 4000026003, "name": "Nudj", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638363} +{"stream": "sources", "data": {"id": 4000027003, "name": "Handshake", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638363} +{"stream": "sources", "data": {"id": 4000028003, "name": "goldi", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638363} +{"stream": "sources", "data": {"id": 4000029003, "name": "Honeypot.io", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638363} +{"stream": "sources", "data": {"id": 4000030003, "name": "Joonko", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638363} +{"stream": "sources", "data": {"id": 4000031003, "name": "Untapped", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638363} +{"stream": "sources", "data": {"id": 4000032003, "name": "Bubblesort", "type": {"id": 4000007003, "name": "Agencies"}}, "emitted_at": 1664285638363} +{"stream": "sources", "data": {"id": 4000033003, "name": "Fetcher", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638363} +{"stream": "sources", "data": {"id": 4000034003, "name": "WorksHub", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638363} +{"stream": "sources", "data": {"id": 4000035003, "name": "CareerBuilder Quick Apply", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638363} +{"stream": "sources", "data": {"id": 4000036003, "name": "BountyJobs", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638363} +{"stream": "sources", "data": {"id": 4000037003, "name": "SmartDreamers", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638363} +{"stream": "sources", "data": {"id": 4000038003, "name": "Gloat", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638364} +{"stream": "sources", "data": {"id": 4000039003, "name": "Selected", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638364} +{"stream": "sources", "data": {"id": 4000040003, "name": "SeekOut", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638364} +{"stream": "sources", "data": {"id": 4000041003, "name": "Jobmailer", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638364} +{"stream": "sources", "data": {"id": 4000042003, "name": "MindMatch", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638364} +{"stream": "sources", "data": {"id": 4000043003, "name": "Hackajob", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638364} +{"stream": "sources", "data": {"id": 4000044003, "name": "Snap.hr", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638364} +{"stream": "sources", "data": {"id": 4000045003, "name": "JamieAI", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638364} +{"stream": "sources", "data": {"id": 4000046003, "name": "Visage", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638364} +{"stream": "sources", "data": {"id": 4000047003, "name": "XING ReferralManager", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638364} +{"stream": "sources", "data": {"id": 4000048003, "name": "WorkShape.io", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638364} +{"stream": "sources", "data": {"id": 4000049003, "name": "Workey", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638364} +{"stream": "sources", "data": {"id": 4000050003, "name": "Uncommon", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638365} +{"stream": "sources", "data": {"id": 4000051003, "name": "Talentful", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638365} +{"stream": "sources", "data": {"id": 4000052003, "name": "TalentBin\u00ae by Monster", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638365} +{"stream": "sources", "data": {"id": 4000053003, "name": "Riviera Partners", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638365} +{"stream": "sources", "data": {"id": 4000054003, "name": "SingleSprout", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638365} +{"stream": "sources", "data": {"id": 4000055003, "name": "ScoutSavvy", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638365} +{"stream": "sources", "data": {"id": 4000056003, "name": "RippleMatch", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638365} +{"stream": "sources", "data": {"id": 4000057003, "name": "Project: \u201cOdin\u201d", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638365} +{"stream": "sources", "data": {"id": 4000058003, "name": "PowerToFly", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638365} +{"stream": "sources", "data": {"id": 4000059003, "name": "OneWire", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638365} +{"stream": "sources", "data": {"id": 4000060003, "name": "OfferZen", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638365} +{"stream": "sources", "data": {"id": 4000061003, "name": "Netin", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638365} +{"stream": "sources", "data": {"id": 4000062003, "name": "Meritocracy", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638366} +{"stream": "sources", "data": {"id": 4000063003, "name": "Jopwell", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638366} +{"stream": "sources", "data": {"id": 4000064003, "name": "Jobjet", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638366} +{"stream": "sources", "data": {"id": 4000065003, "name": "Interviewing.io", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638366} +{"stream": "sources", "data": {"id": 4000066003, "name": "Interseller", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638366} +{"stream": "sources", "data": {"id": 4000067003, "name": "HRMARKET", "type": {"id": 4000007003, "name": "Agencies"}}, "emitted_at": 1664285638366} +{"stream": "sources", "data": {"id": 4000068003, "name": "hireEZ", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638366} +{"stream": "sources", "data": {"id": 4000069003, "name": "HeyJobs", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638366} +{"stream": "sources", "data": {"id": 4000070003, "name": "Hachi", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638366} +{"stream": "sources", "data": {"id": 4000071003, "name": "getTalent", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638366} +{"stream": "sources", "data": {"id": 4000072003, "name": "Functional Works", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638367} +{"stream": "sources", "data": {"id": 4000073003, "name": "Firstbird", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638367} +{"stream": "sources", "data": {"id": 4000074003, "name": "Crowded", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638367} +{"stream": "sources", "data": {"id": 4000075003, "name": "CrediBLL", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638367} +{"stream": "sources", "data": {"id": 4000076003, "name": "Celential.ai", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638367} +{"stream": "sources", "data": {"id": 4000077003, "name": "AmazingHiring", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638367} +{"stream": "sources", "data": {"id": 4000078003, "name": "Teamable", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638368} +{"stream": "sources", "data": {"id": 4000079003, "name": "HubSpot Marketing", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638368} +{"stream": "sources", "data": {"id": 4000080003, "name": "VolkScience", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638368} +{"stream": "sources", "data": {"id": 4000081003, "name": "LeapMind", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638368} +{"stream": "sources", "data": {"id": 4000082003, "name": "Woo", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638368} +{"stream": "sources", "data": {"id": 4000083003, "name": "ReferralMob", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638368} +{"stream": "sources", "data": {"id": 4000084003, "name": "Maildrop", "type": {"id": 4000004003, "name": "Other"}}, "emitted_at": 1664285638368} +{"stream": "sources", "data": {"id": 4000085003, "name": "Thumbtack Technology", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638368} +{"stream": "sources", "data": {"id": 4000086003, "name": "Wendy", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638368} +{"stream": "sources", "data": {"id": 4000087003, "name": "Stella", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638369} +{"stream": "sources", "data": {"id": 4000088003, "name": "Resource", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638369} +{"stream": "sources", "data": {"id": 4000089003, "name": "Talentseer", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638369} +{"stream": "sources", "data": {"id": 4000090003, "name": "Door of Clubs", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638369} +{"stream": "sources", "data": {"id": 4000091003, "name": "untapt", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638369} +{"stream": "sources", "data": {"id": 4000092003, "name": "vsource", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638369} +{"stream": "sources", "data": {"id": 4000093003, "name": "Ideal", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638369} +{"stream": "sources", "data": {"id": 4000094003, "name": "Indeed Prime", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638369} +{"stream": "sources", "data": {"id": 4000095003, "name": "Predikt", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638369} +{"stream": "sources", "data": {"id": 4000096003, "name": "Beamery", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638369} +{"stream": "sources", "data": {"id": 4000097003, "name": "RippleHire", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638370} +{"stream": "sources", "data": {"id": 4000098003, "name": "LinkedIn (Prospecting)", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638370} +{"stream": "sources", "data": {"id": 4000099003, "name": "LinkedIn (Ad Posting)", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638370} +{"stream": "sources", "data": {"id": 4000100003, "name": "FirstJob", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638527} +{"stream": "sources", "data": {"id": 4000101003, "name": "Bsharp", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638528} +{"stream": "sources", "data": {"id": 4000102003, "name": "Landing.jobs", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638528} +{"stream": "sources", "data": {"id": 4000103003, "name": "Vettery", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638528} +{"stream": "sources", "data": {"id": 4000104003, "name": "RAKUNA Recruit", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638528} +{"stream": "sources", "data": {"id": 4000105003, "name": "E-SS portal", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638528} +{"stream": "sources", "data": {"id": 4000106003, "name": "Recsolu", "type": {"id": 4000001003, "name": "In person event"}}, "emitted_at": 1664285638528} +{"stream": "sources", "data": {"id": 4000107003, "name": "SocialReferral [DEV]", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638528} +{"stream": "sources", "data": {"id": 4000108003, "name": "WayUp", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638528} +{"stream": "sources", "data": {"id": 4000109003, "name": "SocialReferral", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638528} +{"stream": "sources", "data": {"id": 4000110003, "name": "HumanPredictions", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638528} +{"stream": "sources", "data": {"id": 4000111003, "name": "Gogohire", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638529} +{"stream": "sources", "data": {"id": 4000112003, "name": "TalentIQ", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638529} +{"stream": "sources", "data": {"id": 4000113003, "name": "DoWeKnowThisGuy", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638529} +{"stream": "sources", "data": {"id": 4000114003, "name": "The Muse", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638529} +{"stream": "sources", "data": {"id": 4000115003, "name": "HackerRank", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638529} +{"stream": "sources", "data": {"id": 4000116003, "name": "Whitetruffle", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638529} +{"stream": "sources", "data": {"id": 4000117003, "name": "LinkedIn Limited Listing", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638529} +{"stream": "sources", "data": {"id": 4000118003, "name": "SpringRole", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638529} +{"stream": "sources", "data": {"id": 4000119003, "name": "StrongIntro", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638529} +{"stream": "sources", "data": {"id": 4000120003, "name": "CodeFights", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638529} +{"stream": "sources", "data": {"id": 4000121003, "name": "Citadel", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638529} +{"stream": "sources", "data": {"id": 4000122003, "name": "Savvy", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638529} +{"stream": "sources", "data": {"id": 4000123003, "name": "SmashFly", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638530} +{"stream": "sources", "data": {"id": 4000124003, "name": "Stack", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638530} +{"stream": "sources", "data": {"id": 4000125003, "name": "Network Monkey", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638530} +{"stream": "sources", "data": {"id": 4000126003, "name": "Triplebyte", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638530} +{"stream": "sources", "data": {"id": 4000127003, "name": "HireArt", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638530} +{"stream": "sources", "data": {"id": 4000128003, "name": "Hirecanvas", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638530} +{"stream": "sources", "data": {"id": 4000129003, "name": "CloserIQ", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638530} +{"stream": "sources", "data": {"id": 4000130003, "name": "Codeity", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638530} +{"stream": "sources", "data": {"id": 4000131003, "name": "ZipRecruiter", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638530} +{"stream": "sources", "data": {"id": 4000132003, "name": "Drafted", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638530} +{"stream": "sources", "data": {"id": 4000133003, "name": "ROIKOI", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638530} +{"stream": "sources", "data": {"id": 4000134003, "name": "DoWeKnowThisGuy Staging", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638531} +{"stream": "sources", "data": {"id": 4000135003, "name": "AngelList", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638531} +{"stream": "sources", "data": {"id": 4000136003, "name": "Archively", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638531} +{"stream": "sources", "data": {"id": 4000137003, "name": "Hired", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638531} +{"stream": "sources", "data": {"id": 4000138003, "name": "EmployeeReferrals.com", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638531} +{"stream": "sources", "data": {"id": 4000139003, "name": "Aevy", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638531} +{"stream": "sources", "data": {"id": 4000140003, "name": "Connectifier", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638531} +{"stream": "sources", "data": {"id": 4000141003, "name": "Simppler", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638531} +{"stream": "sources", "data": {"id": 4000142003, "name": "Internal Applicant", "type": {"id": 4000006003, "name": "Company marketing"}}, "emitted_at": 1664285638531} +{"stream": "sources", "data": {"id": 4000143003, "name": "Clinch", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638531} +{"stream": "sources", "data": {"id": 4000144003, "name": "SwoopTalent", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638531} +{"stream": "sources", "data": {"id": 4000145003, "name": "Pymetrics", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638531} +{"stream": "sources", "data": {"id": 4000146003, "name": "YBorder", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638532} +{"stream": "sources", "data": {"id": 4000147003, "name": "Hirable", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638532} +{"stream": "sources", "data": {"id": 4000148003, "name": "RecruitiFi", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638532} +{"stream": "sources", "data": {"id": 4000149003, "name": "RolePoint", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638532} +{"stream": "sources", "data": {"id": 4000150003, "name": "Entelo", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638532} +{"stream": "sources", "data": {"id": 4000151003, "name": "Piazza", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638532} +{"stream": "sources", "data": {"id": 4000152003, "name": "Setter", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638532} +{"stream": "sources", "data": {"id": 4000153003, "name": "Other", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638532} +{"stream": "sources", "data": {"id": 4000154003, "name": "Coroflot", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638532} +{"stream": "sources", "data": {"id": 4000155003, "name": "Startuply", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638532} +{"stream": "sources", "data": {"id": 4000156003, "name": "Behance", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638532} +{"stream": "sources", "data": {"id": 4000157003, "name": "Dribbble", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638533} +{"stream": "sources", "data": {"id": 4000158003, "name": "Glassdoor", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638533} +{"stream": "sources", "data": {"id": 4000159003, "name": "Careers2.0 by StackOverflow", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638533} +{"stream": "sources", "data": {"id": 4000160003, "name": "LinkedIn (Social Media)", "type": {"id": 4000005003, "name": "Social media"}}, "emitted_at": 1664285638533} +{"stream": "sources", "data": {"id": 4000161003, "name": "Referral", "type": {"id": 4000002003, "name": "Referral"}}, "emitted_at": 1664285638533} +{"stream": "sources", "data": {"id": 4000162003, "name": "Beyond.com", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638533} +{"stream": "sources", "data": {"id": 4000163003, "name": "CareerBuilder", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638533} +{"stream": "sources", "data": {"id": 4000164003, "name": "Monster", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638533} +{"stream": "sources", "data": {"id": 4000165003, "name": "CareerBuilder", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638533} +{"stream": "sources", "data": {"id": 4000166003, "name": "Monster", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638533} +{"stream": "sources", "data": {"id": 4000167003, "name": "craigslist", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638533} +{"stream": "sources", "data": {"id": 4000168003, "name": "Dice", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638533} +{"stream": "sources", "data": {"id": 4000169003, "name": "GitHub Jobs", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638534} +{"stream": "sources", "data": {"id": 4000170003, "name": "Dribbble", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638534} +{"stream": "sources", "data": {"id": 4000171003, "name": "Social media presence", "type": {"id": 4000006003, "name": "Company marketing"}}, "emitted_at": 1664285638534} +{"stream": "sources", "data": {"id": 4000172003, "name": "Customer newsletter", "type": {"id": 4000006003, "name": "Company marketing"}}, "emitted_at": 1664285638534} +{"stream": "sources", "data": {"id": 4000173003, "name": "Use BountyJobs", "type": {"id": 4000007003, "name": "Agencies"}}, "emitted_at": 1664285638534} +{"stream": "sources", "data": {"id": 4000174003, "name": "Google", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638534} +{"stream": "sources", "data": {"id": 4000175003, "name": "Indeed", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638534} +{"stream": "sources", "data": {"id": 4000176003, "name": "Meetups", "type": {"id": 4000001003, "name": "In person event"}}, "emitted_at": 1664285638534} +{"stream": "sources", "data": {"id": 4000177003, "name": "Jobs page on your website", "type": {"id": 4000006003, "name": "Company marketing"}}, "emitted_at": 1664285638534} +{"stream": "sources", "data": {"id": 4000178003, "name": "Twitter", "type": {"id": 4000005003, "name": "Social media"}}, "emitted_at": 1664285638534} +{"stream": "sources", "data": {"id": 4000179003, "name": "Facebook", "type": {"id": 4000005003, "name": "Social media"}}, "emitted_at": 1664285638534} +{"stream": "sources", "data": {"id": 4000180003, "name": "SimplyHired", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638534} +{"stream": "sources", "data": {"id": 4000181003, "name": "Indeed", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638535} +{"stream": "sources", "data": {"id": 4000182003, "name": "Job fairs/Conferences/Trade shows", "type": {"id": 4000001003, "name": "In person event"}}, "emitted_at": 1664285638535} +{"stream": "sources", "data": {"id": 4000183003, "name": "Campus recruiting", "type": {"id": 4000001003, "name": "In person event"}}, "emitted_at": 1664285638535} +{"stream": "sources", "data": {"id": 4000184003, "name": "Uncubed", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638535} +{"stream": "sources", "data": {"id": 4000185003, "name": "Ladders", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638535} +{"stream": "sources", "data": {"id": 4000186003, "name": "Splash", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638535} +{"stream": "sources", "data": {"id": 4000187003, "name": "Recruiter.AI", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638535} +{"stream": "sources", "data": {"id": 4000188003, "name": "Underdog.io", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638535} +{"stream": "sources", "data": {"id": 4000189003, "name": "UpScored", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638535} +{"stream": "sources", "data": {"id": 4000190003, "name": "LinkMatch", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638535} +{"stream": "sources", "data": {"id": 4000191003, "name": "Jobbatical", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638535} +{"stream": "sources", "data": {"id": 4000192003, "name": "Upsider", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638535} +{"stream": "sources", "data": {"id": 4000195003, "name": "zealpath", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638535} +{"stream": "sources", "data": {"id": 4000198003, "name": "AppDirect Connector", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638536} +{"stream": "sources", "data": {"id": 4000213003, "name": "Helm", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638536} +{"stream": "sources", "data": {"id": 4000226003, "name": "Betts", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638536} +{"stream": "sources", "data": {"id": 4000228003, "name": "Circular", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638536} +{"stream": "sources", "data": {"id": 4000241003, "name": "Tempo", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638536} +{"stream": "sources", "data": {"id": 4000264003, "name": "Woo Auto-sourcer", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638536} +{"stream": "sources", "data": {"id": 4000344003, "name": "TopFunnel", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638712} +{"stream": "sources", "data": {"id": 4000538003, "name": "Greenhouse Test", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638712} +{"stream": "sources", "data": {"id": 4000619003, "name": "Wepow Staging", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638712} +{"stream": "sources", "data": {"id": 4001253003, "name": "Showcase Jobs", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638713} +{"stream": "sources", "data": {"id": 4001321003, "name": "Showcase QA", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638713} +{"stream": "sources", "data": {"id": 4001322003, "name": "Showcase Demo", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638713} +{"stream": "sources", "data": {"id": 4001506003, "name": "Indeed - Sponsored", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638713} +{"stream": "sources", "data": {"id": 4001507003, "name": "Indeed - Targeted Ad", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638713} +{"stream": "sources", "data": {"id": 4002280003, "name": "Dash by Dashworks", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638713} +{"stream": "sources", "data": {"id": 4002742003, "name": "Aleph", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638713} +{"stream": "sources", "data": {"id": 4002891003, "name": "Xing", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638713} +{"stream": "sources", "data": {"id": 4003381003, "name": "Talroo", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638713} +{"stream": "sources", "data": {"id": 4004993003, "name": "include.ai", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638713} +{"stream": "sources", "data": {"id": 4005607003, "name": "AppDirect", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638713} +{"stream": "sources", "data": {"id": 4006015003, "name": "Otta", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638713} +{"stream": "sources", "data": {"id": 4006251003, "name": "Revelo", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638714} +{"stream": "sources", "data": {"id": 4006550003, "name": "Rainmakers", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638714} +{"stream": "sources", "data": {"id": 4007366003, "name": "Relode", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638714} +{"stream": "sources", "data": {"id": 4007690003, "name": "JOIN", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638714} +{"stream": "sources", "data": {"id": 4008420003, "name": "ERIN", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638714} +{"stream": "sources", "data": {"id": 4008511003, "name": "VentureBeat Careers, powered by Jobbio", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638714} +{"stream": "sources", "data": {"id": 4008617003, "name": "Monster Organic", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638714} +{"stream": "sources", "data": {"id": 4009581003, "name": "Sourcediv dev", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638714} +{"stream": "sources", "data": {"id": 4009582003, "name": "Sourcediv", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638714} +{"stream": "sources", "data": {"id": 4009906003, "name": "PandoLogic", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638714} +{"stream": "sources", "data": {"id": 4010052003, "name": "JOBfindah", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638714} +{"stream": "sources", "data": {"id": 4010066003, "name": "Cord", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638714} +{"stream": "sources", "data": {"id": 4010238003, "name": "purpose.jobs", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638715} +{"stream": "sources", "data": {"id": 4010715003, "name": "Scout Hires", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638715} +{"stream": "sources", "data": {"id": 4011228003, "name": "CBREX", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638715} +{"stream": "sources", "data": {"id": 4012144003, "name": "Talent By Blind (dev)", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638715} +{"stream": "sources", "data": {"id": 4012145003, "name": "Talent By Blind (test)", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638715} +{"stream": "sources", "data": {"id": 4012387003, "name": "RecruitBot", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638715} +{"stream": "sources", "data": {"id": 4012474003, "name": "RepVue", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638715} +{"stream": "sources", "data": {"id": 4012953003, "name": "Jobplanner", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638715} +{"stream": "sources", "data": {"id": 4013544003, "name": "Test agency", "type": {"id": 4000007003, "name": "Agencies"}}, "emitted_at": 1664285638715} +{"stream": "sources", "data": {"id": 4013712003, "name": "Jobstep", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638715} +{"stream": "sources", "data": {"id": 4014208003, "name": "Relyance", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638715} +{"stream": "sources", "data": {"id": 4014433003, "name": "Intrro", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638715} +{"stream": "sources", "data": {"id": 4014636003, "name": "DataFrenzy", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638715} +{"stream": "sources", "data": {"id": 4015128003, "name": "Talent By Blind", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638716} +{"stream": "sources", "data": {"id": 4015567003, "name": "Real Links", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638716} +{"stream": "sources", "data": {"id": 4016305003, "name": "Scouted", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638716} +{"stream": "sources", "data": {"id": 4016428003, "name": "Talentry", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638716} +{"stream": "sources", "data": {"id": 4016532003, "name": "Careerjet", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638716} +{"stream": "sources", "data": {"id": 4018722003, "name": "ProvenBase", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638716} +{"stream": "sources", "data": {"id": 4018841003, "name": "Homi", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638716} +{"stream": "sources", "data": {"id": 4019367003, "name": "10x10", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638716} +{"stream": "sources", "data": {"id": 4019617003, "name": "Secret Tel Aviv", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638716} +{"stream": "sources", "data": {"id": 4020294003, "name": "BuiltIn", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638716} +{"stream": "sources", "data": {"id": 4020736003, "name": "Tobu", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638716} +{"stream": "sources", "data": {"id": 4020760003, "name": "Velents", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638716} +{"stream": "sources", "data": {"id": 4021421003, "name": "Upward", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638717} +{"stream": "sources", "data": {"id": 4022468003, "name": "ZoomInfo for Recruiters", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638717} +{"stream": "sources", "data": {"id": 4023661003, "name": "Prentus", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638717} +{"stream": "sources", "data": {"id": 4024133003, "name": "Arbeitnow", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638717} +{"stream": "sources", "data": {"id": 4024421003, "name": "GuidedCompass", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638717} +{"stream": "sources", "data": {"id": 4024601003, "name": "Elpha", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638717} +{"stream": "sources", "data": {"id": 4025746003, "name": "Indeed Hiring Platform", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638717} +{"stream": "sources", "data": {"id": 4025759003, "name": "Elpha Dev", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638717} +{"stream": "sources", "data": {"id": 4026726003, "name": "Careerpuck", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638717} +{"stream": "sources", "data": {"id": 4027676003, "name": "Appcast", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638717} +{"stream": "sources", "data": {"id": 4027972003, "name": "SV Academy", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638717} +{"stream": "sources", "data": {"id": 4028644003, "name": "Tolstoy", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638717} +{"stream": "sources", "data": {"id": 4028649003, "name": "TolstoyDev", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638718} +{"stream": "sources", "data": {"id": 4028668003, "name": "OutScout", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638718} +{"stream": "sources", "data": {"id": 4028669003, "name": "OutScoutDev", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638718} +{"stream": "sources", "data": {"id": 4028843003, "name": "signNow", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638718} +{"stream": "sources", "data": {"id": 4029781003, "name": "TitanHouse", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638718} +{"stream": "sources", "data": {"id": 4030665003, "name": "Reprograma", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638718} +{"stream": "sources", "data": {"id": 4031825003, "name": "Trinsly", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638718} +{"stream": "sources", "data": {"id": 4032805003, "name": "Brazen", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638718} +{"stream": "sources", "data": {"id": 4034611003, "name": "Wednesday Talent", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638718} +{"stream": "sources", "data": {"id": 4034771003, "name": "HRMarket 2", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638718} +{"stream": "sources", "data": {"id": 4035349003, "name": "Phenom People", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638718} +{"stream": "sources", "data": {"id": 4036053003, "name": "TalentMarketplace", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638718} +{"stream": "sources", "data": {"id": 4036865003, "name": "Spleadly", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638719} +{"stream": "sources", "data": {"id": 4037066003, "name": "PathMatch", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638719} +{"stream": "sources", "data": {"id": 4037068003, "name": "Bevov", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638719} +{"stream": "sources", "data": {"id": 4037490003, "name": "LinkedIn", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638719} +{"stream": "sources", "data": {"id": 4038632003, "name": "Reflr", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638719} +{"stream": "sources", "data": {"id": 4039580003, "name": "HeroHunt", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638719} +{"stream": "sources", "data": {"id": 4040831003, "name": "Inclusively", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638719} +{"stream": "sources", "data": {"id": 4040946003, "name": "Us in Technology", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638719} +{"stream": "sources", "data": {"id": 4041406003, "name": "Retorio", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638719} +{"stream": "sources", "data": {"id": 4041695003, "name": "Waldo Labs Relay", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638719} +{"stream": "sources", "data": {"id": 4041720003, "name": "Supercharge", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638719} +{"stream": "sources", "data": {"id": 4041898003, "name": "matched.io", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638719} +{"stream": "sources", "data": {"id": 4042739003, "name": "headhuntr.io", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638719} +{"stream": "sources", "data": {"id": 4042896003, "name": "JobVyne", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638720} +{"stream": "sources", "data": {"id": 4042898003, "name": "Celential.ai Recruiting", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638720} +{"stream": "sources", "data": {"id": 4043028003, "name": "EzHire Cannabis", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1664285638720} +{"stream": "sources", "data": {"id": 4043066003, "name": "Programmatic Sourcing", "type": {"id": 4000000003, "name": "Third-party boards"}}, "emitted_at": 1664285638720} +{"stream": "rejection_reasons", "data": {"id": 4014678003, "name": "reason1", "type": {"id": 4000000003, "name": "We rejected them"}}, "emitted_at": 1664285639282} +{"stream": "rejection_reasons", "data": {"id": 4014679003, "name": "reason2", "type": {"id": 4000001003, "name": "They rejected us"}}, "emitted_at": 1664285639283} +{"stream": "rejection_reasons", "data": {"id": 4014680003, "name": "reason3", "type": {"id": 4000002003, "name": "None specified"}}, "emitted_at": 1664285639283} +{"stream": "jobs_openings", "data": {"id": 4320015003, "opening_id": "3-1", "status": "open", "opened_at": "2020-11-24T23:27:11.723Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1664285640400} +{"stream": "jobs_openings", "data": {"id": 4320018003, "opening_id": "4-1", "status": "open", "opened_at": "2020-11-24T23:27:45.665Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1664285640752} +{"stream": "jobs_openings", "data": {"id": 4926182003, "opening_id": "5-1", "status": "open", "opened_at": "2021-10-08T08:19:42.457Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1664285641263} +{"stream": "jobs_openings", "data": {"id": 4928186003, "opening_id": "5-1", "status": "open", "opened_at": "2021-10-10T16:38:57.407Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1664285641263} +{"stream": "jobs_openings", "data": {"id": 4928187003, "opening_id": "5-2", "status": "open", "opened_at": "2021-10-10T16:39:08.365Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1664285641264} +{"stream": "jobs_openings", "data": {"id": 4928188003, "opening_id": "5-2", "status": "open", "opened_at": "2021-10-10T16:39:24.949Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1664285641264} +{"stream": "jobs_openings", "data": {"id": 4926183003, "opening_id": "5-2", "status": "open", "opened_at": "2021-10-08T08:19:42.457Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1664285641264} +{"stream": "jobs_openings", "data": {"id": 4928186003, "opening_id": "5-1", "status": "open", "opened_at": "2021-10-10T16:38:57.407Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1664285641443} +{"stream": "jobs_openings", "data": {"id": 4928187003, "opening_id": "5-2", "status": "open", "opened_at": "2021-10-10T16:39:08.365Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1664285641443} +{"stream": "jobs_openings", "data": {"id": 4928188003, "opening_id": "5-2", "status": "open", "opened_at": "2021-10-10T16:39:24.949Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1664285641443} +{"stream": "jobs_openings", "data": {"id": 4970166003, "opening_id": "6-1", "status": "open", "opened_at": "2021-11-30T01:00:00.000Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1664285641784} +{"stream": "job_stages", "data": {"id": 5245803003, "name": "Application Review", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 0, "interviews": [{"id": 5628614003, "name": "Application Review", "schedulable": false, "interview_kit": {"id": 5628609003, "content": null, "questions": []}, "estimated_minutes": 1, "default_interviewer_users": []}]}, "emitted_at": 1664285642801} +{"stream": "job_stages", "data": {"id": 5245804003, "name": "Preliminary Phone Screen", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 1, "interviews": [{"id": 5628615003, "name": "Preliminary Screening Call", "schedulable": true, "interview_kit": {"id": 5628610003, "content": null, "questions": []}, "estimated_minutes": 20, "default_interviewer_users": []}]}, "emitted_at": 1664285642803} +{"stream": "job_stages", "data": {"id": 5245805003, "name": "Phone Interview", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 2, "interviews": [{"id": 5628616003, "name": "Behavioral Phone Interview", "schedulable": true, "interview_kit": {"id": 5628611003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1664285642803} +{"stream": "job_stages", "data": {"id": 5245806003, "name": "Face to Face", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 3, "interviews": [{"id": 5628617003, "name": "Cultural Add Interview", "schedulable": true, "interview_kit": {"id": 5628612003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}, {"id": 5628618003, "name": "Peer Panel Interview", "schedulable": true, "interview_kit": {"id": 5628613003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}, {"id": 5628619003, "name": "Case Study", "schedulable": true, "interview_kit": {"id": 5628614003, "content": null, "questions": []}, "estimated_minutes": 45, "default_interviewer_users": []}, {"id": 5628620003, "name": "Executive Interview", "schedulable": true, "interview_kit": {"id": 5628615003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1664285642804} +{"stream": "job_stages", "data": {"id": 5245807003, "name": "Reference Check", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 4, "interviews": [{"id": 5628621003, "name": "Former Manager", "schedulable": false, "interview_kit": {"id": 5628616003, "content": null, "questions": []}, "estimated_minutes": 15, "default_interviewer_users": []}]}, "emitted_at": 1664285642804} +{"stream": "job_stages", "data": {"id": 5245808003, "name": "Offer", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 5, "interviews": []}, "emitted_at": 1664285642804} +{"stream": "job_stages", "data": {"id": 5245818003, "name": "Application Review", "created_at": "2020-11-24T23:27:45.704Z", "updated_at": "2020-11-24T23:27:45.704Z", "active": true, "job_id": 4177048003, "priority": 0, "interviews": [{"id": 5628634003, "name": "Application Review", "schedulable": false, "interview_kit": {"id": 5628629003, "content": null, "questions": []}, "estimated_minutes": 1, "default_interviewer_users": []}]}, "emitted_at": 1664285642805} +{"stream": "job_stages", "data": {"id": 5245819003, "name": "Preliminary Phone Screen", "created_at": "2020-11-24T23:27:45.704Z", "updated_at": "2020-11-24T23:27:45.704Z", "active": true, "job_id": 4177048003, "priority": 1, "interviews": [{"id": 5628635003, "name": "Preliminary Screening Call", "schedulable": true, "interview_kit": {"id": 5628630003, "content": null, "questions": []}, "estimated_minutes": 20, "default_interviewer_users": []}]}, "emitted_at": 1664285642805} +{"stream": "job_stages", "data": {"id": 5245820003, "name": "Phone Interview", "created_at": "2020-11-24T23:27:45.704Z", "updated_at": "2020-11-24T23:27:45.704Z", "active": true, "job_id": 4177048003, "priority": 2, "interviews": [{"id": 5628636003, "name": "Behavioral Phone Interview", "schedulable": true, "interview_kit": {"id": 5628631003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1664285642806} +{"stream": "job_stages", "data": {"id": 5245821003, "name": "Face to Face", "created_at": "2020-11-24T23:27:45.704Z", "updated_at": "2020-11-24T23:27:45.704Z", "active": true, "job_id": 4177048003, "priority": 3, "interviews": [{"id": 5628637003, "name": "Cultural Add Interview", "schedulable": true, "interview_kit": {"id": 5628632003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}, {"id": 5628638003, "name": "Peer Panel Interview", "schedulable": true, "interview_kit": {"id": 5628633003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}, {"id": 5628639003, "name": "Case Study", "schedulable": true, "interview_kit": {"id": 5628634003, "content": null, "questions": []}, "estimated_minutes": 45, "default_interviewer_users": []}, {"id": 5628640003, "name": "Executive Interview", "schedulable": true, "interview_kit": {"id": 5628635003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1664285642806} +{"stream": "job_stages", "data": {"id": 5245822003, "name": "Reference Check", "created_at": "2020-11-24T23:27:45.704Z", "updated_at": "2020-11-24T23:27:45.704Z", "active": true, "job_id": 4177048003, "priority": 4, "interviews": [{"id": 5628641003, "name": "Former Manager", "schedulable": false, "interview_kit": {"id": 5628636003, "content": null, "questions": []}, "estimated_minutes": 15, "default_interviewer_users": []}]}, "emitted_at": 1664285642807} +{"stream": "job_stages", "data": {"id": 5245823003, "name": "Offer", "created_at": "2020-11-24T23:27:45.704Z", "updated_at": "2020-11-24T23:27:45.704Z", "active": true, "job_id": 4177048003, "priority": 5, "interviews": []}, "emitted_at": 1664285642807} +{"stream": "job_stages", "data": {"id": 7179760003, "name": "Application Review", "created_at": "2021-10-08T08:19:42.584Z", "updated_at": "2021-10-08T08:19:42.584Z", "active": true, "job_id": 4446240003, "priority": 0, "interviews": [{"id": 8010560003, "name": "Application Review", "schedulable": false, "interview_kit": {"id": 8010544003, "content": "

\r\n
Triage the inbox
\r\n
    \r\n
  • Quickly knock out any applications that are spam, duplicates, or clearly not worth your time
    • \r\n
  • Pass along qualified applicants to the next stage for assessment
    • \r\n
  • Reject any applicants who lack the necessary experience or required skills, or whose applications have poor grammar, spelling, or formatting 
    • \r\n
\r\n

", "questions": []}, "estimated_minutes": 1, "default_interviewer_users": []}]}, "emitted_at": 1664285642807} +{"stream": "job_stages", "data": {"id": 7179761003, "name": "Preliminary Phone Screen", "created_at": "2021-10-08T08:19:42.606Z", "updated_at": "2021-10-08T08:19:42.606Z", "active": true, "job_id": 4446240003, "priority": 1, "interviews": [{"id": 8010561003, "name": "Preliminary Screening Call", "schedulable": true, "interview_kit": {"id": 8010545003, "content": "
Purpose
\r\n
    \r\n
  • High-level screening call to make sure the candidate meets the basic requirements
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Explain the company and the job to the candidate: is it what he/she is looking for?
    • \r\n
  • Walk through his/her resume – are there any red flags?
    • \r\n
  • Find out what the candidate is looking for in his/her ideal role. Write this down so you can refer to this later!
    • \r\n
  • Get the candidate's desired salary range: is it in line with the compensation package?
    • \r\n
  • Tell the candidate what to expect next in your process
    • \r\n
\r\n

", "questions": []}, "estimated_minutes": 20, "default_interviewer_users": []}]}, "emitted_at": 1664285642808} +{"stream": "job_stages", "data": {"id": 7179762003, "name": "Phone Interview", "created_at": "2021-10-08T08:19:42.620Z", "updated_at": "2021-10-08T08:19:42.620Z", "active": true, "job_id": 4446240003, "priority": 2, "interviews": [{"id": 8010562003, "name": "Behavioral Phone Interview", "schedulable": true, "interview_kit": {"id": 8010546003, "content": "
Purpose
\r\n
    \r\n
  • Dig in deeper with a set of behavioral questions that target your desired skill set
    • \r\n
  • Specifically look for the following attributes: [Insert desired attributes here]
    • \r\n
  • Determine whether the candidate should be brought in for a face-to-face interview
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Have the candidate give you concrete examples of times when they demonstrated strengths in your desired skill set
    • \r\n
  • Ask a few open-ended “why” questions
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
", "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1664285642808} +{"stream": "job_stages", "data": {"id": 7179763003, "name": "Face to Face", "created_at": "2021-10-08T08:19:42.632Z", "updated_at": "2021-10-08T08:19:42.632Z", "active": true, "job_id": 4446240003, "priority": 3, "interviews": [{"id": 8010563003, "name": "Cultural Add Interview", "schedulable": true, "interview_kit": {"id": 8010547003, "content": "
Purpose
\r\n
    \r\n
  • Determine whether or not the candidate would be a strong addition to the organization
    • \r\n
  • Do they live by your company values? [Insert Company Values Here]
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Ask the candidate for specific examples of times when they demonstrated your company values, e.g., “Tell me about a time when you took ownership of a project from start to finish”
    • \r\n
  • Ask about their motivations in their work - what excites them, what worries them, how do they work best, etc.
    • \r\n
  • Let the candidate know what they should expect next in your process
    • \r\n
", "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}, {"id": 8010564003, "name": "Peer Panel Interview", "schedulable": true, "interview_kit": {"id": 8010548003, "content": "
    \r\n
  • Ask the candidate behavioral questions that target the skills and personality traits you're looking for
    • \r\n
  • Specifically dig in on the following skills: [Insert skills and traits here]
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Have the candidate give you concrete examples of times when they demonstrated strengths in your desired skill set
    • \r\n
  • Ask a few open-ended “why” questions
    • \r\n
  • Is this someone who you want to work with and would add value to the team?
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
", "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}, {"id": 8010565003, "name": "Case Study", "schedulable": true, "interview_kit": {"id": 8010549003, "content": "

[FIll in your case study problem here]

\r\n

 

\r\n
    \r\n
  • Prepare a case study or problem for the candidate that mimics a real-world situation that he/she would face in the role
    • \r\n
  • Does the candidate approach the problem analytically and logically?
    • \r\n
  • Ask why he/she made certain decisions. Do they demonstrate a desired way of thinking?
    • \r\n
  • Ask follow-up questions to test the candidate further. (e.g., “What would you do this happened? How would you handle the following objection by a superior?”)
    • \r\n
  • Let the candidate know what they should expect next in your process
    • \r\n
", "questions": []}, "estimated_minutes": 45, "default_interviewer_users": []}, {"id": 8010566003, "name": "Executive Interview", "schedulable": true, "interview_kit": {"id": 8010550003, "content": "
    \r\n
  • Unstructured conversation between Executive or Hiring Manager and the candidate
    • \r\n
  • Find out what motivates the candidate. What would they be most excited about if offered the position? What would they be most nervous about?
    • \r\n
  • Sell the candidate on the vision of the company and the quality of the team
    • \r\n
  • Answer any questions the candidate may have
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
", "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1664285642808} +{"stream": "job_stages", "data": {"id": 7179764003, "name": "Reference Check", "created_at": "2021-10-08T08:19:42.665Z", "updated_at": "2021-10-08T08:19:42.665Z", "active": true, "job_id": 4446240003, "priority": 4, "interviews": [{"id": 8010567003, "name": "Former Manager", "schedulable": false, "interview_kit": {"id": 8010551003, "content": "
    \r\n
  • Have a quick chat with a former boss and confirm what the candidate said. Did the candidate accurately represent his/her responsibilities in their previous job?
    • \r\n
  • What does the boss think the candidate's biggest strengths are? What would the boss be most concerned about if hiring the candidate again?
    • \r\n
  • Ask about any red flags or questions you might have about the candidate
    • \r\n
", "questions": []}, "estimated_minutes": 15, "default_interviewer_users": []}]}, "emitted_at": 1664285642809} +{"stream": "job_stages", "data": {"id": 7179765003, "name": "Offer", "created_at": "2021-10-08T08:19:42.679Z", "updated_at": "2021-10-08T08:19:42.679Z", "active": true, "job_id": 4446240003, "priority": 5, "interviews": []}, "emitted_at": 1664285642809} +{"stream": "job_stages", "data": {"id": 7332462003, "name": "Application Review", "created_at": "2021-11-03T19:46:51.185Z", "updated_at": "2021-11-03T19:46:51.185Z", "active": true, "job_id": 4466310003, "priority": 0, "interviews": [{"id": 8202995003, "name": "Application Review", "schedulable": false, "interview_kit": {"id": 8202979003, "content": null, "questions": []}, "estimated_minutes": 1, "default_interviewer_users": []}]}, "emitted_at": 1664285642809} +{"stream": "job_stages", "data": {"id": 7332463003, "name": "Offer", "created_at": "2021-11-03T19:46:51.185Z", "updated_at": "2021-11-03T19:46:51.185Z", "active": true, "job_id": 4466310003, "priority": 1, "interviews": []}, "emitted_at": 1664285642809} +{"stream": "jobs_stages", "data": {"id": 5245803003, "name": "Application Review", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 0, "interviews": [{"id": 5628614003, "name": "Application Review", "schedulable": false, "interview_kit": {"id": 5628609003, "content": null, "questions": []}, "estimated_minutes": 1, "default_interviewer_users": []}]}, "emitted_at": 1664285644042} +{"stream": "jobs_stages", "data": {"id": 5245804003, "name": "Preliminary Phone Screen", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 1, "interviews": [{"id": 5628615003, "name": "Preliminary Screening Call", "schedulable": true, "interview_kit": {"id": 5628610003, "content": null, "questions": []}, "estimated_minutes": 20, "default_interviewer_users": []}]}, "emitted_at": 1664285644043} +{"stream": "jobs_stages", "data": {"id": 5245805003, "name": "Phone Interview", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 2, "interviews": [{"id": 5628616003, "name": "Behavioral Phone Interview", "schedulable": true, "interview_kit": {"id": 5628611003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1664285644043} +{"stream": "jobs_stages", "data": {"id": 5245806003, "name": "Face to Face", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 3, "interviews": [{"id": 5628617003, "name": "Cultural Add Interview", "schedulable": true, "interview_kit": {"id": 5628612003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}, {"id": 5628618003, "name": "Peer Panel Interview", "schedulable": true, "interview_kit": {"id": 5628613003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}, {"id": 5628619003, "name": "Case Study", "schedulable": true, "interview_kit": {"id": 5628614003, "content": null, "questions": []}, "estimated_minutes": 45, "default_interviewer_users": []}, {"id": 5628620003, "name": "Executive Interview", "schedulable": true, "interview_kit": {"id": 5628615003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1664285644043} +{"stream": "jobs_stages", "data": {"id": 5245807003, "name": "Reference Check", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 4, "interviews": [{"id": 5628621003, "name": "Former Manager", "schedulable": false, "interview_kit": {"id": 5628616003, "content": null, "questions": []}, "estimated_minutes": 15, "default_interviewer_users": []}]}, "emitted_at": 1664285644044} +{"stream": "jobs_stages", "data": {"id": 5245808003, "name": "Offer", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 5, "interviews": []}, "emitted_at": 1664285644044} +{"stream": "jobs_stages", "data": {"id": 5245818003, "name": "Application Review", "created_at": "2020-11-24T23:27:45.704Z", "updated_at": "2020-11-24T23:27:45.704Z", "active": true, "job_id": 4177048003, "priority": 0, "interviews": [{"id": 5628634003, "name": "Application Review", "schedulable": false, "interview_kit": {"id": 5628629003, "content": null, "questions": []}, "estimated_minutes": 1, "default_interviewer_users": []}]}, "emitted_at": 1664285644324} +{"stream": "jobs_stages", "data": {"id": 5245819003, "name": "Preliminary Phone Screen", "created_at": "2020-11-24T23:27:45.704Z", "updated_at": "2020-11-24T23:27:45.704Z", "active": true, "job_id": 4177048003, "priority": 1, "interviews": [{"id": 5628635003, "name": "Preliminary Screening Call", "schedulable": true, "interview_kit": {"id": 5628630003, "content": null, "questions": []}, "estimated_minutes": 20, "default_interviewer_users": []}]}, "emitted_at": 1664285644324} +{"stream": "jobs_stages", "data": {"id": 5245820003, "name": "Phone Interview", "created_at": "2020-11-24T23:27:45.704Z", "updated_at": "2020-11-24T23:27:45.704Z", "active": true, "job_id": 4177048003, "priority": 2, "interviews": [{"id": 5628636003, "name": "Behavioral Phone Interview", "schedulable": true, "interview_kit": {"id": 5628631003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1664285644324} +{"stream": "jobs_stages", "data": {"id": 5245821003, "name": "Face to Face", "created_at": "2020-11-24T23:27:45.704Z", "updated_at": "2020-11-24T23:27:45.704Z", "active": true, "job_id": 4177048003, "priority": 3, "interviews": [{"id": 5628637003, "name": "Cultural Add Interview", "schedulable": true, "interview_kit": {"id": 5628632003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}, {"id": 5628638003, "name": "Peer Panel Interview", "schedulable": true, "interview_kit": {"id": 5628633003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}, {"id": 5628639003, "name": "Case Study", "schedulable": true, "interview_kit": {"id": 5628634003, "content": null, "questions": []}, "estimated_minutes": 45, "default_interviewer_users": []}, {"id": 5628640003, "name": "Executive Interview", "schedulable": true, "interview_kit": {"id": 5628635003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1664285644325} +{"stream": "jobs_stages", "data": {"id": 5245822003, "name": "Reference Check", "created_at": "2020-11-24T23:27:45.704Z", "updated_at": "2020-11-24T23:27:45.704Z", "active": true, "job_id": 4177048003, "priority": 4, "interviews": [{"id": 5628641003, "name": "Former Manager", "schedulable": false, "interview_kit": {"id": 5628636003, "content": null, "questions": []}, "estimated_minutes": 15, "default_interviewer_users": []}]}, "emitted_at": 1664285644325} +{"stream": "jobs_stages", "data": {"id": 5245823003, "name": "Offer", "created_at": "2020-11-24T23:27:45.704Z", "updated_at": "2020-11-24T23:27:45.704Z", "active": true, "job_id": 4177048003, "priority": 5, "interviews": []}, "emitted_at": 1664285644325} +{"stream": "jobs_stages", "data": {"id": 7179760003, "name": "Application Review", "created_at": "2021-10-08T08:19:42.584Z", "updated_at": "2021-10-08T08:19:42.584Z", "active": true, "job_id": 4446240003, "priority": 0, "interviews": [{"id": 8010560003, "name": "Application Review", "schedulable": false, "interview_kit": {"id": 8010544003, "content": "

\r\n
Triage the inbox
\r\n
    \r\n
  • Quickly knock out any applications that are spam, duplicates, or clearly not worth your time
    • \r\n
  • Pass along qualified applicants to the next stage for assessment
    • \r\n
  • Reject any applicants who lack the necessary experience or required skills, or whose applications have poor grammar, spelling, or formatting 
    • \r\n
\r\n

", "questions": []}, "estimated_minutes": 1, "default_interviewer_users": []}]}, "emitted_at": 1664285644542} +{"stream": "jobs_stages", "data": {"id": 7179761003, "name": "Preliminary Phone Screen", "created_at": "2021-10-08T08:19:42.606Z", "updated_at": "2021-10-08T08:19:42.606Z", "active": true, "job_id": 4446240003, "priority": 1, "interviews": [{"id": 8010561003, "name": "Preliminary Screening Call", "schedulable": true, "interview_kit": {"id": 8010545003, "content": "
Purpose
\r\n
    \r\n
  • High-level screening call to make sure the candidate meets the basic requirements
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Explain the company and the job to the candidate: is it what he/she is looking for?
    • \r\n
  • Walk through his/her resume – are there any red flags?
    • \r\n
  • Find out what the candidate is looking for in his/her ideal role. Write this down so you can refer to this later!
    • \r\n
  • Get the candidate's desired salary range: is it in line with the compensation package?
    • \r\n
  • Tell the candidate what to expect next in your process
    • \r\n
\r\n

", "questions": []}, "estimated_minutes": 20, "default_interviewer_users": []}]}, "emitted_at": 1664285644542} +{"stream": "jobs_stages", "data": {"id": 7179762003, "name": "Phone Interview", "created_at": "2021-10-08T08:19:42.620Z", "updated_at": "2021-10-08T08:19:42.620Z", "active": true, "job_id": 4446240003, "priority": 2, "interviews": [{"id": 8010562003, "name": "Behavioral Phone Interview", "schedulable": true, "interview_kit": {"id": 8010546003, "content": "
Purpose
\r\n
    \r\n
  • Dig in deeper with a set of behavioral questions that target your desired skill set
    • \r\n
  • Specifically look for the following attributes: [Insert desired attributes here]
    • \r\n
  • Determine whether the candidate should be brought in for a face-to-face interview
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Have the candidate give you concrete examples of times when they demonstrated strengths in your desired skill set
    • \r\n
  • Ask a few open-ended “why” questions
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
", "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1664285644542} +{"stream": "jobs_stages", "data": {"id": 7179763003, "name": "Face to Face", "created_at": "2021-10-08T08:19:42.632Z", "updated_at": "2021-10-08T08:19:42.632Z", "active": true, "job_id": 4446240003, "priority": 3, "interviews": [{"id": 8010563003, "name": "Cultural Add Interview", "schedulable": true, "interview_kit": {"id": 8010547003, "content": "
Purpose
\r\n
    \r\n
  • Determine whether or not the candidate would be a strong addition to the organization
    • \r\n
  • Do they live by your company values? [Insert Company Values Here]
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Ask the candidate for specific examples of times when they demonstrated your company values, e.g., “Tell me about a time when you took ownership of a project from start to finish”
    • \r\n
  • Ask about their motivations in their work - what excites them, what worries them, how do they work best, etc.
    • \r\n
  • Let the candidate know what they should expect next in your process
    • \r\n
", "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}, {"id": 8010564003, "name": "Peer Panel Interview", "schedulable": true, "interview_kit": {"id": 8010548003, "content": "
    \r\n
  • Ask the candidate behavioral questions that target the skills and personality traits you're looking for
    • \r\n
  • Specifically dig in on the following skills: [Insert skills and traits here]
    • \r\n
\r\n
Sample Questions
\r\n
    \r\n
  • Have the candidate give you concrete examples of times when they demonstrated strengths in your desired skill set
    • \r\n
  • Ask a few open-ended “why” questions
    • \r\n
  • Is this someone who you want to work with and would add value to the team?
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
", "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}, {"id": 8010565003, "name": "Case Study", "schedulable": true, "interview_kit": {"id": 8010549003, "content": "

[FIll in your case study problem here]

\r\n

 

\r\n
    \r\n
  • Prepare a case study or problem for the candidate that mimics a real-world situation that he/she would face in the role
    • \r\n
  • Does the candidate approach the problem analytically and logically?
    • \r\n
  • Ask why he/she made certain decisions. Do they demonstrate a desired way of thinking?
    • \r\n
  • Ask follow-up questions to test the candidate further. (e.g., “What would you do this happened? How would you handle the following objection by a superior?”)
    • \r\n
  • Let the candidate know what they should expect next in your process
    • \r\n
", "questions": []}, "estimated_minutes": 45, "default_interviewer_users": []}, {"id": 8010566003, "name": "Executive Interview", "schedulable": true, "interview_kit": {"id": 8010550003, "content": "
    \r\n
  • Unstructured conversation between Executive or Hiring Manager and the candidate
    • \r\n
  • Find out what motivates the candidate. What would they be most excited about if offered the position? What would they be most nervous about?
    • \r\n
  • Sell the candidate on the vision of the company and the quality of the team
    • \r\n
  • Answer any questions the candidate may have
    • \r\n
  • Let the candidate know what to expect next in your process
    • \r\n
", "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1664285644542} +{"stream": "jobs_stages", "data": {"id": 7179764003, "name": "Reference Check", "created_at": "2021-10-08T08:19:42.665Z", "updated_at": "2021-10-08T08:19:42.665Z", "active": true, "job_id": 4446240003, "priority": 4, "interviews": [{"id": 8010567003, "name": "Former Manager", "schedulable": false, "interview_kit": {"id": 8010551003, "content": "
    \r\n
  • Have a quick chat with a former boss and confirm what the candidate said. Did the candidate accurately represent his/her responsibilities in their previous job?
    • \r\n
  • What does the boss think the candidate's biggest strengths are? What would the boss be most concerned about if hiring the candidate again?
    • \r\n
  • Ask about any red flags or questions you might have about the candidate
    • \r\n
", "questions": []}, "estimated_minutes": 15, "default_interviewer_users": []}]}, "emitted_at": 1664285644543} +{"stream": "jobs_stages", "data": {"id": 7179765003, "name": "Offer", "created_at": "2021-10-08T08:19:42.679Z", "updated_at": "2021-10-08T08:19:42.679Z", "active": true, "job_id": 4446240003, "priority": 5, "interviews": []}, "emitted_at": 1664285644543} +{"stream": "jobs_stages", "data": {"id": 7332462003, "name": "Application Review", "created_at": "2021-11-03T19:46:51.185Z", "updated_at": "2021-11-03T19:46:51.185Z", "active": true, "job_id": 4466310003, "priority": 0, "interviews": [{"id": 8202995003, "name": "Application Review", "schedulable": false, "interview_kit": {"id": 8202979003, "content": null, "questions": []}, "estimated_minutes": 1, "default_interviewer_users": []}]}, "emitted_at": 1664285644747} +{"stream": "jobs_stages", "data": {"id": 7332463003, "name": "Offer", "created_at": "2021-11-03T19:46:51.185Z", "updated_at": "2021-11-03T19:46:51.185Z", "active": true, "job_id": 4466310003, "priority": 1, "interviews": []}, "emitted_at": 1664285644747} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-greenhouse/setup.py b/airbyte-integrations/connectors/source-greenhouse/setup.py index 6707890b19ac6..17489c2260b38 100644 --- a/airbyte-integrations/connectors/source-greenhouse/setup.py +++ b/airbyte-integrations/connectors/source-greenhouse/setup.py @@ -16,7 +16,7 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-cdk~=0.1.79", "dataclasses-jsonschema==2.15.1"], + install_requires=["airbyte-cdk>=0.1.91", "dataclasses-jsonschema==2.15.1"], package_data={"": ["*.json", "*.yaml", "schemas/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/greenhouse.yaml b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/greenhouse.yaml index f0f6e1252bee1..6b54022a5d2c4 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/greenhouse.yaml +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/greenhouse.yaml @@ -13,6 +13,15 @@ definitions: name: "{{ options['name'] }}" url_base: "https://harvest.greenhouse.io/v1/" http_method: "GET" + error_handler: + type: CompositeErrorHandler + # ignore 403 error but retry default retriable http errors (429, 500 - 600) + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [403] + action: IGNORE + - type: DefaultErrorHandler authenticator: type: BasicHttpAuthenticator username: "{{ config['api_key'] }}" @@ -23,13 +32,13 @@ definitions: record_selector: $ref: "*ref(definitions.selector)" paginator: - type: LimitPaginator - page_size: 100 + type: DefaultPaginator pagination_strategy: type: "CursorPagination" cursor_value: "{{ headers['link']['next']['url'] }}" stop_condition: "{{ 'next' not in headers['link'] }}" - limit_option: + page_size: 100 + page_size_option: field_name: "per_page" inject_into: "request_parameter" page_token_option: diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py index 4757ed989fc33..1a23e40c3626f 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py @@ -152,3 +152,20 @@ def test_parse_response_empty_content(applications_stream): records = [record for record in parsed_response] assert records == [] + + +def test_ignore_403(applications_stream): + response = requests.Response() + response.status_code = 403 + response._content = b"" + parsed_response = applications_stream.retriever.parse_response(response, stream_state={}) + records = [record for record in parsed_response] + assert records == [] + + +def test_retry_429(applications_stream): + response = requests.Response() + response.status_code = 429 + response._content = b"{}" + should_retry = applications_stream.retriever.should_retry(response) + assert should_retry is True diff --git a/airbyte-integrations/connectors/source-harvest/Dockerfile b/airbyte-integrations/connectors/source-harvest/Dockerfile index cd03eb7f3907a..3c63ca2bef1b1 100644 --- a/airbyte-integrations/connectors/source-harvest/Dockerfile +++ b/airbyte-integrations/connectors/source-harvest/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-harvest diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-harvest/integration_tests/abnormal_state.json index d5687c658eafc..cd404aa473ec7 100644 --- a/airbyte-integrations/connectors/source-harvest/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/abnormal_state.json @@ -1,86 +1,198 @@ -{ - "contacts": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "clients": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "invoices": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "invoice_messages": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "invoice_payments": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "invoice_item_categories": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "estimates": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "estimate_messages": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "estimate_item_categories": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "expenses": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "expense_categories": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "tasks": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "time_entries": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "user_assignments": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "task_assignments": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "projects": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "roles": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "users": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "project_assignments": { - "updated_at": "2217-06-26T21:20:07Z" - }, - "expenses_clients": { - "to": "22170626" - }, - "expenses_projects": { - "to": "22170626" - }, - "expenses_categories": { - "to": "22170626" - }, - "expenses_team": { - "to": "22170626" - }, - "uninvoiced": { - "to": "22170626" - }, - "time_clients": { - "to": "22170626" - }, - "time_projects": { - "to": "22170626" - }, - "time_tasks": { - "to": "22170626" - }, - "time_team": { - "to": "22170626" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "contacts" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "clients" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "invoices" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "invoice_messages" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "invoice_payments" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "invoice_item_categories" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "estimates" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "estimate_messages" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "estimate_item_categories" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "expenses" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "expense_categories" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "tasks" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "time_entries" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "user_assignments" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "task_assignments" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "projects" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "roles" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2217-06-26T21:20:07Z" }, + "stream_descriptor": { "name": "project_assignments" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "22170626" }, + "stream_descriptor": { "name": "expenses_clients" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "22170626" }, + "stream_descriptor": { "name": "expenses_projects" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "22170626" }, + "stream_descriptor": { "name": "expenses_categories" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "22170626" }, + "stream_descriptor": { "name": "expenses_team" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "22170626" }, + "stream_descriptor": { "name": "uninvoiced" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "22170626" }, + "stream_descriptor": { "name": "time_clients" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "22170626" }, + "stream_descriptor": { "name": "time_projects" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "22170626" }, + "stream_descriptor": { "name": "time_tasks" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "22170626" }, + "stream_descriptor": { "name": "time_team" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-harvest/integration_tests/sample_state.json index ae4bb37a212e5..362ce52b00722 100644 --- a/airbyte-integrations/connectors/source-harvest/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/sample_state.json @@ -1,86 +1,198 @@ -{ - "contacts": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "clients": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "invoices": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "invoice_messages": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "invoice_payments": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "invoice_item_categories": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "estimates": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "estimate_messages": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "estimate_item_categories": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "expenses": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "expense_categories": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "tasks": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "time_entries": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "user_assignments": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "task_assignments": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "projects": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "roles": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "users": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "project_assignments": { - "updated_at": "2021-01-01T00:00:00Z" - }, - "expenses_clients": { - "to": "20210101" - }, - "expenses_projects": { - "to": "20210101" - }, - "expenses_categories": { - "to": "20210101" - }, - "expenses_team": { - "to": "20210101" - }, - "uninvoiced": { - "to": "20210101" - }, - "time_clients": { - "to": "20210101" - }, - "time_projects": { - "to": "20210101" - }, - "time_tasks": { - "to": "20210101" - }, - "time_team": { - "to": "20210101" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "contacts" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "clients" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "invoices" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "invoice_messages" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "invoice_payments" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "invoice_item_categories" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "estimates" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "estimate_messages" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "estimate_item_categories" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "expenses" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "expense_categories" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "tasks" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "time_entries" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "user_assignments" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "task_assignments" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "projects" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "roles" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "users" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2021-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "project_assignments" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "20210101" }, + "stream_descriptor": { "name": "expenses_clients" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "20210101" }, + "stream_descriptor": { "name": "expenses_projects" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "20210101" }, + "stream_descriptor": { "name": "expenses_categories" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "20210101" }, + "stream_descriptor": { "name": "expenses_team" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "20210101" }, + "stream_descriptor": { "name": "uninvoiced" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "20210101" }, + "stream_descriptor": { "name": "time_clients" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "20210101" }, + "stream_descriptor": { "name": "time_projects" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "20210101" }, + "stream_descriptor": { "name": "time_tasks" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "to": "20210101" }, + "stream_descriptor": { "name": "time_team" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 71484d687c4bc..86a7233f7bbbe 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json index 36b6029f0ecb4..9af1bcca18685 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json @@ -1,53 +1,121 @@ -{ - "companies": { - "updatedAt": "2221-10-12T13:37:56.412000+00:00" - }, - "contact_lists": { - "updatedAt": "2221-10-12T13:37:56.412000+00:00" - }, - "contacts": { - "updatedAt": "2221-10-12T13:37:56.412000+00:00" - }, - "deals": { - "updatedAt": "2221-10-12T13:37:56.412000+00:00" - }, - "email_events": { - "created": "2221-10-12T13:37:56.412000+00:00" - }, - "engagements_calls": { - "updatedAt": "2221-10-12T13:37:56.412000+00:00" - }, - "engagements_emails": { - "updatedAt": "2221-10-12T13:37:56.412000+00:00" - }, - "engagements_meetings": { - "updatedAt": "2221-10-12T13:37:56.412000+00:00" - }, - "engagements_notes": { - "updatedAt": "2221-10-12T13:37:56.412000+00:00" - }, - "engagements_tasks": { - "updatedAt": "2221-10-12T13:37:56.412000+00:00" - }, - "engagements": { - "lastUpdated": 7945393076412 - }, - "line_items": { - "updatedAt": "2221-10-12T13:37:56.412000+00:00" - }, - "products": { - "updatedAt": "2221-10-12T13:37:56.412000+00:00" - }, - "quotes": { - "updatedAt": "2221-10-12T13:37:56.412000+00:00" - }, - "subscription_changes": { - "timestamp": "2221-10-12T13:37:56.412000+00:00" - }, - "tickets": { - "updatedAt": "2221-10-12T13:37:56.412000+00:00" - }, - "property_history": { - "timestamp": 7966012325000 +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "companies" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "contact_lists" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "contacts" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "deals" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "email_events" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "engagements_calls" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "engagements_emails" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "engagements_meetings" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "engagements_notes" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "engagements_tasks" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastUpdated": 7945393076412 }, + "stream_descriptor": { "name": "engagements" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "line_items" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "products" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "quotes" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "timestamp": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "subscription_changes" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2221-10-12T13:37:56.412000+00:00" }, + "stream_descriptor": { "name": "tickets" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "timestamp": 7966012325000 }, + "stream_descriptor": { "name": "property_history" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 9c67ccd5a4dab..db286e7fd7e5f 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -2,14 +2,14 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -import copy import logging -from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union import requests from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog +from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.utils.schema_helpers import split_config from airbyte_cdk.utils.event_timing import create_timer @@ -137,17 +137,21 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: return available_streams def read( - self, logger: logging.Logger, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, state: MutableMapping[str, Any] = None + self, + logger: logging.Logger, + config: Mapping[str, Any], + catalog: ConfiguredAirbyteCatalog, + state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None, ) -> Iterator[AirbyteMessage]: """ This method is overridden to check whether the stream `quotes` exists in the source, if not skip reading that stream. """ - connector_state = copy.deepcopy(state or {}) logger.info(f"Starting syncing {self.name}") config, internal_config = split_config(config) # TODO assert all streams exist in the connector # get the streams once in case the connector needs to make any queries to generate them stream_instances = {s.name: s for s in self.streams(config)} + state_manager = ConnectorStateManager(stream_instance_map=stream_instances, state=state) self._stream_to_instance_map = stream_instances with create_timer(self.name) as timer: for configured_stream in catalog.streams: @@ -165,7 +169,7 @@ def read( logger=logger, stream_instance=stream_instance, configured_stream=configured_stream, - connector_state=connector_state, + state_manager=state_manager, internal_config=internal_config, ) except Exception as e: diff --git a/airbyte-integrations/connectors/source-instagram/Dockerfile b/airbyte-integrations/connectors/source-instagram/Dockerfile index af483888c48eb..8dadf21002333 100644 --- a/airbyte-integrations/connectors/source-instagram/Dockerfile +++ b/airbyte-integrations/connectors/source-instagram/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.11 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-instagram diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py index 94ddddec7e565..d29a44d21d357 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py @@ -2,18 +2,10 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -import logging from datetime import datetime -from typing import Any, Iterator, List, Mapping, MutableMapping, Tuple +from typing import Any, List, Mapping, Tuple -from airbyte_cdk.models import ( - AirbyteMessage, - AuthSpecification, - ConfiguredAirbyteCatalog, - ConnectorSpecification, - DestinationSyncMode, - OAuth2Specification, -) +from airbyte_cdk.models import AuthSpecification, ConnectorSpecification, DestinationSyncMode, OAuth2Specification from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from pydantic import BaseModel, Field @@ -58,15 +50,6 @@ def check_connection(self, logger, config: Mapping[str, Any]) -> Tuple[bool, Any return ok, error_msg - def read( - self, logger: logging.Logger, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, state: MutableMapping[str, Any] = None - ) -> Iterator[AirbyteMessage]: - for stream in self.streams(config): - state_key = str(stream.name) - if state and state_key in state and hasattr(stream, "upgrade_state_to_latest_format"): - state[state_key] = stream.upgrade_state_to_latest_format(state[state_key]) - return super().read(logger, config, catalog, state) - def streams(self, config: Mapping[str, Any]) -> List[Stream]: """Discovery method, returns available streams diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py index e77e29164562d..828eaf77f46e6 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py @@ -35,10 +35,6 @@ def fields(self) -> List[str]: fields = list(self.get_json_schema().get("properties", {}).keys()) return list(set(fields) - set(non_object_fields)) - def upgrade_state_to_latest_format(self, state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: - """Upgrade state to latest format and return new state object""" - return copy.deepcopy(state) - def request_params( self, stream_slice: Mapping[str, Any] = None, @@ -251,14 +247,6 @@ def _state_has_legacy_format(self, state: Mapping[str, Any]) -> bool: return True return False - def upgrade_state_to_latest_format(self, state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: - """Upgrade state to latest format and return new state object""" - if self._state_has_legacy_format(state): - self.logger.info(f"The {self.name} state has old format, converting...") - return {account_id: {self.cursor_field: str(cursor_value)} for account_id, cursor_value in state.items()} - - return super().upgrade_state_to_latest_format(state) - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): """Update stream state from latest record""" record_value = latest_record[self.cursor_field] diff --git a/airbyte-integrations/connectors/source-intercom/Dockerfile b/airbyte-integrations/connectors/source-intercom/Dockerfile index 88b1fd2787786..8d46e345b8918 100644 --- a/airbyte-integrations/connectors/source-intercom/Dockerfile +++ b/airbyte-integrations/connectors/source-intercom/Dockerfile @@ -35,5 +35,5 @@ COPY source_intercom ./source_intercom ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.26 +LABEL io.airbyte.version=0.1.27 LABEL io.airbyte.name=airbyte/source-intercom diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json index 6ec574f009ce2..1ce39f2acb3f5 100755 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json @@ -1,26 +1,50 @@ -{ - "companies": { - "updated_at": 7626086649 +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 7626086649 }, + "stream_descriptor": { "name": "companies" } + } }, - "company_segments": { - "updated_at": 7626086649, - "companies": { - "updated_at": 7626086649 + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": 7626086649, + "companies": { "updated_at": 7626086649 } + }, + "stream_descriptor": { "name": "company_segments" } } }, - "conversations": { - "updated_at": 7626086649 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 7626086649 }, + "stream_descriptor": { "name": "conversations" } + } }, - "conversation_parts": { - "updated_at": 7626086649, - "conversations": { - "updated_at": 7626086649 + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": 7626086649, + "conversations": { "updated_at": 7626086649 } + }, + "stream_descriptor": { "name": "conversation_parts" } } }, - "contacts": { - "updated_at": 7626086649 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 7626086649 }, + "stream_descriptor": { "name": "contacts" } + } }, - "segments": { - "updated_at": 7626086649 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 7626086649 }, + "stream_descriptor": { "name": "segments" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json index 95d0974f3d30a..bc8f41c5edf0a 100755 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json @@ -1,26 +1,50 @@ -{ - "companies": { - "updated_at": 1626086649 +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1626086649 }, + "stream_descriptor": { "name": "companies" } + } }, - "company_segments": { - "updated_at": 1648469610, - "companies": { - "updated_at": 1625749675 + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": 1648469610, + "companies": { "updated_at": 1625749675 } + }, + "stream_descriptor": { "name": "company_segments" } } }, - "conversations": { - "updated_at": 1632835061 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1632835061 }, + "stream_descriptor": { "name": "conversations" } + } }, - "conversation_parts": { - "updated_at": 1632835061, - "conversations": { - "updated_at": 1632835061 + { + "type": "STREAM", + "stream": { + "stream_state": { + "updated_at": 1632835061, + "conversations": { "updated_at": 1632835061 } + }, + "stream_descriptor": { "name": "conversation_parts" } } }, - "contacts": { - "updated_at": 1626086649 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1626086649 }, + "stream_descriptor": { "name": "contacts" } + } }, - "segments": { - "updated_at": 1626086649 + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1626086649 }, + "stream_descriptor": { "name": "segments" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-klaviyo/Dockerfile b/airbyte-integrations/connectors/source-klaviyo/Dockerfile index e14a712ab89a3..4e769f06f7328 100644 --- a/airbyte-integrations/connectors/source-klaviyo/Dockerfile +++ b/airbyte-integrations/connectors/source-klaviyo/Dockerfile @@ -34,5 +34,5 @@ COPY source_klaviyo ./source_klaviyo ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.8 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-klaviyo diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json index c9ac3cf70e411..2c4f11268d3c5 100644 --- a/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json @@ -1,11 +1,23 @@ -{ - "events": { - "timestamp": 9621295127 +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "timestamp": 9621295127 }, + "stream_descriptor": { "name": "events" } + } }, - "global_exclusions": { - "timestamp": "2120-10-10T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "timestamp": "2120-10-10T00:00:00Z" }, + "stream_descriptor": { "name": "global_exclusions" } + } }, - "flows": { - "created": "2120-10-10 00:00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { "created": "2120-10-10 00:00:00" }, + "stream_descriptor": { "name": "flows" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile index 4f249aaf7cf33..66e0d7aca734d 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile @@ -33,5 +33,5 @@ COPY source_linkedin_ads ./source_linkedin_ads ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-linkedin-ads diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/abnormal_state.json index d29c225e2f8cb..204168f1d2c66 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/abnormal_state.json @@ -1,23 +1,51 @@ -{ - "account_users": { - "lastModified": "2050-01-01" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModified": "2050-01-01" }, + "stream_descriptor": { "name": "account_users" } + } }, - "campaign_groups": { - "lastModified": "2050-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModified": "2050-01-01" }, + "stream_descriptor": { "name": "campaign_groups" } + } }, - "campaigns": { - "lastModified": "2050-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModified": "2050-01-01" }, + "stream_descriptor": { "name": "campaigns" } + } }, - "creatives": { - "lastModified": "2050-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModified": "2050-01-01" }, + "stream_descriptor": { "name": "creatives" } + } }, - "ad_direct_sponsored_contents": { - "lastModified": "2050-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModified": "2050-01-01" }, + "stream_descriptor": { "name": "ad_direct_sponsored_contents" } + } }, - "ad_campaign_analytics": { - "end_date": "2050-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { "end_date": "2050-01-01" }, + "stream_descriptor": { "name": "ad_campaign_analytics" } + } }, - "ad_creative_analytics": { - "end_date": "2050-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { "end_date": "2050-01-01" }, + "stream_descriptor": { "name": "ad_creative_analytics" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-mailchimp/Dockerfile b/airbyte-integrations/connectors/source-mailchimp/Dockerfile index dd7ef7e9f4e78..671746b197fd1 100644 --- a/airbyte-integrations/connectors/source-mailchimp/Dockerfile +++ b/airbyte-integrations/connectors/source-mailchimp/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.14 +LABEL io.airbyte.version=0.2.15 LABEL io.airbyte.name=airbyte/source-mailchimp diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json b/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json index 7fe6fa04915b3..4abc785402121 100644 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json @@ -1,7 +1,25 @@ -{ - "campaigns": { "create_time": "2020-11-23T05:42:11+00:00" }, - "lists": { "date_created": "2020-09-25T04:47:31+00:00" }, - "email_activity": { - "49d68626f3": { "timestamp": "2020-11-23T05:42:10+00:00" } +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "create_time": "2020-11-23T05:42:11+00:00" }, + "stream_descriptor": { "name": "campaigns" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "date_created": "2020-09-25T04:47:31+00:00" }, + "stream_descriptor": { "name": "lists" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "49d68626f3": { "timestamp": "2020-11-23T05:42:10+00:00" } + }, + "stream_descriptor": { "name": "email_activity" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-marketo/Dockerfile b/airbyte-integrations/connectors/source-marketo/Dockerfile index 240aad644a8ed..2535bd7ba33de 100644 --- a/airbyte-integrations/connectors/source-marketo/Dockerfile +++ b/airbyte-integrations/connectors/source-marketo/Dockerfile @@ -34,5 +34,5 @@ COPY source_marketo ./source_marketo ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.8 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-marketo diff --git a/airbyte-integrations/connectors/source-marketo/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-marketo/integration_tests/abnormal_state.json index a4f7b59a29b9b..8d32d6ac02b82 100644 --- a/airbyte-integrations/connectors/source-marketo/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-marketo/integration_tests/abnormal_state.json @@ -1,17 +1,37 @@ -{ - "programs": { - "updatedAt": "2025-07-15T07:41:30Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2025-07-15T07:41:30Z" }, + "stream_descriptor": { "name": "programs" } + } }, - "campaigns": { - "createdAt": "2025-07-15T07:41:30Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "createdAt": "2025-07-15T07:41:30Z" }, + "stream_descriptor": { "name": "campaigns" } + } }, - "lists": { - "createdAt": "2025-07-15T07:41:30Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "createdAt": "2025-07-15T07:41:30Z" }, + "stream_descriptor": { "name": "lists" } + } }, - "leads": { - "updatedAt": "2025-07-15T07:41:30Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2025-07-15T07:41:30Z" }, + "stream_descriptor": { "name": "leads" } + } }, - "activities_visit_webpage": { - "activityDate": "2025-07-15T07:41:30Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "activityDate": "2025-07-15T07:41:30Z" }, + "stream_descriptor": { "name": "activities_visit_webpage" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-marketo/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-marketo/integration_tests/sample_state.json index be2c4e22ee15c..eed9ca5628fbc 100644 --- a/airbyte-integrations/connectors/source-marketo/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-marketo/integration_tests/sample_state.json @@ -1,17 +1,37 @@ -{ - "programs": { - "updatedAt": "2021-07-15T07:41:30Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2021-07-15T07:41:30Z" }, + "stream_descriptor": { "name": "programs" } + } }, - "campaigns": { - "createdAt": "2021-07-15T07:41:30Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "createdAt": "2021-07-15T07:41:30Z" }, + "stream_descriptor": { "name": "campaigns" } + } }, - "lists": { - "createdAt": "2021-07-15T07:41:30Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "createdAt": "2021-07-15T07:41:30Z" }, + "stream_descriptor": { "name": "lists" } + } }, - "leads": { - "updatedAt": "2021-07-15T07:41:30Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2021-07-15T07:41:30Z" }, + "stream_descriptor": { "name": "leads" } + } }, - "activities_visit_webpage": { - "activityDate": "2021-07-15T07:41:30Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "activityDate": "2021-07-15T07:41:30Z" }, + "stream_descriptor": { "name": "activities_visit_webpage" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-marketo/setup.py b/airbyte-integrations/connectors/source-marketo/setup.py index 054f5677afb32..1130809377d91 100644 --- a/airbyte-integrations/connectors/source-marketo/setup.py +++ b/airbyte-integrations/connectors/source-marketo/setup.py @@ -11,6 +11,7 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", + "pytest-faker==2.0.0", "pytest-mock~=3.6.1", "requests-mock", "source-acceptance-test", diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py index d886fddf150bd..aef902242d439 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py @@ -4,7 +4,6 @@ import csv import datetime -import io import json from abc import ABC from time import sleep @@ -228,8 +227,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp default_prop = {"type": ["null", "string"]} schema = self.get_json_schema()["properties"] - fp = io.StringIO(response.text) - reader = csv.DictReader(fp) + reader = csv.DictReader(response.iter_lines(chunk_size=1024, decode_unicode=True)) for record in reader: new_record = {**record} attributes = json.loads(new_record.pop("attributes", "{}")) diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py b/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py index 3659a10ac789b..e163081e43482 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py @@ -2,6 +2,10 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import os.path +import sys +import time + import pendulum import pytest from source_marketo.source import Activities, MarketoAuthenticator @@ -60,3 +64,30 @@ def send_email_stream(config, activity): stream_name = f"activities_{activity['name']}" cls = type(stream_name, (Activities,), {"activity": activity}) return cls(config) + + +@pytest.fixture +def file_generator(faker): + def _generator(min_size: int): + print(f"Generating a test file of {min_size // 1024 ** 2} MB, this could take some time") + + def fake_records_gen(): + new_line = "\n" + for i in range(1000): + yield f"{str(faker.random_int())},{faker.random_int()},{faker.date_of_birth()},{faker.random_int()}," f"{faker.random_int()},{faker.email()},{faker.postcode()}{new_line}" + + size, records = 0, 0 + path = os.path.realpath(str(time.time())) + with open(path, "w") as output: + output.write("marketoGUID,leadId,activityDate,activityTypeId,campaignId,primaryAttributeValueId,primaryAttributeValue\n") + while size < min_size: + frg = fake_records_gen() + print("Writing another 1000 records..") + for person in frg: + output.write(person) + records += 1 + size += sys.getsizeof(person) + print(f"Finished: {records} records written to {path}") + return path, records + + return _generator diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py b/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py index 95e4f0a84bbe5..51768f950d73e 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py @@ -3,6 +3,9 @@ # import logging +import os +import tracemalloc +from functools import partial from unittest.mock import ANY, Mock, patch import pytest @@ -123,7 +126,47 @@ def test_activities_schema(activity, expected_schema, config): ), ) def test_export_parse_response(send_email_stream, response_text, expected_records): - assert list(send_email_stream.parse_response(Mock(text=response_text))) == expected_records + def iter_lines(*args, **kwargs): + yield from response_text.splitlines() + + assert list(send_email_stream.parse_response(Mock(iter_lines=iter_lines, request=Mock(url="/send_email/1")))) == expected_records + + +def test_memory_usage(send_email_stream, file_generator): + min_file_size = 5 * (1024**2) # 5 MB + big_file_path, records_generated = file_generator(min_size=min_file_size) + small_file_path, _ = file_generator(min_size=1) + + def iter_lines(file_path="", **kwargs): + with open(file_path, "r") as file: + for line in file: + yield line + + tracemalloc.start() + records = 0 + + for _ in send_email_stream.parse_response( + Mock(iter_lines=partial(iter_lines, file_path=big_file_path), request=Mock(url="/send_email/1")) + ): + records += 1 + _, big_file_peak = tracemalloc.get_traced_memory() + assert records == records_generated + + tracemalloc.reset_peak() + tracemalloc.clear_traces() + + for _ in send_email_stream.parse_response( + Mock(iter_lines=partial(iter_lines, file_path=small_file_path), request=Mock(url="/send_email/1")) + ): + pass + _, small_file_peak = tracemalloc.get_traced_memory() + + os.remove(big_file_path) + os.remove(small_file_path) + # First we run parse_response() on a large file and track how much memory was consumed. + # Then we do the same with a tiny file. The goal is not to load the whole file into memory when parsing the response, + # so we assert the memory consumed was almost the same for two runs. Allowed delta is 50 KB which is 1% of a big file size. + assert abs(big_file_peak - small_file_peak) < 50 * 1024 @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-mixpanel/Dockerfile b/airbyte-integrations/connectors/source-mixpanel/Dockerfile index a32e593fd6184..7941d35a3b905 100644 --- a/airbyte-integrations/connectors/source-mixpanel/Dockerfile +++ b/airbyte-integrations/connectors/source-mixpanel/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.25 +LABEL io.airbyte.version=0.1.27 LABEL io.airbyte.name=airbyte/source-mixpanel diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json index f1c3d5da07e13..ee368b68b2081 100644 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json @@ -1,22 +1,44 @@ -{ - "funnels": { - "8901755": { - "date": "2030-01-01" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "8901755": { "date": "2030-01-01" } }, + "stream_descriptor": { "name": "funnels" } } }, - "revenue": { - "date": "2030-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { "date": "2030-01-01" }, + "stream_descriptor": { "name": "revenue" } + } }, - "cohorts": { - "created": "2030-01-01 00:00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { "created": "2030-01-01 00:00:00" }, + "stream_descriptor": { "name": "cohorts" } + } }, - "export": { - "time": "2030-01-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "time": "2030-01-01T00:00:00Z" }, + "stream_descriptor": { "name": "export" } + } }, - "cohort_members": { - "last_seen": "2030-01-01T00:00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { "last_seen": "2030-01-01T00:00:00" }, + "stream_descriptor": { "name": "cohort_members" } + } }, - "engage": { - "last_seen": "2030-01-01T00:00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { "last_seen": "2030-01-01T00:00:00" }, + "stream_descriptor": { "name": "engage" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/sample_state.json index 667b6a67d26d6..2cb9e7ea0d9fc 100644 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-mixpanel/integration_tests/sample_state.json @@ -1,12 +1,26 @@ -{ - "funnels": { - "8901755": { "date": "2021-07-13" }, - "10463655": { "date": "2021-07-13" } +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "8901755": { "date": "2021-07-13" }, + "10463655": { "date": "2021-07-13" } + }, + "stream_descriptor": { "name": "funnels" } + } }, - "revenue": { - "date": "2021-07-01" + { + "type": "STREAM", + "stream": { + "stream_state": { "date": "2021-07-01" }, + "stream_descriptor": { "name": "revenue" } + } }, - "export": { - "date": "2021-06-16T12:00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { "date": "2021-06-16T12:00:00" }, + "stream_descriptor": { "name": "export" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py index 3967523c1f076..baae4c86b3348 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py @@ -103,13 +103,16 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: streams = [ Annotations(authenticator=auth, **config), Cohorts(authenticator=auth, **config), - CohortMembers(authenticator=auth, **config), Funnels(authenticator=auth, **config), Revenue(authenticator=auth, **config), ] # streams with dynamically generated schema - for stream in [Engage(authenticator=auth, **config), Export(authenticator=auth, **config)]: + for stream in [ + CohortMembers(authenticator=auth, **config), + Engage(authenticator=auth, **config), + Export(authenticator=auth, **config), + ]: try: stream.get_json_schema() except requests.HTTPError as e: diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py index 302d6c471981b..3047257a9da14 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py @@ -72,4 +72,4 @@ def test_streams_disabled_402(requests_mock, config_raw): requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/engage/properties", setup_response(402, {})) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/events/properties/top", setup_response(402, {})) streams = SourceMixpanel().streams(config_raw) - assert {s.name for s in streams} == {"cohort_members", "funnels", "revenue", "annotations", "cohorts"} + assert {s.name for s in streams} == {"funnels", "revenue", "annotations", "cohorts"} diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index fc2801800ee2c..37f3f2fcdde14 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,7 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.14 + +LABEL io.airbyte.version=1.0.1 + LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java index fdd2b802deb75..18a6ce917c57b 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java @@ -62,7 +62,7 @@ public AirbyteConnectionStatus check(final JsonNode config) throws Exception { //Fail in case SSL mode is preferred return new AirbyteConnectionStatus() .withStatus(Status.FAILED) - .withMessage("Unsecured connection not allowed"); + .withMessage("Unsecured connection not allowed. If no SSH Tunnel set up, please use one of the following SSL modes: required, verify-ca, verify-identity"); } } } diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java index bda54cd7aab35..09d346761ffca 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java @@ -6,8 +6,9 @@ import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -253,7 +254,7 @@ void testStrictSSLUnsecuredNoTunnel() throws Exception { final AirbyteConnectionStatus actual = source.check(config); assertEquals(Status.FAILED, actual.getStatus()); - assertEquals("Unsecured connection not allowed", actual.getMessage()); + assertTrue(actual.getMessage().contains("Unsecured connection not allowed")); } @Test @@ -278,7 +279,7 @@ void testStrictSSLSecuredNoTunnel() throws Exception { final AirbyteConnectionStatus actual = source.check(config); assertEquals(Status.FAILED, actual.getStatus()); - assertNotEquals("Unsecured connection not allowed", actual.getMessage()); + assertFalse(actual.getMessage().contains("Unsecured connection not allowed")); } @Test diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json index c964d26ecad99..bae09c192446b 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json @@ -42,8 +42,8 @@ "order": 4 }, "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", + "title": "JDBC URL Parameters (Advanced)", "type": "string", "order": 5 }, diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index d1c21df03ab90..9698b4b8ba86c 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,5 +16,6 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.14 +LABEL io.airbyte.version=1.0.1 + LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index 08797b63673fe..28b67b7cbf015 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -13,7 +13,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:debezium-v1-9-2') + implementation project(':airbyte-integrations:bases:debezium-v1-9-6') implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-protocol:protocol-models') implementation project(':airbyte-integrations:connectors:source-relational-db') @@ -21,7 +21,7 @@ dependencies { implementation 'mysql:mysql-connector-java:8.0.22' implementation 'org.apache.commons:commons-lang3:3.11' - testImplementation testFixtures(project(':airbyte-integrations:bases:debezium-v1-9-2')) + testImplementation testFixtures(project(':airbyte-integrations:bases:debezium-v1-9-6')) testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation 'org.hamcrest:hamcrest-all:1.3' diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json index c09509ff94984..240d0b1434b35 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json @@ -42,8 +42,8 @@ "order": 4 }, "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", + "title": "JDBC URL Parameters (Advanced)", "type": "string", "order": 5 }, diff --git a/airbyte-integrations/connectors/source-netsuite/Dockerfile b/airbyte-integrations/connectors/source-netsuite/Dockerfile index d9051fb8d52be..322770ae5b557 100644 --- a/airbyte-integrations/connectors/source-netsuite/Dockerfile +++ b/airbyte-integrations/connectors/source-netsuite/Dockerfile @@ -35,5 +35,5 @@ COPY source_netsuite ./source_netsuite ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-netsuite diff --git a/airbyte-integrations/connectors/source-netsuite/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-netsuite/integration_tests/abnormal_state.json index c4b36742d4332..2acb0b0d5278c 100644 --- a/airbyte-integrations/connectors/source-netsuite/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-netsuite/integration_tests/abnormal_state.json @@ -1,44 +1,100 @@ -{ - "customer": { - "lastModifiedDate": "2030-08-19T14:01:00Z" - }, - "customersubsidiaryrelationship": { - "lastModifiedDate": "2030-08-19T14:01:00Z" - }, - "vendorsubsidiaryrelationship": { - "lastModifiedDate": "2030-08-22T09:21:00Z" - }, - "charge": { - "lastModifiedDate": "2030-08-22T09:21:00Z" - }, - "vendorbill": { - "lastModifiedDate": "2030-08-22T09:21:00Z" - }, - "contact": { - "lastModifiedDate": "2030-08-19T14:13:00Z" - }, - "message": { - "lastModifiedDate": "2030-08-22T09:21:00Z" - }, - "subsidiary": { - "lastModifiedDate": "2030-08-22T08:21:00Z" - }, - "task": { - "lastModifiedDate": "2030-08-18T08:11:00Z" - }, - "salesorder": { - "lastModifiedDate": "2030-08-22T09:21:00Z" - }, - "employee": { - "lastModifiedDate": "2030-08-18T12:05:00Z" - }, - "billingaccount": { - "lastModifiedDate": "2030-08-22T09:21:00Z" - }, - "journalentry": { - "lastModifiedDate": "2030-08-22T10:57:00Z" - }, - "vendor": { - "lastModifiedDate": "2030-08-22T08:21:00Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-19T14:01:00Z" }, + "stream_descriptor": { "name": "customer" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-19T14:01:00Z" }, + "stream_descriptor": { "name": "customersubsidiaryrelationship" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-22T09:21:00Z" }, + "stream_descriptor": { "name": "vendorsubsidiaryrelationship" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-22T09:21:00Z" }, + "stream_descriptor": { "name": "charge" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-22T09:21:00Z" }, + "stream_descriptor": { "name": "vendorbill" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-19T14:13:00Z" }, + "stream_descriptor": { "name": "contact" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-22T09:21:00Z" }, + "stream_descriptor": { "name": "message" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-22T08:21:00Z" }, + "stream_descriptor": { "name": "subsidiary" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-18T08:11:00Z" }, + "stream_descriptor": { "name": "task" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-22T09:21:00Z" }, + "stream_descriptor": { "name": "salesorder" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-18T12:05:00Z" }, + "stream_descriptor": { "name": "employee" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-22T09:21:00Z" }, + "stream_descriptor": { "name": "billingaccount" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-22T10:57:00Z" }, + "stream_descriptor": { "name": "journalentry" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "lastModifiedDate": "2030-08-22T08:21:00Z" }, + "stream_descriptor": { "name": "vendor" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-notion/Dockerfile b/airbyte-integrations/connectors/source-notion/Dockerfile index 48c3835c34b13..665c13156c06c 100644 --- a/airbyte-integrations/connectors/source-notion/Dockerfile +++ b/airbyte-integrations/connectors/source-notion/Dockerfile @@ -34,5 +34,5 @@ COPY source_notion ./source_notion ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-notion diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index 5f614a6591b31..db2671ca2005e 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -28,6 +28,11 @@ def __init__(self, config: Mapping[str, Any], **kwargs): super().__init__(**kwargs) self.start_date = config["start_date"] + def backoff_time(self, response: requests.Response) -> Optional[float]: + retry_after = response.headers.get("retry-after") + if retry_after: + return float(retry_after) + def request_headers(self, **kwargs) -> Mapping[str, Any]: params = super().request_headers(**kwargs) # Notion API version, see https://developers.notion.com/reference/versioning diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py index a52d13d481dfe..53407d033baec 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py @@ -74,10 +74,9 @@ def test_should_retry(patch_base_class, http_status, should_retry): def test_backoff_time(patch_base_class): - response_mock = MagicMock() + response_mock = MagicMock(headers={"retry-after": "10"}) stream = NotionStream(config=MagicMock()) - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time + assert stream.backoff_time(response_mock) == 10.0 def test_users_request_params(patch_base_class): diff --git a/airbyte-integrations/connectors/source-pinterest/Dockerfile b/airbyte-integrations/connectors/source-pinterest/Dockerfile index a8436f08f1861..65269be425f6a 100644 --- a/airbyte-integrations/connectors/source-pinterest/Dockerfile +++ b/airbyte-integrations/connectors/source-pinterest/Dockerfile @@ -34,5 +34,5 @@ COPY source_pinterest ./source_pinterest ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/source-pinterest diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py index 11ac4766372b4..f29fe9587a0d4 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py @@ -321,9 +321,18 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: return False, e def streams(self, config: Mapping[str, Any]) -> List[Stream]: + today = pendulum.today() + AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP = 914 + latest_date_allowed_by_api = today.subtract(days=AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP) + start_date = config.get("start_date") if not start_date: - config["start_date"] = "2020-07-28" # Set default start_date if user didn't set it + config["start_date"] = latest_date_allowed_by_api + else: + start_date_formatted = pendulum.from_format(config["start_date"], "YYYY-MM-DD") + delta_today_start_date = today - start_date_formatted + if delta_today_start_date.days > AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP: + config["start_date"] = latest_date_allowed_by_api config["authenticator"] = self.get_authenticator(config) return [ diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json index 031300ddd5da6..d642d95e3e5e0 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json @@ -10,8 +10,8 @@ "start_date": { "type": "string", "title": "Start Date", - "description": "A date in the format YYYY-MM-DD. If you have not set a date, it would be defaulted to 2020-07-28.", - "examples": ["2020-07-28"] + "description": "A date in the format YYYY-MM-DD. If you have not set a date, it would be defaulted to latest allowed date by api (914 days from today).", + "examples": ["2022-07-28"] }, "credentials": { "title": "Authorization Method", diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py index cc343b750ceab..1e78351f19684 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py @@ -14,7 +14,7 @@ def test_config(): "client_secret": "test_client_secret", "refresh_token": "test_refresh_token", "window_in_days": "Sandbox", - "start_date": "2021-05-07T00:00:00Z", + "start_date": "2021-05-07", } @@ -23,7 +23,7 @@ def test_incremental_config(): return { "authenticator": MagicMock(), "window_in_days": 185, - "start_date": "2021-05-07T00:00:00Z", + "start_date": "2021-05-07", } diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index acab80c1214d4..67a8ea2ecb186 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.11 +LABEL io.airbyte.version=1.0.13 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncrypt.java b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncrypt.java index 1827b21d6c589..1329dfab8f75a 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncrypt.java +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncrypt.java @@ -59,7 +59,7 @@ public AirbyteConnectionStatus check(final JsonNode config) throws Exception { //Fail in case SSL mode is disable, allow or prefer return new AirbyteConnectionStatus() .withStatus(Status.FAILED) - .withMessage("Unsecured connection not allowed"); + .withMessage("Unsecured connection not allowed. If no SSH Tunnel set up, please use one of the following SSL modes: require, verify-ca, verify-full"); } } } diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index c35a4c1169253..64e10cd5aab9c 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.11 +LABEL io.airbyte.version=1.0.13 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index a780b5ced643b..535b2209088df 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -13,7 +13,7 @@ application { dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:debezium-v1-9-2') + implementation project(':airbyte-integrations:bases:debezium-v1-9-6') implementation project(':airbyte-protocol:protocol-models') implementation project(':airbyte-integrations:connectors:source-jdbc') implementation project(':airbyte-integrations:connectors:source-relational-db') @@ -21,7 +21,7 @@ dependencies { implementation 'org.apache.commons:commons-lang3:3.11' implementation libs.postgresql - testImplementation testFixtures(project(':airbyte-integrations:bases:debezium-v1-9-2')) + testImplementation testFixtures(project(':airbyte-integrations:bases:debezium-v1-9-6')) testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation project(":airbyte-json-validation") testImplementation project(':airbyte-test-utils') diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java index 564f443ba4b69..719ccf147e8fc 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java @@ -10,7 +10,6 @@ import static io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest.setEnv; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -21,7 +20,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; @@ -45,23 +43,16 @@ import io.airbyte.protocol.models.AirbyteRecordMessage; import io.airbyte.protocol.models.AirbyteStateMessage; import io.airbyte.protocol.models.AirbyteStream; -import io.airbyte.protocol.models.AirbyteStreamState; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.StreamDescriptor; import io.airbyte.protocol.models.SyncMode; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterEach; @@ -278,72 +269,6 @@ public String createSchemaQuery(final String schemaName) { return "CREATE SCHEMA " + schemaName + ";"; } - @Override - @Test - public void testRecordsProducedDuringAndAfterSync() throws Exception { - - final int recordsToCreate = 20; - // first batch of records. 20 created here and 6 created in setup method. - for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { - final JsonNode record = - Jsons.jsonNode(ImmutableMap - .of(COL_ID, 100 + recordsCreated, COL_MAKE_ID, 1, COL_MODEL, - "F-" + recordsCreated)); - writeModelRecord(record); - } - - final AutoCloseableIterator firstBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, null); - final List dataFromFirstBatch = AutoCloseableIterators - .toListAndClose(firstBatchIterator); - final List stateAfterFirstBatch = extractStateMessages(dataFromFirstBatch); - assertEquals(1, stateAfterFirstBatch.size()); - assertNotNull(stateAfterFirstBatch.get(0).getData()); - assertExpectedStateMessages(stateAfterFirstBatch); - final Set recordsFromFirstBatch = extractRecordMessages( - dataFromFirstBatch); - assertEquals((MODEL_RECORDS.size() + recordsToCreate), recordsFromFirstBatch.size()); - - // second batch of records again 20 being created - for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { - final JsonNode record = - Jsons.jsonNode(ImmutableMap - .of(COL_ID, 200 + recordsCreated, COL_MAKE_ID, 1, COL_MODEL, - "F-" + recordsCreated)); - writeModelRecord(record); - } - - final JsonNode state = Jsons.jsonNode(stateAfterFirstBatch); - final AutoCloseableIterator secondBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, state); - final List dataFromSecondBatch = AutoCloseableIterators - .toListAndClose(secondBatchIterator); - - final List stateAfterSecondBatch = extractStateMessages(dataFromSecondBatch); - assertEquals(1, stateAfterSecondBatch.size()); - assertNotNull(stateAfterSecondBatch.get(0).getData()); - assertExpectedStateMessages(stateAfterSecondBatch); - - final Set recordsFromSecondBatch = extractRecordMessages( - dataFromSecondBatch); - assertEquals(recordsToCreate * 2, recordsFromSecondBatch.size(), - "Expected 40 records to be replicated in the second sync."); - - // sometimes there can be more than one of these at the end of the snapshot and just before the - // first incremental. - final Set recordsFromFirstBatchWithoutDuplicates = removeDuplicates( - recordsFromFirstBatch); - final Set recordsFromSecondBatchWithoutDuplicates = removeDuplicates( - recordsFromSecondBatch); - - final int recordsCreatedBeforeTestCount = MODEL_RECORDS.size(); - assertTrue(recordsCreatedBeforeTestCount < recordsFromFirstBatchWithoutDuplicates.size(), - "Expected first sync to include records created while the test was running."); - assertEquals((recordsToCreate * 3) + recordsCreatedBeforeTestCount, - recordsFromFirstBatchWithoutDuplicates.size() + recordsFromSecondBatchWithoutDuplicates - .size()); - } - @Override protected String randomTableSchema() { return MODELS_SCHEMA + "_random"; diff --git a/airbyte-integrations/connectors/source-recharge/Dockerfile b/airbyte-integrations/connectors/source-recharge/Dockerfile index 07007990355b4..ea85b01d2dcfb 100644 --- a/airbyte-integrations/connectors/source-recharge/Dockerfile +++ b/airbyte-integrations/connectors/source-recharge/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/source-recharge diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json index ae9cea634a97e..0e0a42363b5d8 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json @@ -1,23 +1,51 @@ -{ - "addresses": { - "updated_at": "2050-05-18T00:00:00" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_descriptor": { "name": "addresses" } + } }, - "charges": { - "updated_at": "2050-05-18T00:00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_descriptor": { "name": "charges" } + } }, - "customers": { - "updated_at": "2050-05-18T00:00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_descriptor": { "name": "customers" } + } }, - "discounts": { - "updated_at": "2050-05-18T00:00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_descriptor": { "name": "discounts" } + } }, - "onetimes": { - "updated_at": "2050-05-18T00:00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_descriptor": { "name": "onetimes" } + } }, - "orders": { - "updated_at": "2050-05-18T00:00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_descriptor": { "name": "orders" } + } }, - "subscriptions": { - "updated_at": "2050-05-18T00:00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_descriptor": { "name": "subscriptions" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-s3/Dockerfile b/airbyte-integrations/connectors/source-s3/Dockerfile index bf0fa7e4fc0b7..9459dc5752d51 100644 --- a/airbyte-integrations/connectors/source-s3/Dockerfile +++ b/airbyte-integrations/connectors/source-s3/Dockerfile @@ -17,5 +17,5 @@ COPY source_s3 ./source_s3 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.21 +LABEL io.airbyte.version=0.1.22 LABEL io.airbyte.name=airbyte/source-s3 diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-s3/integration_tests/abnormal_state.json index d587ed9686069..1a18a33307b39 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/abnormal_state.json @@ -1,5 +1,11 @@ -{ - "test": { - "_ab_source_file_last_modified": "2999-01-01T00:00:00+0000" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "_ab_source_file_last_modified": "2999-01-01T00:00:00+0000" + }, + "stream_descriptor": { "name": "test" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-salesforce/Dockerfile b/airbyte-integrations/connectors/source-salesforce/Dockerfile index 3a317a3087ee0..0c49fbe795eb9 100644 --- a/airbyte-integrations/connectors/source-salesforce/Dockerfile +++ b/airbyte-integrations/connectors/source-salesforce/Dockerfile @@ -13,5 +13,6 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.17 +LABEL io.airbyte.version=1.0.20 + LABEL io.airbyte.name=airbyte/source-salesforce diff --git a/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml b/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml index 95fde01f89995..87ae3d60ce8b7 100644 --- a/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml @@ -11,7 +11,6 @@ tests: status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" - discovery: - config_path: "secrets/config.json" basic_read: diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/future_state.json b/airbyte-integrations/connectors/source-salesforce/integration_tests/future_state.json index 4746d134fa021..8854977f91b66 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/future_state.json @@ -1,26 +1,58 @@ -{ - "Account": { - "SystemModstamp": "2122-01-18T21:18:20.000Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "SystemModstamp": "2122-01-18T21:18:20.000Z" }, + "stream_descriptor": { "name": "Account" } + } }, - "ActiveFeatureLicenseMetric": { - "SystemModstamp": "2122-08-22T05:08:29.000Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "SystemModstamp": "2122-08-22T05:08:29.000Z" }, + "stream_descriptor": { "name": "ActiveFeatureLicenseMetric" } + } }, - "ActivePermSetLicenseMetric": { - "SystemModstamp": "2122-08-22T05:03:27.000Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "SystemModstamp": "2122-08-22T05:03:27.000Z" }, + "stream_descriptor": { "name": "ActivePermSetLicenseMetric" } + } }, - "ActiveProfileMetric": { - "SystemModstamp": "2121-08-22T06:17:11.000Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "SystemModstamp": "2121-08-22T06:17:11.000Z" }, + "stream_descriptor": { "name": "ActiveProfileMetric" } + } }, - "Asset": { - "SystemModstamp": "2121-08-22T06:17:11.000Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "SystemModstamp": "2121-08-22T06:17:11.000Z" }, + "stream_descriptor": { "name": "Asset" } + } }, - "ObjectPermissions": { - "SystemModstamp": "2121-08-23T10:27:22.000Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "SystemModstamp": "2121-08-23T10:27:22.000Z" }, + "stream_descriptor": { "name": "ObjectPermissions" } + } }, - "PermissionSetTabSetting": { - "SystemModstamp": "2121-08-23T10:27:22.000Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "SystemModstamp": "2121-08-23T10:27:22.000Z" }, + "stream_descriptor": { "name": "PermissionSetTabSetting" } + } }, - "LeadHistory": { - "CreatedDate": "2121-08-23T10:27:22.000Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "CreatedDate": "2121-08-23T10:27:22.000Z" }, + "stream_descriptor": { "name": "LeadHistory" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-salesforce/integration_tests/sample_state.json index f07148900e62b..cc73a22cd2c92 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/sample_state.json @@ -1,23 +1,51 @@ -{ - "Account": { - "SystemModstamp": "2021-01-18T21:18:20.000Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "SystemModstamp": "2021-01-18T21:18:20.000Z" }, + "stream_descriptor": { "name": "Account" } + } }, - "ActiveFeatureLicenseMetric": { - "SystemModstamp": "2021-08-22T05:08:29.000Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "SystemModstamp": "2021-08-22T05:08:29.000Z" }, + "stream_descriptor": { "name": "ActiveFeatureLicenseMetric" } + } }, - "ActivePermSetLicenseMetric": { - "SystemModstamp": "2021-08-22T05:03:27.000Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "SystemModstamp": "2021-08-22T05:03:27.000Z" }, + "stream_descriptor": { "name": "ActivePermSetLicenseMetric" } + } }, - "ActiveProfileMetric": { - "SystemModstamp": "2021-08-22T06:17:11.000Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "SystemModstamp": "2021-08-22T06:17:11.000Z" }, + "stream_descriptor": { "name": "ActiveProfileMetric" } + } }, - "ObjectPermissions": { - "SystemModstamp": "2021-08-23T10:27:22.000Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "SystemModstamp": "2021-08-23T10:27:22.000Z" }, + "stream_descriptor": { "name": "ObjectPermissions" } + } }, - "PermissionSetTabSetting": { - "SystemModstamp": "2021-08-23T10:27:22.000Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "SystemModstamp": "2021-08-23T10:27:22.000Z" }, + "stream_descriptor": { "name": "PermissionSetTabSetting" } + } }, - "LeadHistory": { - "CreatedDate": "2021-08-23T10:27:22.000Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "CreatedDate": "2021-08-23T10:27:22.000Z" }, + "stream_descriptor": { "name": "LeadHistory" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py index 7ddd709943bc0..0232c804d6d1e 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py @@ -127,15 +127,13 @@ "UserRecordAccess", ] +# The following objects are not supported by the Bulk API. Listed objects are version specific. UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS = [ "AcceptedEventRelation", - "AssetTokenEvent", - "AttachedContentNote", "Attachment", "CaseStatus", "ContractStatus", "DeclinedEventRelation", - "EventWhoRelation", "FieldSecurityClassification", "KnowledgeArticle", "KnowledgeArticleVersion", @@ -144,13 +142,12 @@ "KnowledgeArticleVoteStat", "OrderStatus", "PartnerRole", - "QuoteTemplateRichTextData", "RecentlyViewed", "ServiceAppointmentStatus", + "ShiftStatus", "SolutionStatus", "TaskPriority", "TaskStatus", - "TaskWhoRelation", "UndecidedEventRelation", ] diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index b3a9363938636..c22a79e12f1f7 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -48,23 +48,6 @@ def __init__( self.schema: Mapping[str, Any] = schema # type: ignore[assignment] self.sobject_options = sobject_options - def decode(self, chunk): - """ - Most Salesforce instances use UTF-8, but some use ISO-8859-1. - By default, we'll decode using UTF-8, and fallback to ISO-8859-1 if it doesn't work. - See implementation considerations for more details https://developer.salesforce.com/docs/atlas.en-us.api.meta/api/implementation_considerations.htm - """ - if self.encoding == DEFAULT_ENCODING: - try: - decoded = chunk.decode(self.encoding) - return decoded - except UnicodeDecodeError as e: - self.encoding = "ISO-8859-1" - self.logger.info(f"Could not decode chunk. Falling back to {self.encoding} encoding. Error: {e}") - return self.decode(chunk) - else: - return chunk.decode(self.encoding) - @property def name(self) -> str: return self.stream_name @@ -206,6 +189,11 @@ def create_stream_job(self, query: str, url: str) -> Optional[str]: f"Cannot receive data for stream '{self.name}' ," f"sobject options: {self.sobject_options}, error message: '{error_message}'" ) + elif error.response.status_code == codes.FORBIDDEN and error_code == "REQUEST_LIMIT_EXCEEDED": + self.logger.error( + f"Cannot receive data for stream '{self.name}' ," + f"sobject options: {self.sobject_options}, Error message: '{error_data.get('message')}'" + ) elif error.response.status_code == codes.BAD_REQUEST and error_message.endswith("does not support query"): self.logger.error( f"The stream '{self.name}' is not queryable, " @@ -281,34 +269,34 @@ def filter_null_bytes(self, b: bytes): self.logger.warning("Filter 'null' bytes from string, size reduced %d -> %d chars", len(b), len(res)) return res - def download_data(self, url: str, chunk_size: float = 1024) -> os.PathLike: + def download_data(self, url: str, chunk_size: int = 1024) -> tuple[str, str]: """ - Retrieves binary data result from successfully `executed_job`, using chunks, to avoid local memory limitaions. + Retrieves binary data result from successfully `executed_job`, using chunks, to avoid local memory limitations. @ url: string - the url of the `executed_job` - @ chunk_size: float - the buffer size for each chunk to fetch from stream, in bytes, default: 1024 bytes - - Returns the string with file path of downloaded binary data. Saved temporarily. + @ chunk_size: int - the buffer size for each chunk to fetch from stream, in bytes, default: 1024 bytes + Return the tuple containing string with file path of downloaded binary data (Saved temporarily) and file encoding. """ # set filepath for binary data from response tmp_file = os.path.realpath(os.path.basename(url)) - with closing(self._send_http_request("GET", f"{url}/results", stream=True)) as response: - with open(tmp_file, "wb") as data_file: - for chunk in response.iter_content(chunk_size=chunk_size): - data_file.write(self.filter_null_bytes(chunk)) + with closing(self._send_http_request("GET", f"{url}/results", stream=True)) as response, open(tmp_file, "wb") as data_file: + response_encoding = response.encoding or response.apparent_encoding or self.encoding + for chunk in response.iter_content(chunk_size=chunk_size): + data_file.write(self.filter_null_bytes(chunk)) # check the file exists if os.path.isfile(tmp_file): - return tmp_file + return tmp_file, response_encoding else: raise TmpFileIOError(f"The IO/Error occured while verifying binary data. Stream: {self.name}, file {tmp_file} doesn't exist.") - def read_with_chunks(self, path: str = None, chunk_size: int = 100) -> Iterable[Tuple[int, Mapping[str, Any]]]: + def read_with_chunks(self, path: str, file_encoding: str, chunk_size: int = 100) -> Iterable[Tuple[int, Mapping[str, Any]]]: """ Reads the downloaded binary data, using lines chunks, set by `chunk_size`. @ path: string - the path to the downloaded temporarily binary data. + @ file_encoding: string - encoding for binary data file according to Standard Encodings from codecs module @ chunk_size: int - the number of lines to read at a time, default: 100 lines / time. """ try: - with open(path, "r", encoding=self.encoding) as data: + with open(path, "r", encoding=file_encoding) as data: chunks = pd.read_csv(data, chunksize=chunk_size, iterator=True, dialect="unix") for chunk in chunks: chunk = chunk.replace({nan: None}).to_dict(orient="records") @@ -382,7 +370,7 @@ def read_records( count = 0 record: Mapping[str, Any] = {} - for record in self.read_with_chunks(self.download_data(url=job_full_url)): + for record in self.read_with_chunks(*self.download_data(url=job_full_url)): count += 1 yield record self.delete_job(url=job_full_url) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py index 083b85e812f82..2334c4945ba0e 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py @@ -12,7 +12,7 @@ import pytest import requests_mock from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode, Type -from conftest import generate_stream +from conftest import encoding_symbols_parameters, generate_stream from requests.exceptions import HTTPError from source_salesforce.source import SourceSalesforce from source_salesforce.streams import ( @@ -184,14 +184,29 @@ def test_download_data_filter_null_bytes(stream_config, stream_api): with requests_mock.Mocker() as m: m.register_uri("GET", f"{job_full_url}/results", content=b"\x00") - res = list(stream.read_with_chunks(stream.download_data(url=job_full_url))) + res = list(stream.read_with_chunks(*stream.download_data(url=job_full_url))) assert res == [] m.register_uri("GET", f"{job_full_url}/results", content=b'"Id","IsDeleted"\n\x00"0014W000027f6UwQAI","false"\n\x00\x00') - res = list(stream.read_with_chunks(stream.download_data(url=job_full_url))) + res = list(stream.read_with_chunks(*stream.download_data(url=job_full_url))) assert res == [{"Id": "0014W000027f6UwQAI", "IsDeleted": False}] +@pytest.mark.parametrize( + "chunk_size, content_type, content, expected_result", + encoding_symbols_parameters(), + ids=[f"charset: {x[1]}, chunk_size: {x[0]}" for x in encoding_symbols_parameters()], +) +def test_encoding_symbols(stream_config, stream_api, chunk_size, content_type, content, expected_result): + job_full_url: str = "https://fase-account.salesforce.com/services/data/v52.0/jobs/query/7504W00000bkgnpQAA" + stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api) + + with requests_mock.Mocker() as m: + m.register_uri("GET", f"{job_full_url}/results", headers={"Content-Type": f"text/html; charset={content_type}"}, content=content) + res = list(stream.read_with_chunks(*stream.download_data(url=job_full_url, chunk_size=chunk_size))) + assert res == expected_result + + @pytest.mark.parametrize( "login_status_code, login_json_resp, discovery_status_code, discovery_resp_json, expected_error_msg", ( @@ -415,7 +430,7 @@ def test_csv_reader_dialect_unix(): with requests_mock.Mocker() as m: m.register_uri("GET", url + "/results", text=text) - result = [i for i in stream.read_with_chunks(stream.download_data(url))] + result = [i for i in stream.read_with_chunks(*stream.download_data(url))] assert result == data @@ -513,10 +528,3 @@ def test_convert_to_standard_instance(stream_config, stream_api): bulk_stream = generate_stream("Account", stream_config, stream_api) rest_stream = bulk_stream.get_standard_instance() assert isinstance(rest_stream, IncrementalSalesforceStream) - - -def test_decoding(stream_config, stream_api): - stream_name = "AcceptedEventRelation" - stream = generate_stream(stream_name, stream_config, stream_api) - assert stream.decode(b"\xe9\x97\xb4\xe5\x8d\x95\xe7\x9a\x84\xe8\xaf\xb4 \xf0\x9f\xaa\x90") == "间单的说 🪐" - assert stream.decode(b"0\xe5") == "0å" diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py index e5996e13284c6..e0de6179f98f3 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py @@ -101,3 +101,15 @@ def stream_api_v2(stream_config): def generate_stream(stream_name, stream_config, stream_api): return SourceSalesforce.generate_streams(stream_config, {stream_name: None}, stream_api)[0] + + +def encoding_symbols_parameters(): + return [(x, "ISO-8859-1", b'"\xc4"\n,"4"\n\x00,"\xca \xfc"', [{"Ä": "4"}, {"Ä": "Ê ü"}]) for x in range(1, 11)] + [ + ( + x, + "utf-8", + b'"\xd5\x80"\n "\xd5\xaf","\xd5\xaf"\n\x00,"\xe3\x82\x82 \xe3\x83\xa4 \xe3\x83\xa4 \xf0\x9d\x9c\xb5"', + [{"Հ": "կ"}, {"Հ": "も ヤ ヤ 𝜵"}], + ) + for x in range(1, 11) + ] diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_memory.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_memory.py index fcd05203dbbc9..47546c34517f8 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_memory.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_memory.py @@ -16,8 +16,8 @@ ( (1000, 0.4, 1), (10000, 1, 2), - (100000, 4, 7), - (200000, 7, 12), + (100000, 4, 9), + (200000, 7, 19), ), ids=[ "1k recods", @@ -36,7 +36,7 @@ def test_memory_download_data(stream_config, stream_api, n_records, first_size, with requests_mock.Mocker() as m: m.register_uri("GET", f"{job_full_url}/results", content=content) tracemalloc.start() - for x in stream.read_with_chunks(stream.download_data(url=job_full_url)): + for x in stream.read_with_chunks(*stream.download_data(url=job_full_url)): pass fs, fp = tracemalloc.get_traced_memory() first_size_in_mb, first_peak_in_mb = fs / 1024**2, fp / 1024**2 diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/sendgrid.yaml b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/sendgrid.yaml index fa183c0ffda72..a7fa390cbd4ac 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/sendgrid.yaml +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/sendgrid.yaml @@ -16,7 +16,7 @@ definitions: type: "BearerAuthenticator" api_token: "{{ config.apikey }}" cursor_paginator: - type: LimitPaginator + type: DefaultPaginator url_base: "*ref(definitions.requester.url_base)" page_size: "*ref(definitions.page_size)" limit_option: @@ -28,7 +28,7 @@ definitions: type: "CursorPagination" cursor_value: "{{ response._metadata.next }}" offset_paginator: - type: LimitPaginator + type: DefaultPaginator $options: url_base: "*ref(definitions.requester.url_base)" page_size: "*ref(definitions.page_size)" diff --git a/airbyte-integrations/connectors/source-sentry/Dockerfile b/airbyte-integrations/connectors/source-sentry/Dockerfile index 3ebc64803579f..a21817b7bf66b 100644 --- a/airbyte-integrations/connectors/source-sentry/Dockerfile +++ b/airbyte-integrations/connectors/source-sentry/Dockerfile @@ -34,5 +34,5 @@ COPY source_sentry ./source_sentry ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/source-sentry diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml index fde93fc2a9516..8ebc766387b0b 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml @@ -18,7 +18,7 @@ definitions: type: "BearerAuthenticator" api_token: "{{ config.auth_token }}" paginator: - type: LimitPaginator + type: DefaultPaginator url_base: "*ref(definitions.requester.url_base)" page_size: "*ref(definitions.page_size)" limit_option: diff --git a/airbyte-integrations/connectors/source-slack/Dockerfile b/airbyte-integrations/connectors/source-slack/Dockerfile index 60742b61e3fb3..5adf219652ca6 100644 --- a/airbyte-integrations/connectors/source-slack/Dockerfile +++ b/airbyte-integrations/connectors/source-slack/Dockerfile @@ -17,5 +17,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.17 +LABEL io.airbyte.version=0.1.18 LABEL io.airbyte.name=airbyte/source-slack diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.txt index f0b9a604d4d97..23d1bbd4a2340 100644 --- a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.txt @@ -1,23 +1,23 @@ {"stream": "users", "data": {"id": "USLACKBOT", "team_id": "T01AB4DDR2N", "name": "slackbot", "deleted": false, "color": "757575", "real_name": "Slackbot", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Slackbot", "real_name_normalized": "Slackbot", "display_name": "Slackbot", "display_name_normalized": "Slackbot", "fields": {}, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "sv41d8cd98f0", "always_active": true, "first_name": "slackbot", "last_name": "", "image_24": "https://a.slack-edge.com/80588/img/slackbot_24.png", "image_32": "https://a.slack-edge.com/80588/img/slackbot_32.png", "image_48": "https://a.slack-edge.com/80588/img/slackbot_48.png", "image_72": "https://a.slack-edge.com/80588/img/slackbot_72.png", "image_192": "https://a.slack-edge.com/80588/marketing/img/avatars/slackbot/avatar-slackbot.png", "image_512": "https://a.slack-edge.com/80588/img/slackbot_512.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 0, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397747} {"stream": "users", "data": {"id": "U019WF8P78X", "team_id": "T01AB4DDR2N", "name": "google_calendar", "deleted": false, "color": "3c989f", "real_name": "Google Calendar", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Google Calendar", "real_name_normalized": "Google Calendar", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "27bf72bdda29", "api_app_id": "ADZ494LHY", "always_active": true, "image_original": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_original.png", "is_custom_image": true, "bot_id": "B01APS4667K", "first_name": "Google", "last_name": "Calendar", "image_24": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_24.png", "image_32": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_32.png", "image_48": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_48.png", "image_72": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_72.png", "image_192": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_192.png", "image_512": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_512.png", "image_1024": "https://avatars.slack-edge.com/2021-06-01/2118434220774_27bf72bdda29cc1bd6ed_1024.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": true, "is_app_user": false, "updated": 1622570515, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397751} {"stream": "users", "data": {"id": "U019WFB2G31", "team_id": "T01AB4DDR2N", "name": "simplepoll", "deleted": false, "color": "674b1b", "real_name": "Simple Poll", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Simple Poll", "real_name_normalized": "Simple Poll", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "0eee8341dcfc", "api_app_id": "A0HFW7MR6", "always_active": true, "image_original": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_original.png", "is_custom_image": true, "bot_id": "B01ABDTVBJ7", "first_name": "Simple", "last_name": "Poll", "image_24": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_24.png", "image_32": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_32.png", "image_48": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_48.png", "image_72": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_72.png", "image_192": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_192.png", "image_512": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_512.png", "image_1024": "https://avatars.slack-edge.com/2022-08-26/3998313146786_0eee8341dcfc8b0303c5_1024.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": true, "is_app_user": false, "updated": 1661533967, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397752} -{"stream": "users", "data": {"id": "U01A6TEQXRU", "team_id": "T01AB4DDR2N", "name": "sherif", "deleted": false, "color": "5b89d5", "real_name": "sherif (please use other account)", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "Connectors Engineering", "phone": "", "skype": "", "real_name": "sherif (please use other account)", "real_name_normalized": "sherif (please use other account)", "display_name": "s", "display_name_normalized": "s", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "6613ce572ac2", "image_original": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_original.jpg", "is_custom_image": true, "email": "sherif@airbyte.io", "huddle_state": "default_unset", "first_name": "sherif", "last_name": "(please use other account)", "image_24": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_24.jpg", "image_32": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_32.jpg", "image_48": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_48.jpg", "image_72": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_72.jpg", "image_192": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_192.jpg", "image_512": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_512.jpg", "image_1024": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1656352044, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397753} -{"stream": "users", "data": {"id": "U01A81VHD45", "team_id": "T01AB4DDR2N", "name": "michel", "deleted": false, "color": "9f69e7", "real_name": "Michel Tricot", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "Co-Founder & CEO", "phone": "", "skype": "", "real_name": "Michel Tricot", "real_name_normalized": "Michel Tricot", "display_name": "Michel", "display_name_normalized": "Michel", "fields": null, "status_text": "", "status_emoji": ":airbyte:", "status_emoji_display_info": [{"emoji_name": "airbyte", "display_url": "https://emoji.slack-edge.com/T01AB4DDR2N/airbyte/2ba84475f2e46e08.png"}], "status_expiration": 0, "avatar_hash": "9cec7ecdfa05", "image_original": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_original.jpg", "is_custom_image": true, "email": "michel@airbyte.io", "huddle_state": "default_unset", "huddle_state_expiration_ts": 0, "first_name": "Michel", "last_name": "Tricot", "image_24": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_24.jpg", "image_32": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_32.jpg", "image_48": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_48.jpg", "image_72": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_72.jpg", "image_192": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_192.jpg", "image_512": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_512.jpg", "image_1024": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": true, "is_primary_owner": true, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1658760270, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397754} -{"stream": "users", "data": {"id": "U01AB6V6NMQ", "team_id": "T01AB4DDR2N", "name": "john", "deleted": false, "color": "4bbe2e", "real_name": "John Lafleur", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "Co-founder", "phone": "4153597503", "skype": "", "real_name": "John Lafleur", "real_name_normalized": "John Lafleur", "display_name": "John (Airbyte)", "display_name_normalized": "John (Airbyte)", "fields": null, "status_text": "", "status_emoji": ":airbyte:", "status_emoji_display_info": [{"emoji_name": "airbyte", "display_url": "https://emoji.slack-edge.com/T01AB4DDR2N/airbyte/2ba84475f2e46e08.png"}], "status_expiration": 0, "avatar_hash": "6a64a9108cd2", "image_original": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_original.jpg", "is_custom_image": true, "email": "john@airbyte.io", "huddle_state": "default_unset", "huddle_state_expiration_ts": 0, "first_name": "John", "last_name": "Lafleur", "image_24": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_24.jpg", "image_32": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_32.jpg", "image_48": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_48.jpg", "image_72": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_72.jpg", "image_192": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_192.jpg", "image_512": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_512.jpg", "image_1024": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": true, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1661288562, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397755} +{"stream": "users", "data": {"id": "U01A6TEQXRU", "team_id": "T01AB4DDR2N", "name": "sherif", "deleted": false, "color": "5b89d5", "real_name": "sherif (please use other account)", "tz": "Asia/Beirut", "tz_label": "Eastern European Summer Time", "tz_offset": 10800, "profile": {"title": "Connectors Engineering", "phone": "", "skype": "", "real_name": "sherif (please use other account)", "real_name_normalized": "sherif (please use other account)", "display_name": "s", "display_name_normalized": "s", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "6613ce572ac2", "image_original": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_original.jpg", "is_custom_image": true, "email": "sherif@airbyte.io", "huddle_state": "default_unset", "first_name": "sherif", "last_name": "(please use other account)", "image_24": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_24.jpg", "image_32": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_32.jpg", "image_48": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_48.jpg", "image_72": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_72.jpg", "image_192": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_192.jpg", "image_512": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_512.jpg", "image_1024": "https://avatars.slack-edge.com/2022-04-14/3395789907490_6613ce572ac2be21c187_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1664337135, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397753} +{"stream": "users", "data": {"id": "U01A81VHD45", "team_id": "T01AB4DDR2N", "name": "michel", "deleted": false, "color": "9f69e7", "real_name": "Michel Tricot", "tz": "Europe/Brussels", "tz_label": "Central European Summer Time", "tz_offset": 7200, "profile": {"title": "Co-Founder & CEO", "phone": "", "skype": "", "real_name": "Michel Tricot", "real_name_normalized": "Michel Tricot", "display_name": "Michel", "display_name_normalized": "Michel", "fields": null, "status_text": "", "status_emoji": ":airbyte:", "status_emoji_display_info": [{"emoji_name": "airbyte", "display_url": "https://emoji.slack-edge.com/T01AB4DDR2N/airbyte/2ba84475f2e46e08.png"}], "status_expiration": 0, "avatar_hash": "9cec7ecdfa05", "image_original": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_original.jpg", "is_custom_image": true, "email": "michel@airbyte.io", "huddle_state": "default_unset", "huddle_state_expiration_ts": 0, "first_name": "Michel", "last_name": "Tricot", "image_24": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_24.jpg", "image_32": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_32.jpg", "image_48": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_48.jpg", "image_72": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_72.jpg", "image_192": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_192.jpg", "image_512": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_512.jpg", "image_1024": "https://avatars.slack-edge.com/2021-01-21/1662967889443_9cec7ecdfa05e909a50e_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": true, "is_primary_owner": true, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1664190286, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397754} +{"stream": "users", "data": {"id": "U01AB6V6NMQ", "team_id": "T01AB4DDR2N", "name": "john", "deleted": false, "color": "4bbe2e", "real_name": "John Lafleur", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "Co-founder", "phone": "4153597503", "skype": "", "real_name": "John Lafleur", "real_name_normalized": "John Lafleur", "display_name": "John (Airbyte)", "display_name_normalized": "John (Airbyte)", "fields": null, "status_text": "", "status_emoji": ":airbyte:", "status_emoji_display_info": [{"emoji_name": "airbyte", "display_url": "https://emoji.slack-edge.com/T01AB4DDR2N/airbyte/2ba84475f2e46e08.png"}], "status_expiration": 0, "avatar_hash": "6a64a9108cd2", "image_original": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_original.jpg", "is_custom_image": true, "email": "john@airbyte.io", "huddle_state": "default_unset", "huddle_state_expiration_ts": 0, "first_name": "John", "last_name": "Lafleur", "image_24": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_24.jpg", "image_32": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_32.jpg", "image_48": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_48.jpg", "image_72": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_72.jpg", "image_192": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_192.jpg", "image_512": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_512.jpg", "image_1024": "https://avatars.slack-edge.com/2021-01-22/1662578436963_6a64a9108cd2d6fb24aa_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": true, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1663767942, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397755} {"stream": "users", "data": {"id": "U01ABDY04RZ", "team_id": "T01AB4DDR2N", "name": "github-x", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "GitHub (Legacy)", "real_name_normalized": "GitHub (Legacy)", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "de0095e0c631", "api_app_id": "A8GBNUWU8", "always_active": true, "image_original": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_original.png", "is_custom_image": true, "bot_id": "B01A85CCXT7", "first_name": "GitHub", "last_name": "(Legacy)", "image_24": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_24.png", "image_32": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_32.png", "image_48": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_48.png", "image_72": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_72.png", "image_192": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_192.png", "image_512": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_512.png", "image_1024": "https://avatars.slack-edge.com/2021-04-14/1961189182309_de0095e0c6319b75e911_1024.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_bot": true, "is_app_user": true, "updated": 1618854795}, "emitted_at": 1661589397756} -{"stream": "users", "data": {"id": "U01ADKHBJKC", "team_id": "T01AB4DDR2N", "name": "jared", "deleted": false, "color": "e0a729", "real_name": "Jared (message me on other account)", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "Engineering (Platform) @ Airbyte", "phone": "", "skype": "", "real_name": "Jared (message me on other account)", "real_name_normalized": "Jared (message me on other account)", "display_name": "Jared Rhizor (Airbyte)", "display_name_normalized": "Jared Rhizor (Airbyte)", "fields": null, "status_text": "", "status_emoji": ":airbyte:", "status_emoji_display_info": [{"emoji_name": "airbyte", "display_url": "https://emoji.slack-edge.com/T01AB4DDR2N/airbyte/2ba84475f2e46e08.png"}], "status_expiration": 0, "avatar_hash": "438b7f2a6e75", "image_original": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_original.jpg", "is_custom_image": true, "email": "jared@airbyte.io", "first_name": "Jared", "last_name": "(message me on other account)", "image_24": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_24.jpg", "image_32": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_32.jpg", "image_48": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_48.jpg", "image_72": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_72.jpg", "image_192": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_192.jpg", "image_512": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_512.jpg", "image_1024": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1643058749, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397757} -{"stream": "users", "data": {"id": "U01ADQ3FGMC", "team_id": "T01AB4DDR2N", "name": "zapier", "deleted": false, "color": "df3dc0", "real_name": "Zapier", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Zapier", "real_name_normalized": "Zapier", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "302ec95148aa", "api_app_id": "A024R9PQM", "always_active": true, "image_original": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_original.png", "is_custom_image": true, "bot_id": "B01ALN9M9GT", "first_name": "Zapier", "last_name": "", "image_24": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_24.png", "image_32": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_32.png", "image_48": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_48.png", "image_72": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_72.png", "image_192": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_192.png", "image_512": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_512.png", "image_1024": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_1024.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": true, "is_app_user": false, "updated": 1617789592, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397758} -{"stream": "users", "data": {"id": "U01ADSJ1D5H", "team_id": "T01AB4DDR2N", "name": "jamakase54", "deleted": false, "color": "2b6836", "real_name": "Artem Astapenko", "tz": "Europe/Moscow", "tz_label": "Moscow Time", "tz_offset": 10800, "profile": {"title": "CEO and Tech Lead @ Jamakase Technologies", "phone": "", "skype": "", "real_name": "Artem Astapenko", "real_name_normalized": "Artem Astapenko", "display_name": "Artem Astapenko", "display_name_normalized": "Artem Astapenko", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "69713daee7ef", "image_original": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_original.jpg", "is_custom_image": true, "email": "jamakase54@gmail.com", "first_name": "Artem", "last_name": "Astapenko", "image_24": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_24.jpg", "image_32": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_32.jpg", "image_48": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_48.jpg", "image_72": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_72.jpg", "image_192": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_192.jpg", "image_512": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_512.jpg", "image_1024": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1643662749, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397759} +{"stream": "users", "data": {"id": "U01ADKHBJKC", "team_id": "T01AB4DDR2N", "name": "jared", "deleted": false, "color": "e0a729", "real_name": "Jared (message me on other account)", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "Engineering (Platform) @ Airbyte", "phone": "", "skype": "", "real_name": "Jared (message me on other account)", "real_name_normalized": "Jared (message me on other account)", "display_name": "Jared Rhizor (Airbyte)", "display_name_normalized": "Jared Rhizor (Airbyte)", "fields": null, "status_text": "", "status_emoji": ":airbyte:", "status_emoji_display_info": [{"emoji_name": "airbyte", "display_url": "https://emoji.slack-edge.com/T01AB4DDR2N/airbyte/2ba84475f2e46e08.png"}], "status_expiration": 0, "avatar_hash": "438b7f2a6e75", "image_original": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_original.jpg", "is_custom_image": true, "email": "jared@airbyte.io", "first_name": "Jared", "last_name": "(message me on other account)", "image_24": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_24.jpg", "image_32": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_32.jpg", "image_48": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_48.jpg", "image_72": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_72.jpg", "image_192": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_192.jpg", "image_512": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_512.jpg", "image_1024": "https://avatars.slack-edge.com/2022-01-14/2979601530848_438b7f2a6e755022d23f_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": true, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1662549241, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397757} +{"stream": "users", "data": {"id": "U01ADQ3FGMC", "team_id": "T01AB4DDR2N", "name": "zapier", "deleted": false, "color": "df3dc0", "real_name": "Zapier", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Zapier", "real_name_normalized": "Zapier", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "302ec95148aa", "api_app_id": "A024R9PQM", "always_active": true, "image_original": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_original.png", "is_custom_image": true, "bot_id": "B01ALN9M9GT", "first_name": "Zapier", "last_name": "", "image_24": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_24.png", "image_32": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_32.png", "image_48": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_48.png", "image_72": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_72.png", "image_192": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_192.png", "image_512": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_512.png", "image_1024": "https://avatars.slack-edge.com/2021-04-07/1928731185238_302ec95148aaa301b011_1024.png", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": true, "is_app_user": false, "updated": 1662549246, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397758} +{"stream": "users", "data": {"id": "U01ADSJ1D5H", "team_id": "T01AB4DDR2N", "name": "jamakase54", "deleted": false, "color": "2b6836", "real_name": "Artem Astapenko", "tz": "Europe/Moscow", "tz_label": "Moscow Time", "tz_offset": 10800, "profile": {"title": "CEO and Tech Lead @ Jamakase Technologies", "phone": "", "skype": "", "real_name": "Artem Astapenko", "real_name_normalized": "Artem Astapenko", "display_name": "Artem Astapenko", "display_name_normalized": "Artem Astapenko", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "69713daee7ef", "image_original": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_original.jpg", "is_custom_image": true, "email": "jamakase54@gmail.com", "first_name": "Artem", "last_name": "Astapenko", "image_24": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_24.jpg", "image_32": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_32.jpg", "image_48": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_48.jpg", "image_72": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_72.jpg", "image_192": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_192.jpg", "image_512": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_512.jpg", "image_1024": "https://avatars.slack-edge.com/2020-09-10/1356335046275_69713daee7ef29c92197_1024.jpg", "status_text_canonical": "", "team": "T01AB4DDR2N"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1662549243, "is_email_confirmed": true, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1661589397759} {"stream": "channel_messages", "data": {"client_msg_id": "211e97d6-f529-4a61-adaf-65cbedd4c9d3", "type": "message", "text": " Good article from Temporal on use-case oriented docs structure", "user": "U01A6TEQXRU", "ts": "1627014630.175400", "team": "T01AB4DDR2N", "attachments": [{"title": "How we structured docs for user personas | Temporal documentation", "title_link": "https://docs.temporal.io/blog/docs-info-arch-2021/", "text": "A few weeks ago, around the end of March 2021, I set my focus on finally refreshing the information around Temporal's core concepts.", "fallback": "How we structured docs for user personas | Temporal documentation", "image_url": "https://docs.temporal.io/img/documentation-landing-page-april-2021.png", "from_url": "https://docs.temporal.io/blog/docs-info-arch-2021/", "image_width": 351, "image_height": 250, "image_bytes": 173793, "service_icon": "https://docs.temporal.io/img/favicon.png", "service_name": "docs.temporal.io", "id": 1, "original_url": "https://docs.temporal.io/blog/docs-info-arch-2021/"}], "blocks": [{"type": "rich_text", "block_id": "Il3Xn", "elements": [{"type": "rich_text_section", "elements": [{"type": "link", "url": "https://docs.temporal.io/blog/docs-info-arch-2021/"}, {"type": "text", "text": " Good article from Temporal on use-case oriented docs structure"}]}]}], "thread_ts": "1627014630.175400", "reply_count": 2, "reply_users_count": 2, "latest_reply": "1627331109.183100", "reply_users": ["U024VC21YRW", "U01MMSDJGC9"], "is_locked": false, "subscribed": false, "reactions": [{"name": "+1", "users": ["U027WFS9RV5", "U024VC21YRW", "U01MMSDJGC9", "U02QDDL2L74"], "count": 4}, {"name": "+1::skin-tone-4", "users": ["U01RW78A2LE"], "count": 1}], "channel_id": "C01AB7G87NE", "float_ts": 1627014630.1754}, "emitted_at": 1661589426355} {"stream": "channel_messages", "data": {"client_msg_id": "65bf237c-29b0-452b-af93-af7f6c07bdce", "type": "message", "text": "Hi everyone,\nI have tried to replicate 10gb data from postgres as source to destination mysql but status still in progress ...it has been running since 2 hour.\ncan any one tell what will be reason ?", "user": "U026QUA5622", "ts": "1627062331.350600", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "=by4", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hi everyone,\nI have tried to replicate 10gb data from postgres as source to destination mysql but status still in progress ...it has been running since 2 hour.\ncan any one tell what will be reason ?"}]}]}], "channel_id": "C021JANJ6TY", "float_ts": 1627062331.3506}, "emitted_at": 1661589426858} -{"stream": "channel_messages", "data": {"client_msg_id": "aaefcfbc-b33c-4435-aa14-99506353717e", "type": "message", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n\u2022 I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected\n\u2022 I get the following log at the end of the sync (though successful) : \n```2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.persistence.job.tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.persistence.job.tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.persistence.job.tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]```\nThanks !", "user": "U028HUR1TJR", "ts": "1627060754.349200", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "SS7FL", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n"}]}, {"type": "rich_text_list", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "I get the following log at the end of the sync (though successful) : "}]}], "style": "bullet", "indent": 0}, {"type": "rich_text_preformatted", "elements": [{"type": "text", "text": "2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.persistence.job.tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.persistence.job.tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.persistence.job.tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "Thanks !"}]}]}], "thread_ts": "1627060754.349200", "reply_count": 3, "reply_users_count": 2, "latest_reply": "1627062388.350800", "reply_users": ["U01BV1SDQMA", "U028HUR1TJR"], "is_locked": false, "subscribed": false, "channel_id": "C021JANJ6TY", "float_ts": 1627060754.3492}, "emitted_at": 1661589426858} +{"stream": "channel_messages", "data": {"client_msg_id": "aaefcfbc-b33c-4435-aa14-99506353717e", "type": "message", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n\u2022 I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected\n\u2022 I get the following log at the end of the sync (though successful) : \n```2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]```\nThanks !", "user": "U028HUR1TJR", "ts": "1627060754.349200", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "SS7FL", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n"}]}, {"type": "rich_text_list", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "I get the following log at the end of the sync (though successful) : "}]}], "style": "bullet", "indent": 0}, {"type": "rich_text_preformatted", "elements": [{"type": "text", "text": "2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "Thanks !"}]}]}], "thread_ts": "1627060754.349200", "reply_count": 3, "reply_users_count": 2, "latest_reply": "1627062388.350800", "reply_users": ["U01BV1SDQMA", "U028HUR1TJR"], "is_locked": false, "subscribed": false, "channel_id": "C021JANJ6TY", "float_ts": 1627060754.3492}, "emitted_at": 1661589426858} {"stream": "channel_messages", "data": {"client_msg_id": "3e57321f-da00-49dc-94b5-f3a0ef5aa723", "type": "message", "text": "Hey, we are currently gathering experience with our first connection (custom connector > redshift). We can\u2019t find much information on how the namespace customization works. Is it possible to inject own variables into the custom format as ${SOURCE_NAMESPACE}? Are there other variables that are offered by you in the default?", "user": "U027CJETYP8", "ts": "1627033140.340300", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "k=mGP", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hey, we are currently gathering experience with our first connection (custom connector > redshift). We can\u2019t find much information on how the namespace customization works. Is it possible to inject own variables into the custom format as ${SOURCE_NAMESPACE}? Are there other variables that are offered by you in the default?"}]}]}], "thread_ts": "1627033140.340300", "reply_count": 5, "reply_users_count": 3, "latest_reply": "1627934001.030300", "reply_users": ["U01BV1SDQMA", "U027CJETYP8", "U01MMSDJGC9"], "is_locked": false, "subscribed": false, "channel_id": "C021JANJ6TY", "float_ts": 1627033140.3403}, "emitted_at": 1661589426858} {"stream": "channel_messages", "data": {"client_msg_id": "092f4adb-d803-47ae-8c36-f6e2e99141b0", "type": "message", "text": "Hi all, I\u2019m testing airbyte deployment with GKE autopilot and it seems like HostPort/pod affinity is not permited in autopilot as the error below says.\n```kubectl apply -k kube/overlays/stable \n\nserviceaccount/airbyte-admin created\n created\n created\nconfigmap/airbyte-env-m2467g889g created\nconfigmap/airbyte-temporal-dynamicconfig created\nconfigmap/sweep-pod-script created\nsecret/gcs-log-creds created\nservice/airbyte-db-svc created\nservice/airbyte-minio-svc created\nservice/airbyte-server-svc created\nservice/airbyte-temporal-svc created\nservice/airbyte-webapp-svc created\ndeployment.apps/airbyte-db created\ndeployment.apps/airbyte-pod-sweeper created\ndeployment.apps/airbyte-scheduler created\ndeployment.apps/airbyte-temporal created\ndeployment.apps/airbyte-webapp created\npersistentvolumeclaim/airbyte-minio-pv-claim created\npersistentvolumeclaim/airbyte-volume-configs created\npersistentvolumeclaim/airbyte-volume-db created\npersistentvolumeclaim/airbyte-volume-workspace created\nError from server ([denied by autogke-no-host-port] container airbyte-minio specifies a host port; disallowed in Autopilot. error when creating \"kube/overlays/stable\": admission webhook \"validation.gatekeeper.sh\" denied the request: [denied by autogke-no-host-port] container airbyte-minio specifies a host port; disallowed in Autopilot. \nError from server ([denied by autogke-pod-affinity-limitation] pod affinity is not allowed in Autopilot on topologyKeys: <{\"kubernetes.io/hostname\"}>; Autopilot allows pod affinity only on topologyKeys: <[\"topology.kubernetes.io/region\", \"topology.kubernetes.io/zone\", \"failure-domain.beta.kubernetes.io/region\", \"failure-domain.beta.kubernetes.io/zone\"]>. error when creating \"kube/overlays/stable\": admission webhook \"validation.gatekeeper.sh\" denied the request: [denied by autogke-pod-affinity-limitation] pod affinity is not allowed in Autopilot on topologyKeys: <{\"kubernetes.io/hostname\"}>; Autopilot allows pod affinity only on topologyKeys: <[\"topology.kubernetes.io/region\", \"topology.kubernetes.io/zone\", \"failure-domain.beta.kubernetes.io/region\", \"failure-domain.beta.kubernetes.io/zone\"]>. ```\nAre there any workarounds on this issue?", "user": "U024WGT8H3R", "ts": "1627023042.338300", "team": "T01AB4DDR2N", "edited": {"user": "U024WGT8H3R", "ts": "1627023114.000000"}, "blocks": [{"type": "rich_text", "block_id": "4NDy", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hi all, I\u2019m testing airbyte deployment with GKE autopilot and it seems like HostPort/pod affinity is not permited in autopilot as the error below says.\n"}]}, {"type": "rich_text_preformatted", "elements": [{"type": "text", "text": "kubectl apply -k kube/overlays/stable \n\nserviceaccount/airbyte-admin created\nrole.rbac.authorization.k8s.io/airbyte-admin-role created\nrolebinding.rbac.authorization.k8s.io/airbyte-admin-binding created\nconfigmap/airbyte-env-m2467g889g created\nconfigmap/airbyte-temporal-dynamicconfig created\nconfigmap/sweep-pod-script created\nsecret/gcs-log-creds created\nservice/airbyte-db-svc created\nservice/airbyte-minio-svc created\nservice/airbyte-server-svc created\nservice/airbyte-temporal-svc created\nservice/airbyte-webapp-svc created\ndeployment.apps/airbyte-db created\ndeployment.apps/airbyte-pod-sweeper created\ndeployment.apps/airbyte-scheduler created\ndeployment.apps/airbyte-temporal created\ndeployment.apps/airbyte-webapp created\npersistentvolumeclaim/airbyte-minio-pv-claim created\npersistentvolumeclaim/airbyte-volume-configs created\npersistentvolumeclaim/airbyte-volume-db created\npersistentvolumeclaim/airbyte-volume-workspace created\nError from server ([denied by autogke-no-host-port] container airbyte-minio specifies a host port; disallowed in Autopilot. error when creating \"kube/overlays/stable\": admission webhook \"validation.gatekeeper.sh\" denied the request: [denied by autogke-no-host-port] container airbyte-minio specifies a host port; disallowed in Autopilot. \nError from server ([denied by autogke-pod-affinity-limitation] pod affinity is not allowed in Autopilot on topologyKeys: <{\"kubernetes.io/hostname\"}>; Autopilot allows pod affinity only on topologyKeys: <[\"topology.kubernetes.io/region\", \"topology.kubernetes.io/zone\", \"failure-domain.beta.kubernetes.io/region\", \"failure-domain.beta.kubernetes.io/zone\"]>. error when creating \"kube/overlays/stable\": admission webhook \"validation.gatekeeper.sh\" denied the request: [denied by autogke-pod-affinity-limitation] pod affinity is not allowed in Autopilot on topologyKeys: <{\"kubernetes.io/hostname\"}>; Autopilot allows pod affinity only on topologyKeys: <[\"topology.kubernetes.io/region\", \"topology.kubernetes.io/zone\", \"failure-domain.beta.kubernetes.io/region\", \"failure-domain.beta.kubernetes.io/zone\"]>. "}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "Are there any workarounds on this issue?"}]}]}], "thread_ts": "1627023042.338300", "reply_count": 9, "reply_users_count": 4, "latest_reply": "1632724785.457800", "reply_users": ["U01BV1SDQMA", "U01HYQYV1BQ", "U024WGT8H3R", "U02F7M6UD0S"], "is_locked": false, "subscribed": false, "reactions": [{"name": "white_check_mark", "users": ["U01HYQYV1BQ"], "count": 1}], "channel_id": "C021JANJ6TY", "float_ts": 1627023042.3383}, "emitted_at": 1661589426858} {"stream": "threads", "data": {"client_msg_id": "211e97d6-f529-4a61-adaf-65cbedd4c9d3", "type": "message", "text": " Good article from Temporal on use-case oriented docs structure", "user": "U01A6TEQXRU", "ts": "1627014630.175400", "team": "T01AB4DDR2N", "attachments": [{"title": "How we structured docs for user personas | Temporal documentation", "title_link": "https://docs.temporal.io/blog/docs-info-arch-2021/", "text": "A few weeks ago, around the end of March 2021, I set my focus on finally refreshing the information around Temporal's core concepts.", "fallback": "How we structured docs for user personas | Temporal documentation", "image_url": "https://docs.temporal.io/img/documentation-landing-page-april-2021.png", "from_url": "https://docs.temporal.io/blog/docs-info-arch-2021/", "image_width": 351, "image_height": 250, "image_bytes": 173793, "service_icon": "https://docs.temporal.io/img/favicon.png", "service_name": "docs.temporal.io", "id": 1, "original_url": "https://docs.temporal.io/blog/docs-info-arch-2021/"}], "blocks": [{"type": "rich_text", "block_id": "Il3Xn", "elements": [{"type": "rich_text_section", "elements": [{"type": "link", "url": "https://docs.temporal.io/blog/docs-info-arch-2021/"}, {"type": "text", "text": " Good article from Temporal on use-case oriented docs structure"}]}]}], "thread_ts": "1627014630.175400", "reply_count": 2, "reply_users_count": 2, "latest_reply": "1627331109.183100", "reply_users": ["U024VC21YRW", "U01MMSDJGC9"], "is_locked": false, "subscribed": false, "reactions": [{"name": "+1", "users": ["U027WFS9RV5", "U024VC21YRW", "U01MMSDJGC9", "U02QDDL2L74"], "count": 4}, {"name": "+1::skin-tone-4", "users": ["U01RW78A2LE"], "count": 1}], "channel_id": "C01AB7G87NE", "float_ts": 1627014630.1754}, "emitted_at": 1661589428491} {"stream": "threads", "data": {"client_msg_id": "6ae996d2-85c7-423f-b24e-9f6ce29c109e", "type": "message", "text": "Separating the ops/server persona from the dev persona is a major thing I look for in good vendor docs. I always had a few weeks of ops work to do with vendor adoption before I could hand stuff off to the dev team to actually use. I wanted the details about deployment nuances right up front, but not making a mess of the dev docs. So it\u2019s good to see that reflected in the article.", "user": "U024VC21YRW", "ts": "1627055946.177500", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "4xpJB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Separating the ops/server persona from the dev persona is a major thing I look for in good vendor docs. I always had a few weeks of ops work to do with vendor adoption before I could hand stuff off to the dev team to actually use. I wanted the details about deployment nuances right up front, but not making a mess of the dev docs. So it\u2019s good to see that reflected in the article."}]}]}], "thread_ts": "1627014630.175400", "parent_user_id": "U01A6TEQXRU", "channel_id": "C01AB7G87NE", "float_ts": 1627055946.1775}, "emitted_at": 1661589428494} {"stream": "threads", "data": {"client_msg_id": "00b54cbd-9fce-421d-9c4e-33a5bad08ab7", "type": "message", "text": "<@U01RW78A2LE> this is something that maybe we need to discuss :smile:", "user": "U01MMSDJGC9", "ts": "1627331109.183100", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "xBQ", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U01RW78A2LE"}, {"type": "text", "text": " this is something that maybe we need to discuss "}, {"type": "emoji", "name": "smile", "unicode": "1f604"}]}]}], "thread_ts": "1627014630.175400", "parent_user_id": "U01A6TEQXRU", "reactions": [{"name": "+1::skin-tone-4", "users": ["U01RW78A2LE"], "count": 1}], "channel_id": "C01AB7G87NE", "float_ts": 1627331109.1831}, "emitted_at": 1661589428494} {"stream": "threads", "data": {"client_msg_id": "65bf237c-29b0-452b-af93-af7f6c07bdce", "type": "message", "text": "Hi everyone,\nI have tried to replicate 10gb data from postgres as source to destination mysql but status still in progress ...it has been running since 2 hour.\ncan any one tell what will be reason ?", "user": "U026QUA5622", "ts": "1627062331.350600", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "=by4", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hi everyone,\nI have tried to replicate 10gb data from postgres as source to destination mysql but status still in progress ...it has been running since 2 hour.\ncan any one tell what will be reason ?"}]}]}], "channel_id": "C021JANJ6TY", "float_ts": 1627062331.3506}, "emitted_at": 1661589429206} -{"stream": "threads", "data": {"client_msg_id": "aaefcfbc-b33c-4435-aa14-99506353717e", "type": "message", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n\u2022 I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected\n\u2022 I get the following log at the end of the sync (though successful) : \n```2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.persistence.job.tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.persistence.job.tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.persistence.job.tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]```\nThanks !", "user": "U028HUR1TJR", "ts": "1627060754.349200", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "SS7FL", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n"}]}, {"type": "rich_text_list", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "I get the following log at the end of the sync (though successful) : "}]}], "style": "bullet", "indent": 0}, {"type": "rich_text_preformatted", "elements": [{"type": "text", "text": "2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.persistence.job.tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.persistence.job.tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.persistence.job.tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "Thanks !"}]}]}], "thread_ts": "1627060754.349200", "reply_count": 3, "reply_users_count": 2, "latest_reply": "1627062388.350800", "reply_users": ["U01BV1SDQMA", "U028HUR1TJR"], "is_locked": false, "subscribed": false, "channel_id": "C021JANJ6TY", "float_ts": 1627060754.3492}, "emitted_at": 1661589429515} +{"stream": "threads", "data": {"client_msg_id": "aaefcfbc-b33c-4435-aa14-99506353717e", "type": "message", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n\u2022 I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected\n\u2022 I get the following log at the end of the sync (though successful) : \n```2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]```\nThanks !", "user": "U028HUR1TJR", "ts": "1627060754.349200", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "SS7FL", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Hello, I have set up airbyte for the first time with a Postgres > BigQuery connector, and I\u2019m facing a few issues and I\u2019m not sure whether I\u2019m doing it right, or if they are expected\n"}]}, {"type": "rich_text_list", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "I select only one table in the source schema, but once I saved I can\u2019t see anymore the \u201cnot selected\u201d ones, and when I update the source schema all tables get selected so I lose the information of which tables were previously selected"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "I get the following log at the end of the sync (though successful) : "}]}], "style": "bullet", "indent": 0}, {"type": "rich_text_preformatted", "elements": [{"type": "text", "text": "2021-07-23 17:08:06 INFO () EnvConfigs(getEnvOrDefault):302 - WORKER_ENVIRONMENT not found or empty, defaulting to DOCKER\n2021-07-23 17:08:06 INFO () DefaultNormalizationWorker(run):77 - Normalization executed in 0.\n2021-07-23 17:08:06 INFO () TemporalAttemptExecution(get):133 - Stopping cancellation check scheduling...\n2021-07-23 17:08:06 ERROR () Exceptions(swallow):84 - Swallowed error.\njava.lang.NullPointerException: null value in entry: sync_cpu_request=null\n\tat com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:32) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:171) ~[guava-30.1.1-jre.jar:?]\n\tat com.google.common.collect.ImmutableMap$Builder.put(ImmutableMap.java:281) ~[guava-30.1.1-jre.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.TrackingMetadata.generateSyncMetadata(TrackingMetadata.java:62) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.generateSyncMetadata(JobTracker.java:211) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]\n\tat io.airbyte.scheduler.persistence.job_tracker.JobTracker.lambda$trackSync$3(JobTracker.java:134) ~[io.airbyte.airbyte-scheduler-persistence-0.27.5-alpha.jar:?]"}]}, {"type": "rich_text_section", "elements": [{"type": "text", "text": "Thanks !"}]}]}], "thread_ts": "1627060754.349200", "reply_count": 3, "reply_users_count": 2, "latest_reply": "1627062388.350800", "reply_users": ["U01BV1SDQMA", "U028HUR1TJR"], "is_locked": false, "subscribed": false, "channel_id": "C021JANJ6TY", "float_ts": 1627060754.3492}, "emitted_at": 1661589429515} {"stream": "threads", "data": {"client_msg_id": "71a5a159-5e5c-462d-87ce-c202172dbb99", "type": "message", "text": "You can follow these issues, i think they are similar to what you are reporting:\n\u2022 \n\u2022 ", "user": "U01BV1SDQMA", "ts": "1627060931.349300", "team": "T01AB4DDR2N", "attachments": [{"title": "", "footer": "", "id": 1, "footer_icon": "https://slack.github.com/static/img/favicon-neutral.png", "ts": 1621556593, "color": "36a64f", "mrkdwn_in": ["text"], "fallback": "", "bot_id": "B01US8SR7JN", "app_unfurl_url": "https://github.com/airbytehq/airbyte/issues/3520", "is_app_unfurl": true}, {"title": "", "footer": "", "id": 2, "footer_icon": "https://slack.github.com/static/img/favicon-neutral.png", "ts": 1624460581, "color": "36a64f", "mrkdwn_in": ["text"], "fallback": "", "bot_id": "B01US8SR7JN", "app_unfurl_url": "https://github.com/airbytehq/airbyte/issues/4295", "is_app_unfurl": true}], "blocks": [{"type": "rich_text", "block_id": "5BgK", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "You can follow these issues, i think they are similar to what you are reporting:\n"}]}, {"type": "rich_text_list", "elements": [{"type": "rich_text_section", "elements": [{"type": "link", "url": "https://github.com/airbytehq/airbyte/issues/3520"}]}, {"type": "rich_text_section", "elements": [{"type": "link", "url": "https://github.com/airbytehq/airbyte/issues/4295"}]}], "style": "bullet", "indent": 0}]}], "thread_ts": "1627060754.349200", "parent_user_id": "U028HUR1TJR", "channel_id": "C021JANJ6TY", "float_ts": 1627060931.3493}, "emitted_at": 1661589429516} {"stream": "threads", "data": {"client_msg_id": "bea4bda9-286a-494d-b9a0-e21799482588", "type": "message", "text": "For your exception, you don\u2019t have to worry about it but thanks for reporting! we\u2019ll have to address this but it\u2019s only for reporting usage of the app, there would be no impact on the sync job itself", "user": "U01BV1SDQMA", "ts": "1627061101.349600", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "a/t", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "For your exception, you don\u2019t have to worry about it but thanks for reporting! we\u2019ll have to address this but it\u2019s only for reporting usage of the app, there would be no impact on the sync job itself"}]}]}], "thread_ts": "1627060754.349200", "parent_user_id": "U028HUR1TJR", "reactions": [{"name": "+1", "users": ["U028HUR1TJR"], "count": 1}], "channel_id": "C021JANJ6TY", "float_ts": 1627061101.3496}, "emitted_at": 1661589429517} {"stream": "threads", "data": {"client_msg_id": "ba278400-8e9a-4602-9a9e-8d19b4119d5b", "type": "message", "text": "Thanks !", "user": "U028HUR1TJR", "ts": "1627062388.350800", "team": "T01AB4DDR2N", "blocks": [{"type": "rich_text", "block_id": "vXY6K", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Thanks !"}]}]}], "thread_ts": "1627060754.349200", "parent_user_id": "U028HUR1TJR", "channel_id": "C021JANJ6TY", "float_ts": 1627062388.3508}, "emitted_at": 1661589429518} diff --git a/airbyte-integrations/connectors/source-slack/setup.py b/airbyte-integrations/connectors/source-slack/setup.py index 6c94c5d9cbf54..2bc854bc51223 100644 --- a/airbyte-integrations/connectors/source-slack/setup.py +++ b/airbyte-integrations/connectors/source-slack/setup.py @@ -15,7 +15,7 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-cdk~=0.1", "pendulum>=2,<3"], + install_requires=["airbyte-cdk", "pendulum>=2,<3"], package_data={"": ["*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, diff --git a/airbyte-integrations/connectors/source-slack/source_slack/spec.json b/airbyte-integrations/connectors/source-slack/source_slack/spec.json index 7fce7bbe35427..816d2738860bb 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/spec.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/spec.json @@ -60,7 +60,8 @@ "title": "Client ID", "description": "Slack client_id. See our docs if you need help finding this id.", "type": "string", - "examples": ["slack-client-id-example"] + "examples": ["slack-client-id-example"], + "airbyte_secret": true }, "client_secret": { "title": "Client Secret", diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile b/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile index 8f7fe1505c762..c120b5bd1f45b 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile @@ -25,5 +25,5 @@ COPY source_snapchat_marketing ./source_snapchat_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/source-snapchat-marketing diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile index f0af63030f290..b18ea514d130d 100644 --- a/airbyte-integrations/connectors/source-stripe/Dockerfile +++ b/airbyte-integrations/connectors/source-stripe/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.38 +LABEL io.airbyte.version=0.1.39 LABEL io.airbyte.name=airbyte/source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json index aac5fd711ab02..ddcbbd1019c69 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json @@ -1,20 +1,128 @@ -{ - "charges": { "created": 10000000000 }, - "coupons": { "created": 10000000000 }, - "events": { "created": 10000000000 }, - "customers": { "created": 10000000000 }, - "plans": { "created": 10000000000 }, - "invoices": { "created": 10000000000 }, - "invoice_items": { "date": 10000000000 }, - "transfers": { "created": 10000000000 }, - "subscriptions": { "created": 10000000000 }, - "balance_transactions": { "created": 10000000000 }, - "payouts": { "created": 10000000000 }, - "disputes": { "created": 10000000000 }, - "products": { "created": 10000000000 }, - "refunds": { "created": 10000000000 }, - "payment_intents": { "created": 10000000000 }, - "promotion_codes": { "created": 10000000000 }, - "checkout_sessions": { "expires_at": 10000000000 }, - "checkout_sessions_line_items": { "checkout_session_expires_at": 10000000000 } -} +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "charges" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "coupons" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "events" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "customers" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "plans" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "invoices" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "date": 10000000000 }, + "stream_descriptor": { "name": "invoice_items" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "transfers" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "subscriptions" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "balance_transactions" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "payouts" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "disputes" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "products" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "refunds" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "payment_intents" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "promotion_codes" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "expires_at": 10000000000 }, + "stream_descriptor": { "name": "checkout_sessions" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "checkout_session_expires_at": 10000000000 }, + "stream_descriptor": { "name": "checkout_sessions_line_items" } + } + } +] diff --git a/airbyte-integrations/connectors/source-surveymonkey/Dockerfile b/airbyte-integrations/connectors/source-surveymonkey/Dockerfile index 3e1d4e395cddc..b30aba08a3032 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/Dockerfile +++ b/airbyte-integrations/connectors/source-surveymonkey/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-surveymonkey diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/abnormal_state.json index 629a07a2805e2..6f48fdeb56b94 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/abnormal_state.json @@ -1,46 +1,30 @@ -{ - "surveys": { - "date_modified": "2022-06-10T11:02:01" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "date_modified": "2022-06-10T11:02:01" }, + "stream_descriptor": { "name": "surveys" } + } }, - "survey_responses": { - "306079584": { - "date_modified": "2022-06-08T18:17:18+00:00" - }, - "307785429": { - "date_modified": "2022-06-10T10:59:43+00:00" - }, - "307785444": { - "date_modified": "2022-06-10T11:00:19+00:00" - }, - "307785394": { - "date_modified": "2022-06-10T11:00:59+00:00" - }, - "307785402": { - "date_modified": "2022-06-10T11:01:31+00:00" - }, - "307785408": { - "date_modified": "2022-06-10T11:02:08+00:00" - }, - "307785448": { - "date_modified": "2022-06-10T11:02:49+00:00" - }, - "307784834": { - "date_modified": "2022-06-10T11:03:45+00:00" - }, - "307784863": { - "date_modified": "2022-06-10T11:04:29+00:00" - }, - "307784846": { - "date_modified": "2022-06-10T11:05:05+00:00" - }, - "307784856": { - "date_modified": "2022-06-10T11:05:44+00:00" - }, - "307785388": { - "date_modified": "2022-06-10T11:06:20+00:00" - }, - "307785415": { - "date_modified": "2022-06-10T11:06:43+00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { + "306079584": { "date_modified": "2022-06-08T18:17:18+00:00" }, + "307785429": { "date_modified": "2022-06-10T10:59:43+00:00" }, + "307785444": { "date_modified": "2022-06-10T11:00:19+00:00" }, + "307785394": { "date_modified": "2022-06-10T11:00:59+00:00" }, + "307785402": { "date_modified": "2022-06-10T11:01:31+00:00" }, + "307785408": { "date_modified": "2022-06-10T11:02:08+00:00" }, + "307785448": { "date_modified": "2022-06-10T11:02:49+00:00" }, + "307784834": { "date_modified": "2022-06-10T11:03:45+00:00" }, + "307784863": { "date_modified": "2022-06-10T11:04:29+00:00" }, + "307784846": { "date_modified": "2022-06-10T11:05:05+00:00" }, + "307784856": { "date_modified": "2022-06-10T11:05:44+00:00" }, + "307785388": { "date_modified": "2022-06-10T11:06:20+00:00" }, + "307785415": { "date_modified": "2022-06-10T11:06:43+00:00" } + }, + "stream_descriptor": { "name": "survey_responses" } } } -} +] diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile index de930c15736cb..67ad709b1674f 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile @@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.15 +LABEL io.airbyte.version=0.1.16 LABEL io.airbyte.name=airbyte/source-tiktok-marketing diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json index 2ebacb3c5f073..a4cd2bd35c2e0 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json @@ -1,61 +1,105 @@ -{ - "ads": { - "modify_time": "2030-08-30 09:16:10" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "modify_time": "2030-08-30 09:16:10" }, + "stream_descriptor": { "name": "ads" } + } }, - "ad_groups": { - "modify_time": "2030-08-30 09:16:10" + { + "type": "STREAM", + "stream": { + "stream_state": { "modify_time": "2030-08-30 09:16:10" }, + "stream_descriptor": { "name": "ad_groups" } + } }, - "campaigns": { - "modify_time": "2030-08-30 09:16:10" + { + "type": "STREAM", + "stream": { + "stream_state": { "modify_time": "2030-08-30 09:16:10" }, + "stream_descriptor": { "name": "campaigns" } + } }, - - "ads_reports_daily": { - "stat_time_day": "2030-01-01", - "dimensions": { - "stat_time_day": "2030-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { + "stat_time_day": "2030-01-01", + "dimensions": { "stat_time_day": "2030-01-01" } + }, + "stream_descriptor": { "name": "ads_reports_daily" } } }, - "advertisers_reports_daily": { - "stat_time_day": "2030-01-01", - "dimensions": { - "stat_time_day": "2030-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { + "stat_time_day": "2030-01-01", + "dimensions": { "stat_time_day": "2030-01-01" } + }, + "stream_descriptor": { "name": "advertisers_reports_daily" } } }, - "ad_groups_reports_daily": { - "stat_time_day": "2030-01-01", - "dimensions": { - "stat_time_day": "2030-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { + "stat_time_day": "2030-01-01", + "dimensions": { "stat_time_day": "2030-01-01" } + }, + "stream_descriptor": { "name": "ad_groups_reports_daily" } } }, - "campaigns_reports_daily": { - "stat_time_day": "2030-01-01", - "dimensions": { - "stat_time_day": "2030-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { + "stat_time_day": "2030-01-01", + "dimensions": { "stat_time_day": "2030-01-01" } + }, + "stream_descriptor": { "name": "campaigns_reports_daily" } } }, - - "ads_audience_reports_daily": { - "stat_time_day": "2030-01-01", - "dimensions": { - "stat_time_day": "2030-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { + "stat_time_day": "2030-01-01", + "dimensions": { "stat_time_day": "2030-01-01" } + }, + "stream_descriptor": { "name": "ads_audience_reports_daily" } } }, - "advertisers_audience_reports_daily": { - "stat_time_day": "2030-01-01", - "dimensions": { - "stat_time_day": "2030-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { + "stat_time_day": "2030-01-01", + "dimensions": { "stat_time_day": "2030-01-01" } + }, + "stream_descriptor": { "name": "advertisers_audience_reports_daily" } } }, - "ad_group_audience_reports_daily": { - "stat_time_day": "2030-01-01", - "dimensions": { - "stat_time_day": "2030-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { + "stat_time_day": "2030-01-01", + "dimensions": { "stat_time_day": "2030-01-01" } + }, + "stream_descriptor": { "name": "ad_group_audience_reports_daily" } } }, - "campaigns_audience_reports_by_country_daily": { - "stat_time_day": "2030-01-01", - "dimensions": { - "stat_time_day": "2030-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { + "stat_time_day": "2030-01-01", + "dimensions": { "stat_time_day": "2030-01-01" } + }, + "stream_descriptor": { + "name": "campaigns_audience_reports_by_country_daily" + } } } -} +] diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/setup.py b/airbyte-integrations/connectors/source-tiktok-marketing/setup.py index 85924a9961754..fd635e42316e0 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/setup.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1.42"] +MAIN_REQUIREMENTS = ["airbyte-cdk"] TEST_REQUIREMENTS = ["pytest~=6.1", "source-acceptance-test", "requests-mock==1.9.3", "timeout-decorator==0.5.0"] diff --git a/airbyte-integrations/connectors/source-twilio/Dockerfile b/airbyte-integrations/connectors/source-twilio/Dockerfile index cb05acaace51b..314473194d62d 100644 --- a/airbyte-integrations/connectors/source-twilio/Dockerfile +++ b/airbyte-integrations/connectors/source-twilio/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-twilio diff --git a/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml b/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml index 6cdf4538cba5c..6acb71ad3500f 100644 --- a/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml @@ -2,30 +2,32 @@ connector_image: airbyte/source-twilio:dev tests: spec: - spec_path: "source_twilio/spec.json" - connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - discovery: - - config_path: "secrets/config.json" - basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/no_empty_streams_catalog.json" - expect_records: - path: "integration_tests/expected_records.txt" - incremental: - - config_path: "secrets/config.json" - # usage records stream produces and error if cursor date gte than current date - configured_catalog_path: "integration_tests/no_empty_streams_no_usage_records_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - - config_path: "secrets/config_with_lookback.json" - # usage records stream produces and error if cursor date gte than current date - configured_catalog_path: "integration_tests/no_empty_streams_no_usage_records_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - threshold_days: 30 - full_refresh: - - config_path: "secrets/config.json" - # `constant_records_catalog.json` does not contain the available phone numbers streams, - # as they may change on each request - configured_catalog_path: "integration_tests/constant_records_catalog.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.10" +# connection: +# - config_path: "secrets/config.json" +# status: "succeed" +# - config_path: "integration_tests/invalid_config.json" +# status: "failed" +# discovery: +# - config_path: "secrets/config.json" +# basic_read: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/no_empty_streams_catalog.json" +# expect_records: +# path: "integration_tests/expected_records.txt" +# incremental: +# - config_path: "secrets/config.json" +# # usage records stream produces and error if cursor date gte than current date +# configured_catalog_path: "integration_tests/no_empty_streams_no_usage_records_catalog.json" +# future_state_path: "integration_tests/abnormal_state.json" +# - config_path: "secrets/config_with_lookback.json" +# # usage records stream produces and error if cursor date gte than current date +# configured_catalog_path: "integration_tests/no_empty_streams_no_usage_records_catalog.json" +# future_state_path: "integration_tests/abnormal_state.json" +# threshold_days: 30 +# full_refresh: +# - config_path: "secrets/config.json" +# # `constant_records_catalog.json` does not contain the available phone numbers streams, +# # as they may change on each request +# configured_catalog_path: "integration_tests/constant_records_catalog.json" diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-twilio/integration_tests/abnormal_state.json index 7da15e996072b..f9d01d539b97c 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/abnormal_state.json @@ -1,20 +1,68 @@ -{ - "calls": { - "end_time": "2030-10-01T00:00:00Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "end_time": "2030-10-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "calls" + } + } }, - "conferences": { - "date_created": "2030-10-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_created": "2030-10-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "conferences" + } + } }, - "recordings": { - "date_created": "2030-10-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_created": "2030-10-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "recordings" + } + } }, - "messages": { - "date_sent": "2030-10-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_sent": "2030-10-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "messages" + } + } }, - "message_media": { - "date_created": "2030-10-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_created": "2030-10-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "message_media" + } + } }, - "alerts": { - "date_generated": "2030-10-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_generated": "2030-10-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "alerts" + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.txt index ab688f37c9267..f3353926de87a 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.txt @@ -9,146 +9,140 @@ {"stream": "applications", "data": {"sms_status_callback": null, "voice_caller_id_lookup": false, "voice_fallback_url": null, "date_updated": "2020-11-25T09:47:31Z", "sms_fallback_method": "POST", "friendly_name": "Test friendly name", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Applications/AP731b039bbb9103a1ae2f0afbe85949d4.json", "sms_fallback_url": null, "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "voice_method": "GET", "voice_url": "http://demo.twilio.com/docs/voice.xml", "sms_method": "POST", "public_application_connect_enabled": false, "status_callback_method": "POST", "sid": "AP731b039bbb9103a1ae2f0afbe85949d4", "date_created": "2020-11-25T09:47:31Z", "sms_url": null, "status_callback": null, "voice_fallback_method": "POST", "api_version": "2010-04-01", "message_status_callback": null}, "emitted_at": 1655893073768} {"stream": "applications", "data": {"sms_status_callback": null, "voice_caller_id_lookup": false, "voice_fallback_url": null, "date_updated": "2020-11-25T09:47:31Z", "sms_fallback_method": "POST", "friendly_name": "Test friendly name", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Applications/AP1c10c50172412d3a65dfd7395d11640f.json", "sms_fallback_url": null, "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "voice_method": "GET", "voice_url": "http://demo.twilio.com/docs/voice.xml", "sms_method": "POST", "public_application_connect_enabled": false, "status_callback_method": "POST", "sid": "AP1c10c50172412d3a65dfd7395d11640f", "date_created": "2020-11-25T09:47:31Z", "sms_url": null, "status_callback": null, "voice_fallback_method": "POST", "api_version": "2010-04-01", "message_status_callback": null}, "emitted_at": 1655893073769} {"stream": "applications", "data": {"sms_status_callback": null, "voice_caller_id_lookup": false, "voice_fallback_url": null, "date_updated": "2020-11-25T09:47:31Z", "sms_fallback_method": "POST", "friendly_name": "Test friendly name", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Applications/AP9370b66dc53499e2459d82d75d21c6f8.json", "sms_fallback_url": null, "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "voice_method": "GET", "voice_url": "http://demo.twilio.com/docs/voice.xml", "sms_method": "POST", "public_application_connect_enabled": false, "status_callback_method": "POST", "sid": "AP9370b66dc53499e2459d82d75d21c6f8", "date_created": "2020-11-25T09:47:31Z", "sms_url": null, "status_callback": null, "voice_fallback_method": "POST", "api_version": "2010-04-01", "message_status_callback": null}, "emitted_at": 1655893073769} -{"stream": "available_phone_number_countries", "data": {"country_code": "IE", "country": "Ireland", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IE/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IE/Mobile.json"}}, "emitted_at": 1655893076197} -{"stream": "available_phone_number_countries", "data": {"country_code": "SG", "country": "Singapore", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SG.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SG/Mobile.json"}}, "emitted_at": 1655893076200} -{"stream": "available_phone_number_countries", "data": {"country_code": "BE", "country": "Belgium", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BE.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BE/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BE/Mobile.json"}}, "emitted_at": 1655893076202} -{"stream": "available_phone_number_countries", "data": {"country_code": "CA", "country": "Canada", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CA/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CA/TollFree.json"}}, "emitted_at": 1655893076203} -{"stream": "available_phone_number_countries", "data": {"country_code": "PT", "country": "Portugal", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PT/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PT/Mobile.json"}}, "emitted_at": 1655893076204} -{"stream": "available_phone_number_countries", "data": {"country_code": "SE", "country": "Sweden", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SE/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SE/Mobile.json"}}, "emitted_at": 1655893076205} -{"stream": "available_phone_number_countries", "data": {"country_code": "RO", "country": "Romania", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/RO.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/RO/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/RO/TollFree.json"}}, "emitted_at": 1655893076206} -{"stream": "available_phone_number_countries", "data": {"country_code": "AE", "country": "United Arab Emirates", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AE.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AE/TollFree.json"}}, "emitted_at": 1655893076207} -{"stream": "available_phone_number_countries", "data": {"country_code": "FI", "country": "Finland", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/FI.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/FI/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/FI/Mobile.json"}}, "emitted_at": 1655893076208} -{"stream": "available_phone_number_countries", "data": {"country_code": "GB", "country": "United Kingdom", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GB.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GB/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GB/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GB/Mobile.json"}}, "emitted_at": 1655893076209} -{"stream": "available_phone_number_countries", "data": {"country_code": "PA", "country": "Panama", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PA/Local.json"}}, "emitted_at": 1655893076210} -{"stream": "available_phone_number_countries", "data": {"country_code": "PE", "country": "Peru", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PE.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PE/TollFree.json"}}, "emitted_at": 1655893076211} -{"stream": "available_phone_number_countries", "data": {"country_code": "FR", "country": "France", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/FR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/FR/Local.json"}}, "emitted_at": 1655893076212} -{"stream": "available_phone_number_countries", "data": {"country_code": "CZ", "country": "Czech Republic", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CZ/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CZ/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CZ/Mobile.json"}}, "emitted_at": 1655893076213} -{"stream": "available_phone_number_countries", "data": {"country_code": "DE", "country": "Germany", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DE/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DE/Mobile.json"}}, "emitted_at": 1655893076215} -{"stream": "available_phone_number_countries", "data": {"country_code": "GH", "country": "Ghana", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GH.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GH/Mobile.json"}}, "emitted_at": 1655893076216} -{"stream": "available_phone_number_countries", "data": {"country_code": "DK", "country": "Denmark", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DK.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DK/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DK/Mobile.json"}}, "emitted_at": 1655893076217} -{"stream": "available_phone_number_countries", "data": {"country_code": "UG", "country": "Uganda", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/UG.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/UG/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/UG/TollFree.json"}}, "emitted_at": 1655893076219} -{"stream": "available_phone_number_countries", "data": {"country_code": "PL", "country": "Poland", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PL.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PL/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PL/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PL/Mobile.json"}}, "emitted_at": 1655893076220} -{"stream": "available_phone_number_countries", "data": {"country_code": "MX", "country": "Mexico", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MX.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MX/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MX/TollFree.json"}}, "emitted_at": 1655893076221} -{"stream": "available_phone_number_countries", "data": {"country_code": "IS", "country": "Iceland", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IS.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IS/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IS/Mobile.json"}}, "emitted_at": 1655893076222} -{"stream": "available_phone_number_countries", "data": {"country_code": "DZ", "country": "Algeria", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DZ/Local.json"}}, "emitted_at": 1655893076223} -{"stream": "available_phone_number_countries", "data": {"country_code": "ZA", "country": "South Africa", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ZA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ZA/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ZA/Mobile.json"}}, "emitted_at": 1655893076223} -{"stream": "available_phone_number_countries", "data": {"country_code": "JP", "country": "Japan", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/JP.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/JP/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/JP/TollFree.json"}}, "emitted_at": 1655893076224} -{"stream": "available_phone_number_countries", "data": {"country_code": "HR", "country": "Croatia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HR/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HR/TollFree.json"}}, "emitted_at": 1655893076225} -{"stream": "available_phone_number_countries", "data": {"country_code": "ID", "country": "Indonesia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ID.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ID/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ID/TollFree.json"}}, "emitted_at": 1655893076225} -{"stream": "available_phone_number_countries", "data": {"country_code": "BR", "country": "Brazil", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BR/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BR/TollFree.json"}}, "emitted_at": 1655893076226} -{"stream": "available_phone_number_countries", "data": {"country_code": "AT", "country": "Austria", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AT/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AT/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AT/Mobile.json"}}, "emitted_at": 1655893076227} -{"stream": "available_phone_number_countries", "data": {"country_code": "US", "country": "United States", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/US.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/US/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/US/TollFree.json"}}, "emitted_at": 1655893076228} -{"stream": "available_phone_number_countries", "data": {"country_code": "VI", "country": "Virgin Islands, U.S.", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/VI.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/VI/Local.json"}}, "emitted_at": 1655893076228} -{"stream": "available_phone_number_countries", "data": {"country_code": "EC", "country": "Ecuador", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/EC.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/EC/Local.json"}}, "emitted_at": 1655893076229} -{"stream": "available_phone_number_countries", "data": {"country_code": "KE", "country": "Kenya", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/KE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/KE/Local.json"}}, "emitted_at": 1655893076229} -{"stream": "available_phone_number_countries", "data": {"country_code": "NL", "country": "Netherlands", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NL.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NL/Mobile.json"}}, "emitted_at": 1655893076230} -{"stream": "available_phone_number_countries", "data": {"country_code": "CL", "country": "Chile", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CL.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CL/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CL/Mobile.json"}}, "emitted_at": 1655893076230} -{"stream": "available_phone_number_countries", "data": {"country_code": "CH", "country": "Switzerland", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CH.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CH/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CH/Mobile.json"}}, "emitted_at": 1655893076231} -{"stream": "available_phone_number_countries", "data": {"country_code": "TN", "country": "Tunisia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TN.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TN/Local.json"}}, "emitted_at": 1655893076232} -{"stream": "available_phone_number_countries", "data": {"country_code": "TT", "country": "Trinidad and Tobago", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TT/Local.json"}}, "emitted_at": 1655893076232} -{"stream": "available_phone_number_countries", "data": {"country_code": "TH", "country": "Thailand", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TH.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TH/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TH/TollFree.json"}}, "emitted_at": 1655893076233} -{"stream": "available_phone_number_countries", "data": {"country_code": "SI", "country": "Slovenia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SI.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SI/Local.json"}}, "emitted_at": 1655893076233} -{"stream": "available_phone_number_countries", "data": {"country_code": "SK", "country": "Slovakia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SK.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SK/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SK/TollFree.json"}}, "emitted_at": 1655893076233} -{"stream": "available_phone_number_countries", "data": {"country_code": "PR", "country": "Puerto Rico", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PR/Local.json"}}, "emitted_at": 1655893076234} -{"stream": "available_phone_number_countries", "data": {"country_code": "PH", "country": "Philippines", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PH.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PH/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PH/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PH/Mobile.json"}}, "emitted_at": 1655893076234} -{"stream": "available_phone_number_countries", "data": {"country_code": "NZ", "country": "New Zealand", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NZ/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NZ/TollFree.json"}}, "emitted_at": 1655893076235} -{"stream": "available_phone_number_countries", "data": {"country_code": "NA", "country": "Namibia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NA/Local.json"}}, "emitted_at": 1655893076235} -{"stream": "available_phone_number_countries", "data": {"country_code": "MU", "country": "Mauritius", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MU.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MU/Mobile.json"}}, "emitted_at": 1655893076236} -{"stream": "available_phone_number_countries", "data": {"country_code": "ML", "country": "Mali", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ML.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ML/Local.json"}}, "emitted_at": 1655893076236} -{"stream": "available_phone_number_countries", "data": {"country_code": "MO", "country": "Macau", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MO.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MO/Mobile.json"}}, "emitted_at": 1655893076236} -{"stream": "available_phone_number_countries", "data": {"country_code": "LU", "country": "Luxembourg", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/LU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/LU/Local.json"}}, "emitted_at": 1655893076237} -{"stream": "available_phone_number_countries", "data": {"country_code": "LT", "country": "Lithuania", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/LT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/LT/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/LT/Mobile.json"}}, "emitted_at": 1655893076237} -{"stream": "available_phone_number_countries", "data": {"country_code": "JM", "country": "Jamaica", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/JM.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/JM/Local.json"}}, "emitted_at": 1655893076238} -{"stream": "available_phone_number_countries", "data": {"country_code": "IL", "country": "Israel", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IL.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IL/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IL/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IL/Mobile.json"}}, "emitted_at": 1655893076238} -{"stream": "available_phone_number_countries", "data": {"country_code": "HU", "country": "Hungary", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HU/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HU/Mobile.json"}}, "emitted_at": 1655893076238} -{"stream": "available_phone_number_countries", "data": {"country_code": "HK", "country": "Hong Kong", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HK.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HK/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HK/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HK/Mobile.json"}}, "emitted_at": 1655893076239} -{"stream": "available_phone_number_countries", "data": {"country_code": "GN", "country": "Guinea", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GN.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GN/Mobile.json"}}, "emitted_at": 1655893076239} -{"stream": "available_phone_number_countries", "data": {"country_code": "GD", "country": "Grenada", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GD.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GD/Local.json"}}, "emitted_at": 1655893076240} -{"stream": "available_phone_number_countries", "data": {"country_code": "GR", "country": "Greece", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GR/Local.json"}}, "emitted_at": 1655893076241} -{"stream": "available_phone_number_countries", "data": {"country_code": "GE", "country": "Georgia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GE/Local.json"}}, "emitted_at": 1655893076241} -{"stream": "available_phone_number_countries", "data": {"country_code": "EE", "country": "Estonia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/EE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/EE/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/EE/Mobile.json"}}, "emitted_at": 1655893076242} -{"stream": "available_phone_number_countries", "data": {"country_code": "SV", "country": "El Salvador", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SV.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SV/Local.json"}}, "emitted_at": 1655893076242} -{"stream": "available_phone_number_countries", "data": {"country_code": "DO", "country": "Dominican Republic", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DO.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DO/Local.json"}}, "emitted_at": 1655893076242} -{"stream": "available_phone_number_countries", "data": {"country_code": "CY", "country": "Cyprus", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CY.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CY/Local.json"}}, "emitted_at": 1655893076243} -{"stream": "available_phone_number_countries", "data": {"country_code": "CO", "country": "Colombia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CO.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CO/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CO/TollFree.json"}}, "emitted_at": 1655893076243} -{"stream": "available_phone_number_countries", "data": {"country_code": "KY", "country": "Cayman Islands", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/KY.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/KY/Local.json"}}, "emitted_at": 1655893076243} -{"stream": "available_phone_number_countries", "data": {"country_code": "BG", "country": "Bulgaria", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BG.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BG/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BG/TollFree.json"}}, "emitted_at": 1655893076244} -{"stream": "available_phone_number_countries", "data": {"country_code": "BW", "country": "Botswana", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BW.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BW/TollFree.json"}}, "emitted_at": 1655893076244} -{"stream": "available_phone_number_countries", "data": {"country_code": "BA", "country": "Bosnia and Herzegovina", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BA/Local.json"}}, "emitted_at": 1655893076244} -{"stream": "available_phone_number_countries", "data": {"country_code": "BJ", "country": "Benin", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BJ.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BJ/Mobile.json"}}, "emitted_at": 1655893076244} -{"stream": "available_phone_number_countries", "data": {"country_code": "BB", "country": "Barbados", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BB.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BB/Local.json"}}, "emitted_at": 1655893076245} -{"stream": "available_phone_number_countries", "data": {"country_code": "AU", "country": "Australia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU/Mobile.json"}}, "emitted_at": 1655893076245} -{"stream": "available_phone_number_countries", "data": {"country_code": "AR", "country": "Argentina", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AR/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AR/TollFree.json"}}, "emitted_at": 1655893076245} -{"stream": "available_phone_number_countries", "data": {"country_code": "IE", "country": "Ireland", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IE/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IE/Mobile.json"}}, "emitted_at": 1655893077076} -{"stream": "available_phone_number_countries", "data": {"country_code": "SG", "country": "Singapore", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SG.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SG/Mobile.json"}}, "emitted_at": 1655893077077} -{"stream": "available_phone_number_countries", "data": {"country_code": "BE", "country": "Belgium", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BE.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BE/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BE/Mobile.json"}}, "emitted_at": 1655893077078} -{"stream": "available_phone_number_countries", "data": {"country_code": "CA", "country": "Canada", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CA/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CA/TollFree.json"}}, "emitted_at": 1655893077079} -{"stream": "available_phone_number_countries", "data": {"country_code": "PT", "country": "Portugal", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PT/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PT/Mobile.json"}}, "emitted_at": 1655893077080} -{"stream": "available_phone_number_countries", "data": {"country_code": "SE", "country": "Sweden", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SE/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SE/Mobile.json"}}, "emitted_at": 1655893077081} -{"stream": "available_phone_number_countries", "data": {"country_code": "RO", "country": "Romania", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/RO.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/RO/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/RO/TollFree.json"}}, "emitted_at": 1655893077083} -{"stream": "available_phone_number_countries", "data": {"country_code": "AE", "country": "United Arab Emirates", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AE.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AE/TollFree.json"}}, "emitted_at": 1655893077084} -{"stream": "available_phone_number_countries", "data": {"country_code": "FI", "country": "Finland", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/FI.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/FI/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/FI/Mobile.json"}}, "emitted_at": 1655893077085} -{"stream": "available_phone_number_countries", "data": {"country_code": "GB", "country": "United Kingdom", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GB.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GB/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GB/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GB/Mobile.json"}}, "emitted_at": 1655893077086} -{"stream": "available_phone_number_countries", "data": {"country_code": "PA", "country": "Panama", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PA/Local.json"}}, "emitted_at": 1655893077087} -{"stream": "available_phone_number_countries", "data": {"country_code": "PE", "country": "Peru", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PE.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PE/TollFree.json"}}, "emitted_at": 1655893077088} -{"stream": "available_phone_number_countries", "data": {"country_code": "FR", "country": "France", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/FR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/FR/Local.json"}}, "emitted_at": 1655893077089} -{"stream": "available_phone_number_countries", "data": {"country_code": "CZ", "country": "Czech Republic", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CZ/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CZ/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CZ/Mobile.json"}}, "emitted_at": 1655893077090} -{"stream": "available_phone_number_countries", "data": {"country_code": "DE", "country": "Germany", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DE/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DE/Mobile.json"}}, "emitted_at": 1655893077091} -{"stream": "available_phone_number_countries", "data": {"country_code": "GH", "country": "Ghana", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GH.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GH/Mobile.json"}}, "emitted_at": 1655893077092} -{"stream": "available_phone_number_countries", "data": {"country_code": "DK", "country": "Denmark", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DK.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DK/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DK/Mobile.json"}}, "emitted_at": 1655893077093} -{"stream": "available_phone_number_countries", "data": {"country_code": "UG", "country": "Uganda", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/UG.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/UG/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/UG/TollFree.json"}}, "emitted_at": 1655893077094} -{"stream": "available_phone_number_countries", "data": {"country_code": "PL", "country": "Poland", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PL.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PL/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PL/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PL/Mobile.json"}}, "emitted_at": 1655893077095} -{"stream": "available_phone_number_countries", "data": {"country_code": "MX", "country": "Mexico", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MX.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MX/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MX/TollFree.json"}}, "emitted_at": 1655893077096} -{"stream": "available_phone_number_countries", "data": {"country_code": "IS", "country": "Iceland", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IS.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IS/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IS/Mobile.json"}}, "emitted_at": 1655893077098} -{"stream": "available_phone_number_countries", "data": {"country_code": "DZ", "country": "Algeria", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DZ/Local.json"}}, "emitted_at": 1655893077099} -{"stream": "available_phone_number_countries", "data": {"country_code": "ZA", "country": "South Africa", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ZA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ZA/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ZA/Mobile.json"}}, "emitted_at": 1655893077100} -{"stream": "available_phone_number_countries", "data": {"country_code": "JP", "country": "Japan", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/JP.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/JP/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/JP/TollFree.json"}}, "emitted_at": 1655893077101} -{"stream": "available_phone_number_countries", "data": {"country_code": "HR", "country": "Croatia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HR/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HR/TollFree.json"}}, "emitted_at": 1655893077102} -{"stream": "available_phone_number_countries", "data": {"country_code": "ID", "country": "Indonesia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ID.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ID/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ID/TollFree.json"}}, "emitted_at": 1655893077103} -{"stream": "available_phone_number_countries", "data": {"country_code": "BR", "country": "Brazil", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BR/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BR/TollFree.json"}}, "emitted_at": 1655893077104} -{"stream": "available_phone_number_countries", "data": {"country_code": "AT", "country": "Austria", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AT/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AT/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AT/Mobile.json"}}, "emitted_at": 1655893077105} -{"stream": "available_phone_number_countries", "data": {"country_code": "US", "country": "United States", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/US.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/US/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/US/TollFree.json"}}, "emitted_at": 1655893077106} -{"stream": "available_phone_number_countries", "data": {"country_code": "VI", "country": "Virgin Islands, U.S.", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/VI.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/VI/Local.json"}}, "emitted_at": 1655893077106} -{"stream": "available_phone_number_countries", "data": {"country_code": "EC", "country": "Ecuador", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/EC.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/EC/Local.json"}}, "emitted_at": 1655893077107} -{"stream": "available_phone_number_countries", "data": {"country_code": "KE", "country": "Kenya", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/KE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/KE/Local.json"}}, "emitted_at": 1655893077108} -{"stream": "available_phone_number_countries", "data": {"country_code": "NL", "country": "Netherlands", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NL.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NL/Mobile.json"}}, "emitted_at": 1655893077108} -{"stream": "available_phone_number_countries", "data": {"country_code": "CL", "country": "Chile", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CL.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CL/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CL/Mobile.json"}}, "emitted_at": 1655893077109} -{"stream": "available_phone_number_countries", "data": {"country_code": "CH", "country": "Switzerland", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CH.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CH/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CH/Mobile.json"}}, "emitted_at": 1655893077110} -{"stream": "available_phone_number_countries", "data": {"country_code": "TN", "country": "Tunisia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TN.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TN/Local.json"}}, "emitted_at": 1655893077110} -{"stream": "available_phone_number_countries", "data": {"country_code": "TT", "country": "Trinidad and Tobago", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TT/Local.json"}}, "emitted_at": 1655893077111} -{"stream": "available_phone_number_countries", "data": {"country_code": "TH", "country": "Thailand", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TH.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TH/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TH/TollFree.json"}}, "emitted_at": 1655893077111} -{"stream": "available_phone_number_countries", "data": {"country_code": "SI", "country": "Slovenia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SI.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SI/Local.json"}}, "emitted_at": 1655893077112} -{"stream": "available_phone_number_countries", "data": {"country_code": "SK", "country": "Slovakia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SK.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SK/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SK/TollFree.json"}}, "emitted_at": 1655893077112} -{"stream": "available_phone_number_countries", "data": {"country_code": "PR", "country": "Puerto Rico", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PR/Local.json"}}, "emitted_at": 1655893077113} -{"stream": "available_phone_number_countries", "data": {"country_code": "PH", "country": "Philippines", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PH.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PH/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PH/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PH/Mobile.json"}}, "emitted_at": 1655893077113} -{"stream": "available_phone_number_countries", "data": {"country_code": "NZ", "country": "New Zealand", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NZ/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NZ/TollFree.json"}}, "emitted_at": 1655893077114} -{"stream": "available_phone_number_countries", "data": {"country_code": "NA", "country": "Namibia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NA/Local.json"}}, "emitted_at": 1655893077114} -{"stream": "available_phone_number_countries", "data": {"country_code": "MU", "country": "Mauritius", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MU.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MU/Mobile.json"}}, "emitted_at": 1655893077115} -{"stream": "available_phone_number_countries", "data": {"country_code": "ML", "country": "Mali", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ML.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ML/Local.json"}}, "emitted_at": 1655893077115} -{"stream": "available_phone_number_countries", "data": {"country_code": "MO", "country": "Macau", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MO.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MO/Mobile.json"}}, "emitted_at": 1655893077115} -{"stream": "available_phone_number_countries", "data": {"country_code": "LU", "country": "Luxembourg", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/LU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/LU/Local.json"}}, "emitted_at": 1655893077116} -{"stream": "available_phone_number_countries", "data": {"country_code": "LT", "country": "Lithuania", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/LT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/LT/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/LT/Mobile.json"}}, "emitted_at": 1655893077116} -{"stream": "available_phone_number_countries", "data": {"country_code": "JM", "country": "Jamaica", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/JM.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/JM/Local.json"}}, "emitted_at": 1655893077117} -{"stream": "available_phone_number_countries", "data": {"country_code": "IL", "country": "Israel", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IL.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IL/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IL/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IL/Mobile.json"}}, "emitted_at": 1655893077117} -{"stream": "available_phone_number_countries", "data": {"country_code": "HU", "country": "Hungary", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HU/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HU/Mobile.json"}}, "emitted_at": 1655893077117} -{"stream": "available_phone_number_countries", "data": {"country_code": "HK", "country": "Hong Kong", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HK.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HK/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HK/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HK/Mobile.json"}}, "emitted_at": 1655893077118} -{"stream": "available_phone_number_countries", "data": {"country_code": "GN", "country": "Guinea", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GN.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GN/Mobile.json"}}, "emitted_at": 1655893077118} -{"stream": "available_phone_number_countries", "data": {"country_code": "GD", "country": "Grenada", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GD.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GD/Local.json"}}, "emitted_at": 1655893077119} -{"stream": "available_phone_number_countries", "data": {"country_code": "GR", "country": "Greece", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GR/Local.json"}}, "emitted_at": 1655893077119} -{"stream": "available_phone_number_countries", "data": {"country_code": "GE", "country": "Georgia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GE/Local.json"}}, "emitted_at": 1655893077119} -{"stream": "available_phone_number_countries", "data": {"country_code": "EE", "country": "Estonia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/EE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/EE/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/EE/Mobile.json"}}, "emitted_at": 1655893077120} -{"stream": "available_phone_number_countries", "data": {"country_code": "SV", "country": "El Salvador", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SV.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SV/Local.json"}}, "emitted_at": 1655893077120} -{"stream": "available_phone_number_countries", "data": {"country_code": "DO", "country": "Dominican Republic", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DO.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DO/Local.json"}}, "emitted_at": 1655893077120} -{"stream": "available_phone_number_countries", "data": {"country_code": "CY", "country": "Cyprus", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CY.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CY/Local.json"}}, "emitted_at": 1655893077121} -{"stream": "available_phone_number_countries", "data": {"country_code": "CO", "country": "Colombia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CO.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CO/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CO/TollFree.json"}}, "emitted_at": 1655893077121} -{"stream": "available_phone_number_countries", "data": {"country_code": "KY", "country": "Cayman Islands", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/KY.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/KY/Local.json"}}, "emitted_at": 1655893077122} -{"stream": "available_phone_number_countries", "data": {"country_code": "BG", "country": "Bulgaria", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BG.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BG/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BG/TollFree.json"}}, "emitted_at": 1655893077122} -{"stream": "available_phone_number_countries", "data": {"country_code": "BW", "country": "Botswana", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BW.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BW/TollFree.json"}}, "emitted_at": 1655893077122} -{"stream": "available_phone_number_countries", "data": {"country_code": "BA", "country": "Bosnia and Herzegovina", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BA/Local.json"}}, "emitted_at": 1655893077123} -{"stream": "available_phone_number_countries", "data": {"country_code": "BJ", "country": "Benin", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BJ.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BJ/Mobile.json"}}, "emitted_at": 1655893077123} -{"stream": "available_phone_number_countries", "data": {"country_code": "BB", "country": "Barbados", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BB.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BB/Local.json"}}, "emitted_at": 1655893077123} -{"stream": "available_phone_number_countries", "data": {"country_code": "AU", "country": "Australia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU/Mobile.json"}}, "emitted_at": 1655893077124} -{"stream": "available_phone_number_countries", "data": {"country_code": "AR", "country": "Argentina", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR/TollFree.json"}}, "emitted_at": 1655893077124} +{"stream": "available_phone_number_countries", "data": {"country_code": "SE", "country": "Sweden", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SE.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SE/Mobile.json"}}, "emitted_at": 1664560270804} +{"stream": "available_phone_number_countries", "data": {"country_code": "SK", "country": "Slovakia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SK.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SK/TollFree.json"}}, "emitted_at": 1664560270809} +{"stream": "available_phone_number_countries", "data": {"country_code": "PL", "country": "Poland", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PL.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PL/Mobile.json"}}, "emitted_at": 1664560270809} +{"stream": "available_phone_number_countries", "data": {"country_code": "LT", "country": "Lithuania", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/LT.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/LT/Mobile.json"}}, "emitted_at": 1664560270810} +{"stream": "available_phone_number_countries", "data": {"country_code": "JP", "country": "Japan", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/JP.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/JP/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/JP/TollFree.json"}}, "emitted_at": 1664560270810} +{"stream": "available_phone_number_countries", "data": {"country_code": "IE", "country": "Ireland", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IE/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IE/Mobile.json"}}, "emitted_at": 1664560270810} +{"stream": "available_phone_number_countries", "data": {"country_code": "SG", "country": "Singapore", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SG.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SG/Mobile.json"}}, "emitted_at": 1664560270810} +{"stream": "available_phone_number_countries", "data": {"country_code": "BE", "country": "Belgium", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BE.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BE/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BE/Mobile.json"}}, "emitted_at": 1664560270811} +{"stream": "available_phone_number_countries", "data": {"country_code": "CA", "country": "Canada", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CA/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CA/TollFree.json"}}, "emitted_at": 1664560270811} +{"stream": "available_phone_number_countries", "data": {"country_code": "PT", "country": "Portugal", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PT/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PT/Mobile.json"}}, "emitted_at": 1664560270811} +{"stream": "available_phone_number_countries", "data": {"country_code": "RO", "country": "Romania", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/RO.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/RO/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/RO/TollFree.json"}}, "emitted_at": 1664560270811} +{"stream": "available_phone_number_countries", "data": {"country_code": "FI", "country": "Finland", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/FI.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/FI/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/FI/Mobile.json"}}, "emitted_at": 1664560270812} +{"stream": "available_phone_number_countries", "data": {"country_code": "GB", "country": "United Kingdom", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GB.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GB/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GB/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GB/Mobile.json"}}, "emitted_at": 1664560270812} +{"stream": "available_phone_number_countries", "data": {"country_code": "PA", "country": "Panama", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PA/Local.json"}}, "emitted_at": 1664560270812} +{"stream": "available_phone_number_countries", "data": {"country_code": "PE", "country": "Peru", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PE.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PE/TollFree.json"}}, "emitted_at": 1664560270812} +{"stream": "available_phone_number_countries", "data": {"country_code": "FR", "country": "France", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/FR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/FR/Local.json"}}, "emitted_at": 1664560270813} +{"stream": "available_phone_number_countries", "data": {"country_code": "CZ", "country": "Czech Republic", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CZ/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CZ/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CZ/Mobile.json"}}, "emitted_at": 1664560270814} +{"stream": "available_phone_number_countries", "data": {"country_code": "DE", "country": "Germany", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DE/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DE/Mobile.json"}}, "emitted_at": 1664560270814} +{"stream": "available_phone_number_countries", "data": {"country_code": "GH", "country": "Ghana", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GH.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GH/Mobile.json"}}, "emitted_at": 1664560270815} +{"stream": "available_phone_number_countries", "data": {"country_code": "DK", "country": "Denmark", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DK.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DK/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DK/Mobile.json"}}, "emitted_at": 1664560270815} +{"stream": "available_phone_number_countries", "data": {"country_code": "UG", "country": "Uganda", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/UG.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/UG/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/UG/TollFree.json"}}, "emitted_at": 1664560270815} +{"stream": "available_phone_number_countries", "data": {"country_code": "MX", "country": "Mexico", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MX.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MX/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MX/TollFree.json"}}, "emitted_at": 1664560270816} +{"stream": "available_phone_number_countries", "data": {"country_code": "IS", "country": "Iceland", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IS.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IS/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IS/Mobile.json"}}, "emitted_at": 1664560270816} +{"stream": "available_phone_number_countries", "data": {"country_code": "DZ", "country": "Algeria", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/DZ/Local.json"}}, "emitted_at": 1664560270816} +{"stream": "available_phone_number_countries", "data": {"country_code": "ZA", "country": "South Africa", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ZA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ZA/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ZA/Mobile.json"}}, "emitted_at": 1664560270816} +{"stream": "available_phone_number_countries", "data": {"country_code": "HR", "country": "Croatia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HR/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HR/TollFree.json"}}, "emitted_at": 1664560270816} +{"stream": "available_phone_number_countries", "data": {"country_code": "ID", "country": "Indonesia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ID.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ID/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ID/TollFree.json"}}, "emitted_at": 1664560270817} +{"stream": "available_phone_number_countries", "data": {"country_code": "BR", "country": "Brazil", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BR/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BR/TollFree.json"}}, "emitted_at": 1664560270817} +{"stream": "available_phone_number_countries", "data": {"country_code": "AT", "country": "Austria", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AT/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AT/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AT/Mobile.json"}}, "emitted_at": 1664560270817} +{"stream": "available_phone_number_countries", "data": {"country_code": "US", "country": "United States", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/US.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/US/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/US/TollFree.json"}}, "emitted_at": 1664560270817} +{"stream": "available_phone_number_countries", "data": {"country_code": "VI", "country": "Virgin Islands, U.S.", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/VI.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/VI/Local.json"}}, "emitted_at": 1664560270817} +{"stream": "available_phone_number_countries", "data": {"country_code": "EC", "country": "Ecuador", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/EC.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/EC/Local.json"}}, "emitted_at": 1664560270817} +{"stream": "available_phone_number_countries", "data": {"country_code": "KE", "country": "Kenya", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/KE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/KE/Local.json"}}, "emitted_at": 1664560270817} +{"stream": "available_phone_number_countries", "data": {"country_code": "NL", "country": "Netherlands", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NL.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NL/Mobile.json"}}, "emitted_at": 1664560270817} +{"stream": "available_phone_number_countries", "data": {"country_code": "CL", "country": "Chile", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CL.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CL/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CL/Mobile.json"}}, "emitted_at": 1664560270817} +{"stream": "available_phone_number_countries", "data": {"country_code": "CH", "country": "Switzerland", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CH.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CH/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CH/Mobile.json"}}, "emitted_at": 1664560270817} +{"stream": "available_phone_number_countries", "data": {"country_code": "TN", "country": "Tunisia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TN.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TN/Local.json"}}, "emitted_at": 1664560270817} +{"stream": "available_phone_number_countries", "data": {"country_code": "TT", "country": "Trinidad and Tobago", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TT/Local.json"}}, "emitted_at": 1664560270818} +{"stream": "available_phone_number_countries", "data": {"country_code": "TH", "country": "Thailand", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TH.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TH/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/TH/TollFree.json"}}, "emitted_at": 1664560270818} +{"stream": "available_phone_number_countries", "data": {"country_code": "SI", "country": "Slovenia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SI.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SI/Local.json"}}, "emitted_at": 1664560270818} +{"stream": "available_phone_number_countries", "data": {"country_code": "PR", "country": "Puerto Rico", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PR/Local.json"}}, "emitted_at": 1664560270818} +{"stream": "available_phone_number_countries", "data": {"country_code": "PH", "country": "Philippines", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PH.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PH/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PH/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/PH/Mobile.json"}}, "emitted_at": 1664560270818} +{"stream": "available_phone_number_countries", "data": {"country_code": "NZ", "country": "New Zealand", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NZ/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NZ/TollFree.json"}}, "emitted_at": 1664560270818} +{"stream": "available_phone_number_countries", "data": {"country_code": "NA", "country": "Namibia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/NA/Local.json"}}, "emitted_at": 1664560270818} +{"stream": "available_phone_number_countries", "data": {"country_code": "MU", "country": "Mauritius", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MU.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MU/Mobile.json"}}, "emitted_at": 1664560270818} +{"stream": "available_phone_number_countries", "data": {"country_code": "ML", "country": "Mali", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ML.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/ML/Local.json"}}, "emitted_at": 1664560270818} +{"stream": "available_phone_number_countries", "data": {"country_code": "MO", "country": "Macau", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MO.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/MO/Mobile.json"}}, "emitted_at": 1664560270818} +{"stream": "available_phone_number_countries", "data": {"country_code": "JM", "country": "Jamaica", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/JM.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/JM/Local.json"}}, "emitted_at": 1664560270818} +{"stream": "available_phone_number_countries", "data": {"country_code": "IL", "country": "Israel", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IL.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IL/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IL/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/IL/Mobile.json"}}, "emitted_at": 1664560270819} +{"stream": "available_phone_number_countries", "data": {"country_code": "HU", "country": "Hungary", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HU/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HU/Mobile.json"}}, "emitted_at": 1664560270819} +{"stream": "available_phone_number_countries", "data": {"country_code": "HK", "country": "Hong Kong", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HK.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HK/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HK/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/HK/Mobile.json"}}, "emitted_at": 1664560270819} +{"stream": "available_phone_number_countries", "data": {"country_code": "GN", "country": "Guinea", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GN.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GN/Mobile.json"}}, "emitted_at": 1664560270819} +{"stream": "available_phone_number_countries", "data": {"country_code": "GD", "country": "Grenada", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GD.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GD/Local.json"}}, "emitted_at": 1664560270819} +{"stream": "available_phone_number_countries", "data": {"country_code": "GR", "country": "Greece", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GR/Local.json"}}, "emitted_at": 1664560270819} +{"stream": "available_phone_number_countries", "data": {"country_code": "GE", "country": "Georgia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/GE/Local.json"}}, "emitted_at": 1664560270819} +{"stream": "available_phone_number_countries", "data": {"country_code": "EE", "country": "Estonia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/EE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/EE/Local.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/EE/Mobile.json"}}, "emitted_at": 1664560270819} +{"stream": "available_phone_number_countries", "data": {"country_code": "SV", "country": "El Salvador", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SV.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SV/Local.json"}}, "emitted_at": 1664560270819} +{"stream": "available_phone_number_countries", "data": {"country_code": "CY", "country": "Cyprus", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CY.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CY/Local.json"}}, "emitted_at": 1664560270819} +{"stream": "available_phone_number_countries", "data": {"country_code": "CO", "country": "Colombia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CO.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CO/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/CO/TollFree.json"}}, "emitted_at": 1664560270820} +{"stream": "available_phone_number_countries", "data": {"country_code": "KY", "country": "Cayman Islands", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/KY.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/KY/Local.json"}}, "emitted_at": 1664560270820} +{"stream": "available_phone_number_countries", "data": {"country_code": "BG", "country": "Bulgaria", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BG.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BG/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BG/TollFree.json"}}, "emitted_at": 1664560270820} +{"stream": "available_phone_number_countries", "data": {"country_code": "BW", "country": "Botswana", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BW.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BW/TollFree.json"}}, "emitted_at": 1664560270820} +{"stream": "available_phone_number_countries", "data": {"country_code": "BA", "country": "Bosnia and Herzegovina", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BA/Local.json"}}, "emitted_at": 1664560270820} +{"stream": "available_phone_number_countries", "data": {"country_code": "BJ", "country": "Benin", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BJ.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BJ/Mobile.json"}}, "emitted_at": 1664560270821} +{"stream": "available_phone_number_countries", "data": {"country_code": "BB", "country": "Barbados", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BB.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BB/Local.json"}}, "emitted_at": 1664560270821} +{"stream": "available_phone_number_countries", "data": {"country_code": "AU", "country": "Australia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU/Mobile.json"}}, "emitted_at": 1664560270821} +{"stream": "available_phone_number_countries", "data": {"country_code": "AR", "country": "Argentina", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AR/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AR/TollFree.json"}}, "emitted_at": 1664560270821} +{"stream": "available_phone_number_countries", "data": {"country_code": "SE", "country": "Sweden", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SE.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SE/Mobile.json"}}, "emitted_at": 1664560271154} +{"stream": "available_phone_number_countries", "data": {"country_code": "SK", "country": "Slovakia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SK.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SK/TollFree.json"}}, "emitted_at": 1664560271154} +{"stream": "available_phone_number_countries", "data": {"country_code": "PL", "country": "Poland", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PL.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PL/Mobile.json"}}, "emitted_at": 1664560271155} +{"stream": "available_phone_number_countries", "data": {"country_code": "LT", "country": "Lithuania", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/LT.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/LT/Mobile.json"}}, "emitted_at": 1664560271155} +{"stream": "available_phone_number_countries", "data": {"country_code": "JP", "country": "Japan", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/JP.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/JP/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/JP/TollFree.json"}}, "emitted_at": 1664560271155} +{"stream": "available_phone_number_countries", "data": {"country_code": "IE", "country": "Ireland", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IE/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IE/Mobile.json"}}, "emitted_at": 1664560271155} +{"stream": "available_phone_number_countries", "data": {"country_code": "SG", "country": "Singapore", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SG.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SG/Mobile.json"}}, "emitted_at": 1664560271156} +{"stream": "available_phone_number_countries", "data": {"country_code": "BE", "country": "Belgium", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BE.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BE/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BE/Mobile.json"}}, "emitted_at": 1664560271156} +{"stream": "available_phone_number_countries", "data": {"country_code": "CA", "country": "Canada", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CA/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CA/TollFree.json"}}, "emitted_at": 1664560271156} +{"stream": "available_phone_number_countries", "data": {"country_code": "PT", "country": "Portugal", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PT/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PT/Mobile.json"}}, "emitted_at": 1664560271156} +{"stream": "available_phone_number_countries", "data": {"country_code": "RO", "country": "Romania", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/RO.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/RO/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/RO/TollFree.json"}}, "emitted_at": 1664560271156} +{"stream": "available_phone_number_countries", "data": {"country_code": "FI", "country": "Finland", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/FI.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/FI/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/FI/Mobile.json"}}, "emitted_at": 1664560271157} +{"stream": "available_phone_number_countries", "data": {"country_code": "GB", "country": "United Kingdom", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GB.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GB/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GB/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GB/Mobile.json"}}, "emitted_at": 1664560271157} +{"stream": "available_phone_number_countries", "data": {"country_code": "PA", "country": "Panama", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PA/Local.json"}}, "emitted_at": 1664560271157} +{"stream": "available_phone_number_countries", "data": {"country_code": "PE", "country": "Peru", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PE.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PE/TollFree.json"}}, "emitted_at": 1664560271157} +{"stream": "available_phone_number_countries", "data": {"country_code": "FR", "country": "France", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/FR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/FR/Local.json"}}, "emitted_at": 1664560271157} +{"stream": "available_phone_number_countries", "data": {"country_code": "CZ", "country": "Czech Republic", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CZ/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CZ/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CZ/Mobile.json"}}, "emitted_at": 1664560271157} +{"stream": "available_phone_number_countries", "data": {"country_code": "DE", "country": "Germany", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DE/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DE/Mobile.json"}}, "emitted_at": 1664560271157} +{"stream": "available_phone_number_countries", "data": {"country_code": "GH", "country": "Ghana", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GH.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GH/Mobile.json"}}, "emitted_at": 1664560271157} +{"stream": "available_phone_number_countries", "data": {"country_code": "DK", "country": "Denmark", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DK.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DK/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DK/Mobile.json"}}, "emitted_at": 1664560271158} +{"stream": "available_phone_number_countries", "data": {"country_code": "UG", "country": "Uganda", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/UG.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/UG/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/UG/TollFree.json"}}, "emitted_at": 1664560271158} +{"stream": "available_phone_number_countries", "data": {"country_code": "MX", "country": "Mexico", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MX.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MX/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MX/TollFree.json"}}, "emitted_at": 1664560271158} +{"stream": "available_phone_number_countries", "data": {"country_code": "IS", "country": "Iceland", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IS.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IS/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IS/Mobile.json"}}, "emitted_at": 1664560271158} +{"stream": "available_phone_number_countries", "data": {"country_code": "DZ", "country": "Algeria", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/DZ/Local.json"}}, "emitted_at": 1664560271158} +{"stream": "available_phone_number_countries", "data": {"country_code": "ZA", "country": "South Africa", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ZA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ZA/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ZA/Mobile.json"}}, "emitted_at": 1664560271158} +{"stream": "available_phone_number_countries", "data": {"country_code": "HR", "country": "Croatia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HR/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HR/TollFree.json"}}, "emitted_at": 1664560271158} +{"stream": "available_phone_number_countries", "data": {"country_code": "ID", "country": "Indonesia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ID.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ID/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ID/TollFree.json"}}, "emitted_at": 1664560271158} +{"stream": "available_phone_number_countries", "data": {"country_code": "BR", "country": "Brazil", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BR/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BR/TollFree.json"}}, "emitted_at": 1664560271159} +{"stream": "available_phone_number_countries", "data": {"country_code": "AT", "country": "Austria", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AT/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AT/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AT/Mobile.json"}}, "emitted_at": 1664560271159} +{"stream": "available_phone_number_countries", "data": {"country_code": "US", "country": "United States", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/US.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/US/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/US/TollFree.json"}}, "emitted_at": 1664560271159} +{"stream": "available_phone_number_countries", "data": {"country_code": "VI", "country": "Virgin Islands, U.S.", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/VI.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/VI/Local.json"}}, "emitted_at": 1664560271159} +{"stream": "available_phone_number_countries", "data": {"country_code": "EC", "country": "Ecuador", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/EC.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/EC/Local.json"}}, "emitted_at": 1664560271159} +{"stream": "available_phone_number_countries", "data": {"country_code": "KE", "country": "Kenya", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/KE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/KE/Local.json"}}, "emitted_at": 1664560271159} +{"stream": "available_phone_number_countries", "data": {"country_code": "NL", "country": "Netherlands", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NL.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NL/Mobile.json"}}, "emitted_at": 1664560271159} +{"stream": "available_phone_number_countries", "data": {"country_code": "CL", "country": "Chile", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CL.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CL/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CL/Mobile.json"}}, "emitted_at": 1664560271159} +{"stream": "available_phone_number_countries", "data": {"country_code": "CH", "country": "Switzerland", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CH.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CH/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CH/Mobile.json"}}, "emitted_at": 1664560271159} +{"stream": "available_phone_number_countries", "data": {"country_code": "TN", "country": "Tunisia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TN.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TN/Local.json"}}, "emitted_at": 1664560271160} +{"stream": "available_phone_number_countries", "data": {"country_code": "TT", "country": "Trinidad and Tobago", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TT.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TT/Local.json"}}, "emitted_at": 1664560271160} +{"stream": "available_phone_number_countries", "data": {"country_code": "TH", "country": "Thailand", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TH.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TH/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/TH/TollFree.json"}}, "emitted_at": 1664560271160} +{"stream": "available_phone_number_countries", "data": {"country_code": "SI", "country": "Slovenia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SI.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SI/Local.json"}}, "emitted_at": 1664560271160} +{"stream": "available_phone_number_countries", "data": {"country_code": "PR", "country": "Puerto Rico", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PR/Local.json"}}, "emitted_at": 1664560271160} +{"stream": "available_phone_number_countries", "data": {"country_code": "PH", "country": "Philippines", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PH.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PH/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PH/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/PH/Mobile.json"}}, "emitted_at": 1664560271160} +{"stream": "available_phone_number_countries", "data": {"country_code": "NZ", "country": "New Zealand", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NZ.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NZ/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NZ/TollFree.json"}}, "emitted_at": 1664560271160} +{"stream": "available_phone_number_countries", "data": {"country_code": "NA", "country": "Namibia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/NA/Local.json"}}, "emitted_at": 1664560271160} +{"stream": "available_phone_number_countries", "data": {"country_code": "MU", "country": "Mauritius", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MU.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MU/Mobile.json"}}, "emitted_at": 1664560271160} +{"stream": "available_phone_number_countries", "data": {"country_code": "ML", "country": "Mali", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ML.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/ML/Local.json"}}, "emitted_at": 1664560271161} +{"stream": "available_phone_number_countries", "data": {"country_code": "MO", "country": "Macau", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MO.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/MO/Mobile.json"}}, "emitted_at": 1664560271161} +{"stream": "available_phone_number_countries", "data": {"country_code": "JM", "country": "Jamaica", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/JM.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/JM/Local.json"}}, "emitted_at": 1664560271161} +{"stream": "available_phone_number_countries", "data": {"country_code": "IL", "country": "Israel", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IL.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IL/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IL/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/IL/Mobile.json"}}, "emitted_at": 1664560271161} +{"stream": "available_phone_number_countries", "data": {"country_code": "HU", "country": "Hungary", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HU/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HU/Mobile.json"}}, "emitted_at": 1664560271161} +{"stream": "available_phone_number_countries", "data": {"country_code": "HK", "country": "Hong Kong", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HK.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HK/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HK/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/HK/Mobile.json"}}, "emitted_at": 1664560271161} +{"stream": "available_phone_number_countries", "data": {"country_code": "GN", "country": "Guinea", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GN.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GN/Mobile.json"}}, "emitted_at": 1664560271161} +{"stream": "available_phone_number_countries", "data": {"country_code": "GD", "country": "Grenada", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GD.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GD/Local.json"}}, "emitted_at": 1664560271161} +{"stream": "available_phone_number_countries", "data": {"country_code": "GR", "country": "Greece", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GR/Local.json"}}, "emitted_at": 1664560271161} +{"stream": "available_phone_number_countries", "data": {"country_code": "GE", "country": "Georgia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/GE/Local.json"}}, "emitted_at": 1664560271162} +{"stream": "available_phone_number_countries", "data": {"country_code": "EE", "country": "Estonia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/EE.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/EE/Local.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/EE/Mobile.json"}}, "emitted_at": 1664560271162} +{"stream": "available_phone_number_countries", "data": {"country_code": "SV", "country": "El Salvador", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SV.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/SV/Local.json"}}, "emitted_at": 1664560271162} +{"stream": "available_phone_number_countries", "data": {"country_code": "CY", "country": "Cyprus", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CY.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CY/Local.json"}}, "emitted_at": 1664560271162} +{"stream": "available_phone_number_countries", "data": {"country_code": "CO", "country": "Colombia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CO.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CO/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/CO/TollFree.json"}}, "emitted_at": 1664560271162} +{"stream": "available_phone_number_countries", "data": {"country_code": "KY", "country": "Cayman Islands", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/KY.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/KY/Local.json"}}, "emitted_at": 1664560271162} +{"stream": "available_phone_number_countries", "data": {"country_code": "BG", "country": "Bulgaria", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BG.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BG/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BG/TollFree.json"}}, "emitted_at": 1664560271162} +{"stream": "available_phone_number_countries", "data": {"country_code": "BW", "country": "Botswana", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BW.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BW/TollFree.json"}}, "emitted_at": 1664560271162} +{"stream": "available_phone_number_countries", "data": {"country_code": "BA", "country": "Bosnia and Herzegovina", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BA/Local.json"}}, "emitted_at": 1664560271162} +{"stream": "available_phone_number_countries", "data": {"country_code": "BJ", "country": "Benin", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BJ.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BJ/Mobile.json"}}, "emitted_at": 1664560271162} +{"stream": "available_phone_number_countries", "data": {"country_code": "BB", "country": "Barbados", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BB.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BB/Local.json"}}, "emitted_at": 1664560271163} +{"stream": "available_phone_number_countries", "data": {"country_code": "AU", "country": "Australia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU/Mobile.json"}}, "emitted_at": 1664560271163} +{"stream": "available_phone_number_countries", "data": {"country_code": "AR", "country": "Argentina", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR/TollFree.json"}}, "emitted_at": 1664560271163} {"stream": "incoming_phone_numbers", "data": {"sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "friendly_name": "2FA Number - PLEASE DO NOT TOUCH. Use another number for anythin", "phone_number": "+12056561170", "voice_url": "https://handler.twilio.com/twiml/EH7af811843f38093d724a5c2e80b3eabe", "voice_method": "POST", "voice_fallback_url": "", "voice_fallback_method": "POST", "voice_caller_id_lookup": false, "date_created": "2020-12-11T04:28:40Z", "date_updated": "2022-09-23T14:47:41Z", "sms_url": "https://webhooks.twilio.com/v1/Accounts/ACdade166c12e160e9ed0a6088226718fb/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "sms_method": "POST", "sms_fallback_url": "", "sms_fallback_method": "POST", "address_requirements": "none", "beta": false, "capabilities": {"voice": true, "sms": true, "mms": true}, "status_callback": "", "status_callback_method": "POST", "api_version": "2010-04-01", "voice_application_sid": "", "sms_application_sid": "", "origin": "twilio", "trunk_sid": null, "emergency_status": "Active", "emergency_address_sid": null, "emergency_address_status": "unregistered", "address_sid": null, "identity_sid": null, "bundle_sid": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PNe40bd7f3ac343b32fd51275d2d5b3dcc.json", "status": "in-use"}, "emitted_at": 1655893245291} {"stream": "keys", "data": {"date_updated": "2021-02-01T07:30:21Z", "date_created": "2021-02-01T07:30:21Z", "friendly_name": "Studio API Key", "sid": "SK60085e9cfc3d94aa1b987b25c78067a9"}, "emitted_at": 1655893247168} {"stream": "calls", "data": {"date_updated": "2022-06-17T22:28:34Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 61, "from": "+15312726629", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAe71d3c7533543b5c81b1be3fc5affa2b", "queue_time": 0, "price": -0.017, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-06-17T22:27:33Z", "date_created": "2022-06-17T22:27:32Z", "from_formatted": "(531) 272-6629", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-06-17T22:28:34Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1655893249727} @@ -533,4 +527,4 @@ {"stream": "message_media", "data": {"sid": "MEa79c9e10f96b3c6018caa5e9a62567cc", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "parent_sid": "MM9170a757ee3976ecb8ebeb4e9580f9c0", "content_type": "image/png", "date_created": "2022-09-23T20:17:19Z", "date_updated": "2022-09-23T20:17:20Z", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM9170a757ee3976ecb8ebeb4e9580f9c0/Media/MEa79c9e10f96b3c6018caa5e9a62567cc.json"}, "emitted_at": 1663964973993} {"stream": "usage_triggers", "data": {"sid": "UT33bd2bf238d94863a609133da897d676", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2020-11-25T10:02:19Z", "date_updated": "2020-11-25T10:02:19Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 1000.0, "current_value": 130.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UT33bd2bf238d94863a609133da897d676.json", "api_version": "2010-04-01"}, "emitted_at": 1655893322691} {"stream": "usage_triggers", "data": {"sid": "UT3c3c157dcaf347829d5a0f75e97b572e", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2020-11-25T10:02:34Z", "date_updated": "2020-11-25T10:02:34Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 999.0, "current_value": 130.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UT3c3c157dcaf347829d5a0f75e97b572e.json", "api_version": "2010-04-01"}, "emitted_at": 1655893322698} -{"stream": "usage_triggers", "data": {"sid": "UT7170996eff504647ac9f215222ee296f", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2020-11-25T10:02:41Z", "date_updated": "2020-11-25T10:02:41Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 943.0, "current_value": 130.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UT7170996eff504647ac9f215222ee296f.json", "api_version": "2010-04-01"}, "emitted_at": 1655893322701} +{"stream": "usage_triggers", "data": {"sid": "UT7170996eff504647ac9f215222ee296f", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2020-11-25T10:02:41Z", "date_updated": "2020-11-25T10:02:41Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 943.0, "current_value": 130.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UT7170996eff504647ac9f215222ee296f.json", "api_version": "2010-04-01"}, "emitted_at": 1655893322701} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-twilio/integration_tests/sample_state.json index 5b57d952ba89e..075389e3382c1 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/sample_state.json @@ -1,23 +1,79 @@ -{ - "calls": { - "end_time": "2022-06-11T00:00:00Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "end_time": "2022-06-11T00:00:00Z" + }, + "stream_descriptor": { + "name": "calls" + } + } }, - "conferences": { - "date_updated": "2020-01-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_updated": "2020-01-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "conferences" + } + } }, - "recordings": { - "date_created": "2020-01-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_created": "2020-01-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "recordings" + } + } }, - "messages": { - "date_sent": "2020-01-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_sent": "2020-01-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "messages" + } + } }, - "message_media": { - "date_created": "2020-01-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_created": "2020-01-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "message_media" + } + } }, - "usage_records": { - "start_date": "2020-01-01" + { + "type": "STREAM", + "stream": { + "stream_state": { + "start_date": "2020-01-01" + }, + "stream_descriptor": { + "name": "usage_records" + } + } }, - "alerts": { - "date_updated": "2020-01-01T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_updated": "2020-01-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "alerts" + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py index da781c0ae92cc..274f60e347c7b 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py @@ -68,7 +68,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: incremental_stream_kwargs = { "authenticator": auth, "start_date": config["start_date"], - "lookback_window": config["lookback_window"], + "lookback_window": config.get("lookback_window", 0), } # Fix for `Date range specified in query is partially or entirely outside of retention window of 400 days` diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/spec.json b/airbyte-integrations/connectors/source-twilio/source_twilio/spec.json index 4d80f059920b9..8d94200a1d043 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/spec.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/spec.json @@ -34,6 +34,8 @@ "description": "How far into the past to look for records. (in minutes)", "examples": [60], "default": 0, + "minimum": 0, + "maximum": 576000, "type": "integer", "order": 4 } diff --git a/airbyte-integrations/connectors/source-youtube-analytics/Dockerfile b/airbyte-integrations/connectors/source-youtube-analytics/Dockerfile index f2e903c8f82af..5f59df26be359 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/Dockerfile +++ b/airbyte-integrations/connectors/source-youtube-analytics/Dockerfile @@ -34,5 +34,5 @@ COPY source_youtube_analytics ./source_youtube_analytics ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-youtube-analytics diff --git a/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/source.py b/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/source.py index a79633dbf76c6..302bb747397a5 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/source.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/source.py @@ -36,12 +36,22 @@ class JobsResource(HttpStream): name = None primary_key = None http_method = None + raise_on_http_errors = True url_base = "https://youtubereporting.googleapis.com/v1/" JOB_NAME = "Airbyte reporting job" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None + def should_retry(self, response: requests.Response) -> bool: + # if the connected Google account is not bounded with target Youtube account, + # we receive `401: UNAUTHENTICATED` + if response.status_code == 401: + setattr(self, "raise_on_http_errors", False) + return False + else: + return super().should_retry(response) + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: return [response.json()] @@ -169,13 +179,14 @@ def get_authenticator(config): def check_connection(self, logger, config) -> Tuple[bool, any]: authenticator = self.get_authenticator(config) jobs_resource = JobsResource(authenticator=authenticator) - - try: - jobs_resource.list() - except Exception as e: - return False, str(e) - - return True, None + result = jobs_resource.list() + if result: + return True, None + else: + return ( + False, + "The Youtube account is not valid. Please make sure you're trying to use the active Youtube Account connected to your Google Account.", + ) def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = self.get_authenticator(config) diff --git a/airbyte-integrations/connectors/source-youtube-analytics/unit_tests/test_source.py b/airbyte-integrations/connectors/source-youtube-analytics/unit_tests/test_source.py index ffc1b68305588..48aa6975787a0 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/unit_tests/test_source.py @@ -14,7 +14,7 @@ def test_check_connection(requests_mock): access_token = "token" mock_oauth_call = requests_mock.post("https://oauth2.googleapis.com/token", json={"access_token": access_token, "expires_in": 0}) - mock_jobs_call = requests_mock.get("https://youtubereporting.googleapis.com/v1/jobs", json={}) + mock_jobs_call = requests_mock.get("https://youtubereporting.googleapis.com/v1/jobs", json={"jobs": [1, 2, 3]}) source = SourceYoutubeAnalytics() logger_mock, config_mock = MagicMock(), MagicMock() diff --git a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile index 9efaf68d888f9..ed53b37416024 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile @@ -16,5 +16,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main_dev.py"] -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-zendesk-chat diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_source.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_source.py index eab78bbdc04c1..29cd0ba33ab58 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_source.py @@ -3,11 +3,12 @@ # +from unittest.mock import patch + import pytest import requests -from unittest.mock import patch from airbyte_cdk import AirbyteLogger -from source_zendesk_chat.source import ZendeskAuthentication, SourceZendeskChat +from source_zendesk_chat.source import SourceZendeskChat, ZendeskAuthentication from source_zendesk_chat.streams import ( Accounts, Agents, @@ -31,10 +32,11 @@ def test_get_auth(): - expected = {'Authorization': 'Bearer access_token'} + expected = {"Authorization": "Bearer access_token"} result = ZendeskAuthentication(TEST_CONFIG).get_auth().get_auth_header() assert expected == result - + + @pytest.mark.parametrize( "response, check_passed", [ @@ -48,7 +50,8 @@ def test_check(response, check_passed): result = TEST_INSTANCE.check_connection(logger=AirbyteLogger, config=TEST_CONFIG) mock_method.assert_called() assert check_passed == result[0] - + + @pytest.mark.parametrize( "stream_cls", [ @@ -71,5 +74,3 @@ def test_streams(stream_cls): for stream in streams: if stream_cls in streams: assert isinstance(stream, stream_cls) - - diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_streams.py index 767bb232dd991..acfac7e8e6753 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_streams.py @@ -29,7 +29,7 @@ class TestFullRefreshStreams: """ - STREAMS: + STREAMS: Accounts, Shortcuts, Triggers, Departments, Goals, Skills, Roles, RoutingSettings """ @@ -50,7 +50,7 @@ def test_request_kwargs(self, stream_cls): stream = stream_cls(TEST_CONFIG) expected = {"timeout": 60} assert expected == stream.request_kwargs(stream_state=None) - + @pytest.mark.parametrize( "stream_cls, expected", [ @@ -66,7 +66,7 @@ def test_request_kwargs(self, stream_cls): ) def test_backoff_time(self, requests_mock, stream_cls, expected): stream = stream_cls(TEST_CONFIG) - url =f"{stream.url_base}{stream.path()}" + url = f"{stream.url_base}{stream.path()}" test_headers = {"Retry-After": expected} requests_mock.get(url, headers=test_headers) response = requests.get(url) @@ -90,7 +90,7 @@ def test_path(self, stream_cls, expected): stream = stream_cls(TEST_CONFIG) result = stream.path() assert result == expected - + @pytest.mark.parametrize( "stream_cls, expected_cursor", [ @@ -113,25 +113,25 @@ def test_next_page_token(self, requests_mock, stream_cls, expected_cursor): response = requests.get(url) result = stream.next_page_token(response) assert result == {"cursor": [expected_cursor]} - + @pytest.mark.parametrize( "stream_cls, next_page_token, expected", [ - (Accounts, {"cursor": "MTU4MD"}, {'limit': 100, 'cursor': 'MTU4MD'}), - (Departments, {"cursor": "c1Mzc"}, {'limit': 100, 'cursor': 'c1Mzc'}), - (Goals, {"cursor": "wfHw0MzJ8"}, {'limit': 100, 'cursor': 'wfHw0MzJ8'}), - (Roles, {"cursor": "0MzJ8"}, {'limit': 100, 'cursor': '0MzJ8'}), - (RoutingSettings, {"cursor": "MTUC4wJ8"}, {'limit': 100, 'cursor': 'MTUC4wJ8'}), - (Shortcuts, {"cursor": "MTU4MD"}, {'limit': 100, 'cursor': 'MTU4MD'}), - (Skills, {"cursor": "c1Mzc"}, {'limit': 100, 'cursor': 'c1Mzc'}), - (Triggers, {"cursor": "0MzJ8"}, {'limit': 100, 'cursor': '0MzJ8'}), + (Accounts, {"cursor": "MTU4MD"}, {"limit": 100, "cursor": "MTU4MD"}), + (Departments, {"cursor": "c1Mzc"}, {"limit": 100, "cursor": "c1Mzc"}), + (Goals, {"cursor": "wfHw0MzJ8"}, {"limit": 100, "cursor": "wfHw0MzJ8"}), + (Roles, {"cursor": "0MzJ8"}, {"limit": 100, "cursor": "0MzJ8"}), + (RoutingSettings, {"cursor": "MTUC4wJ8"}, {"limit": 100, "cursor": "MTUC4wJ8"}), + (Shortcuts, {"cursor": "MTU4MD"}, {"limit": 100, "cursor": "MTU4MD"}), + (Skills, {"cursor": "c1Mzc"}, {"limit": 100, "cursor": "c1Mzc"}), + (Triggers, {"cursor": "0MzJ8"}, {"limit": 100, "cursor": "0MzJ8"}), ], ) def test_request_params(self, stream_cls, next_page_token, expected): stream = stream_cls(TEST_CONFIG) result = stream.request_params(stream_state=None, next_page_token=next_page_token) assert result == expected - + @pytest.mark.parametrize( "stream_cls, test_response, expected", [ @@ -152,11 +152,11 @@ def test_parse_response(self, requests_mock, stream_cls, test_response, expected response = requests.get(url) result = stream.parse_response(response) assert list(result) == expected - + class TestTimeIncrementalStreams: """ - STREAMS: + STREAMS: AgentTimelines, Chats """ @@ -171,7 +171,7 @@ def test_state_checkpoint_interval(self, stream_cls, expected): stream = stream_cls(start_date=TEST_CONFIG["start_date"]) result = stream.state_checkpoint_interval assert result == expected - + @pytest.mark.parametrize( "stream_cls, expected", [ @@ -183,14 +183,14 @@ def test_cursor_field(self, stream_cls, expected): stream = stream_cls(start_date=TEST_CONFIG["start_date"]) result = stream.cursor_field assert result == expected - + @pytest.mark.parametrize( "stream_cls, test_response, expected", [ - (AgentTimelines, {"end_time": "123"}, {'start_time': '123'}), - (Chats, {"end_time": "123"}, {'start_time': '123'}), + (AgentTimelines, {"end_time": "123"}, {"start_time": "123"}), + (Chats, {"end_time": "123"}, {"start_time": "123"}), ], - ) + ) def test_next_page_token(self, requests_mock, stream_cls, test_response, expected): stream = stream_cls(start_date=TEST_CONFIG["start_date"]) test_response.update(**{"count": stream.limit}) @@ -199,46 +199,46 @@ def test_next_page_token(self, requests_mock, stream_cls, test_response, expecte response = requests.get(url) result = stream.next_page_token(response) assert result == expected - + @pytest.mark.parametrize( "stream_cls, current_state, last_record, expected", [ - (AgentTimelines, {}, {'start_time': '2021-01-01'}, {'start_time': '2021-01-01T00:00:00Z'}), - (Chats, {"update_timestamp": "2022-02-02"}, {'update_timestamp': '2022-03-03'}, {'update_timestamp': '2022-03-03T00:00:00Z'}), + (AgentTimelines, {}, {"start_time": "2021-01-01"}, {"start_time": "2021-01-01T00:00:00Z"}), + (Chats, {"update_timestamp": "2022-02-02"}, {"update_timestamp": "2022-03-03"}, {"update_timestamp": "2022-03-03T00:00:00Z"}), ], - ) + ) def test_get_updated_state(self, stream_cls, current_state, last_record, expected): stream = stream_cls(start_date=TEST_CONFIG["start_date"]) result = stream.get_updated_state(current_state, last_record) assert result == expected - + @pytest.mark.parametrize( "stream_cls, stream_state, next_page_token, expected", [ - (AgentTimelines, {}, {'start_time': '123'}, {'limit': 1000, 'start_time': '123', 'fields': 'agent_timeline(*)'}), - (Chats, {"update_timestamp": "2022-02-02"}, {'start_time': '234'}, {'limit': 1000, 'start_time': '234', 'fields': 'chats(*)'}), + (AgentTimelines, {}, {"start_time": "123"}, {"limit": 1000, "start_time": "123", "fields": "agent_timeline(*)"}), + (Chats, {"update_timestamp": "2022-02-02"}, {"start_time": "234"}, {"limit": 1000, "start_time": "234", "fields": "chats(*)"}), ], ) def test_request_params(self, stream_cls, stream_state, next_page_token, expected): stream = stream_cls(start_date=TEST_CONFIG["start_date"]) result = stream.request_params(stream_state=stream_state, next_page_token=next_page_token) assert result == expected - + @pytest.mark.parametrize( "stream_cls, test_response, expected", [ ( AgentTimelines, - {"agent_timeline" : {"id": "123", "agent_id": "test_id", "start_time": "2021-01-01"}}, - [{'id': 'test_id|2021-01-01T00:00:00Z', 'agent_id': 'test_id', 'start_time': '2021-01-01T00:00:00Z'}], + {"agent_timeline": {"id": "123", "agent_id": "test_id", "start_time": "2021-01-01"}}, + [{"id": "test_id|2021-01-01T00:00:00Z", "agent_id": "test_id", "start_time": "2021-01-01T00:00:00Z"}], ), ( Chats, - {"chats" : {"id": "234", "agent_id": "test_id", "update_timestamp": "2022-01-01"}}, - [{'id': '234', 'agent_id': 'test_id', 'update_timestamp': '2022-01-01T00:00:00Z'}], + {"chats": {"id": "234", "agent_id": "test_id", "update_timestamp": "2022-01-01"}}, + [{"id": "234", "agent_id": "test_id", "update_timestamp": "2022-01-01T00:00:00Z"}], ), ], - ) + ) def test_parse_response(self, requests_mock, stream_cls, test_response, expected): stream = stream_cls(start_date=TEST_CONFIG["start_date"]) url = f"{stream.url_base}{stream.path()}" @@ -246,7 +246,6 @@ def test_parse_response(self, requests_mock, stream_cls, test_response, expected response = requests.get(url) result = stream.parse_response(response) assert list(result) == expected - @pytest.mark.parametrize( "stream_cls, expected", @@ -259,11 +258,11 @@ def test_path(self, stream_cls, expected): stream = stream_cls(start_date=TEST_CONFIG["start_date"]) result = stream.path() assert result == expected - + class TestIdIncrementalStreams: """ - STREAMS: + STREAMS: Agents, Bans """ @@ -278,7 +277,7 @@ def test_path(self, stream_cls, expected): stream = stream_cls(TEST_CONFIG) result = stream.path() assert result == expected - + @pytest.mark.parametrize( "stream_cls, expected", [ @@ -290,25 +289,25 @@ def test_cursor_field(self, stream_cls, expected): stream = stream_cls(TEST_CONFIG) result = stream.cursor_field assert result == expected - + @pytest.mark.parametrize( "stream_cls, current_state, last_record, expected", [ - (Agents, {}, {'id': '1'}, {'id': '1'}), - (Bans, {"id": "1"}, {'id': '2'}, {'id': '2'}), + (Agents, {}, {"id": "1"}, {"id": "1"}), + (Bans, {"id": "1"}, {"id": "2"}, {"id": "2"}), ], - ) + ) def test_get_updated_state(self, stream_cls, current_state, last_record, expected): stream = stream_cls(TEST_CONFIG) result = stream.get_updated_state(current_state, last_record) assert result == expected - + @pytest.mark.parametrize( "stream_cls, test_response, expected", [ - (Agents, [{"id": "2"}], {'since_id': '2'}), + (Agents, [{"id": "2"}], {"since_id": "2"}), ], - ) + ) def test_next_page_token(self, requests_mock, stream_cls, test_response, expected): stream = stream_cls(TEST_CONFIG) stream.limit = 1 @@ -317,13 +316,13 @@ def test_next_page_token(self, requests_mock, stream_cls, test_response, expecte response = requests.get(url) result = stream.next_page_token(response) assert result == expected - + @pytest.mark.parametrize( "stream_cls, test_response, expected", [ (Agents, {"id": "2"}, [{"id": "2"}]), ], - ) + ) def test_parse_response(self, requests_mock, stream_cls, test_response, expected): stream = stream_cls(TEST_CONFIG) url = f"{stream.url_base}{stream.path()}" @@ -331,16 +330,15 @@ def test_parse_response(self, requests_mock, stream_cls, test_response, expected response = requests.get(url) result = stream.parse_response(response) assert list(result) == expected - + @pytest.mark.parametrize( "stream_cls, stream_state, next_page_token, expected", [ - (Agents, {}, {'since_id': '1'}, {'limit': 100, 'since_id': '1'}), - (Bans, {"id": "1"}, {'since_id': '2'}, {'limit': 100, 'since_id': '2'}), + (Agents, {}, {"since_id": "1"}, {"limit": 100, "since_id": "1"}), + (Bans, {"id": "1"}, {"since_id": "2"}, {"limit": 100, "since_id": "2"}), ], ) def test_request_params(self, stream_cls, stream_state, next_page_token, expected): stream = stream_cls(TEST_CONFIG) result = stream.request_params(stream_state=stream_state, next_page_token=next_page_token) assert result == expected - diff --git a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile index 76d0301dc9911..6f8396d9e2f20 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile @@ -25,5 +25,5 @@ COPY source_zendesk_support ./source_zendesk_support ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.15 +LABEL io.airbyte.version=0.2.16 LABEL io.airbyte.name=airbyte/source-zendesk-support diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json index 1d4e33a89834d..00365044a1e74 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json @@ -1,41 +1,93 @@ -{ - "users": { - "updated_at": "2222-07-19T22:21:37Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:37Z" }, + "stream_descriptor": { "name": "users" } + } }, - "groups": { - "updated_at": "2222-07-15T22:19:01Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-15T22:19:01Z" }, + "stream_descriptor": { "name": "groups" } + } }, - "organizations": { - "updated_at": "2222-07-15T19:29:14Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-15T19:29:14Z" }, + "stream_descriptor": { "name": "organizations" } + } }, - "satisfaction_ratings": { - "updated_at": "2222-07-20T10:05:18Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-20T10:05:18Z" }, + "stream_descriptor": { "name": "satisfaction_ratings" } + } }, - "tickets": { - "updated_at": "2222-07-20T10:05:18Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-20T10:05:18Z" }, + "stream_descriptor": { "name": "tickets" } + } }, - "group_memberships": { - "updated_at": "2222-04-23T15:34:20Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-04-23T15:34:20Z" }, + "stream_descriptor": { "name": "group_memberships" } + } }, - "ticket_fields": { - "updated_at": "2222-12-11T19:34:05Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-12-11T19:34:05Z" }, + "stream_descriptor": { "name": "ticket_fields" } + } }, - "ticket_forms": { - "updated_at": "2222-12-11T20:34:37Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-12-11T20:34:37Z" }, + "stream_descriptor": { "name": "ticket_forms" } + } }, - "ticket_metrics": { - "updated_at": "2222-07-19T22:21:26Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "ticket_metrics" } + } }, - "ticket_metric_events": { - "time": "2222-07-19T22:21:26Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "time": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "ticket_metric_events" } + } }, - "macros": { - "updated_at": "2222-12-11T19:34:06Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-12-11T19:34:06Z" }, + "stream_descriptor": { "name": "macros" } + } }, - "ticket_comments": { - "created_at": "2222-07-19T22:21:26Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "created_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "ticket_comments" } + } }, - "ticket_audits": { - "created_at": "2222-07-19T22:21:26Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "created_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "ticket_audits" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile b/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile index cb02e69e25896..2e633bb8fdfbd 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/source-zendesk-talk diff --git a/airbyte-integrations/connectors/source-zendesk-talk/setup.py b/airbyte-integrations/connectors/source-zendesk-talk/setup.py index e8445f68c0fae..1bc17ab2e0d10 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"] +MAIN_REQUIREMENTS = ["airbyte-cdk"] TEST_REQUIREMENTS = [ "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/spec.json b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/spec.json index c4c0e368a3f85..7a6a98c98eef4 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/spec.json +++ b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/spec.json @@ -6,21 +6,16 @@ "type": "object", "required": ["start_date", "subdomain"], "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date from which you'd like to replicate data for Zendesk Talk API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "examples": ["2020-10-15T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, "subdomain": { "type": "string", "title": "Subdomain", + "order": 0, "description": "This is your Zendesk subdomain that can be found in your account URL. For example, in https://{MY_SUBDOMAIN}.zendesk.com/, where MY_SUBDOMAIN is the value of your subdomain." }, "credentials": { "title": "Authentication", "type": "object", + "order": 1, "description": "Zendesk service provides two authentication methods. Choose between: `OAuth2.0` or `API token`.", "oneOf": [ { @@ -66,6 +61,14 @@ } } ] + }, + "start_date": { + "type": "string", + "title": "Start Date", + "order": 2, + "description": "The date from which you'd like to replicate data for Zendesk Talk API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", + "examples": ["2020-10-15T00:00:00Z"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" } } }, diff --git a/airbyte-integrations/connectors/source-zenloop/Dockerfile b/airbyte-integrations/connectors/source-zenloop/Dockerfile index 8d78427e583fe..9e50a910966cb 100644 --- a/airbyte-integrations/connectors/source-zenloop/Dockerfile +++ b/airbyte-integrations/connectors/source-zenloop/Dockerfile @@ -34,5 +34,5 @@ COPY source_zenloop ./source_zenloop ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-zenloop diff --git a/airbyte-integrations/connectors/source-zenloop/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-zenloop/integration_tests/abnormal_state.json index 681b893354634..ad6fd553b49ad 100644 --- a/airbyte-integrations/connectors/source-zenloop/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-zenloop/integration_tests/abnormal_state.json @@ -1,8 +1,16 @@ -{ - "answers": { - "inserted_at": "2099-08-18T08:35:49.540Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "inserted_at": "2099-08-18T08:35:49.540Z" }, + "stream_descriptor": { "name": "answers" } + } }, - "answers_survey_group": { - "inserted_at": "2099-08-18T08:35:49.540Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "inserted_at": "2099-08-18T08:35:49.540Z" }, + "stream_descriptor": { "name": "answers_survey_group" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-zenloop/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-zenloop/integration_tests/sample_state.json index cbc8e3dfa4ee1..9cebc455f98f6 100644 --- a/airbyte-integrations/connectors/source-zenloop/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-zenloop/integration_tests/sample_state.json @@ -1,8 +1,16 @@ -{ - "answers": { - "inserted_at": "2021-08-18T08:35:49.540Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "inserted_at": "2021-08-18T08:35:49.540Z" }, + "stream_descriptor": { "name": "answers" } + } }, - "answers_survey_group": { - "inserted_at": "2021-08-18T08:35:49.540Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "inserted_at": "2021-08-18T08:35:49.540Z" }, + "stream_descriptor": { "name": "answers_survey_group" } + } } -} +] diff --git a/airbyte-metrics/reporter/Dockerfile b/airbyte-metrics/reporter/Dockerfile index c1d18d9b51399..62ea5e695c60f 100644 --- a/airbyte-metrics/reporter/Dockerfile +++ b/airbyte-metrics/reporter/Dockerfile @@ -2,7 +2,7 @@ ARG JDK_VERSION=17.0.4 ARG JDK_IMAGE=amazoncorretto:${JDK_VERSION} FROM ${JDK_IMAGE} AS metrics-reporter -ARG VERSION=0.40.9 +ARG VERSION=0.40.10 ENV APPLICATION airbyte-metrics-reporter ENV VERSION ${VERSION} diff --git a/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ReporterApp.java b/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ReporterApp.java index 461cd8afdffa4..17d8331dcc7bb 100644 --- a/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ReporterApp.java +++ b/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ReporterApp.java @@ -18,6 +18,7 @@ import io.airbyte.metrics.lib.MetricClientFactory; import io.airbyte.metrics.lib.MetricEmittingApps; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import javax.sql.DataSource; import lombok.extern.slf4j.Slf4j; import org.flywaydb.core.Flyway; @@ -59,7 +60,7 @@ public static void main(final String[] args) throws DatabaseCheckException { log.info("Scheduling {} metrics for emission..", toEmits.length); for (final ToEmit toEmit : toEmits) { - pollers.scheduleAtFixedRate(toEmit.emitRunnable, 0, toEmit.period, toEmit.timeUnit); + pollers.scheduleAtFixedRate(toEmit.emit, 0, toEmit.duration.getSeconds(), TimeUnit.SECONDS); } } } diff --git a/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ToEmit.java b/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ToEmit.java index 790890335270f..3313a73da4bc6 100644 --- a/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ToEmit.java +++ b/airbyte-metrics/reporter/src/main/java/io/airbyte/metrics/reporter/ToEmit.java @@ -4,75 +4,91 @@ package io.airbyte.metrics.reporter; -import io.airbyte.commons.lang.Exceptions.Procedure; import io.airbyte.db.instance.jobs.jooq.generated.enums.JobStatus; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClientFactory; import io.airbyte.metrics.lib.MetricQueries; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; -import java.util.concurrent.TimeUnit; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; +import java.lang.invoke.MethodHandles; +import java.time.Duration; +import java.util.concurrent.Callable; import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class contains all metrics emitted by the {@link ReporterApp}. */ -@Slf4j -@AllArgsConstructor public enum ToEmit { NUM_PENDING_JOBS(countMetricEmission(() -> { final var pendingJobs = ReporterApp.configDatabase.query(MetricQueries::numberOfPendingJobs); MetricClientFactory.getMetricClient().gauge(OssMetricsRegistry.NUM_PENDING_JOBS, pendingJobs); + return null; })), NUM_RUNNING_JOBS(countMetricEmission(() -> { final var runningJobs = ReporterApp.configDatabase.query(MetricQueries::numberOfRunningJobs); MetricClientFactory.getMetricClient().gauge(OssMetricsRegistry.NUM_RUNNING_JOBS, runningJobs); + return null; })), NUM_ORPHAN_RUNNING_JOB(countMetricEmission(() -> { final var orphanRunningJobs = ReporterApp.configDatabase.query(MetricQueries::numberOfOrphanRunningJobs); MetricClientFactory.getMetricClient().gauge(OssMetricsRegistry.NUM_ORPHAN_RUNNING_JOBS, orphanRunningJobs); + return null; })), OLDEST_RUNNING_JOB_AGE_SECS(countMetricEmission(() -> { final var age = ReporterApp.configDatabase.query(MetricQueries::oldestRunningJobAgeSecs); MetricClientFactory.getMetricClient().gauge(OssMetricsRegistry.OLDEST_RUNNING_JOB_AGE_SECS, age); + return null; })), OLDEST_PENDING_JOB_AGE_SECS(countMetricEmission(() -> { final var age = ReporterApp.configDatabase.query(MetricQueries::oldestPendingJobAgeSecs); MetricClientFactory.getMetricClient().gauge(OssMetricsRegistry.OLDEST_PENDING_JOB_AGE_SECS, age); + return null; })), NUM_ACTIVE_CONN_PER_WORKSPACE(countMetricEmission(() -> { final var age = ReporterApp.configDatabase.query(MetricQueries::numberOfActiveConnPerWorkspace); - for (long count : age) { + for (final long count : age) { MetricClientFactory.getMetricClient().distribution(OssMetricsRegistry.NUM_ACTIVE_CONN_PER_WORKSPACE, count); } + return null; })), - NUM_ABNORMAL_SCHEDULED_SYNCS_LAST_DAY(countMetricEmission(() -> { + NUM_ABNORMAL_SCHEDULED_SYNCS_LAST_DAY(Duration.ofHours(1), countMetricEmission(() -> { final var count = ReporterApp.configDatabase.query(MetricQueries::numberOfJobsNotRunningOnScheduleInLastDay); MetricClientFactory.getMetricClient().gauge(OssMetricsRegistry.NUM_ABNORMAL_SCHEDULED_SYNCS_IN_LAST_DAY, count); - }), 1, TimeUnit.HOURS), - NUM_TOTAL_SCHEDULED_SYNCS_LAST_DAY(countMetricEmission(() -> { + return null; + })), + NUM_TOTAL_SCHEDULED_SYNCS_LAST_DAY(Duration.ofHours(1), countMetricEmission(() -> { final var count = ReporterApp.configDatabase.query(MetricQueries::numScheduledActiveConnectionsInLastDay); MetricClientFactory.getMetricClient().gauge(OssMetricsRegistry.NUM_TOTAL_SCHEDULED_SYNCS_IN_LAST_DAY, count); - }), 1, TimeUnit.HOURS), - OVERALL_JOB_RUNTIME_IN_LAST_HOUR_BY_TERMINAL_STATE_SECS(countMetricEmission(() -> { + return null; + })), + OVERALL_JOB_RUNTIME_IN_LAST_HOUR_BY_TERMINAL_STATE_SECS(Duration.ofHours(1), countMetricEmission(() -> { final var times = ReporterApp.configDatabase.query(MetricQueries::overallJobRuntimeForTerminalJobsInLastHour); - for (Pair pair : times) { + for (final Pair pair : times) { MetricClientFactory.getMetricClient().distribution( OssMetricsRegistry.OVERALL_JOB_RUNTIME_IN_LAST_HOUR_BY_TERMINAL_STATE_SECS, pair.getRight(), new MetricAttribute(MetricTags.JOB_STATUS, MetricTags.getJobStatus(pair.getLeft()))); } - }), 1, TimeUnit.HOURS); + return null; + })); + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); // default constructor - final public Runnable emitRunnable; - final public long period; - final public TimeUnit timeUnit; + /** A runnable that emits a metric. */ + final public Runnable emit; + /** How often this metric would emit data. */ + final public Duration duration; + + ToEmit(final Runnable emit) { + this(Duration.ofSeconds(15), emit); + } - ToEmit(final Runnable toEmit) { - this(toEmit, 15, TimeUnit.SECONDS); + ToEmit(final Duration duration, final Runnable emit) { + this.duration = duration; + this.emit = emit; } /** @@ -82,7 +98,7 @@ public enum ToEmit { * @param metricQuery * @return */ - private static Runnable countMetricEmission(final Procedure metricQuery) { + private static Runnable countMetricEmission(final Callable metricQuery) { return () -> { try { metricQuery.call(); diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobPersistence.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobPersistence.java index 810bff38b7784..dfa93dd2efd1d 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobPersistence.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobPersistence.java @@ -23,7 +23,9 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.text.Names; import io.airbyte.commons.text.Sqls; +import io.airbyte.commons.version.AirbyteProtocolVersion; import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.commons.version.Version; import io.airbyte.config.AttemptFailureSummary; import io.airbyte.config.FailureReason; import io.airbyte.config.JobConfig; @@ -705,65 +707,35 @@ private static long getEpoch(final Record record, final String fieldName) { @Override public boolean isSecretMigrated() throws IOException { - final Result result = jobDatabase.query(ctx -> ctx.select() - .from(AIRBYTE_METADATA_TABLE) - .where(DSL.field(METADATA_KEY_COL).eq(SECRET_MIGRATION_STATUS)) - .fetch()); - - return result.stream().count() == 1; + return getMetadata(SECRET_MIGRATION_STATUS).count() == 1; } @Override public void setSecretMigrationDone() throws IOException { - jobDatabase.query(ctx -> ctx.execute(String.format( - "INSERT INTO %s(%s, %s) VALUES('%s', '%s') ON CONFLICT (%s) DO UPDATE SET %s = '%s'", - AIRBYTE_METADATA_TABLE, - METADATA_KEY_COL, - METADATA_VAL_COL, - SECRET_MIGRATION_STATUS, - true, - METADATA_KEY_COL, - METADATA_VAL_COL, - true))); + setMetadata(SECRET_MIGRATION_STATUS, "true"); } private final String SCHEDULER_MIGRATION_STATUS = "schedulerMigration"; @Override public boolean isSchedulerMigrated() throws IOException { - final Result result = jobDatabase.query(ctx -> ctx.select() - .from(AIRBYTE_METADATA_TABLE) - .where(DSL.field(METADATA_KEY_COL).eq(SCHEDULER_MIGRATION_STATUS)) - .fetch()); - - return result.stream().count() == 1; + return getMetadata(SCHEDULER_MIGRATION_STATUS).count() == 1; } @Override public void setSchedulerMigrationDone() throws IOException { - jobDatabase.query(ctx -> ctx.execute(String.format( - "INSERT INTO %s(%s, %s) VALUES('%s', '%s') ON CONFLICT (%s) DO UPDATE SET %s = '%s'", - AIRBYTE_METADATA_TABLE, - METADATA_KEY_COL, - METADATA_VAL_COL, - SCHEDULER_MIGRATION_STATUS, - true, - METADATA_KEY_COL, - METADATA_VAL_COL, - true))); + setMetadata(SCHEDULER_MIGRATION_STATUS, "true"); } @Override public Optional getVersion() throws IOException { - final Result result = jobDatabase.query(ctx -> ctx.select() - .from(AIRBYTE_METADATA_TABLE) - .where(DSL.field(METADATA_KEY_COL).eq(AirbyteVersion.AIRBYTE_VERSION_KEY_NAME)) - .fetch()); - return result.stream().findFirst().map(r -> r.getValue(METADATA_VAL_COL, String.class)); + return getMetadata(AirbyteVersion.AIRBYTE_VERSION_KEY_NAME).findFirst(); } @Override public void setVersion(final String airbyteVersion) throws IOException { + // This is not using setMetadata due to the extra (s_init_db, airbyteVersion) that is + // added to the metadata table jobDatabase.query(ctx -> ctx.execute(String.format( "INSERT INTO %s(%s, %s) VALUES('%s', '%s'), ('%s_init_db', '%s') ON CONFLICT (%s) DO UPDATE SET %s = '%s'", AIRBYTE_METADATA_TABLE, @@ -776,6 +748,45 @@ public void setVersion(final String airbyteVersion) throws IOException { METADATA_KEY_COL, METADATA_VAL_COL, airbyteVersion))); + + } + + @Override + public Optional getAirbyteProtocolVersionMax() throws IOException { + return getMetadata(AirbyteProtocolVersion.AIRBYTE_PROTOCOL_VERSION_MAX_KEY_NAME).findFirst().map(Version::new); + } + + @Override + public void setAirbyteProtocolVersionMax(final Version version) throws IOException { + setMetadata(AirbyteProtocolVersion.AIRBYTE_PROTOCOL_VERSION_MAX_KEY_NAME, version.serialize()); + } + + @Override + public Optional getAirbyteProtocolVersionMin() throws IOException { + return getMetadata(AirbyteProtocolVersion.AIRBYTE_PROTOCOL_VERSION_MIN_KEY_NAME).findFirst().map(Version::new); + } + + @Override + public void setAirbyteProtocolVersionMin(final Version version) throws IOException { + setMetadata(AirbyteProtocolVersion.AIRBYTE_PROTOCOL_VERSION_MIN_KEY_NAME, version.serialize()); + } + + private Stream getMetadata(final String keyName) throws IOException { + return jobDatabase.query(ctx -> ctx.select() + .from(AIRBYTE_METADATA_TABLE) + .where(DSL.field(METADATA_KEY_COL).eq(keyName)) + .fetch()).stream().map(r -> r.getValue(METADATA_VAL_COL, String.class)); + } + + private void setMetadata(final String keyName, final String value) throws IOException { + jobDatabase.query(ctx -> ctx + .insertInto(DSL.table(AIRBYTE_METADATA_TABLE)) + .columns(DSL.field(METADATA_KEY_COL), DSL.field(METADATA_VAL_COL)) + .values(keyName, value) + .onConflict(DSL.field(METADATA_KEY_COL)) + .doUpdate() + .set(DSL.field(METADATA_VAL_COL), value) + .execute()); } @Override diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/JobPersistence.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/JobPersistence.java index fea5a9c036578..06554863eff0a 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/JobPersistence.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/JobPersistence.java @@ -5,6 +5,7 @@ package io.airbyte.persistence.job; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.version.Version; import io.airbyte.config.AttemptFailureSummary; import io.airbyte.config.JobConfig; import io.airbyte.config.JobConfig.ConfigType; @@ -235,6 +236,26 @@ List listJobStatusAndTimestampWithConnection(UUID con */ void setVersion(String airbyteVersion) throws IOException; + /** + * Get the max supported Airbyte Protocol Version + */ + Optional getAirbyteProtocolVersionMax() throws IOException; + + /** + * Set the max supported Airbyte Protocol Version + */ + void setAirbyteProtocolVersionMax(Version version) throws IOException; + + /** + * Get the min supported Airbyte Protocol Version + */ + Optional getAirbyteProtocolVersionMin() throws IOException; + + /** + * Set the min supported Airbyte Protocol Version + */ + void setAirbyteProtocolVersionMin(Version version) throws IOException; + /** * Returns a deployment UUID. */ diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobPersistenceTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobPersistenceTest.java index d706f8e5d565d..310ecbdbbb852 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobPersistenceTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobPersistenceTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.Sets; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.text.Sqls; +import io.airbyte.commons.version.Version; import io.airbyte.config.AttemptFailureSummary; import io.airbyte.config.FailureReason; import io.airbyte.config.FailureReason.FailureOrigin; @@ -580,6 +581,36 @@ void testSchedulerMigrationMetadata() throws IOException { assertTrue(isMigrated); } + @Test + void testAirbyteProtocolVersionMaxMetadata() throws IOException { + assertTrue(jobPersistence.getAirbyteProtocolVersionMax().isEmpty()); + + final Version maxVersion1 = new Version("0.1.0"); + jobPersistence.setAirbyteProtocolVersionMax(maxVersion1); + final Optional maxVersion1read = jobPersistence.getAirbyteProtocolVersionMax(); + assertEquals(maxVersion1, maxVersion1read.orElseThrow()); + + final Version maxVersion2 = new Version("1.2.1"); + jobPersistence.setAirbyteProtocolVersionMax(maxVersion2); + final Optional maxVersion2read = jobPersistence.getAirbyteProtocolVersionMax(); + assertEquals(maxVersion2, maxVersion2read.orElseThrow()); + } + + @Test + void testAirbyteProtocolVersionMinMetadata() throws IOException { + assertTrue(jobPersistence.getAirbyteProtocolVersionMin().isEmpty()); + + final Version minVersion1 = new Version("1.1.0"); + jobPersistence.setAirbyteProtocolVersionMin(minVersion1); + final Optional minVersion1read = jobPersistence.getAirbyteProtocolVersionMin(); + assertEquals(minVersion1, minVersion1read.orElseThrow()); + + final Version minVersion2 = new Version("3.0.1"); + jobPersistence.setAirbyteProtocolVersionMin(minVersion2); + final Optional minVersion2read = jobPersistence.getAirbyteProtocolVersionMin(); + assertEquals(minVersion2, minVersion2read.orElseThrow()); + } + private long createJobAt(final Instant created_at) throws IOException { when(timeSupplier.get()).thenReturn(created_at); return jobPersistence.enqueueJob(SCOPE, SPEC_JOB_CONFIG).orElseThrow(); diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index 83adcf2c84833..b53841e633bc7 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -4,7 +4,7 @@ FROM ${JDK_IMAGE} AS server EXPOSE 8000 -ARG VERSION=0.40.9 +ARG VERSION=0.40.10 ENV APPLICATION airbyte-server ENV VERSION ${VERSION} diff --git a/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpExporter.java b/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpExporter.java deleted file mode 100644 index b9e6a0ab09161..0000000000000 --- a/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpExporter.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.server; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.io.Archives; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.yaml.Yamls; -import io.airbyte.config.ConfigSchema; -import io.airbyte.config.DestinationConnection; -import io.airbyte.config.SourceConnection; -import io.airbyte.config.StandardDestinationDefinition; -import io.airbyte.config.StandardSourceDefinition; -import io.airbyte.config.StandardSync; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.config.persistence.SecretsRepositoryReader; -import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.persistence.job.WorkspaceHelper; -import io.airbyte.validation.json.JsonValidationException; -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.commons.io.FileUtils; - -// TODO: Write a test case which compares the output dump with the output of ArchiveHandler export -// for the same data - -/** - * This class acts like export method of ArchiveHandler but the difference is 1. It takes a full - * dump of whatever is available in the config directory without any schema validation. We dont want - * schema validation because in case of automatic migration, the code that is going to do the schema - * validation is from new version but the data in the config files is old. Thus schema validation - * would fail. 2. Unlike ArchiveHandler, this doesn't take the dump of specific files but looks at - * the config directory and takes the full dump of whatever is available - */ -public class ConfigDumpExporter { - - private static final String ARCHIVE_FILE_NAME = "airbyte_config_dump"; - private static final String CONFIG_FOLDER_NAME = "airbyte_config"; - private static final String VERSION_FILE_NAME = "VERSION"; - private final ConfigRepository configRepository; - private final SecretsRepositoryReader secretsRepositoryReader; - private final JobPersistence jobPersistence; - private final WorkspaceHelper workspaceHelper; - - public ConfigDumpExporter(final ConfigRepository configRepository, - final SecretsRepositoryReader secretsRepositoryReader, - final JobPersistence jobPersistence, - final WorkspaceHelper workspaceHelper) { - this.configRepository = configRepository; - this.secretsRepositoryReader = secretsRepositoryReader; - this.jobPersistence = jobPersistence; - this.workspaceHelper = workspaceHelper; - } - - public File dump() { - try { - final Path tempFolder = Files.createTempDirectory(Path.of("/tmp"), ARCHIVE_FILE_NAME); - final File dump = Files.createTempFile(ARCHIVE_FILE_NAME, ".tar.gz").toFile(); - exportVersionFile(tempFolder); - dumpConfigsDatabase(tempFolder); - - Archives.createArchive(tempFolder, dump.toPath()); - return dump; - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - - private void exportVersionFile(final Path tempFolder) throws IOException { - final String version = jobPersistence.getVersion().orElseThrow(); - final File versionFile = Files.createFile(tempFolder.resolve(VERSION_FILE_NAME)).toFile(); - FileUtils.writeStringToFile(versionFile, version, Charset.defaultCharset()); - } - - private void dumpConfigsDatabase(final Path parentFolder) throws IOException { - for (final Map.Entry> configEntry : secretsRepositoryReader.dumpConfigsWithSecrets().entrySet()) { - writeConfigsToArchive(parentFolder, configEntry.getKey(), configEntry.getValue()); - } - } - - private static void writeConfigsToArchive(final Path storageRoot, - final String schemaType, - final Stream configs) - throws IOException { - writeConfigsToArchive(storageRoot, schemaType, configs.collect(Collectors.toList())); - } - - private static void writeConfigsToArchive(final Path storageRoot, - final String schemaType, - final List configList) - throws IOException { - final Path configPath = buildConfigPath(storageRoot, schemaType); - Files.createDirectories(configPath.getParent()); - if (!configList.isEmpty()) { - final List sortedConfigs = configList.stream() - .sorted(Comparator.comparing(JsonNode::toString)).collect( - Collectors.toList()); - Files.writeString(configPath, Yamls.serialize(sortedConfigs)); - } else { - // Create empty file - Files.createFile(configPath); - } - } - - private static Path buildConfigPath(final Path storageRoot, final String schemaType) { - return storageRoot.resolve(CONFIG_FOLDER_NAME) - .resolve(String.format("%s.yaml", schemaType)); - } - - public File exportWorkspace(final UUID workspaceId) throws JsonValidationException, IOException, ConfigNotFoundException { - final Path tempFolder = Files.createTempDirectory(Path.of("/tmp"), ARCHIVE_FILE_NAME); - final File dump = Files.createTempFile(ARCHIVE_FILE_NAME, ".tar.gz").toFile(); - exportVersionFile(tempFolder); - exportConfigsDatabase(tempFolder, workspaceId); - - Archives.createArchive(tempFolder, dump.toPath()); - return dump; - } - - private void exportConfigsDatabase(final Path parentFolder, final UUID workspaceId) - throws IOException, JsonValidationException, ConfigNotFoundException { - final Collection sourceConnections = writeConfigsToArchive( - parentFolder, - ConfigSchema.SOURCE_CONNECTION.name(), - secretsRepositoryReader::listSourceConnectionWithSecrets, - (sourceConnection) -> workspaceId.equals(sourceConnection.getWorkspaceId())); - writeConfigsToArchive(parentFolder, ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), - () -> listSourceDefinition(sourceConnections), - (config) -> true); - - final Collection destinationConnections = writeConfigsToArchive( - parentFolder, - ConfigSchema.DESTINATION_CONNECTION.name(), - secretsRepositoryReader::listDestinationConnectionWithSecrets, - (destinationConnection) -> workspaceId.equals(destinationConnection.getWorkspaceId())); - writeConfigsToArchive(parentFolder, ConfigSchema.STANDARD_DESTINATION_DEFINITION.name(), - () -> listDestinationDefinition(destinationConnections), - (config) -> true); - - writeConfigsToArchive( - parentFolder, - ConfigSchema.STANDARD_SYNC_OPERATION.name(), - configRepository::listStandardSyncOperations, - (operation) -> workspaceId.equals(operation.getWorkspaceId())); - - final List standardSyncs = new ArrayList<>(); - for (final StandardSync standardSync : configRepository.listStandardSyncs()) { - if (workspaceHelper != null && - workspaceId.equals(workspaceHelper.getWorkspaceForConnection(standardSync.getSourceId(), standardSync.getDestinationId()))) { - standardSyncs.add(standardSync); - } - } - writeConfigsToArchive(parentFolder, ConfigSchema.STANDARD_SYNC.name(), standardSyncs.stream().map(Jsons::jsonNode)); - } - - private Collection writeConfigsToArchive(final Path parentFolder, - final String configSchemaName, - final ListConfigCall listConfigCall, - final Function filterConfigCall) - throws JsonValidationException, ConfigNotFoundException, IOException { - final Collection configs = listConfigCall.apply().stream().filter(filterConfigCall::apply).collect(Collectors.toList()); - writeConfigsToArchive(parentFolder, configSchemaName, configs.stream().map(Jsons::jsonNode)); - return configs; - } - - private Collection listSourceDefinition(final Collection sourceConnections) - throws JsonValidationException, ConfigNotFoundException, IOException { - final Map sourceDefinitionMap = new HashMap<>(); - for (final SourceConnection sourceConnection : sourceConnections) { - if (!sourceDefinitionMap.containsKey(sourceConnection.getSourceDefinitionId())) { - sourceDefinitionMap - .put(sourceConnection.getSourceDefinitionId(), - configRepository.getStandardSourceDefinition(sourceConnection.getSourceDefinitionId())); - } - } - return sourceDefinitionMap.values(); - } - - private Collection listDestinationDefinition(final Collection destinationConnections) - throws JsonValidationException, ConfigNotFoundException, IOException { - final Map destinationDefinitionMap = new HashMap<>(); - for (final DestinationConnection destinationConnection : destinationConnections) { - if (!destinationDefinitionMap.containsKey(destinationConnection.getDestinationDefinitionId())) { - destinationDefinitionMap - .put(destinationConnection.getDestinationDefinitionId(), - configRepository.getStandardDestinationDefinition(destinationConnection.getDestinationDefinitionId())); - } - } - return destinationDefinitionMap.values(); - } - - /** - * List all configurations of type @param <T> that already exists - */ - public interface ListConfigCall { - - Collection apply() throws IOException, JsonValidationException, ConfigNotFoundException; - - } - -} diff --git a/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImporter.java b/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImporter.java deleted file mode 100644 index a3c6dbcfa1a89..0000000000000 --- a/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImporter.java +++ /dev/null @@ -1,576 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.server; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Streams; -import io.airbyte.analytics.TrackingClientSingleton; -import io.airbyte.api.model.generated.UploadRead; -import io.airbyte.commons.enums.Enums; -import io.airbyte.commons.io.Archives; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.AirbyteVersion; -import io.airbyte.commons.yaml.Yamls; -import io.airbyte.config.AirbyteConfig; -import io.airbyte.config.ConfigSchema; -import io.airbyte.config.DestinationConnection; -import io.airbyte.config.SourceConnection; -import io.airbyte.config.StandardDestinationDefinition; -import io.airbyte.config.StandardSourceDefinition; -import io.airbyte.config.StandardSync; -import io.airbyte.config.StandardSyncOperation; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigPersistence; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.config.persistence.SecretsRepositoryWriter; -import io.airbyte.persistence.job.DefaultJobPersistence; -import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.persistence.job.WorkspaceHelper; -import io.airbyte.server.errors.IdNotFoundKnownException; -import io.airbyte.validation.json.JsonSchemaValidator; -import io.airbyte.validation.json.JsonValidationException; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collection; -import java.util.Comparator; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.UUID; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@SuppressWarnings("PMD.AvoidReassigningLoopVariables") -public class ConfigDumpImporter { - - private static final Logger LOGGER = LoggerFactory.getLogger(ConfigDumpImporter.class); - private static final String CONFIG_FOLDER_NAME = "airbyte_config"; - private static final String VERSION_FILE_NAME = "VERSION"; - private static final Path TMP_AIRBYTE_STAGED_RESOURCES = Path.of("/tmp/airbyte_staged_resources"); - - private final ConfigRepository configRepository; - private final SecretsRepositoryWriter secretsRepositoryWriter; - private final WorkspaceHelper workspaceHelper; - private final JsonSchemaValidator jsonSchemaValidator; - private final JobPersistence jobPersistence; - private final boolean importDefinitions; - - public ConfigDumpImporter(final ConfigRepository configRepository, - final SecretsRepositoryWriter secretsRepositoryWriter, - final JobPersistence jobPersistence, - final WorkspaceHelper workspaceHelper, - final boolean importDefinitions) { - this(configRepository, secretsRepositoryWriter, jobPersistence, workspaceHelper, new JsonSchemaValidator(), importDefinitions); - } - - @VisibleForTesting - public ConfigDumpImporter(final ConfigRepository configRepository, - final SecretsRepositoryWriter secretsRepositoryWriter, - final JobPersistence jobPersistence, - final WorkspaceHelper workspaceHelper, - final JsonSchemaValidator jsonSchemaValidator, - final boolean importDefinitions) { - this.jsonSchemaValidator = jsonSchemaValidator; - this.jobPersistence = jobPersistence; - this.configRepository = configRepository; - this.secretsRepositoryWriter = secretsRepositoryWriter; - this.workspaceHelper = workspaceHelper; - this.importDefinitions = importDefinitions; - } - - /** - * Re-initialize the staged resource folder that contains uploaded artifacts when importing - * workspaces. This is because they need to be done in two steps (two API endpoints), upload - * resource first then import. When server starts, we flush the content of this folder, deleting - * previously staged resources that were not imported yet. - */ - public static void initStagedResourceFolder() { - try { - final File stagedResourceRoot = TMP_AIRBYTE_STAGED_RESOURCES.toFile(); - if (stagedResourceRoot.exists()) { - FileUtils.forceDelete(stagedResourceRoot); - } - FileUtils.forceMkdir(stagedResourceRoot); - FileUtils.forceDeleteOnExit(stagedResourceRoot); - } catch (final IOException e) { - throw new RuntimeException("Failed to create staging resource folder", e); - } - } - - public void importDataWithSeed(final AirbyteVersion targetVersion, final File archive, final ConfigPersistence seedPersistence) - throws IOException, JsonValidationException { - final Path sourceRoot = Files.createTempDirectory(Path.of("/tmp"), "airbyte_archive"); - try { - // 1. Unzip source - Archives.extractArchive(archive.toPath(), sourceRoot); - - // 2. dry run - try { - checkImport(targetVersion, sourceRoot); - importConfigsFromArchive(sourceRoot, true); - } catch (final Exception e) { - LOGGER.error("Dry run failed.", e); - throw e; - } - - // 4. Import Configs and update connector definitions - importConfigsFromArchive(sourceRoot, false); - configRepository.loadDataNoSecrets(seedPersistence); - - // 5. Set DB version - LOGGER.info("Setting the DB Airbyte version to : " + targetVersion); - jobPersistence.setVersion(targetVersion.serialize()); - - // 6. check db version - checkDBVersion(targetVersion); - } finally { - FileUtils.deleteDirectory(sourceRoot.toFile()); - FileUtils.deleteQuietly(archive); - } - - // identify this instance as the new customer id. - configRepository.listStandardWorkspaces(true).forEach(workspace -> TrackingClientSingleton.get().identify(workspace.getWorkspaceId())); - } - - private void checkImport(final AirbyteVersion targetVersion, final Path tempFolder) throws IOException { - final Path versionFile = tempFolder.resolve(VERSION_FILE_NAME); - final AirbyteVersion importVersion = new AirbyteVersion(Files - .readString(versionFile, Charset.defaultCharset()) - .replace("\n", "") - .strip()); - LOGGER.info(String.format("Checking Airbyte Version to import %s", importVersion)); - if (!AirbyteVersion.isCompatible(targetVersion, importVersion)) { - throw new IOException(String - .format("Imported VERSION (%s) is incompatible with current Airbyte version (%s).\n" + - "Please upgrade your Airbyte Archive, see more at https://docs.airbyte.com/operator-guides/upgrading-airbyte\n", - importVersion, targetVersion)); - } - } - - // Config - private List listDirectories(final Path sourceRoot) throws IOException { - try (final Stream files = Files.list(sourceRoot.resolve(CONFIG_FOLDER_NAME))) { - return files.map(c -> c.getFileName().toString()) - .collect(Collectors.toList()); - } - } - - private void importConfigsFromArchive(final Path sourceRoot, final boolean dryRun) throws IOException { - final List directories = listDirectories(sourceRoot); - final Map> data = new LinkedHashMap<>(); - - for (final String directory : directories) { - final Optional configSchemaOptional = Enums.toEnum(directory.replace(".yaml", ""), ConfigSchema.class); - - if (configSchemaOptional.isEmpty()) { - continue; - } - - final ConfigSchema configSchema = configSchemaOptional.get(); - data.put(configSchema, readConfigsFromArchive(sourceRoot, configSchema)); - } - secretsRepositoryWriter.replaceAllConfigs(data, dryRun); - } - - private Stream readConfigsFromArchive(final Path storageRoot, final ConfigSchema schemaType) - throws IOException { - - final Path configPath = buildConfigPath(storageRoot, schemaType); - if (configPath.toFile().exists()) { - final String configStr = Files.readString(configPath); - final JsonNode node = Yamls.deserialize(configStr); - return StreamSupport - .stream(Spliterators.spliteratorUnknownSize(node.elements(), Spliterator.ORDERED), false) - .map(element -> { - final T config = Jsons.object(element, schemaType.getClassName()); - try { - validateJson(config, schemaType); - return config; - } catch (final JsonValidationException e) { - throw new RuntimeException(e); - } - }); - - } else { - throw new FileNotFoundException( - String.format("Airbyte Configuration %s was not found in the archive", schemaType)); - } - } - - private void validateJson(final T config, final ConfigSchema configType) throws JsonValidationException { - final JsonNode schema = JsonSchemaValidator.getSchema(configType.getConfigSchemaFile()); - jsonSchemaValidator.ensure(schema, Jsons.jsonNode(config)); - } - - protected static Path buildConfigPath(final Path storageRoot, final ConfigSchema schemaType) { - return storageRoot.resolve(CONFIG_FOLDER_NAME) - .resolve(String.format("%s.yaml", schemaType.name())); - } - - /** - * The deployment concept is specific to the environment that Airbyte is running in (not the data - * being imported). Thus, if there is a deployment in the imported data, we filter it out. In - * addition, before running the import, we look up the current deployment id, and make sure that - * that id is inserted when we run the import. - * - * @param postgresPersistence - database that we are importing into. - * @param metadataTableStream - stream of records to be imported into the metadata table. - * @return modified stream with old deployment id removed and correct deployment id inserted. - * @throws IOException - you never know when you IO. - */ - static Stream replaceDeploymentMetadata(final JobPersistence postgresPersistence, - final Stream metadataTableStream) - throws IOException { - // filter out the deployment record from the import data, if it exists. - Stream stream = metadataTableStream - .filter(record -> !DefaultJobPersistence.DEPLOYMENT_ID_KEY.equals(record.get(DefaultJobPersistence.METADATA_KEY_COL).asText())); - - // insert the current deployment id, if it exists. - final Optional deploymentOptional = postgresPersistence.getDeployment(); - if (deploymentOptional.isPresent()) { - final JsonNode deploymentRecord = Jsons.jsonNode(ImmutableMap.builder() - .put(DefaultJobPersistence.METADATA_KEY_COL, DefaultJobPersistence.DEPLOYMENT_ID_KEY) - .put(DefaultJobPersistence.METADATA_VAL_COL, deploymentOptional.get().toString()) - .build()); - stream = Streams.concat(stream, Stream.of(deploymentRecord)); - } - return stream; - } - - private void checkDBVersion(final AirbyteVersion airbyteVersion) throws IOException { - final Optional airbyteDatabaseVersion = jobPersistence.getVersion().map(AirbyteVersion::new); - airbyteDatabaseVersion - .ifPresent(dbVersion -> AirbyteVersion.assertIsCompatible(airbyteVersion, dbVersion)); - } - - public UploadRead uploadArchiveResource(final File archive) { - try { - final UUID resourceId = UUID.randomUUID(); - FileUtils.moveFile(archive, TMP_AIRBYTE_STAGED_RESOURCES.resolve(resourceId.toString()).toFile()); - return new UploadRead() - .status(UploadRead.StatusEnum.SUCCEEDED) - .resourceId(resourceId); - } catch (final IOException e) { - LOGGER.error("Failed to upload archive resource", e); - return new UploadRead().status(UploadRead.StatusEnum.FAILED); - } - } - - public File getArchiveResource(final UUID resourceId) { - final File archive = TMP_AIRBYTE_STAGED_RESOURCES.resolve(resourceId.toString()).toFile(); - if (!archive.exists()) { - throw new IdNotFoundKnownException("Archive Resource not found", resourceId.toString()); - } - return archive; - } - - public void deleteArchiveResource(final UUID resourceId) { - final File archive = getArchiveResource(resourceId); - FileUtils.deleteQuietly(archive); - } - - public void importIntoWorkspace(final AirbyteVersion targetVersion, final UUID workspaceId, final File archive) - throws IOException, JsonValidationException, ConfigNotFoundException { - final Path sourceRoot = Files.createTempDirectory(Path.of("/tmp"), "airbyte_archive"); - try { - // 1. Unzip source - Archives.extractArchive(archive.toPath(), sourceRoot); - - // TODO: Auto-migrate archive? - - // 2. dry run - try { - checkImport(targetVersion, sourceRoot); - importConfigsIntoWorkspace(sourceRoot, workspaceId, true); - } catch (final Exception e) { - LOGGER.error("Dry run failed.", e); - throw e; - } - - // 3. import configs - importConfigsIntoWorkspace(sourceRoot, workspaceId, false); - } finally { - FileUtils.deleteDirectory(sourceRoot.toFile()); - } - } - - private void importConfigsIntoWorkspace(final Path sourceRoot, final UUID workspaceId, final boolean dryRun) - throws IOException, JsonValidationException, ConfigNotFoundException { - // Keep maps of any re-assigned ids - final Map sourceIdMap = new HashMap<>(); - final Map destinationIdMap = new HashMap<>(); - final Map operationIdMap = new HashMap<>(); - - final List directories = listDirectories(sourceRoot); - // We sort the directories because we want to process SOURCE_CONNECTION after - // STANDARD_SOURCE_DEFINITION and DESTINATION_CONNECTION after STANDARD_DESTINATION_DEFINITION - // so that we can identify which connectors should not be imported because the definitions are not - // existing - directories.sort(Comparator.reverseOrder()); - Stream standardSyncs = null; - - for (final String directory : directories) { - final Optional configSchemaOptional = Enums.toEnum(directory.replace(".yaml", ""), ConfigSchema.class); - - if (configSchemaOptional.isEmpty()) { - continue; - } - final ConfigSchema configSchema = configSchemaOptional.get(); - final Stream configs = readConfigsFromArchive(sourceRoot, configSchema); - - if (dryRun) { - continue; - } - - switch (configSchema) { - case STANDARD_SOURCE_DEFINITION -> { - if (canImportDefinitions()) { - importSourceDefinitionIntoWorkspace(configs); - } - } - case SOURCE_CONNECTION -> sourceIdMap.putAll(importIntoWorkspace( - ConfigSchema.SOURCE_CONNECTION, - configs.map(c -> (SourceConnection) c), - configRepository::listSourceConnection, - (sourceConnection) -> !workspaceId.equals(sourceConnection.getWorkspaceId()), - (sourceConnection, sourceId) -> { - sourceConnection.setSourceId(sourceId); - sourceConnection.setWorkspaceId(workspaceId); - return sourceConnection; - }, - (sourceConnection) -> { - // make sure connector definition exists - try { - final StandardSourceDefinition sourceDefinition = - configRepository.getStandardSourceDefinition(sourceConnection.getSourceDefinitionId()); - if (sourceDefinition == null) { - return; - } - if (sourceDefinition.getTombstone() != null && sourceDefinition.getTombstone()) { - return; - } - secretsRepositoryWriter.writeSourceConnection(sourceConnection, sourceDefinition.getSpec()); - } catch (final ConfigNotFoundException e) { - return; - } - })); - case STANDARD_DESTINATION_DEFINITION -> { - if (canImportDefinitions()) { - importDestinationDefinitionIntoWorkspace(configs); - } - } - case DESTINATION_CONNECTION -> destinationIdMap.putAll(importIntoWorkspace( - ConfigSchema.DESTINATION_CONNECTION, - configs.map(c -> (DestinationConnection) c), - configRepository::listDestinationConnection, - (destinationConnection) -> !workspaceId.equals(destinationConnection.getWorkspaceId()), - (destinationConnection, destinationId) -> { - destinationConnection.setDestinationId(destinationId); - destinationConnection.setWorkspaceId(workspaceId); - return destinationConnection; - }, - (destinationConnection) -> { - // make sure connector definition exists - try { - final StandardDestinationDefinition destinationDefinition = configRepository.getStandardDestinationDefinition( - destinationConnection.getDestinationDefinitionId()); - if (destinationDefinition == null) { - return; - } - if (destinationDefinition.getTombstone() != null && destinationDefinition.getTombstone()) { - return; - } - secretsRepositoryWriter.writeDestinationConnection(destinationConnection, destinationDefinition.getSpec()); - } catch (final ConfigNotFoundException e) { - return; - } - })); - case STANDARD_SYNC -> standardSyncs = configs; - case STANDARD_SYNC_OPERATION -> operationIdMap.putAll(importIntoWorkspace( - ConfigSchema.STANDARD_SYNC_OPERATION, - configs.map(c -> (StandardSyncOperation) c), - configRepository::listStandardSyncOperations, - (operation) -> !workspaceId.equals(operation.getWorkspaceId()), - (operation, operationId) -> { - operation.setOperationId(operationId); - operation.setWorkspaceId(workspaceId); - return operation; - }, - configRepository::writeStandardSyncOperation)); - default -> {} - } - } - - if (standardSyncs != null) { - // we import connections (standard sync) last to update reference to modified ids - importIntoWorkspace( - ConfigSchema.STANDARD_SYNC, - standardSyncs.map(c -> (StandardSync) c), - configRepository::listStandardSyncs, - (standardSync) -> { - try { - return !workspaceId.equals(workspaceHelper.getWorkspaceForConnection(standardSync.getSourceId(), standardSync.getDestinationId())); - } catch (final JsonValidationException | ConfigNotFoundException e) { - return true; - } - }, - (standardSync, connectionId) -> { - standardSync.setConnectionId(connectionId); - standardSync.setSourceId(sourceIdMap.get(standardSync.getSourceId())); - standardSync.setDestinationId(destinationIdMap.get(standardSync.getDestinationId())); - standardSync.setOperationIds(standardSync.getOperationIds() - .stream() - .map(operationIdMap::get) - .collect(Collectors.toList())); - return standardSync; - }, - (standardSync) -> { - // make sure connectors definition exists - try { - if (configRepository.getSourceConnection(standardSync.getSourceId()) == null || - configRepository.getDestinationConnection(standardSync.getDestinationId()) == null) { - return; - } - for (final UUID operationId : standardSync.getOperationIds()) { - if (configRepository.getStandardSyncOperation(operationId) == null) { - return; - } - } - } catch (final ConfigNotFoundException e) { - return; - } - configRepository.writeStandardSync(standardSync); - }); - } - } - - /** - * Method that @return if this importer will import standard connector definitions or not - */ - public boolean canImportDefinitions() { - return importDefinitions; - } - - protected void importSourceDefinitionIntoWorkspace(final Stream configs) - throws JsonValidationException, ConfigNotFoundException, IOException { - importIntoWorkspace( - ConfigSchema.STANDARD_SOURCE_DEFINITION, - configs.map(c -> (StandardSourceDefinition) c), - () -> configRepository.listStandardSourceDefinitions(false), - (config) -> true, - (config, id) -> { - if (id.equals(config.getSourceDefinitionId())) { - return config; - } else { - // a newId has been generated for this definition as it is in conflict with an existing one - // here we return null, so we don't do anything to the old definition - return null; - } - }, - (config) -> { - if (config != null) { - configRepository.writeStandardSourceDefinition(config); - } - }); - } - - protected void importDestinationDefinitionIntoWorkspace(final Stream configs) - throws JsonValidationException, ConfigNotFoundException, IOException { - importIntoWorkspace( - ConfigSchema.STANDARD_DESTINATION_DEFINITION, - configs.map(c -> (StandardDestinationDefinition) c), - () -> configRepository.listStandardDestinationDefinitions(false), - (config) -> true, - (config, id) -> { - if (id.equals(config.getDestinationDefinitionId())) { - return config; - } else { - // a newId has been generated for this definition as it is in conflict with an existing one - // here we return null, so we don't do anything to the old definition - return null; - } - }, - (config) -> { - if (config != null) { - configRepository.writeStandardDestinationDefinition(config); - } - }); - } - - private Map importIntoWorkspace(final ConfigSchema configSchema, - final Stream configs, - final ListConfigCall listConfigCall, - final Function filterConfigCall, - final MutateConfigCall mutateConfig, - final PersistConfigCall persistConfig) - throws JsonValidationException, ConfigNotFoundException, IOException { - final Map idsMap = new HashMap<>(); - // To detect conflicts, we retrieve ids already in use by others for this ConfigSchema (ids from the - // current workspace can be safely updated) - final Set idsInUse = listConfigCall.apply() - .stream() - .filter(filterConfigCall::apply) - .map(configSchema::getId) - .map(UUID::fromString) - .collect(Collectors.toSet()); - for (T config : configs.collect(Collectors.toList())) { - final UUID configId = UUID.fromString(configSchema.getId(config)); - final UUID configIdToPersist = idsInUse.contains(configId) ? UUID.randomUUID() : configId; - config = mutateConfig.apply(config, configIdToPersist); - if (config != null) { - idsMap.put(configId, UUID.fromString(configSchema.getId(config))); - persistConfig.apply(config); - } else { - idsMap.put(configId, configId); - } - } - return idsMap; - } - - /** - * List all configurations of type @param <T> that already exists (we'll be using this to know - * which ids are already in use) - */ - public interface ListConfigCall { - - Collection apply() throws IOException, JsonValidationException, ConfigNotFoundException; - - } - - /** - * Apply some modifications to the configuration with new ids - */ - public interface MutateConfigCall { - - T apply(T config, UUID newId) throws IOException, JsonValidationException, ConfigNotFoundException; - - } - - /** - * Persist the configuration - */ - public interface PersistConfigCall { - - void apply(T config) throws JsonValidationException, IOException; - - } - -} diff --git a/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java index bc2b2703ae8f5..4ed1af3866126 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ConfigurationApiFactory.java @@ -5,11 +5,9 @@ package io.airbyte.server; import io.airbyte.analytics.TrackingClient; -import io.airbyte.commons.io.FileTtlManager; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.helpers.LogConfigs; -import io.airbyte.config.persistence.ConfigPersistence; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.SecretsRepositoryReader; import io.airbyte.config.persistence.SecretsRepositoryWriter; @@ -30,11 +28,9 @@ public class ConfigurationApiFactory implements Factory { private static ConfigRepository configRepository; private static JobPersistence jobPersistence; - private static ConfigPersistence seed; private static SecretsRepositoryReader secretsRepositoryReader; private static SecretsRepositoryWriter secretsRepositoryWriter; private static SynchronousSchedulerClient synchronousSchedulerClient; - private static FileTtlManager archiveTtlManager; private static StatePersistence statePersistence; private static Map mdc; private static Database configsDatabase; @@ -54,9 +50,7 @@ public static void setValues( final SecretsRepositoryReader secretsRepositoryReader, final SecretsRepositoryWriter secretsRepositoryWriter, final JobPersistence jobPersistence, - final ConfigPersistence seed, final SynchronousSchedulerClient synchronousSchedulerClient, - final FileTtlManager archiveTtlManager, final StatePersistence statePersistence, final Map mdc, final Database configsDatabase, @@ -72,11 +66,9 @@ public static void setValues( final Flyway jobsFlyway) { ConfigurationApiFactory.configRepository = configRepository; ConfigurationApiFactory.jobPersistence = jobPersistence; - ConfigurationApiFactory.seed = seed; ConfigurationApiFactory.secretsRepositoryReader = secretsRepositoryReader; ConfigurationApiFactory.secretsRepositoryWriter = secretsRepositoryWriter; ConfigurationApiFactory.synchronousSchedulerClient = synchronousSchedulerClient; - ConfigurationApiFactory.archiveTtlManager = archiveTtlManager; ConfigurationApiFactory.mdc = mdc; ConfigurationApiFactory.configsDatabase = configsDatabase; ConfigurationApiFactory.jobsDatabase = jobsDatabase; @@ -99,11 +91,9 @@ public ConfigurationApi provide() { return new ConfigurationApi( ConfigurationApiFactory.configRepository, ConfigurationApiFactory.jobPersistence, - ConfigurationApiFactory.seed, ConfigurationApiFactory.secretsRepositoryReader, ConfigurationApiFactory.secretsRepositoryWriter, ConfigurationApiFactory.synchronousSchedulerClient, - ConfigurationApiFactory.archiveTtlManager, ConfigurationApiFactory.configsDatabase, ConfigurationApiFactory.jobsDatabase, ConfigurationApiFactory.statePersistence, diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index b304e2c008bbb..c50b443e203da 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -8,8 +8,6 @@ import io.airbyte.analytics.Deployment; import io.airbyte.analytics.TrackingClient; import io.airbyte.analytics.TrackingClientSingleton; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.lang.CloseableShutdownHook; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.temporal.TemporalUtils; @@ -17,9 +15,9 @@ import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.Configs; import io.airbyte.config.EnvConfigs; +import io.airbyte.config.StandardSync; import io.airbyte.config.StandardSync.Status; import io.airbyte.config.helpers.LogClientSingleton; -import io.airbyte.config.init.YamlSeedConfigPersistence; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigPersistence; import io.airbyte.config.persistence.ConfigRepository; @@ -167,7 +165,6 @@ private static void assertDatabasesReady(final Configs configs, } public static ServerRunnable getServer(final ServerFactory apiFactory, - final ConfigPersistence seed, final Configs configs, final DSLContext configsDslContext, final Flyway configsFlyway, @@ -182,14 +179,9 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, LOGGER.info("Checking databases.."); assertDatabasesReady(configs, configsDslContext, configsFlyway, jobsDslContext, jobsFlyway); - LOGGER.info("Creating Staged Resource folder..."); - ConfigDumpImporter.initStagedResourceFolder(); - LOGGER.info("Creating config repository..."); final Database configsDatabase = new Database(configsDslContext); - final FeatureFlags featureFlags = new EnvVariableFeatureFlags(); final JsonSecretsProcessor jsonSecretsProcessor = JsonSecretsProcessor.builder() - .maskSecrets(!featureFlags.exposeSecretsInExport()) .copySecrets(false) .build(); final ConfigPersistence configPersistence = DatabaseConfigPersistence.createWithValidation(configsDatabase, jsonSecretsProcessor); @@ -271,7 +263,6 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, secretsRepositoryReader, secretsRepositoryWriter, jobPersistence, - seed, configsDatabase, jobsDatabase, trackingClient, @@ -300,13 +291,14 @@ static void migrateExistingConnectionsToTemporalScheduler(final ConfigRepository final Set connectionIds = configRepository.listStandardSyncs().stream() .filter(standardSync -> standardSync.getStatus() == Status.ACTIVE || standardSync.getStatus() == Status.INACTIVE) - .map(standardSync -> standardSync.getConnectionId()).collect(Collectors.toSet()); + .map(StandardSync::getConnectionId) + .collect(Collectors.toSet()); eventRunner.migrateSyncIfNeeded(connectionIds); jobPersistence.setSchedulerMigrationDone(); LOGGER.info("Done migrating to the new scheduler..."); } - public static void main(final String[] args) throws Exception { + public static void main(final String[] args) { try { final Configs configs = new EnvConfigs(); @@ -327,10 +319,8 @@ public static void main(final String[] args) throws Exception { ConfigsDatabaseMigrator.DB_IDENTIFIER, ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); final Flyway jobsFlyway = FlywayFactory.create(jobsDataSource, DbMigrationHandler.class.getSimpleName(), JobsDatabaseMigrator.DB_IDENTIFIER, JobsDatabaseMigrator.MIGRATION_FILE_LOCATION); - final ConfigPersistence yamlSeedConfigPersistence = - new YamlSeedConfigPersistence(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); - getServer(new ServerFactory.Api(), yamlSeedConfigPersistence, configs, configsDslContext, configsFlyway, jobsDslContext, jobsFlyway).start(); + getServer(new ServerFactory.Api(), configs, configsDslContext, configsFlyway, jobsDslContext, jobsFlyway).start(); } } catch (final Throwable e) { LOGGER.error("Server failed", e); diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java index aa0af731c63ed..a86486d53961e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerFactory.java @@ -5,11 +5,9 @@ package io.airbyte.server; import io.airbyte.analytics.TrackingClient; -import io.airbyte.commons.io.FileTtlManager; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.helpers.LogConfigs; -import io.airbyte.config.persistence.ConfigPersistence; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.SecretsRepositoryReader; import io.airbyte.config.persistence.SecretsRepositoryWriter; @@ -22,7 +20,6 @@ import java.net.http.HttpClient; import java.nio.file.Path; import java.util.Set; -import java.util.concurrent.TimeUnit; import org.flywaydb.core.Flyway; import org.slf4j.MDC; @@ -33,7 +30,6 @@ ServerRunnable create(SynchronousSchedulerClient cachingSchedulerClient, SecretsRepositoryReader secretsRepositoryReader, SecretsRepositoryWriter secretsRepositoryWriter, JobPersistence jobPersistence, - ConfigPersistence seed, Database configsDatabase, Database jobsDatabase, TrackingClient trackingClient, @@ -54,7 +50,6 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul final SecretsRepositoryReader secretsRepositoryReader, final SecretsRepositoryWriter secretsRepositoryWriter, final JobPersistence jobPersistence, - final ConfigPersistence seed, final Database configsDatabase, final Database jobsDatabase, final TrackingClient trackingClient, @@ -72,9 +67,7 @@ public ServerRunnable create(final SynchronousSchedulerClient synchronousSchedul secretsRepositoryReader, secretsRepositoryWriter, jobPersistence, - seed, synchronousSchedulerClient, - new FileTtlManager(10, TimeUnit.MINUTES, 10), new StatePersistence(configsDatabase), MDC.getCopyOfContextMap(), configsDatabase, diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 09b68f029a249..ba39d020aaaab 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -99,12 +99,10 @@ import io.airbyte.api.model.generated.WorkspaceReadList; import io.airbyte.api.model.generated.WorkspaceUpdate; import io.airbyte.api.model.generated.WorkspaceUpdateName; -import io.airbyte.commons.io.FileTtlManager; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.helpers.LogConfigs; import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigPersistence; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.SecretsRepositoryReader; import io.airbyte.config.persistence.SecretsRepositoryWriter; @@ -114,7 +112,6 @@ import io.airbyte.persistence.job.WorkspaceHelper; import io.airbyte.server.errors.BadObjectSchemaKnownException; import io.airbyte.server.errors.IdNotFoundKnownException; -import io.airbyte.server.handlers.ArchiveHandler; import io.airbyte.server.handlers.AttemptHandler; import io.airbyte.server.handlers.ConnectionsHandler; import io.airbyte.server.handlers.DbMigrationHandler; @@ -160,7 +157,6 @@ public class ConfigurationApi implements io.airbyte.api.generated.V1Api { private final JobHistoryHandler jobHistoryHandler; private final WebBackendConnectionsHandler webBackendConnectionsHandler; private final HealthCheckHandler healthCheckHandler; - private final ArchiveHandler archiveHandler; private final LogsHandler logsHandler; private final OpenApiConfigHandler openApiConfigHandler; private final DbMigrationHandler dbMigrationHandler; @@ -172,11 +168,9 @@ public class ConfigurationApi implements io.airbyte.api.generated.V1Api { public ConfigurationApi(final ConfigRepository configRepository, final JobPersistence jobPersistence, - final ConfigPersistence seed, final SecretsRepositoryReader secretsRepositoryReader, final SecretsRepositoryWriter secretsRepositoryWriter, final SynchronousSchedulerClient synchronousSchedulerClient, - final FileTtlManager archiveTtlManager, final Database configsDatabase, final Database jobsDatabase, final StatePersistence statePersistence, @@ -243,16 +237,6 @@ public ConfigurationApi(final ConfigRepository configRepository, eventRunner, configRepository); healthCheckHandler = new HealthCheckHandler(configRepository); - archiveHandler = new ArchiveHandler( - airbyteVersion, - configRepository, - secretsRepositoryReader, - secretsRepositoryWriter, - jobPersistence, - seed, - workspaceHelper, - archiveTtlManager, - true); logsHandler = new LogsHandler(); openApiConfigHandler = new OpenApiConfigHandler(); dbMigrationHandler = new DbMigrationHandler(configsDatabase, configsFlyway, jobsDatabase, jobsFlyway); @@ -842,10 +826,6 @@ public InternalOperationResult setWorkflowInAttempt(final SetWorkflowInAttemptRe return execute(() -> attemptHandler.setWorkflowInAttempt(requestBody)); } - public boolean canImportDefinitions() { - return archiveHandler.canImportDefinitions(); - } - private static T execute(final HandlerCall call) { try { return call.call(); diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java b/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java index 6a3aeae05799e..73d9e0f61220a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java @@ -4,7 +4,6 @@ package io.airbyte.server.converters; -import io.airbyte.api.client.model.generated.ConnectionScheduleType; import io.airbyte.api.model.generated.ActorDefinitionResourceRequirements; import io.airbyte.api.model.generated.ConnectionRead; import io.airbyte.api.model.generated.ConnectionSchedule; @@ -19,7 +18,6 @@ import io.airbyte.config.BasicSchedule; import io.airbyte.config.Schedule; import io.airbyte.config.StandardSync; -import io.airbyte.config.StandardSync.ScheduleType; import io.airbyte.server.handlers.helpers.CatalogConverter; import java.util.stream.Collectors; @@ -153,63 +151,94 @@ public static ConnectionScheduleDataBasicSchedule.TimeUnitEnum toApiBasicSchedul return Enums.convertTo(timeUnit, ConnectionScheduleDataBasicSchedule.TimeUnitEnum.class); } - public static void populateConnectionReadSchedule(final StandardSync standardSync, final ConnectionRead connectionRead) { - // TODO(https://github.com/airbytehq/airbyte/issues/11432): only return new schema once frontend is - // ready. + public static io.airbyte.api.model.generated.ConnectionScheduleType toApiConnectionScheduleType(final StandardSync standardSync) { if (standardSync.getScheduleType() != null) { - // Populate everything based on the new schema. switch (standardSync.getScheduleType()) { case MANUAL -> { - connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL); + return io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL; } case BASIC_SCHEDULE -> { - connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.BASIC); - connectionRead.scheduleData(new ConnectionScheduleData() + return io.airbyte.api.model.generated.ConnectionScheduleType.BASIC; + } + case CRON -> { + return io.airbyte.api.model.generated.ConnectionScheduleType.CRON; + } + default -> throw new RuntimeException("Unexpected scheduleType " + standardSync.getScheduleType()); + } + } else if (standardSync.getManual()) { + // Legacy schema, manual sync. + return io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL; + } else { + // Legacy schema, basic schedule. + return io.airbyte.api.model.generated.ConnectionScheduleType.BASIC; + } + } + + public static io.airbyte.api.model.generated.ConnectionScheduleData toApiConnectionScheduleData(final StandardSync standardSync) { + if (standardSync.getScheduleType() != null) { + switch (standardSync.getScheduleType()) { + case MANUAL -> { + return null; + } + case BASIC_SCHEDULE -> { + return new ConnectionScheduleData() .basicSchedule(new ConnectionScheduleDataBasicSchedule() .timeUnit(toApiBasicScheduleTimeUnit(standardSync.getScheduleData().getBasicSchedule().getTimeUnit())) - .units(standardSync.getScheduleData().getBasicSchedule().getUnits()))); - connectionRead.schedule(new ConnectionSchedule() - .timeUnit(toApiTimeUnit(standardSync.getScheduleData().getBasicSchedule().getTimeUnit())) - .units(standardSync.getScheduleData().getBasicSchedule().getUnits())); + .units(standardSync.getScheduleData().getBasicSchedule().getUnits())); } case CRON -> { - // We don't populate any legacy data here. - connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.CRON); - connectionRead.scheduleData(new ConnectionScheduleData() + return new ConnectionScheduleData() .cron(new ConnectionScheduleDataCron() .cronExpression(standardSync.getScheduleData().getCron().getCronExpression()) - .cronTimeZone(standardSync.getScheduleData().getCron().getCronTimeZone()))); + .cronTimeZone(standardSync.getScheduleData().getCron().getCronTimeZone())); } + default -> throw new RuntimeException("Unexpected scheduleType " + standardSync.getScheduleType()); } } else if (standardSync.getManual()) { // Legacy schema, manual sync. - connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL); + return null; } else { // Legacy schema, basic schedule. - connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.BASIC); - connectionRead.schedule(new ConnectionSchedule() - .timeUnit(toApiTimeUnit(standardSync.getSchedule().getTimeUnit())) - .units(standardSync.getSchedule().getUnits())); - connectionRead.scheduleData(new ConnectionScheduleData() + return new ConnectionScheduleData() .basicSchedule(new ConnectionScheduleDataBasicSchedule() .timeUnit(toApiBasicScheduleTimeUnit(standardSync.getSchedule().getTimeUnit())) - .units(standardSync.getSchedule().getUnits()))); + .units(standardSync.getSchedule().getUnits())); } } - public static ConnectionScheduleType toApiScheduleType(final ScheduleType scheduleType) { - switch (scheduleType) { - case MANUAL -> { - return ConnectionScheduleType.MANUAL; - } - case BASIC_SCHEDULE -> { - return ConnectionScheduleType.BASIC; - } - case CRON -> { - return ConnectionScheduleType.CRON; + public static ConnectionSchedule toLegacyConnectionSchedule(final StandardSync standardSync) { + if (standardSync.getScheduleType() != null) { + // Populate everything based on the new schema. + switch (standardSync.getScheduleType()) { + case MANUAL, CRON -> { + // We don't populate any legacy data here. + return null; + } + case BASIC_SCHEDULE -> { + return new ConnectionSchedule() + .timeUnit(toApiTimeUnit(standardSync.getScheduleData().getBasicSchedule().getTimeUnit())) + .units(standardSync.getScheduleData().getBasicSchedule().getUnits()); + } + default -> throw new RuntimeException("Unexpected scheduleType " + standardSync.getScheduleType()); } + } else if (standardSync.getManual()) { + // Legacy schema, manual sync. + return null; + } else { + // Legacy schema, basic schedule. + return new ConnectionSchedule() + .timeUnit(toApiTimeUnit(standardSync.getSchedule().getTimeUnit())) + .units(standardSync.getSchedule().getUnits()); } - throw new RuntimeException("Unexpected schedule type"); + } + + public static void populateConnectionReadSchedule(final StandardSync standardSync, final ConnectionRead connectionRead) { + connectionRead.scheduleType(toApiConnectionScheduleType(standardSync)); + connectionRead.scheduleData(toApiConnectionScheduleData(standardSync)); + + // TODO(https://github.com/airbytehq/airbyte/issues/11432): only return new schema once frontend is + // ready. + connectionRead.schedule(toLegacyConnectionSchedule(standardSync)); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/ConfigurationUpdate.java b/airbyte-server/src/main/java/io/airbyte/server/converters/ConfigurationUpdate.java index 18888169e2c55..f4ac339423cd1 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/ConfigurationUpdate.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/ConfigurationUpdate.java @@ -27,7 +27,6 @@ public class ConfigurationUpdate { public ConfigurationUpdate(final ConfigRepository configRepository, final SecretsRepositoryReader secretsRepositoryReader) { this(configRepository, secretsRepositoryReader, JsonSecretsProcessor.builder() - .maskSecrets(true) .copySecrets(true) .build()); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/ArchiveHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/ArchiveHandler.java deleted file mode 100644 index adfb4dbfd852d..0000000000000 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/ArchiveHandler.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.server.handlers; - -import io.airbyte.api.model.generated.ImportRead; -import io.airbyte.api.model.generated.ImportRead.StatusEnum; -import io.airbyte.api.model.generated.ImportRequestBody; -import io.airbyte.api.model.generated.UploadRead; -import io.airbyte.api.model.generated.WorkspaceIdRequestBody; -import io.airbyte.commons.io.FileTtlManager; -import io.airbyte.commons.version.AirbyteVersion; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigPersistence; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.config.persistence.SecretsRepositoryReader; -import io.airbyte.config.persistence.SecretsRepositoryWriter; -import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.persistence.job.WorkspaceHelper; -import io.airbyte.server.ConfigDumpExporter; -import io.airbyte.server.ConfigDumpImporter; -import io.airbyte.server.errors.InternalServerKnownException; -import io.airbyte.validation.json.JsonValidationException; -import java.io.File; -import java.io.IOException; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ArchiveHandler { - - private static final Logger LOGGER = LoggerFactory.getLogger(ArchiveHandler.class); - - private final AirbyteVersion version; - private final ConfigDumpExporter configDumpExporter; - private final ConfigDumpImporter configDumpImporter; - private final ConfigPersistence seed; - private final FileTtlManager fileTtlManager; - - public ArchiveHandler(final AirbyteVersion version, - final ConfigRepository configRepository, - final SecretsRepositoryReader secretsRepositoryReader, - final SecretsRepositoryWriter secretsRepositoryWriter, - final JobPersistence jobPersistence, - final ConfigPersistence seed, - final WorkspaceHelper workspaceHelper, - final FileTtlManager fileTtlManager, - final boolean importDefinitions) { - this( - version, - fileTtlManager, - new ConfigDumpExporter(configRepository, secretsRepositoryReader, jobPersistence, workspaceHelper), - new ConfigDumpImporter(configRepository, secretsRepositoryWriter, jobPersistence, workspaceHelper, importDefinitions), - seed); - } - - public ArchiveHandler(final AirbyteVersion version, - final FileTtlManager fileTtlManager, - final ConfigDumpExporter configDumpExporter, - final ConfigDumpImporter configDumpImporter, - final ConfigPersistence seed) { - this.version = version; - this.configDumpExporter = configDumpExporter; - this.configDumpImporter = configDumpImporter; - this.seed = seed; - this.fileTtlManager = fileTtlManager; - } - - /** - * Creates an archive tarball file using Gzip compression of internal Airbyte Data - * - * @return that tarball File. - */ - public File exportData() { - final File archive = configDumpExporter.dump(); - fileTtlManager.register(archive.toPath()); - return archive; - } - - /** - * Creates an archive tarball file using Gzip compression of only configurations tied to - * - * @param workspaceIdRequestBody which is the target workspace to export - * @return that lightweight tarball file - */ - public File exportWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { - final File archive; - try { - archive = configDumpExporter.exportWorkspace(workspaceIdRequestBody.getWorkspaceId()); - fileTtlManager.register(archive.toPath()); - return archive; - } catch (final JsonValidationException | IOException | ConfigNotFoundException e) { - throw new InternalServerKnownException(String.format("Failed to export Workspace configuration due to: %s", e.getMessage()), e); - } - } - - /** - * Extract internal Airbyte data from the @param archive tarball file (using Gzip compression) as - * produced by {@link #exportData()}. Note that the provided archived file will be deleted. - * - * @return a status object describing if import was successful or not. - */ - public ImportRead importData(final File archive) { - try { - return importInternal(() -> configDumpImporter.importDataWithSeed(version, archive, seed)); - } finally { - FileUtils.deleteQuietly(archive); - } - } - - public UploadRead uploadArchiveResource(final File archive) { - return configDumpImporter.uploadArchiveResource(archive); - } - - /** - * Extract Airbyte configuration data from the archive tarball file (using Gzip compression) as - * produced by {@link #exportWorkspace(WorkspaceIdRequestBody)}. The configurations from the tarball - * may get mutated to be safely included into the current workspace. (the exact same tarball could - * be imported into 2 different workspaces) Note that the provided archived file will be deleted. - * - * @return a status object describing if import was successful or not. - */ - public ImportRead importIntoWorkspace(final ImportRequestBody importRequestBody) { - final File archive = configDumpImporter.getArchiveResource(importRequestBody.getResourceId()); - try { - return importInternal( - () -> configDumpImporter.importIntoWorkspace(version, importRequestBody.getWorkspaceId(), archive)); - } finally { - configDumpImporter.deleteArchiveResource(importRequestBody.getResourceId()); - } - } - - private ImportRead importInternal(final importCall importCall) { - ImportRead result; - try { - importCall.importData(); - result = new ImportRead().status(StatusEnum.SUCCEEDED); - } catch (final Exception e) { - LOGGER.error("Import failed", e); - result = new ImportRead().status(StatusEnum.FAILED).reason(e.getMessage()); - } - - return result; - } - - public interface importCall { - - void importData() throws IOException, JsonValidationException, ConfigNotFoundException; - - } - - public boolean canImportDefinitions() { - return configDumpImporter.canImportDefinitions(); - } - -} diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java index bf64ceeb271c2..32fab92ad6d76 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java @@ -24,7 +24,7 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.util.MoreLists; import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; @@ -200,7 +200,7 @@ private StandardDestinationDefinition destinationDefinitionFromCreate(final Dest destinationDefCreate.getDockerRepository(), destinationDefCreate.getDockerImageTag()); - final AirbyteVersion airbyteProtocolVersion = AirbyteProtocolVersion.getWithDefault(spec.getProtocolVersion()); + final Version airbyteProtocolVersion = AirbyteProtocolVersion.getWithDefault(spec.getProtocolVersion()); final UUID id = uuidSupplier.get(); final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() @@ -234,7 +234,7 @@ public DestinationDefinitionRead updateDestinationDefinition(final DestinationDe ? ApiPojoConverters.actorDefResourceReqsToInternal(destinationDefinitionUpdate.getResourceRequirements()) : currentDestination.getResourceRequirements(); - final AirbyteVersion airbyteProtocolVersion = AirbyteProtocolVersion.getWithDefault(spec.getProtocolVersion()); + final Version airbyteProtocolVersion = AirbyteProtocolVersion.getWithDefault(spec.getProtocolVersion()); final StandardDestinationDefinition newDestination = new StandardDestinationDefinition() .withDestinationDefinitionId(currentDestination.getDestinationDefinitionId()) diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java index d6a2111eded8d..27ec317b83c57 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java @@ -58,7 +58,7 @@ public class DestinationHandler { this.configRepository = configRepository; this.secretsRepositoryReader = secretsRepositoryReader; this.secretsRepositoryWriter = secretsRepositoryWriter; - this.validator = integrationSchemaValidation; + validator = integrationSchemaValidation; this.connectionsHandler = connectionsHandler; this.uuidGenerator = uuidGenerator; this.configurationUpdate = configurationUpdate; @@ -78,7 +78,6 @@ public DestinationHandler(final ConfigRepository configRepository, connectionsHandler, UUID::randomUUID, JsonSecretsProcessor.builder() - .maskSecrets(true) .copySecrets(true) .build(), new ConfigurationUpdate(configRepository, secretsRepositoryReader)); @@ -309,7 +308,8 @@ protected static DestinationRead toDestinationRead(final DestinationConnection d .destinationDefinitionId(destinationConnection.getDestinationDefinitionId()) .connectionConfiguration(destinationConnection.getConfiguration()) .name(destinationConnection.getName()) - .destinationName(standardDestinationDefinition.getName()); + .destinationName(standardDestinationDefinition.getName()) + .icon(DestinationDefinitionsHandler.loadIcon(standardDestinationDefinition.getIcon())); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java index 027eb3c3c4f46..2b092bf525421 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java @@ -25,7 +25,7 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.util.MoreLists; import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; @@ -204,7 +204,7 @@ private StandardSourceDefinition sourceDefinitionFromCreate(final SourceDefiniti throws IOException { final ConnectorSpecification spec = getSpecForImage(sourceDefinitionCreate.getDockerRepository(), sourceDefinitionCreate.getDockerImageTag()); - final AirbyteVersion airbyteProtocolVersion = AirbyteProtocolVersion.getWithDefault(spec.getProtocolVersion()); + final Version airbyteProtocolVersion = AirbyteProtocolVersion.getWithDefault(spec.getProtocolVersion()); final UUID id = uuidSupplier.get(); return new StandardSourceDefinition() @@ -237,7 +237,7 @@ public SourceDefinitionRead updateSourceDefinition(final SourceDefinitionUpdate ? ApiPojoConverters.actorDefResourceReqsToInternal(sourceDefinitionUpdate.getResourceRequirements()) : currentSourceDefinition.getResourceRequirements(); - final AirbyteVersion airbyteProtocolVersion = AirbyteProtocolVersion.getWithDefault(spec.getProtocolVersion()); + final Version airbyteProtocolVersion = AirbyteProtocolVersion.getWithDefault(spec.getProtocolVersion()); final StandardSourceDefinition newSource = new StandardSourceDefinition() .withSourceDefinitionId(currentSourceDefinition.getSourceDefinitionId()) diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceHandler.java index dd21d18ab80d0..57c751e3e589c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceHandler.java @@ -56,7 +56,7 @@ public class SourceHandler { this.configRepository = configRepository; this.secretsRepositoryReader = secretsRepositoryReader; this.secretsRepositoryWriter = secretsRepositoryWriter; - this.validator = integrationSchemaValidation; + validator = integrationSchemaValidation; this.connectionsHandler = connectionsHandler; this.uuidGenerator = uuidGenerator; this.configurationUpdate = configurationUpdate; @@ -76,7 +76,6 @@ public SourceHandler(final ConfigRepository configRepository, connectionsHandler, UUID::randomUUID, JsonSecretsProcessor.builder() - .maskSecrets(true) .copySecrets(true) .build(), new ConfigurationUpdate(configRepository, secretsRepositoryReader)); @@ -316,7 +315,8 @@ protected static SourceRead toSourceRead(final SourceConnection sourceConnection .workspaceId(sourceConnection.getWorkspaceId()) .sourceDefinitionId(sourceConnection.getSourceDefinitionId()) .connectionConfiguration(sourceConnection.getConfiguration()) - .name(sourceConnection.getName()); + .name(sourceConnection.getName()) + .icon(SourceDefinitionsHandler.loadIcon(standardSourceDefinition.getIcon())); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java index 27bd3e9b4de3f..28293cfadce13 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @@ -31,6 +31,7 @@ import io.airbyte.api.model.generated.StreamDescriptor; import io.airbyte.api.model.generated.StreamTransform; import io.airbyte.api.model.generated.WebBackendConnectionCreate; +import io.airbyte.api.model.generated.WebBackendConnectionListItem; import io.airbyte.api.model.generated.WebBackendConnectionRead; import io.airbyte.api.model.generated.WebBackendConnectionReadList; import io.airbyte.api.model.generated.WebBackendConnectionRequestBody; @@ -42,9 +43,11 @@ import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.MoreBooleans; +import io.airbyte.config.StandardSync; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.server.converters.ApiPojoConverters; import io.airbyte.server.handlers.helpers.CatalogConverter; import io.airbyte.server.scheduler.EventRunner; import io.airbyte.validation.json.JsonValidationException; @@ -96,17 +99,20 @@ public ConnectionStateType getStateType(final ConnectionIdRequestBody connection public WebBackendConnectionReadList webBackendListConnectionsForWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) throws ConfigNotFoundException, IOException, JsonValidationException { - final List reads = Lists.newArrayList(); - for (final ConnectionRead connection : connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody).getConnections()) { - reads.add(buildWebBackendConnectionRead(connection)); + final List connectionItems = Lists.newArrayList(); + + // passing 'false' so that deleted connections are not included + for (final StandardSync standardSync : configRepository.listWorkspaceStandardSyncs(workspaceIdRequestBody.getWorkspaceId(), false)) { + connectionItems.add(buildWebBackendConnectionListItem(standardSync)); } - return new WebBackendConnectionReadList().connections(reads); + + return new WebBackendConnectionReadList().connections(connectionItems); } private WebBackendConnectionRead buildWebBackendConnectionRead(final ConnectionRead connectionRead) throws ConfigNotFoundException, IOException, JsonValidationException { - final SourceRead source = getSourceRead(connectionRead); - final DestinationRead destination = getDestinationRead(connectionRead); + final SourceRead source = getSourceRead(connectionRead.getSourceId()); + final DestinationRead destination = getDestinationRead(connectionRead.getDestinationId()); final OperationReadList operations = getOperationReadList(connectionRead); final Optional latestSyncJob = jobHistoryHandler.getLatestSyncJob(connectionRead.getConnectionId()); final Optional latestRunningSyncJob = jobHistoryHandler.getLatestRunningSyncJob(connectionRead.getConnectionId()); @@ -124,14 +130,42 @@ private WebBackendConnectionRead buildWebBackendConnectionRead(final ConnectionR return webBackendConnectionRead; } - private SourceRead getSourceRead(final ConnectionRead connectionRead) throws JsonValidationException, IOException, ConfigNotFoundException { - final SourceIdRequestBody sourceIdRequestBody = new SourceIdRequestBody().sourceId(connectionRead.getSourceId()); + private WebBackendConnectionListItem buildWebBackendConnectionListItem(final StandardSync standardSync) + throws JsonValidationException, ConfigNotFoundException, IOException { + final SourceRead source = getSourceRead(standardSync.getSourceId()); + final DestinationRead destination = getDestinationRead(standardSync.getDestinationId()); + final Optional latestSyncJob = jobHistoryHandler.getLatestSyncJob(standardSync.getConnectionId()); + final Optional latestRunningSyncJob = jobHistoryHandler.getLatestRunningSyncJob(standardSync.getConnectionId()); + + final WebBackendConnectionListItem listItem = new WebBackendConnectionListItem() + .connectionId(standardSync.getConnectionId()) + .sourceId(standardSync.getSourceId()) + .destinationId(standardSync.getDestinationId()) + .status(ApiPojoConverters.toApiStatus(standardSync.getStatus())) + .name(standardSync.getName()) + .scheduleType(ApiPojoConverters.toApiConnectionScheduleType(standardSync)) + .scheduleData(ApiPojoConverters.toApiConnectionScheduleData(standardSync)) + .source(source) + .destination(destination); + + listItem.setIsSyncing(latestRunningSyncJob.isPresent()); + + latestSyncJob.ifPresent(job -> { + listItem.setLatestSyncJobCreatedAt(job.getCreatedAt()); + listItem.setLatestSyncJobStatus(job.getStatus()); + }); + + return listItem; + } + + private SourceRead getSourceRead(final UUID sourceId) throws JsonValidationException, IOException, ConfigNotFoundException { + final SourceIdRequestBody sourceIdRequestBody = new SourceIdRequestBody().sourceId(sourceId); return sourceHandler.getSource(sourceIdRequestBody); } - private DestinationRead getDestinationRead(final ConnectionRead connectionRead) + private DestinationRead getDestinationRead(final UUID destinationId) throws JsonValidationException, IOException, ConfigNotFoundException { - final DestinationIdRequestBody destinationIdRequestBody = new DestinationIdRequestBody().destinationId(connectionRead.getDestinationId()); + final DestinationIdRequestBody destinationIdRequestBody = new DestinationIdRequestBody().destinationId(destinationId); return destinationHandler.getDestination(destinationIdRequestBody); } diff --git a/airbyte-server/src/test/java/io/airbyte/server/ConfigDumpImporterTest.java b/airbyte-server/src/test/java/io/airbyte/server/ConfigDumpImporterTest.java deleted file mode 100644 index 3f86267159ae5..0000000000000 --- a/airbyte-server/src/test/java/io/airbyte/server/ConfigDumpImporterTest.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.server; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.AdditionalMatchers.not; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.AirbyteVersion; -import io.airbyte.config.DestinationConnection; -import io.airbyte.config.SourceConnection; -import io.airbyte.config.StandardDestinationDefinition; -import io.airbyte.config.StandardSourceDefinition; -import io.airbyte.config.StandardSync; -import io.airbyte.config.StandardSync.Status; -import io.airbyte.config.StandardSyncOperation; -import io.airbyte.config.StandardSyncOperation.OperatorType; -import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.config.persistence.SecretsRepositoryReader; -import io.airbyte.config.persistence.SecretsRepositoryWriter; -import io.airbyte.persistence.job.DefaultJobPersistence; -import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.persistence.job.WorkspaceHelper; -import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.validation.json.JsonSchemaValidator; -import io.airbyte.validation.json.JsonValidationException; -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class ConfigDumpImporterTest { - - static final AirbyteVersion TEST_VERSION = new AirbyteVersion("0.0.1-test-version"); - - private ConfigRepository configRepository; - private SecretsRepositoryReader secretsRepositoryReader; - private SecretsRepositoryWriter secretsRepositoryWriter; - private ConfigDumpImporter configDumpImporter; - private ConfigDumpExporter configDumpExporter; - - private UUID workspaceId; - private SourceConnection sourceConnection; - private DestinationConnection destinationConnection; - private StandardSyncOperation operation; - private StandardSync connection; - private ConnectorSpecification emptyConnectorSpec; - - @BeforeEach - void setup() throws IOException, JsonValidationException, ConfigNotFoundException { - configRepository = mock(ConfigRepository.class); - secretsRepositoryReader = mock(SecretsRepositoryReader.class); - secretsRepositoryWriter = mock(SecretsRepositoryWriter.class); - final JobPersistence jobPersistence = mock(JobPersistence.class); - final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); - - emptyConnectorSpec = new ConnectorSpecification().withConnectionSpecification(Jsons.emptyObject()); - - configDumpImporter = new ConfigDumpImporter( - configRepository, - secretsRepositoryWriter, - jobPersistence, - workspaceHelper, - mock(JsonSchemaValidator.class), - true); - - configDumpExporter = new ConfigDumpExporter(configRepository, secretsRepositoryReader, jobPersistence, workspaceHelper); - - workspaceId = UUID.randomUUID(); - when(jobPersistence.getVersion()).thenReturn(Optional.of(TEST_VERSION.serialize())); - - final StandardSourceDefinition standardSourceDefinition = new StandardSourceDefinition() - .withSourceDefinitionId(UUID.randomUUID()) - .withName("test-standard-source") - .withDockerRepository("test") - .withDocumentationUrl("http://doc") - .withIcon("hello") - .withDockerImageTag("dev") - .withSpec(emptyConnectorSpec); - sourceConnection = new SourceConnection() - .withSourceId(UUID.randomUUID()) - .withSourceDefinitionId(standardSourceDefinition.getSourceDefinitionId()) - .withConfiguration(Jsons.emptyObject()) - .withName("test-source") - .withTombstone(false) - .withWorkspaceId(workspaceId); - when(configRepository.listStandardSourceDefinitions(false)) - .thenReturn(List.of(standardSourceDefinition)); - when(configRepository.getStandardSourceDefinition(standardSourceDefinition.getSourceDefinitionId())) - .thenReturn(standardSourceDefinition); - when(configRepository.getSourceConnection(any())) - .thenReturn(sourceConnection); - - final StandardDestinationDefinition standardDestinationDefinition = new StandardDestinationDefinition() - .withDestinationDefinitionId(UUID.randomUUID()) - .withName("test-standard-destination") - .withDockerRepository("test") - .withDocumentationUrl("http://doc") - .withIcon("hello") - .withDockerImageTag("dev") - .withSpec(emptyConnectorSpec); - destinationConnection = new DestinationConnection() - .withDestinationId(UUID.randomUUID()) - .withDestinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) - .withConfiguration(Jsons.emptyObject()) - .withName("test-source") - .withTombstone(false) - .withWorkspaceId(workspaceId); - when(configRepository.listStandardDestinationDefinitions(false)) - .thenReturn(List.of(standardDestinationDefinition)); - when(configRepository.getStandardDestinationDefinition(standardDestinationDefinition.getDestinationDefinitionId())) - .thenReturn(standardDestinationDefinition); - when(configRepository.getDestinationConnection(any())) - .thenReturn(destinationConnection); - - operation = new StandardSyncOperation() - .withOperationId(UUID.randomUUID()) - .withName("test-operation") - .withWorkspaceId(workspaceId) - .withTombstone(false) - .withOperatorType(OperatorType.DBT); - when(configRepository.getStandardSyncOperation(any())) - .thenReturn(operation); - - connection = new StandardSync() - .withConnectionId(UUID.randomUUID()) - .withSourceId(sourceConnection.getSourceId()) - .withDestinationId(destinationConnection.getDestinationId()) - .withOperationIds(List.of(operation.getOperationId())) - .withName("test-sync") - .withStatus(Status.ACTIVE); - - when(workspaceHelper.getWorkspaceForConnection(sourceConnection.getSourceId(), destinationConnection.getDestinationId())) - .thenReturn(workspaceId); - } - - @Test - void testImportIntoWorkspaceWithConflicts() throws JsonValidationException, ConfigNotFoundException, IOException { - when(secretsRepositoryReader.listSourceConnectionWithSecrets()) - .thenReturn(List.of(sourceConnection, - new SourceConnection() - .withSourceId(UUID.randomUUID()) - .withWorkspaceId(UUID.randomUUID()))); - when(secretsRepositoryReader.listDestinationConnectionWithSecrets()) - .thenReturn(List.of(destinationConnection, - new DestinationConnection() - .withDestinationId(UUID.randomUUID()) - .withWorkspaceId(UUID.randomUUID()))); - when(configRepository.listStandardSyncOperations()) - .thenReturn(List.of(operation, - new StandardSyncOperation() - .withOperationId(UUID.randomUUID()) - .withWorkspaceId(UUID.randomUUID()))); - when(configRepository.listStandardSyncs()) - .thenReturn(List.of(connection)); - final File archive = configDumpExporter.exportWorkspace(workspaceId); - - final UUID newWorkspaceId = UUID.randomUUID(); - configDumpImporter.importIntoWorkspace(TEST_VERSION, newWorkspaceId, archive); - - verify(secretsRepositoryWriter) - .writeSourceConnection( - Jsons.clone(sourceConnection).withWorkspaceId(newWorkspaceId).withSourceId(not(eq(sourceConnection.getSourceId()))), - eq(emptyConnectorSpec)); - verify(secretsRepositoryWriter).writeDestinationConnection( - Jsons.clone(destinationConnection).withWorkspaceId(newWorkspaceId).withDestinationId(not(eq(destinationConnection.getDestinationId()))), - eq(emptyConnectorSpec)); - verify(configRepository) - .writeStandardSyncOperation(Jsons.clone(operation).withWorkspaceId(newWorkspaceId).withOperationId(not(eq(operation.getOperationId())))); - verify(configRepository).writeStandardSync(Jsons.clone(connection).withConnectionId(not(eq(connection.getConnectionId())))); - } - - @Test - void testImportIntoWorkspaceWithoutConflicts() throws JsonValidationException, ConfigNotFoundException, IOException { - when(secretsRepositoryReader.listSourceConnectionWithSecrets()) - // First called for export - .thenReturn(List.of(sourceConnection, - new SourceConnection() - .withSourceId(UUID.randomUUID()) - .withWorkspaceId(UUID.randomUUID()))) - // then called for import - .thenReturn(List.of(new SourceConnection() - .withSourceId(UUID.randomUUID()) - .withWorkspaceId(UUID.randomUUID()))); - when(secretsRepositoryReader.listDestinationConnectionWithSecrets()) - // First called for export - .thenReturn(List.of(destinationConnection, - new DestinationConnection() - .withDestinationId(UUID.randomUUID()) - .withWorkspaceId(UUID.randomUUID()))) - // then called for import - .thenReturn(List.of(new DestinationConnection() - .withDestinationId(UUID.randomUUID()) - .withWorkspaceId(UUID.randomUUID()))); - when(configRepository.listStandardSyncOperations()) - // First called for export - .thenReturn(List.of(operation, - new StandardSyncOperation() - .withOperationId(UUID.randomUUID()) - .withWorkspaceId(UUID.randomUUID()))) - // then called for import - .thenReturn(List.of(new StandardSyncOperation() - .withOperationId(UUID.randomUUID()) - .withWorkspaceId(UUID.randomUUID()))); - when(configRepository.listStandardSyncs()) - // First called for export - .thenReturn(List.of(connection)) - // then called for import - .thenReturn(List.of()); - final File archive = configDumpExporter.exportWorkspace(workspaceId); - - final UUID newWorkspaceId = UUID.randomUUID(); - configDumpImporter.importIntoWorkspace(TEST_VERSION, newWorkspaceId, archive); - - verify(secretsRepositoryWriter) - .writeSourceConnection(Jsons.clone(sourceConnection).withWorkspaceId(newWorkspaceId), emptyConnectorSpec); - verify(secretsRepositoryWriter) - .writeDestinationConnection(Jsons.clone(destinationConnection).withWorkspaceId(newWorkspaceId), emptyConnectorSpec); - verify(configRepository).writeStandardSyncOperation(Jsons.clone(operation).withWorkspaceId(newWorkspaceId)); - verify(configRepository).writeStandardSync(connection); - } - - @Test - void testReplaceDeploymentMetadata() throws Exception { - final UUID oldDeploymentUuid = UUID.randomUUID(); - final UUID newDeploymentUuid = UUID.randomUUID(); - - final JsonNode airbyteVersion = Jsons.deserialize("{\"key\":\"airbyte_version\",\"value\":\"dev\"}"); - final JsonNode serverUuid = Jsons.deserialize("{\"key\":\"server_uuid\",\"value\":\"e895a584-7dbf-48ce-ace6-0bc9ea570c34\"}"); - final JsonNode date = Jsons.deserialize("{\"key\":\"date\",\"value\":\"1956-08-17\"}"); - final JsonNode oldDeploymentId = Jsons.deserialize( - String.format("{\"key\":\"%s\",\"value\":\"%s\"}", DefaultJobPersistence.DEPLOYMENT_ID_KEY, oldDeploymentUuid)); - final JsonNode newDeploymentId = Jsons.deserialize( - String.format("{\"key\":\"%s\",\"value\":\"%s\"}", DefaultJobPersistence.DEPLOYMENT_ID_KEY, newDeploymentUuid)); - - final JobPersistence jobPersistence = mock(JobPersistence.class); - - // when new deployment id does not exist, the old deployment id is removed - when(jobPersistence.getDeployment()).thenReturn(Optional.empty()); - final Stream inputStream1 = Stream.of(airbyteVersion, serverUuid, date, oldDeploymentId); - final Stream outputStream1 = ConfigDumpImporter.replaceDeploymentMetadata(jobPersistence, inputStream1); - final Stream expectedStream1 = Stream.of(airbyteVersion, serverUuid, date); - assertEquals(expectedStream1.collect(Collectors.toList()), outputStream1.collect(Collectors.toList())); - - // when new deployment id exists, the old deployment id is replaced with the new one - when(jobPersistence.getDeployment()).thenReturn(Optional.of(newDeploymentUuid)); - final Stream inputStream2 = Stream.of(airbyteVersion, serverUuid, date, oldDeploymentId); - final Stream outputStream2 = ConfigDumpImporter.replaceDeploymentMetadata(jobPersistence, inputStream2); - final Stream expectedStream2 = Stream.of(airbyteVersion, serverUuid, date, newDeploymentId); - assertEquals(expectedStream2.collect(Collectors.toList()), outputStream2.collect(Collectors.toList())); - } - -} diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java index af371faf2ffd2..0bba057a25aaa 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/ConfigurationApiTest.java @@ -4,17 +4,15 @@ package io.airbyte.server.apis; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import io.airbyte.analytics.TrackingClient; -import io.airbyte.commons.io.FileTtlManager; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.Configs; import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.helpers.LogConfigs; -import io.airbyte.config.persistence.ConfigPersistence; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.SecretsRepositoryReader; import io.airbyte.config.persistence.SecretsRepositoryWriter; @@ -39,11 +37,9 @@ void testImportDefinitions() { final ConfigurationApi configurationApi = new ConfigurationApi( mock(ConfigRepository.class), mock(JobPersistence.class), - mock(ConfigPersistence.class), mock(SecretsRepositoryReader.class), mock(SecretsRepositoryWriter.class), mock(SynchronousSchedulerClient.class), - mock(FileTtlManager.class), mock(Database.class), mock(Database.class), mock(StatePersistence.class), @@ -56,7 +52,8 @@ void testImportDefinitions() { mock(EventRunner.class), mock(Flyway.class), mock(Flyway.class)); - assertTrue(configurationApi.canImportDefinitions()); + + assertFalse(configurationApi.getHealthCheck().getAvailable()); } } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java deleted file mode 100644 index e68c6aa7ce6ca..0000000000000 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java +++ /dev/null @@ -1,454 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.server.handlers; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.api.model.generated.ImportRead; -import io.airbyte.api.model.generated.ImportRead.StatusEnum; -import io.airbyte.api.model.generated.ImportRequestBody; -import io.airbyte.api.model.generated.UploadRead; -import io.airbyte.api.model.generated.WorkspaceIdRequestBody; -import io.airbyte.commons.io.FileTtlManager; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; -import io.airbyte.commons.version.AirbyteVersion; -import io.airbyte.config.ActorCatalog; -import io.airbyte.config.ConfigSchema; -import io.airbyte.config.DestinationConnection; -import io.airbyte.config.Notification; -import io.airbyte.config.Notification.NotificationType; -import io.airbyte.config.SlackNotificationConfiguration; -import io.airbyte.config.SourceConnection; -import io.airbyte.config.StandardDestinationDefinition; -import io.airbyte.config.StandardSourceDefinition; -import io.airbyte.config.StandardSourceDefinition.SourceType; -import io.airbyte.config.StandardSync; -import io.airbyte.config.StandardWorkspace; -import io.airbyte.config.init.YamlSeedConfigPersistence; -import io.airbyte.config.persistence.ConfigPersistence; -import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.config.persistence.DatabaseConfigPersistence; -import io.airbyte.config.persistence.SecretsRepositoryReader; -import io.airbyte.config.persistence.SecretsRepositoryWriter; -import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; -import io.airbyte.config.persistence.split_secrets.NoOpSecretsHydrator; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.instance.test.TestDatabaseProviders; -import io.airbyte.persistence.job.DefaultJobPersistence; -import io.airbyte.persistence.job.JobPersistence; -import io.airbyte.persistence.job.WorkspaceHelper; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.test.utils.DatabaseConnectionHelper; -import io.airbyte.validation.json.JsonValidationException; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import javax.sql.DataSource; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; -import org.testcontainers.shaded.org.apache.commons.io.FileUtils; - -class ArchiveHandlerTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(ArchiveHandlerTest.class); - - private static final AirbyteVersion VERSION = new AirbyteVersion("0.6.8"); - private static PostgreSQLContainer container; - - private DataSource dataSource; - private DSLContext dslContext; - private Database jobDatabase; - private Database configDatabase; - private JobPersistence jobPersistence; - private SecretsRepositoryReader secretsRepositoryReader; - private SecretsRepositoryWriter secretsRepositoryWriter; - private ConfigPersistence configPersistence; - private ConfigPersistence seedPersistence; - private JsonSecretsProcessor jsonSecretsProcessor; - private ConfigRepository configRepository; - private ArchiveHandler archiveHandler; - private WorkspaceHelper workspaceHelper; - - private static class NoOpFileTtlManager extends FileTtlManager { - - public NoOpFileTtlManager() { - super(1L, TimeUnit.MINUTES, 1L); - } - - @Override - public void register(final Path path) {} - - } - - @BeforeAll - static void dbSetup() { - container = new PostgreSQLContainer<>("postgres:13-alpine") - .withDatabaseName("airbyte") - .withUsername("docker") - .withPassword("docker"); - container.start(); - } - - @AfterAll - static void dbDown() { - container.close(); - } - - @BeforeEach - void setup() throws Exception { - dataSource = DatabaseConnectionHelper.createDataSource(container); - dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); - final TestDatabaseProviders databaseProviders = new TestDatabaseProviders(dataSource, dslContext); - jobDatabase = databaseProviders.createNewJobsDatabase(); - configDatabase = databaseProviders.createNewConfigsDatabase(); - jobPersistence = new DefaultJobPersistence(jobDatabase); - seedPersistence = new YamlSeedConfigPersistence(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS); - jsonSecretsProcessor = JsonSecretsProcessor.builder() - .maskSecrets(false) - .copySecrets(false) - .build(); - configPersistence = new DatabaseConfigPersistence(jobDatabase, jsonSecretsProcessor); - configPersistence.replaceAllConfigs(Collections.emptyMap(), false); - configPersistence.loadData(seedPersistence); - configRepository = new ConfigRepository(configPersistence, configDatabase); - secretsRepositoryReader = new SecretsRepositoryReader(configRepository, new NoOpSecretsHydrator()); - secretsRepositoryWriter = new SecretsRepositoryWriter(configRepository, Optional.empty(), Optional.empty()); - - jobPersistence.setVersion(VERSION.serialize()); - - workspaceHelper = new WorkspaceHelper(configRepository, jobPersistence); - - archiveHandler = new ArchiveHandler( - VERSION, - configRepository, - secretsRepositoryReader, - secretsRepositoryWriter, - jobPersistence, - seedPersistence, - workspaceHelper, - new NoOpFileTtlManager(), - true); - } - - @AfterEach - void tearDown() throws Exception { - dslContext.close(); - DataSourceFactory.close(dataSource); - } - - /** - * After exporting and importing, the configs should remain the same. - */ - @Test - void testFullExportImportRoundTrip() throws Exception { - assertSameConfigDump(seedPersistence.dumpConfigs(), secretsRepositoryReader.dumpConfigsWithSecrets()); - - // Export the configs. - File archive = archiveHandler.exportData(); - - // After deleting the configs, the dump becomes empty. - configPersistence.replaceAllConfigs(Collections.emptyMap(), false); - assertSameConfigDump(Collections.emptyMap(), secretsRepositoryReader.dumpConfigsWithSecrets()); - - // After importing the configs, the dump is restored. - assertTrue(archive.exists()); - final ImportRead importResult = archiveHandler.importData(archive); - assertFalse(archive.exists()); - assertEquals(StatusEnum.SUCCEEDED, importResult.getStatus()); - assertSameConfigDump(seedPersistence.dumpConfigs(), secretsRepositoryReader.dumpConfigsWithSecrets()); - - // When a connector definition is in use, it will not be updated. - final UUID sourceS3DefinitionId = UUID.fromString("69589781-7828-43c5-9f63-8925b1c1ccc2"); - final String sourceS3DefinitionVersion = "0.0.0"; - final StandardSourceDefinition sourceS3Definition = seedPersistence.getConfig( - ConfigSchema.STANDARD_SOURCE_DEFINITION, - sourceS3DefinitionId.toString(), - StandardSourceDefinition.class) - // This source definition is on an old version - .withDockerImageTag(sourceS3DefinitionVersion) - .withTombstone(false); - final Notification notification = new Notification() - .withNotificationType(NotificationType.SLACK) - .withSendOnFailure(true) - .withSendOnSuccess(true) - .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook("webhook-url")); - final StandardWorkspace workspace = new StandardWorkspace() - .withWorkspaceId(UUID.randomUUID()) - .withCustomerId(UUID.randomUUID()) - .withName("test-workspace") - .withSlug("random-string") - .withEmail("abc@xyz.com") - .withInitialSetupComplete(true) - .withAnonymousDataCollection(true) - .withNews(true) - .withSecurityUpdates(true) - .withDisplaySetupWizard(true) - .withTombstone(false) - .withNotifications(Collections.singletonList(notification)) - .withFirstCompletedSync(true) - .withFeedbackDone(true); - final SourceConnection source = new SourceConnection() - .withSourceDefinitionId(sourceS3DefinitionId) - .withSourceId(UUID.randomUUID()) - .withWorkspaceId(workspace.getWorkspaceId()) - .withName("Test source") - .withConfiguration(Jsons.deserialize("{}")) - .withTombstone(false); - - final StandardDestinationDefinition DESTINATION_S3 = new StandardDestinationDefinition() - .withName("S3") - .withDestinationDefinitionId(UUID.fromString("4816b78f-1489-44c1-9060-4b19d5fa9362")) - .withDockerRepository("airbyte/destination-s3") - .withDockerImageTag("0.1.12") - .withSpec(sourceS3Definition.getSpec()) - .withDocumentationUrl("https://docs.airbyte.io/integrations/destinations/s3") - .withTombstone(false); - - final DestinationConnection destination = new DestinationConnection() - .withName("Destination") - .withDestinationId(UUID.randomUUID()) - .withDestinationDefinitionId(DESTINATION_S3.getDestinationDefinitionId()) - .withConfiguration(Jsons.deserialize("{}")) - .withWorkspaceId(workspace.getWorkspaceId()); - - final ActorCatalog actorCatalog = new ActorCatalog() - .withId(UUID.randomUUID()) - .withCatalog(Jsons.deserialize("{}")) - .withCatalogHash(""); - - final StandardSync sync = new StandardSync() - .withName("Connection") - .withConnectionId(UUID.randomUUID()) - .withSourceId(source.getSourceId()) - .withDestinationId(destination.getDestinationId()) - .withCatalog(new ConfiguredAirbyteCatalog().withStreams(List.of())) - .withSourceCatalogId(actorCatalog.getId()) - .withManual(true); - - // Write source connection and an old source definition. - configPersistence.writeConfig(ConfigSchema.STANDARD_WORKSPACE, workspace.getWorkspaceId().toString(), workspace); - configPersistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, sourceS3DefinitionId.toString(), sourceS3Definition); - configPersistence.writeConfig(ConfigSchema.SOURCE_CONNECTION, source.getSourceId().toString(), source); - - configPersistence.writeConfig(ConfigSchema.ACTOR_CATALOG, actorCatalog.getId().toString(), actorCatalog); - configPersistence.writeConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, DESTINATION_S3.getDestinationDefinitionId().toString(), - DESTINATION_S3); - configPersistence.writeConfig(ConfigSchema.DESTINATION_CONNECTION, destination.getDestinationId().toString(), destination); - - configPersistence.writeConfig(ConfigSchema.STANDARD_SYNC, sync.getConnectionId().toString(), sync); - - // Export, wipe, and import the configs. - archive = archiveHandler.exportData(); - configPersistence.replaceAllConfigs(Collections.emptyMap(), false); - archiveHandler.importData(archive); - - // The version has not changed. - final StandardSourceDefinition actualS3Definition = configPersistence.getConfig( - ConfigSchema.STANDARD_SOURCE_DEFINITION, - sourceS3DefinitionId.toString(), - StandardSourceDefinition.class); - assertEquals(sourceS3DefinitionVersion, actualS3Definition.getDockerImageTag()); - } - - @Test - void testLightWeightExportImportRoundTrip() throws Exception { - assertSameConfigDump(seedPersistence.dumpConfigs(), secretsRepositoryReader.dumpConfigsWithSecrets()); - - // Insert some workspace data - final UUID workspaceId = UUID.randomUUID(); - setupTestData(workspaceId); - final Map> workspaceDump = secretsRepositoryReader.dumpConfigsWithSecrets(); - - // Insert some other workspace data - setupTestData(UUID.randomUUID()); - - // Export the first workspace configs - File archive = archiveHandler.exportWorkspace(new WorkspaceIdRequestBody().workspaceId(workspaceId)); - final File secondArchive = Files.createTempFile("tests", "archive").toFile(); - FileUtils.copyFile(archive, secondArchive); - - // After deleting all the configs, the dump becomes empty. - configPersistence.replaceAllConfigs(Collections.emptyMap(), false); - assertSameConfigDump(Collections.emptyMap(), secretsRepositoryReader.dumpConfigsWithSecrets()); - - // Restore default seed data - configPersistence.loadData(seedPersistence); - assertSameConfigDump(seedPersistence.dumpConfigs(), secretsRepositoryReader.dumpConfigsWithSecrets()); - - setupWorkspaceData(workspaceId); - - // After importing the configs, the first workspace is restored. - assertTrue(archive.exists()); - UploadRead uploadRead = archiveHandler.uploadArchiveResource(archive); - assertFalse(archive.exists()); - assertEquals(UploadRead.StatusEnum.SUCCEEDED, uploadRead.getStatus()); - ImportRead importResult = archiveHandler.importIntoWorkspace(new ImportRequestBody() - .resourceId(uploadRead.getResourceId()) - .workspaceId(workspaceId)); - assertEquals(StatusEnum.SUCCEEDED, importResult.getStatus()); - assertSameConfigDump(workspaceDump, secretsRepositoryReader.dumpConfigsWithSecrets()); - - // we modify first workspace - setupTestData(workspaceId); - final Map> secondWorkspaceDump = secretsRepositoryReader.dumpConfigsWithSecrets(); - - final UUID secondWorkspaceId = UUID.randomUUID(); - setupWorkspaceData(secondWorkspaceId); - - // the archive is importing again in another workspace - final UploadRead secondUploadRead = archiveHandler.uploadArchiveResource(secondArchive); - assertEquals(UploadRead.StatusEnum.SUCCEEDED, secondUploadRead.getStatus()); - final ImportRead secondImportResult = archiveHandler.importIntoWorkspace(new ImportRequestBody() - .resourceId(secondUploadRead.getResourceId()) - .workspaceId(secondWorkspaceId)); - assertEquals(StatusEnum.SUCCEEDED, secondImportResult.getStatus()); - - final UUID secondSourceId = secretsRepositoryReader.listSourceConnectionWithSecrets() - .stream() - .filter(sourceConnection -> secondWorkspaceId.equals(sourceConnection.getWorkspaceId())) - .map(SourceConnection::getSourceId) - .collect(Collectors.toList()).get(0); - - final StandardSourceDefinition standardSourceDefinition = new StandardSourceDefinition() - .withSourceDefinitionId(UUID.randomUUID()) - .withSourceType(SourceType.API) - .withName("random-source-1") - .withDockerImageTag("tag-1") - .withDockerRepository("repository-1") - .withDocumentationUrl("documentation-url-1") - .withIcon("icon-1") - .withSpec(new ConnectorSpecification()) - .withTombstone(false); - - final SourceConnection sourceConnection = new SourceConnection() - .withWorkspaceId(secondWorkspaceId) - .withSourceId(secondSourceId) - .withName("Some new names") - .withSourceDefinitionId(standardSourceDefinition.getSourceDefinitionId()) - .withTombstone(false) - .withConfiguration(Jsons.emptyObject()); - - final ConnectorSpecification emptyConnectorSpec = mock(ConnectorSpecification.class); - when(emptyConnectorSpec.getConnectionSpecification()).thenReturn(Jsons.emptyObject()); - - configRepository.writeStandardSourceDefinition(standardSourceDefinition); - secretsRepositoryWriter.writeSourceConnection(sourceConnection, emptyConnectorSpec); - - // check that first workspace is unchanged even though modifications were made to second workspace - // (that contains similar connections from importing the same archive) - archive = archiveHandler.exportWorkspace(new WorkspaceIdRequestBody().workspaceId(workspaceId)); - configPersistence.replaceAllConfigs(Collections.emptyMap(), false); - configPersistence.loadData(seedPersistence); - setupWorkspaceData(workspaceId); - uploadRead = archiveHandler.uploadArchiveResource(archive); - assertEquals(UploadRead.StatusEnum.SUCCEEDED, uploadRead.getStatus()); - importResult = archiveHandler.importIntoWorkspace(new ImportRequestBody() - .resourceId(uploadRead.getResourceId()) - .workspaceId(workspaceId)); - assertEquals(StatusEnum.SUCCEEDED, importResult.getStatus()); - assertSameConfigDump(secondWorkspaceDump, secretsRepositoryReader.dumpConfigsWithSecrets()); - } - - private void setupWorkspaceData(final UUID workspaceId) throws IOException, JsonValidationException { - configPersistence.writeConfig(ConfigSchema.STANDARD_WORKSPACE, workspaceId.toString(), new StandardWorkspace() - .withWorkspaceId(workspaceId) - .withName("test-workspace") - .withSlug(workspaceId.toString()) - .withInitialSetupComplete(false) - .withTombstone(false)); - } - - private void setupTestData(final UUID workspaceId) throws JsonValidationException, IOException { - // Fill up with some configurations - setupWorkspaceData(workspaceId); - final UUID sourceid = UUID.randomUUID(); - configPersistence.writeConfig(ConfigSchema.SOURCE_CONNECTION, sourceid.toString(), new SourceConnection() - .withSourceId(sourceid) - .withWorkspaceId(workspaceId) - .withSourceDefinitionId(UUID.fromString("ef69ef6e-aa7f-4af1-a01d-ef775033524e")) // GitHub source definition - .withName("test-source") - .withConfiguration(Jsons.jsonNode(ImmutableMap.of("start_date", "2021-03-01T00:00:00Z", "repository", "airbytehq/airbyte"))) - .withTombstone(false)); - final UUID destinationId = UUID.randomUUID(); - configPersistence.writeConfig(ConfigSchema.DESTINATION_CONNECTION, destinationId.toString(), new DestinationConnection() - .withDestinationId(destinationId) - .withWorkspaceId(workspaceId) - .withDestinationDefinitionId(UUID.fromString("079d5540-f236-4294-ba7c-ade8fd918496")) // BigQuery destination definition - .withName("test-destination") - .withConfiguration(Jsons.jsonNode(ImmutableMap.of("project_id", "project", "dataset_id", "dataset"))) - .withTombstone(false)); - } - - private void assertSameConfigDump(final Map> expected, final Map> actual) { - assertEquals(expected.keySet(), actual.keySet(), - String.format("The expected (%s) vs actual (%s) streams does not match", expected.size(), actual.size())); - for (final String stream : expected.keySet()) { - LOGGER.info("Checking stream {}", stream); - // assertEquals cannot correctly check the equality of two maps with stream values, - // so streams are converted to sets before being compared. - final Set expectedRecords = expected.get(stream).collect(Collectors.toSet()); - final Set actualRecords = actual.get(stream).collect(Collectors.toSet()); - for (final var expectedRecord : expectedRecords) { - assertTrue( - backwardCompatibleContains(actualRecords, expectedRecord), - String.format( - "\n Expected record was not found:\n%s\n Actual records were:\n%s\n", - expectedRecord, - Strings.join(actualRecords, "\n"))); - } - assertEquals(expectedRecords.size(), actualRecords.size(), - String.format( - "The expected vs actual records does not match:\n expected records:\n%s\n actual records\n%s\n", - Strings.join(expectedRecords, "\n"), - Strings.join(actualRecords, "\n"))); - } - } - - /* - * The protocol version is currently optional and defaults to 0.2.0 To reflect that today we need to - * support connectors without protocol version in the spec, we add a secondary check with the - * default protocol version - */ - private boolean backwardCompatibleContains(final Set actualRecords, final JsonNode expectedRecord) { - return actualRecords.contains(expectedRecord) || - (!expectedRecord.has("protocolVersion") && actualRecords.contains(cloneWithDefaultVersion(expectedRecord))); - } - - private JsonNode cloneWithDefaultVersion(final JsonNode json) { - final ObjectNode clonedJson = json.deepCopy(); - clonedJson.put("protocolVersion", "0.2.0"); - return clonedJson; - } - -} diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java index 41f9c4561bb4c..6dc096c862e11 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java @@ -5,6 +5,7 @@ package io.airbyte.server.handlers; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -58,6 +59,10 @@ class DestinationHandlerTest { private JsonSecretsProcessor secretsProcessor; private ConnectorSpecification connectorSpecification; + // needs to match name of file in src/test/resources/icons + private static final String ICON = "test-destination.svg"; + private static final String LOADED_ICON = DestinationDefinitionsHandler.loadIcon(ICON); + @SuppressWarnings("unchecked") @BeforeEach void setUp() throws IOException { @@ -78,7 +83,8 @@ void setUp() throws IOException { .withDockerRepository("thebestrepo") .withDockerImageTag("thelatesttag") .withDocumentationUrl("https://wikipedia.org") - .withSpec(connectorSpecification); + .withSpec(connectorSpecification) + .withIcon(ICON); destinationDefinitionSpecificationRead = new DestinationDefinitionSpecificationRead() .connectionSpecification(connectorSpecification.getConnectionSpecification()) @@ -127,7 +133,8 @@ void testCreateDestination() throws JsonValidationException, ConfigNotFoundExcep .workspaceId(destinationConnection.getWorkspaceId()) .destinationId(destinationConnection.getDestinationId()) .connectionConfiguration(DestinationHelpers.getTestDestinationJson()) - .destinationName(standardDestinationDefinition.getName()); + .destinationName(standardDestinationDefinition.getName()) + .icon(LOADED_ICON); assertEquals(expectedDestinationRead, actualDestinationRead); @@ -187,7 +194,8 @@ void testGetDestination() throws JsonValidationException, ConfigNotFoundExceptio .workspaceId(destinationConnection.getWorkspaceId()) .destinationId(destinationConnection.getDestinationId()) .connectionConfiguration(destinationConnection.getConfiguration()) - .destinationName(standardDestinationDefinition.getName()); + .destinationName(standardDestinationDefinition.getName()) + .icon(LOADED_ICON); final DestinationIdRequestBody destinationIdRequestBody = new DestinationIdRequestBody().destinationId(expectedDestinationRead.getDestinationId()); @@ -201,6 +209,10 @@ void testGetDestination() throws JsonValidationException, ConfigNotFoundExceptio final DestinationRead actualDestinationRead = destinationHandler.getDestination(destinationIdRequestBody); assertEquals(expectedDestinationRead, actualDestinationRead); + + // make sure the icon was loaded into actual svg content + assertTrue(expectedDestinationRead.getIcon().startsWith("")); + verify(secretsProcessor) .prepareSecretsForOutput(destinationConnection.getConfiguration(), destinationDefinitionSpecificationRead.getConnectionSpecification()); } @@ -213,7 +225,8 @@ void testListDestinationForWorkspace() throws JsonValidationException, ConfigNot .workspaceId(destinationConnection.getWorkspaceId()) .destinationId(destinationConnection.getDestinationId()) .connectionConfiguration(destinationConnection.getConfiguration()) - .destinationName(standardDestinationDefinition.getName()); + .destinationName(standardDestinationDefinition.getName()) + .icon(LOADED_ICON); final WorkspaceIdRequestBody workspaceIdRequestBody = new WorkspaceIdRequestBody().workspaceId(destinationConnection.getWorkspaceId()); when(configRepository.getDestinationConnection(destinationConnection.getDestinationId())).thenReturn(destinationConnection); @@ -239,7 +252,8 @@ void testSearchDestinations() throws JsonValidationException, ConfigNotFoundExce .workspaceId(destinationConnection.getWorkspaceId()) .destinationId(destinationConnection.getDestinationId()) .connectionConfiguration(destinationConnection.getConfiguration()) - .destinationName(standardDestinationDefinition.getName()); + .destinationName(standardDestinationDefinition.getName()) + .icon(LOADED_ICON); when(configRepository.getDestinationConnection(destinationConnection.getDestinationId())).thenReturn(destinationConnection); when(configRepository.listDestinationConnection()).thenReturn(Lists.newArrayList(destinationConnection)); @@ -270,7 +284,8 @@ void testCloneDestinationWithConfiguration() throws JsonValidationException, Con .workspaceId(clonedConnection.getWorkspaceId()) .destinationId(clonedConnection.getDestinationId()) .connectionConfiguration(clonedConnection.getConfiguration()) - .destinationName(standardDestinationDefinition.getName()); + .destinationName(standardDestinationDefinition.getName()) + .icon(LOADED_ICON); final DestinationRead destinationRead = new DestinationRead() .name(destinationConnection.getName()) .destinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) @@ -309,7 +324,8 @@ void testCloneDestinationWithoutConfiguration() throws JsonValidationException, .workspaceId(clonedConnection.getWorkspaceId()) .destinationId(clonedConnection.getDestinationId()) .connectionConfiguration(clonedConnection.getConfiguration()) - .destinationName(standardDestinationDefinition.getName()); + .destinationName(standardDestinationDefinition.getName()) + .icon(LOADED_ICON); final DestinationRead destinationRead = new DestinationRead() .name(destinationConnection.getName()) .destinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceHandlerTest.java index 664e81bbb1077..c8157a8ae493b 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceHandlerTest.java @@ -5,6 +5,7 @@ package io.airbyte.server.handlers; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -64,6 +65,9 @@ class SourceHandlerTest { private JsonSecretsProcessor secretsProcessor; private ConnectorSpecification connectorSpecification; + // needs to match name of file in src/test/resources/icons + private static final String ICON = "test-source.svg"; + @SuppressWarnings("unchecked") @BeforeEach void setUp() throws IOException { @@ -84,7 +88,8 @@ void setUp() throws IOException { .withDockerRepository("thebestrepo") .withDockerImageTag("thelatesttag") .withDocumentationUrl("https://wikipedia.org") - .withSpec(connectorSpecification); + .withSpec(connectorSpecification) + .withIcon(ICON); sourceDefinitionSpecificationRead = new SourceDefinitionSpecificationRead() .sourceDefinitionId(standardSourceDefinition.getSourceDefinitionId()) @@ -190,6 +195,10 @@ void testGetSource() throws JsonValidationException, ConfigNotFoundException, IO final SourceRead actualSourceRead = sourceHandler.getSource(sourceIdRequestBody); assertEquals(expectedSourceRead, actualSourceRead); + + // make sure the icon was loaded into actual svg content + assertTrue(expectedSourceRead.getIcon().startsWith("")); + verify(secretsProcessor).prepareSecretsForOutput(sourceConnection.getConfiguration(), sourceDefinitionSpecificationRead.getConnectionSpecification()); } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java index a99bac040a274..37e79fed6e93a 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java @@ -26,7 +26,6 @@ import io.airbyte.api.model.generated.ConnectionCreate; import io.airbyte.api.model.generated.ConnectionIdRequestBody; import io.airbyte.api.model.generated.ConnectionRead; -import io.airbyte.api.model.generated.ConnectionReadList; import io.airbyte.api.model.generated.ConnectionSchedule; import io.airbyte.api.model.generated.ConnectionSchedule.TimeUnitEnum; import io.airbyte.api.model.generated.ConnectionState; @@ -56,6 +55,7 @@ import io.airbyte.api.model.generated.SyncMode; import io.airbyte.api.model.generated.SynchronousJobRead; import io.airbyte.api.model.generated.WebBackendConnectionCreate; +import io.airbyte.api.model.generated.WebBackendConnectionListItem; import io.airbyte.api.model.generated.WebBackendConnectionRead; import io.airbyte.api.model.generated.WebBackendConnectionReadList; import io.airbyte.api.model.generated.WebBackendConnectionRequestBody; @@ -106,6 +106,7 @@ class WebBackendConnectionsHandlerTest { private SourceRead sourceRead; private ConnectionRead connectionRead; + private WebBackendConnectionListItem expectedListItem; private OperationReadList operationReadList; private WebBackendConnectionRead expected; private WebBackendConnectionRead expectedWithNewSchema; @@ -119,6 +120,11 @@ class WebBackendConnectionsHandlerTest { private static final String FIELD3 = "field3"; private static final String FIELD5 = "field5"; + // needs to match name of file in src/test/resources/icons + private static final String SOURCE_ICON = "test-source.svg"; + private static final String DESTINATION_ICON = "test-destination.svg"; + private static final String SVG = ""; + @BeforeEach void setup() throws IOException, JsonValidationException, ConfigNotFoundException { connectionsHandler = mock(ConnectionsHandler.class); @@ -142,14 +148,18 @@ void setup() throws IOException, JsonValidationException, ConfigNotFoundExceptio configRepository); final StandardSourceDefinition standardSourceDefinition = SourceDefinitionHelpers.generateSourceDefinition(); + standardSourceDefinition.setIcon(SOURCE_ICON); final SourceConnection source = SourceHelpers.generateSource(UUID.randomUUID()); sourceRead = SourceHelpers.getSourceRead(source, standardSourceDefinition); final StandardDestinationDefinition destinationDefinition = DestinationDefinitionHelpers.generateDestination(); + destinationDefinition.setIcon(DESTINATION_ICON); final DestinationConnection destination = DestinationHelpers.generateDestination(UUID.randomUUID()); final DestinationRead destinationRead = DestinationHelpers.getDestinationRead(destination, destinationDefinition); final StandardSync standardSync = ConnectionHelpers.generateSyncWithSourceId(source.getSourceId()); + when(configRepository.listWorkspaceStandardSyncs(sourceRead.getWorkspaceId(), false)) + .thenReturn(Collections.singletonList(standardSync)); connectionRead = ConnectionHelpers.generateExpectedConnectionRead(standardSync); operationReadList = new OperationReadList() .operations(List.of(new OperationRead() @@ -184,6 +194,14 @@ void setup() throws IOException, JsonValidationException, ConfigNotFoundExceptio when(jobHistoryHandler.getLatestSyncJob(connectionRead.getConnectionId())).thenReturn(Optional.of(jobRead.getJob())); + expectedListItem = ConnectionHelpers.generateExpectedWebBackendConnectionListItem( + standardSync, + sourceRead, + destinationRead, + false, + jobRead.getJob().getCreatedAt(), + jobRead.getJob().getStatus()); + expected = new WebBackendConnectionRead() .connectionId(connectionRead.getConnectionId()) .sourceId(connectionRead.getSourceId()) @@ -281,20 +299,20 @@ void testGetWorkspaceStateEmpty() throws IOException { } @Test - void testWebBackendListConnectionsForWorkspace() throws ConfigNotFoundException, IOException, JsonValidationException { + void testWebBackendListConnectionsForWorkspace() throws ConfigNotFoundException, IOException, + JsonValidationException { final WorkspaceIdRequestBody workspaceIdRequestBody = new WorkspaceIdRequestBody(); workspaceIdRequestBody.setWorkspaceId(sourceRead.getWorkspaceId()); - final ConnectionReadList connectionReadList = new ConnectionReadList(); - connectionReadList.setConnections(Collections.singletonList(connectionRead)); - final ConnectionIdRequestBody connectionIdRequestBody = new ConnectionIdRequestBody(); - connectionIdRequestBody.setConnectionId(connectionRead.getConnectionId()); - when(connectionsHandler.listConnectionsForWorkspace(workspaceIdRequestBody)).thenReturn(connectionReadList); - when(operationsHandler.listOperationsForConnection(connectionIdRequestBody)).thenReturn(operationReadList); + final WebBackendConnectionReadList WebBackendConnectionReadList = + wbHandler.webBackendListConnectionsForWorkspace(workspaceIdRequestBody); - final WebBackendConnectionReadList WebBackendConnectionReadList = wbHandler.webBackendListConnectionsForWorkspace(workspaceIdRequestBody); assertEquals(1, WebBackendConnectionReadList.getConnections().size()); - assertEquals(expected, WebBackendConnectionReadList.getConnections().get(0)); + assertEquals(expectedListItem, WebBackendConnectionReadList.getConnections().get(0)); + + // make sure the icons were loaded into actual svg content + assertTrue(expectedListItem.getSource().getIcon().startsWith(SVG)); + assertTrue(expectedListItem.getDestination().getIcon().startsWith(SVG)); } @Test @@ -311,6 +329,10 @@ void testWebBackendGetConnection() throws ConfigNotFoundException, IOException, final WebBackendConnectionRead WebBackendConnectionRead = wbHandler.webBackendGetConnection(webBackendConnectionRequestBody); assertEquals(expected, WebBackendConnectionRead); + + // make sure the icons were loaded into actual svg content + assertTrue(expected.getSource().getIcon().startsWith(SVG)); + assertTrue(expected.getDestination().getIcon().startsWith(SVG)); } WebBackendConnectionRead testWebBackendGetConnection(final boolean withCatalogRefresh) diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java index 07f3adf0ca9a1..042b537dc1ed1 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java @@ -17,8 +17,12 @@ import io.airbyte.api.model.generated.ConnectionScheduleDataBasicSchedule; import io.airbyte.api.model.generated.ConnectionScheduleType; import io.airbyte.api.model.generated.ConnectionStatus; +import io.airbyte.api.model.generated.DestinationRead; +import io.airbyte.api.model.generated.JobStatus; import io.airbyte.api.model.generated.ResourceRequirements; +import io.airbyte.api.model.generated.SourceRead; import io.airbyte.api.model.generated.SyncMode; +import io.airbyte.api.model.generated.WebBackendConnectionListItem; import io.airbyte.commons.text.Names; import io.airbyte.config.BasicSchedule; import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType; @@ -201,6 +205,31 @@ public static ConnectionRead connectionReadFromStandardSync(final StandardSync s return connectionRead; } + public static WebBackendConnectionListItem generateExpectedWebBackendConnectionListItem( + final StandardSync standardSync, + final SourceRead source, + final DestinationRead destination, + final boolean isSyncing, + final Long latestSyncJobCreatedAt, + final JobStatus latestSynJobStatus) { + + final WebBackendConnectionListItem connectionListItem = new WebBackendConnectionListItem() + .connectionId(standardSync.getConnectionId()) + .name(standardSync.getName()) + .sourceId(standardSync.getSourceId()) + .destinationId(standardSync.getDestinationId()) + .source(source) + .destination(destination) + .status(ApiPojoConverters.toApiStatus(standardSync.getStatus())) + .isSyncing(isSyncing) + .latestSyncJobCreatedAt(latestSyncJobCreatedAt) + .latestSyncJobStatus(latestSynJobStatus) + .scheduleType(ApiPojoConverters.toApiConnectionScheduleType(standardSync)) + .scheduleData(ApiPojoConverters.toApiConnectionScheduleData(standardSync)); + + return connectionListItem; + } + public static JsonNode generateBasicJsonSchema() { return CatalogHelpers.fieldsToJsonSchema(Field.of(FIELD_NAME, JsonSchemaType.STRING)); } diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationHelpers.java index ffdca4655cbf7..a3eb07edee60d 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationHelpers.java @@ -9,6 +9,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.config.DestinationConnection; import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.server.handlers.DestinationDefinitionsHandler; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -61,7 +62,8 @@ public static DestinationRead getDestinationRead(final DestinationConnection des .destinationId(destination.getDestinationId()) .connectionConfiguration(destination.getConfiguration()) .name(destination.getName()) - .destinationName(standardDestinationDefinition.getName()); + .destinationName(standardDestinationDefinition.getName()) + .icon(DestinationDefinitionsHandler.loadIcon(standardDestinationDefinition.getIcon())); } } diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceHelpers.java index 964d6476c99e0..975256c751a7f 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceHelpers.java @@ -9,6 +9,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.server.handlers.SourceDefinitionsHandler; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -58,7 +59,8 @@ public static SourceRead getSourceRead(final SourceConnection source, final Stan .sourceId(source.getSourceId()) .connectionConfiguration(source.getConfiguration()) .name(source.getName()) - .sourceName(standardSourceDefinition.getName()); + .sourceName(standardSourceDefinition.getName()) + .icon(SourceDefinitionsHandler.loadIcon(standardSourceDefinition.getIcon())); } } diff --git a/airbyte-server/src/test/resources/icons/test-destination.svg b/airbyte-server/src/test/resources/icons/test-destination.svg new file mode 100644 index 0000000000000..29fb9f8e862e5 --- /dev/null +++ b/airbyte-server/src/test/resources/icons/test-destination.svg @@ -0,0 +1,3 @@ + + destination + diff --git a/airbyte-server/src/test/resources/icons/test-source.svg b/airbyte-server/src/test/resources/icons/test-source.svg new file mode 100644 index 0000000000000..7d81478ebd00a --- /dev/null +++ b/airbyte-server/src/test/resources/icons/test-source.svg @@ -0,0 +1,3 @@ + + source + diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json index 1192c3b7051e5..6348c7d826657 100644 --- a/airbyte-webapp/package-lock.json +++ b/airbyte-webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "airbyte-webapp", - "version": "0.40.9", + "version": "0.40.10", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "airbyte-webapp", - "version": "0.40.9", + "version": "0.40.10", "dependencies": { "@floating-ui/react-dom": "^1.0.0", "@fortawesome/fontawesome-svg-core": "^6.1.1", @@ -29724,11 +29724,6 @@ "node": ">=8" } }, - "node_modules/js-cookie": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-2.2.1.tgz", - "integrity": "sha512-HvdH2LzI/EAZcUwA8+0nKNtWHqS+ZmijLA30RwZA0bo7ToCckjK5MkGhjED9KoRcXO6BaGI3I9UIzSA1FKFPOQ==" - }, "node_modules/js-sha3": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.8.0.tgz", @@ -39535,6 +39530,11 @@ "react-dom": ">=16.8.0" } }, + "node_modules/react-use/node_modules/js-cookie": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-2.2.1.tgz", + "integrity": "sha512-HvdH2LzI/EAZcUwA8+0nKNtWHqS+ZmijLA30RwZA0bo7ToCckjK5MkGhjED9KoRcXO6BaGI3I9UIzSA1FKFPOQ==" + }, "node_modules/react-use/node_modules/tslib": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.1.0.tgz", @@ -70482,11 +70482,6 @@ } } }, - "js-cookie": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-2.2.1.tgz", - "integrity": "sha512-HvdH2LzI/EAZcUwA8+0nKNtWHqS+ZmijLA30RwZA0bo7ToCckjK5MkGhjED9KoRcXO6BaGI3I9UIzSA1FKFPOQ==" - }, "js-sha3": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.8.0.tgz", @@ -77590,6 +77585,11 @@ "tslib": "^2.1.0" }, "dependencies": { + "js-cookie": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-2.2.1.tgz", + "integrity": "sha512-HvdH2LzI/EAZcUwA8+0nKNtWHqS+ZmijLA30RwZA0bo7ToCckjK5MkGhjED9KoRcXO6BaGI3I9UIzSA1FKFPOQ==" + }, "tslib": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.1.0.tgz", diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 36b359eaf81cd..f88d88c686aff 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -1,6 +1,6 @@ { "name": "airbyte-webapp", - "version": "0.40.9", + "version": "0.40.10", "private": true, "engines": { "node": ">=16.0.0" diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx index 22baa371665f4..e8f6760144aab 100644 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx @@ -39,7 +39,7 @@ const EditorHeader: React.FC = ({ {mainTitle || } {mode !== "readonly" && ( -
{name || id}
+ icon={} + /> + icon={} + />
); diff --git a/airbyte-webapp/src/components/CenteredPageComponents/BigButton.tsx b/airbyte-webapp/src/components/CenteredPageComponents/BigButton.tsx deleted file mode 100644 index 582eaa9deda39..0000000000000 --- a/airbyte-webapp/src/components/CenteredPageComponents/BigButton.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import styled from "styled-components"; - -import { Button } from "components"; -const BigButton = styled(Button)<{ shadow?: boolean }>` - font-size: 16px; - line-height: 19px; - padding: 10px 27px; - font-weight: 500; - box-shadow: ${({ shadow }) => (shadow ? "0 8px 5px -5px rgba(0, 0, 0, 0.2)" : "none")}; -`; - -export default BigButton; diff --git a/airbyte-webapp/src/components/CenteredPageComponents/PaddedCard.tsx b/airbyte-webapp/src/components/CenteredPageComponents/PaddedCard.tsx index da4759ccb1251..ab9f9cd6d14f5 100644 --- a/airbyte-webapp/src/components/CenteredPageComponents/PaddedCard.tsx +++ b/airbyte-webapp/src/components/CenteredPageComponents/PaddedCard.tsx @@ -1,6 +1,6 @@ import styled from "styled-components"; -import { Card } from "components/base/Card"; +import { Card } from "components/ui/Card"; const PaddedCard = styled(Card)` width: 100%; diff --git a/airbyte-webapp/src/components/CenteredPageComponents/index.tsx b/airbyte-webapp/src/components/CenteredPageComponents/index.tsx index 5bdf3b25f1198..3569e3a632c0c 100644 --- a/airbyte-webapp/src/components/CenteredPageComponents/index.tsx +++ b/airbyte-webapp/src/components/CenteredPageComponents/index.tsx @@ -1,5 +1,4 @@ -import BigButton from "./BigButton"; import PageViewContainer from "./PageViewContainer"; import Subtitle from "./Subtitle"; -export { BigButton, PageViewContainer, Subtitle }; +export { PageViewContainer, Subtitle }; diff --git a/airbyte-webapp/src/components/ConfirmationModal/ConfirmationModal.module.scss b/airbyte-webapp/src/components/ConfirmationModal/ConfirmationModal.module.scss new file mode 100644 index 0000000000000..86081ff47b39b --- /dev/null +++ b/airbyte-webapp/src/components/ConfirmationModal/ConfirmationModal.module.scss @@ -0,0 +1,3 @@ +.buttonWithMargin { + margin-right: 12px; +} diff --git a/airbyte-webapp/src/components/ConfirmationModal/ConfirmationModal.tsx b/airbyte-webapp/src/components/ConfirmationModal/ConfirmationModal.tsx index 33fa911081130..ff87ee8d3d325 100644 --- a/airbyte-webapp/src/components/ConfirmationModal/ConfirmationModal.tsx +++ b/airbyte-webapp/src/components/ConfirmationModal/ConfirmationModal.tsx @@ -2,16 +2,15 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { LoadingButton } from "components"; -import { Button } from "components/base/Button"; +import { Button } from "components/ui/Button"; import { Modal } from "components/ui/Modal"; import useLoadingState from "../../hooks/useLoadingState"; +import styles from "./ConfirmationModal.module.scss"; const Content = styled.div` width: 585px; font-size: 14px; - line-height: 28px; padding: 25px; white-space: pre-line; `; @@ -22,10 +21,6 @@ const ButtonContent = styled.div` justify-content: flex-end; `; -const ButtonWithMargin = styled(Button)` - margin-right: 12px; -`; - export interface ConfirmationModalProps { onClose: () => void; title: string; @@ -53,12 +48,18 @@ export const ConfirmationModal: React.FC = ({ - + + diff --git a/airbyte-webapp/src/components/ConnectionBlock/ConnectionBlock.tsx b/airbyte-webapp/src/components/ConnectionBlock/ConnectionBlock.tsx index efa5c5bdb25d0..8999455935474 100644 --- a/airbyte-webapp/src/components/ConnectionBlock/ConnectionBlock.tsx +++ b/airbyte-webapp/src/components/ConnectionBlock/ConnectionBlock.tsx @@ -3,9 +3,10 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; import React from "react"; -import { Card } from "../base/Card"; -import { ConnectionBlockItem, Content } from "./components/ConnectionBlockItem"; +import { Card } from "components/ui/Card"; + import styles from "./ConnectionBlock.module.scss"; +import { ConnectionBlockItem, Content } from "./ConnectionBlockItem"; interface IProps { itemFrom?: { name: string; icon?: string }; diff --git a/airbyte-webapp/src/components/ConnectionBlock/components/ConnectionBlockItem.tsx b/airbyte-webapp/src/components/ConnectionBlock/ConnectionBlockItem.tsx similarity index 100% rename from airbyte-webapp/src/components/ConnectionBlock/components/ConnectionBlockItem.tsx rename to airbyte-webapp/src/components/ConnectionBlock/ConnectionBlockItem.tsx diff --git a/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx b/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx index 2c73e12475cb4..7c6bec992db95 100644 --- a/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx +++ b/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx @@ -2,9 +2,9 @@ import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { Button, DropDownRow } from "components"; -import { Popout } from "components/base/Popout/Popout"; -import { Text } from "components/base/Text"; import { ReleaseStageBadge } from "components/ReleaseStageBadge"; +import { Popout } from "components/ui/Popout"; +import { Text } from "components/ui/Text"; import { ReleaseStage } from "core/request/AirbyteClient"; import { FeatureItem, useFeature } from "hooks/services/Feature"; diff --git a/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.module.scss b/airbyte-webapp/src/components/CreateConnection/CreateConnection.module.scss similarity index 100% rename from airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.module.scss rename to airbyte-webapp/src/components/CreateConnection/CreateConnection.module.scss diff --git a/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx b/airbyte-webapp/src/components/CreateConnection/CreateConnection.tsx similarity index 90% rename from airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx rename to airbyte-webapp/src/components/CreateConnection/CreateConnection.tsx index 79b04b1192b31..d84727c643bda 100644 --- a/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx +++ b/airbyte-webapp/src/components/CreateConnection/CreateConnection.tsx @@ -9,27 +9,23 @@ import { JobItem } from "components/JobItem/JobItem"; import LoadingSchema from "components/LoadingSchema"; import { LogsRequestError } from "core/request/LogsRequestError"; -import { ConnectionFormServiceProvider } from "hooks/services/Connection/ConnectionFormService"; +import { ConnectionFormServiceProvider } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useChangedFormsById, useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; import { useCreateConnection, ValuesProps } from "hooks/services/useConnectionHook"; import { ConnectionForm } from "views/Connection/ConnectionForm"; import { DestinationRead, SourceRead } from "../../core/request/AirbyteClient"; import { useDiscoverSchema } from "../../hooks/services/useSourceHook"; -import TryAfterErrorBlock from "./components/TryAfterErrorBlock"; -import styles from "./CreateConnectionContent.module.scss"; +import styles from "./CreateConnection.module.scss"; +import TryAfterErrorBlock from "./TryAfterErrorBlock"; -interface CreateConnectionContentProps { +interface CreateConnectionProps { source: SourceRead; destination: DestinationRead; afterSubmitConnection?: () => void; } -const CreateConnectionContent: React.FC = ({ - source, - destination, - afterSubmitConnection, -}) => { +export const CreateConnection: React.FC = ({ source, destination, afterSubmitConnection }) => { const { mutateAsync: createConnection } = useCreateConnection(); const navigate = useNavigate(); @@ -113,5 +109,3 @@ const CreateConnectionContent: React.FC = ({ ); }; - -export default CreateConnectionContent; diff --git a/airbyte-webapp/src/components/CreateConnectionContent/components/TryAfterErrorBlock.module.scss b/airbyte-webapp/src/components/CreateConnection/TryAfterErrorBlock.module.scss similarity index 83% rename from airbyte-webapp/src/components/CreateConnectionContent/components/TryAfterErrorBlock.module.scss rename to airbyte-webapp/src/components/CreateConnection/TryAfterErrorBlock.module.scss index 8907a23b1781c..fae2399b7e238 100644 --- a/airbyte-webapp/src/components/CreateConnectionContent/components/TryAfterErrorBlock.module.scss +++ b/airbyte-webapp/src/components/CreateConnection/TryAfterErrorBlock.module.scss @@ -1,4 +1,4 @@ -@use "../../../scss/variables"; +@use "../../scss/variables"; .container { padding: 40px; diff --git a/airbyte-webapp/src/components/CreateConnectionContent/components/TryAfterErrorBlock.tsx b/airbyte-webapp/src/components/CreateConnection/TryAfterErrorBlock.tsx similarity index 79% rename from airbyte-webapp/src/components/CreateConnectionContent/components/TryAfterErrorBlock.tsx rename to airbyte-webapp/src/components/CreateConnection/TryAfterErrorBlock.tsx index fa68d51d09757..4c7a1145e86d2 100644 --- a/airbyte-webapp/src/components/CreateConnectionContent/components/TryAfterErrorBlock.tsx +++ b/airbyte-webapp/src/components/CreateConnection/TryAfterErrorBlock.tsx @@ -1,9 +1,9 @@ import React from "react"; import { FormattedMessage } from "react-intl"; -import { Button } from "components/base/Button"; -import { Text } from "components/base/Text"; import { StatusIcon } from "components/StatusIcon"; +import { Button } from "components/ui/Button"; +import { Text } from "components/ui/Text"; import styles from "./TryAfterErrorBlock.module.scss"; @@ -18,7 +18,7 @@ const TryAfterErrorBlock: React.FC = ({ message, onClic {message || } - diff --git a/airbyte-webapp/src/components/CreateConnectionContent/index.tsx b/airbyte-webapp/src/components/CreateConnectionContent/index.tsx deleted file mode 100644 index 2c8aae68f1d7f..0000000000000 --- a/airbyte-webapp/src/components/CreateConnectionContent/index.tsx +++ /dev/null @@ -1,3 +0,0 @@ -import CreateConnectionContent from "./CreateConnectionContent"; - -export default CreateConnectionContent; diff --git a/airbyte-webapp/src/components/DeleteBlock/DeleteBlock.tsx b/airbyte-webapp/src/components/DeleteBlock/DeleteBlock.tsx index 866a76e2fd7e5..ca8617e08a50e 100644 --- a/airbyte-webapp/src/components/DeleteBlock/DeleteBlock.tsx +++ b/airbyte-webapp/src/components/DeleteBlock/DeleteBlock.tsx @@ -3,12 +3,12 @@ import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; import styled from "styled-components"; -import { Button, H5 } from "components"; +import { H5 } from "components/base/Titles"; +import { Button } from "components/ui/Button"; +import { Card } from "components/ui/Card"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; -import { Card } from "../base/Card"; - interface IProps { type: "source" | "destination" | "connection"; onDelete: () => Promise; @@ -56,7 +56,7 @@ const DeleteBlock: React.FC = ({ type, onDelete }) => { - diff --git a/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.tsx b/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.tsx index 9723c03ca0a32..4e146026f9107 100644 --- a/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.tsx +++ b/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.tsx @@ -8,8 +8,9 @@ import { useUpdateEffect } from "react-use"; import rehypeSlug from "rehype-slug"; import urls from "rehype-urls"; -import { LoadingPage, PageTitle } from "components"; +import { LoadingPage } from "components"; import { Markdown } from "components/ui/Markdown"; +import { PageHeader } from "components/ui/PageHeader"; import { useConfig } from "config"; import { useDocumentation } from "hooks/services/useDocumentation"; @@ -52,7 +53,7 @@ export const DocumentationPanel: React.FC = () => { ) : (
- } /> + } /> = ({ re resource={resourceType} />
- diff --git a/airbyte-webapp/src/components/EntityTable/ConnectionTable.module.scss b/airbyte-webapp/src/components/EntityTable/ConnectionTable.module.scss new file mode 100644 index 0000000000000..909820b5e1700 --- /dev/null +++ b/airbyte-webapp/src/components/EntityTable/ConnectionTable.module.scss @@ -0,0 +1,10 @@ +.tableHeaderButton { + background-color: inherit; + border: none; + color: inherit; + cursor: pointer; + font-size: inherit; + font-weight: inherit; + text-transform: inherit; + white-space: nowrap; +} diff --git a/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx b/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx index 4089b99d3206a..94507199358c8 100644 --- a/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx +++ b/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx @@ -16,8 +16,9 @@ import ConnectorCell from "./components/ConnectorCell"; import FrequencyCell from "./components/FrequencyCell"; import LastSyncCell from "./components/LastSyncCell"; import NameCell from "./components/NameCell"; -import SortButton from "./components/SortButton"; +import SortIcon from "./components/SortIcon"; import StatusCell from "./components/StatusCell"; +import styles from "./ConnectionTable.module.scss"; import { ITableDataItem, SortOrderEnum } from "./types"; const Content = styled.div` @@ -80,14 +81,10 @@ const ConnectionTable: React.FC = ({ data, entity, onClickRow, onSync }) () => [ { Header: ( - <> + ), headerHighlighted: true, accessor: "name", @@ -98,18 +95,14 @@ const ConnectionTable: React.FC = ({ data, entity, onClickRow, onSync }) }, { Header: ( - <> + ), headerHighlighted: true, accessor: "entityName", @@ -124,18 +117,14 @@ const ConnectionTable: React.FC = ({ data, entity, onClickRow, onSync }) }, { Header: ( - <> + ), accessor: "connectorName", Cell: ({ cell, row }: CellProps) => ( @@ -152,14 +141,10 @@ const ConnectionTable: React.FC = ({ data, entity, onClickRow, onSync }) }, { Header: ( - <> + ), accessor: "lastSync", Cell: ({ cell, row }: CellProps) => ( diff --git a/airbyte-webapp/src/components/EntityTable/ImplementationTable.module.scss b/airbyte-webapp/src/components/EntityTable/ImplementationTable.module.scss index 765abc8a7c300..4102ac450d9ef 100644 --- a/airbyte-webapp/src/components/EntityTable/ImplementationTable.module.scss +++ b/airbyte-webapp/src/components/EntityTable/ImplementationTable.module.scss @@ -5,3 +5,14 @@ padding-left: 32px !important; } } + +.tableHeaderButton { + display: inline-flex; + font-size: inherit; + text-transform: inherit; + color: inherit; + background-color: inherit; + border: none; + font-weight: inherit; + cursor: pointer; +} diff --git a/airbyte-webapp/src/components/EntityTable/ImplementationTable.tsx b/airbyte-webapp/src/components/EntityTable/ImplementationTable.tsx index cf602f3d812cc..156e5d114dd54 100644 --- a/airbyte-webapp/src/components/EntityTable/ImplementationTable.tsx +++ b/airbyte-webapp/src/components/EntityTable/ImplementationTable.tsx @@ -13,7 +13,7 @@ import ConnectEntitiesCell from "./components/ConnectEntitiesCell"; import ConnectorCell from "./components/ConnectorCell"; import LastSyncCell from "./components/LastSyncCell"; import NameCell from "./components/NameCell"; -import SortButton from "./components/SortButton"; +import SortIcon from "./components/SortIcon"; import styles from "./ImplementationTable.module.scss"; import { EntityTableDataItem, SortOrderEnum } from "./types"; @@ -69,14 +69,10 @@ const ImplementationTable: React.FC = ({ data, entity, onClickRow }) => () => [ { Header: ( - <> + ), headerHighlighted: true, accessor: "entityName", @@ -87,14 +83,10 @@ const ImplementationTable: React.FC = ({ data, entity, onClickRow }) => }, { Header: ( - <> + ), accessor: "connectorName", Cell: ({ cell, row }: CellProps) => ( @@ -110,14 +102,10 @@ const ImplementationTable: React.FC = ({ data, entity, onClickRow }) => }, { Header: ( - <> + ), accessor: "lastSync", Cell: ({ cell, row }: CellProps) => ( diff --git a/airbyte-webapp/src/components/EntityTable/components/SortButton.tsx b/airbyte-webapp/src/components/EntityTable/components/SortButton.tsx deleted file mode 100644 index 5ea7f7cc7dc08..0000000000000 --- a/airbyte-webapp/src/components/EntityTable/components/SortButton.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import { faCaretDown, faCaretUp } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import * as React from "react"; -import styled from "styled-components"; - -import { Button } from "components"; - -const SortButtonView = styled(Button)<{ wasActive?: boolean }>` - min-width: 18px; - font-size: 11px; - line-height: 12px; - opacity: ${({ wasActive }) => (wasActive ? 1 : 0.4)}; - pointer-events: all; - background: none; - border: none; - color: inherit; - padding: 0; - - &:hover { - color: inherit; - } -`; - -interface IProps { - lowToLarge?: boolean; - wasActive?: boolean; - onClick: () => void; -} - -const SortButton: React.FC = ({ wasActive, onClick, lowToLarge }) => { - return ( - - - - ); -}; - -export default SortButton; diff --git a/airbyte-webapp/src/components/EntityTable/components/SortIcon.module.scss b/airbyte-webapp/src/components/EntityTable/components/SortIcon.module.scss new file mode 100644 index 0000000000000..7e0dfe796b3a1 --- /dev/null +++ b/airbyte-webapp/src/components/EntityTable/components/SortIcon.module.scss @@ -0,0 +1,3 @@ +.sortIcon { + margin-left: 10px; +} diff --git a/airbyte-webapp/src/components/EntityTable/components/SortIcon.tsx b/airbyte-webapp/src/components/EntityTable/components/SortIcon.tsx new file mode 100644 index 0000000000000..4fa76b0023d86 --- /dev/null +++ b/airbyte-webapp/src/components/EntityTable/components/SortIcon.tsx @@ -0,0 +1,16 @@ +import { faCaretDown, faCaretUp } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import * as React from "react"; + +import styles from "./SortIcon.module.scss"; + +interface SortIconProps { + lowToLarge?: boolean; + wasActive?: boolean; +} + +const SortIcon: React.FC = ({ wasActive, lowToLarge }) => ( + +); + +export default SortIcon; diff --git a/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx b/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx index 54cd0891d6e95..2eb2072fbc2f3 100644 --- a/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx @@ -3,7 +3,7 @@ import { FormattedMessage } from "react-intl"; import { useAsyncFn } from "react-use"; import styled from "styled-components"; -import { LoadingButton, Switch } from "components"; +import { Button, Switch } from "components"; import { useEnableConnection } from "hooks/services/useConnectionHook"; @@ -16,10 +16,6 @@ interface IProps { onSync: (id: string) => void; } -const SmallButton = styled(LoadingButton)` - padding: 6px 8px 7px; -`; - const ProgressMessage = styled.div` padding: 7px 0; `; @@ -62,9 +58,9 @@ const StatusCell: React.FC = ({ enabled, isManual, id, isSyncing, onSync } return ( - + ); }; diff --git a/airbyte-webapp/src/components/EntityTable/hooks.tsx b/airbyte-webapp/src/components/EntityTable/hooks.tsx index 37b1441c4d47e..199be7d7b4f4a 100644 --- a/airbyte-webapp/src/components/EntityTable/hooks.tsx +++ b/airbyte-webapp/src/components/EntityTable/hooks.tsx @@ -1,13 +1,13 @@ import { useSyncConnection } from "hooks/services/useConnectionHook"; -import { WebBackendConnectionRead } from "../../core/request/AirbyteClient"; +import { WebBackendConnectionListItem } from "../../core/request/AirbyteClient"; const useSyncActions = (): { - syncManualConnection: (connection: WebBackendConnectionRead) => Promise; + syncManualConnection: (connection: WebBackendConnectionListItem) => Promise; } => { const { mutateAsync: syncConnection } = useSyncConnection(); - const syncManualConnection = async (connection: WebBackendConnectionRead) => { + const syncManualConnection = async (connection: WebBackendConnectionListItem) => { await syncConnection(connection); }; diff --git a/airbyte-webapp/src/components/EntityTable/utils.tsx b/airbyte-webapp/src/components/EntityTable/utils.tsx index 0b20cc762d288..78744bd67b581 100644 --- a/airbyte-webapp/src/components/EntityTable/utils.tsx +++ b/airbyte-webapp/src/components/EntityTable/utils.tsx @@ -5,7 +5,7 @@ import { JobStatus, SourceDefinitionRead, SourceRead, - WebBackendConnectionRead, + WebBackendConnectionListItem, } from "../../core/request/AirbyteClient"; import { EntityTableDataItem, ITableDataItem, Status as ConnectionSyncStatus } from "./types"; @@ -18,7 +18,7 @@ export function getEntityTableData< S extends "source" | "destination", SoD extends S extends "source" ? SourceRead : DestinationRead, Def extends S extends "source" ? SourceDefinitionRead : DestinationDefinitionRead ->(entities: SoD[], connections: WebBackendConnectionRead[], definitions: Def[], type: S): EntityTableDataItem[] { +>(entities: SoD[], connections: WebBackendConnectionListItem[], definitions: Def[], type: S): EntityTableDataItem[] { const connectType = type === "source" ? "destination" : "source"; const mappedEntities = entities.map((entityItem) => { @@ -76,43 +76,32 @@ export function getEntityTableData< } export const getConnectionTableData = ( - connections: WebBackendConnectionRead[], - sourceDefinitions: SourceDefinitionRead[], - destinationDefinitions: DestinationDefinitionRead[], + connections: WebBackendConnectionListItem[], type: "source" | "destination" | "connection" ): ITableDataItem[] => { const connectType = type === "source" ? "destination" : "source"; - return connections.map((connection) => { - const sourceIcon = sourceDefinitions.find( - (definition) => definition.sourceDefinitionId === connection.source.sourceDefinitionId - )?.icon; - const destinationIcon = destinationDefinitions.find( - (definition) => definition.destinationDefinitionId === connection.destination.destinationDefinitionId - )?.icon; - - return { - connectionId: connection.connectionId, - name: connection.name, - entityName: - type === "connection" - ? `${connection.source?.sourceName} - ${connection.source?.name}` - : connection[connectType]?.name || "", - connectorName: - type === "connection" - ? `${connection.destination?.destinationName} - ${connection.destination?.name}` - : getConnectorTypeName(connection[connectType]), - lastSync: connection.latestSyncJobCreatedAt, - enabled: connection.status === ConnectionStatus.active, - scheduleData: connection.scheduleData, - scheduleType: connection.scheduleType, - status: connection.status, - isSyncing: connection.isSyncing, - lastSyncStatus: getConnectionSyncStatus(connection.status, connection.latestSyncJobStatus), - connectorIcon: type === "destination" ? sourceIcon : destinationIcon, - entityIcon: type === "destination" ? destinationIcon : sourceIcon, - }; - }); + return connections.map((connection) => ({ + connectionId: connection.connectionId, + name: connection.name, + entityName: + type === "connection" + ? `${connection.source?.sourceName} - ${connection.source?.name}` + : connection[connectType]?.name || "", + connectorName: + type === "connection" + ? `${connection.destination?.destinationName} - ${connection.destination?.name}` + : getConnectorTypeName(connection[connectType]), + lastSync: connection.latestSyncJobCreatedAt, + enabled: connection.status === ConnectionStatus.active, + scheduleData: connection.scheduleData, + scheduleType: connection.scheduleType, + status: connection.status, + isSyncing: connection.isSyncing, + lastSyncStatus: getConnectionSyncStatus(connection.status, connection.latestSyncJobStatus), + connectorIcon: type === "destination" ? connection.source.icon : connection.destination.icon, + entityIcon: type === "destination" ? connection.destination.icon : connection.source.icon, + })); }; export const getConnectionSyncStatus = ( diff --git a/airbyte-webapp/src/components/GroupControls/index.stories.tsx b/airbyte-webapp/src/components/GroupControls/index.stories.tsx index a03c8579d49e9..0a318dba94ce1 100644 --- a/airbyte-webapp/src/components/GroupControls/index.stories.tsx +++ b/airbyte-webapp/src/components/GroupControls/index.stories.tsx @@ -1,6 +1,6 @@ import { ComponentStory, ComponentMeta } from "@storybook/react"; -import { Card } from "components/base"; +import { Card } from "components/ui/Card"; import { FormBlock, FormConditionItem } from "core/form/types"; import { GroupLabel } from "views/Connector/ServiceForm/components/Sections/GroupLabel"; diff --git a/airbyte-webapp/src/components/JobItem/JobItem.tsx b/airbyte-webapp/src/components/JobItem/JobItem.tsx index e193254402cc9..a7b5f1a8d1e0a 100644 --- a/airbyte-webapp/src/components/JobItem/JobItem.tsx +++ b/airbyte-webapp/src/components/JobItem/JobItem.tsx @@ -3,7 +3,7 @@ import styled from "styled-components"; import { Spinner } from "components/ui/Spinner"; -import { JobsWithJobs } from "pages/ConnectionPage/pages/ConnectionItemPage/components/JobsList"; +import { JobsWithJobs } from "pages/ConnectionPage/pages/ConnectionItemPage/JobsList"; import { AttemptRead, JobStatus, SynchronousJobRead } from "../../core/request/AirbyteClient"; import { useAttemptLink } from "./attemptLinkUtils"; diff --git a/airbyte-webapp/src/components/JobItem/components/DebugInfoDetailsModal.module.scss b/airbyte-webapp/src/components/JobItem/components/DebugInfoDetailsModal.module.scss new file mode 100644 index 0000000000000..d1566a69a9f94 --- /dev/null +++ b/airbyte-webapp/src/components/JobItem/components/DebugInfoDetailsModal.module.scss @@ -0,0 +1,3 @@ +.buttonWithMargin { + margin-right: 9px; +} diff --git a/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx b/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx index d1113178a25e2..d49df3f3508bc 100644 --- a/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx +++ b/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx @@ -28,13 +28,12 @@ const DownloadButton: React.FC = ({ jobDebugInfo, fileName return ( + icon={} + /> ); }; diff --git a/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx b/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx index afe78d6fb6f77..bb5c2e22e9708 100644 --- a/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx @@ -3,7 +3,7 @@ import React, { useState } from "react"; import { FormattedMessage } from "react-intl"; import { useLocation } from "react-router-dom"; -import { JobsWithJobs } from "pages/ConnectionPage/pages/ConnectionItemPage/components/JobsList"; +import { JobsWithJobs } from "pages/ConnectionPage/pages/ConnectionItemPage/JobsList"; import { useGetDebugInfoJob } from "services/job/JobService"; import { AttemptRead, AttemptStatus, SynchronousJobRead } from "../../../core/request/AirbyteClient"; diff --git a/airbyte-webapp/src/components/JobItem/components/LinkToAttemptButton.tsx b/airbyte-webapp/src/components/JobItem/components/LinkToAttemptButton.tsx index af0bc103af36d..507a2dc97b780 100644 --- a/airbyte-webapp/src/components/JobItem/components/LinkToAttemptButton.tsx +++ b/airbyte-webapp/src/components/JobItem/components/LinkToAttemptButton.tsx @@ -4,8 +4,8 @@ import React, { useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useDebounce } from "react-use"; -import { Button } from "components"; -import { Tooltip } from "components/base/Tooltip"; +import { Button } from "components/ui/Button"; +import { Tooltip } from "components/ui/Tooltip"; import { copyToClipboard } from "utils/clipboard"; @@ -37,13 +37,12 @@ export const LinkToAttemptButton: React.FC = ({ jobId, attemptId }) => { disabled={!showCopyTooltip} control={ + icon={} + /> } > diff --git a/airbyte-webapp/src/components/JobItem/components/MainInfo.tsx b/airbyte-webapp/src/components/JobItem/components/MainInfo.tsx index 48695082a90e7..b0865b145ce7c 100644 --- a/airbyte-webapp/src/components/JobItem/components/MainInfo.tsx +++ b/airbyte-webapp/src/components/JobItem/components/MainInfo.tsx @@ -8,7 +8,7 @@ import { StatusIcon } from "components"; import { Cell, Row } from "components/SimpleTableComponents"; import { AttemptRead, JobStatus, SynchronousJobRead } from "core/request/AirbyteClient"; -import { JobsWithJobs } from "pages/ConnectionPage/pages/ConnectionItemPage/components/JobsList"; +import { JobsWithJobs } from "pages/ConnectionPage/pages/ConnectionItemPage/JobsList"; import { getJobStatus } from "../JobItem"; import AttemptDetails from "./AttemptDetails"; diff --git a/airbyte-webapp/src/components/JobItem/components/ResetStreamDetails.tsx b/airbyte-webapp/src/components/JobItem/components/ResetStreamDetails.tsx index 75572aacd8ec3..ab035f2596b6b 100644 --- a/airbyte-webapp/src/components/JobItem/components/ResetStreamDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/ResetStreamDetails.tsx @@ -1,7 +1,7 @@ import classNames from "classnames"; import React from "react"; -import { Text } from "components/base/Text"; +import { Text } from "components/ui/Text"; import styles from "./ResetStreamDetails.module.scss"; diff --git a/airbyte-webapp/src/components/LabeledControl/ControlLabels.tsx b/airbyte-webapp/src/components/LabeledControl/ControlLabels.tsx index adf8a6c2a22ee..cab7607fdf120 100644 --- a/airbyte-webapp/src/components/LabeledControl/ControlLabels.tsx +++ b/airbyte-webapp/src/components/LabeledControl/ControlLabels.tsx @@ -2,9 +2,9 @@ import className from "classnames"; import React from "react"; import { FormattedMessage } from "react-intl"; -import { Text } from "components/base/Text"; -import { InfoTooltip } from "components/base/Tooltip"; import Label from "components/Label"; +import { Text } from "components/ui/Text"; +import { InfoTooltip } from "components/ui/Tooltip"; import styles from "./ControlLabels.module.scss"; diff --git a/airbyte-webapp/src/components/LabeledInput/LabeledInput.tsx b/airbyte-webapp/src/components/LabeledInput/LabeledInput.tsx index c287c50e269ae..74d93305ab1f8 100644 --- a/airbyte-webapp/src/components/LabeledInput/LabeledInput.tsx +++ b/airbyte-webapp/src/components/LabeledInput/LabeledInput.tsx @@ -1,7 +1,7 @@ import React from "react"; -import { Input, InputProps } from "components/base"; import { ControlLabels, ControlLabelsProps } from "components/LabeledControl"; +import { Input, InputProps } from "components/ui/Input"; type LabeledInputProps = Pick & InputProps; diff --git a/airbyte-webapp/src/components/LabeledSwitch/LabeledSwitch.tsx b/airbyte-webapp/src/components/LabeledSwitch/LabeledSwitch.tsx index fb4b806f30777..31b95f86757ef 100644 --- a/airbyte-webapp/src/components/LabeledSwitch/LabeledSwitch.tsx +++ b/airbyte-webapp/src/components/LabeledSwitch/LabeledSwitch.tsx @@ -1,7 +1,7 @@ import classNames from "classnames"; import React from "react"; -import { CheckBox, Switch } from "components/base"; +import { CheckBox, Switch } from "components/ui"; import styles from "./LabeledSwitch.module.scss"; diff --git a/airbyte-webapp/src/components/PageTitle/PageTitle.tsx b/airbyte-webapp/src/components/PageTitle/PageTitle.tsx deleted file mode 100644 index e21846eb3345b..0000000000000 --- a/airbyte-webapp/src/components/PageTitle/PageTitle.tsx +++ /dev/null @@ -1,66 +0,0 @@ -import React from "react"; -import styled from "styled-components"; - -import { H3 } from "components"; - -interface PageTitleProps { - withLine?: boolean; - middleComponent?: React.ReactNode; - middleTitleBlock?: React.ReactNode; - endComponent?: React.ReactNode; - title: React.ReactNode; -} - -export const MainContainer = styled.div<{ withLine?: boolean }>` - padding: 20px 32px 18px; - border-bottom: ${({ theme, withLine }) => (withLine ? `1px solid ${theme.greyColor20}` : "none")}; - position: relative; - z-index: 2; - color: ${({ theme }) => theme.darkPrimaryColor}; - display: flex; - flex-direction: row; - justify-content: space-between; - align-items: center; -`; - -export const MiddleBlock = styled.div` - flex: 1 0 0; - display: flex; - justify-content: center; -`; - -export const MiddleTitleBlock = styled(H3)` - flex: 1 0 0; - display: flex; - justify-content: center; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; -`; - -export const EndBlock = styled.div` - flex: 1 0 0; - display: flex; - justify-content: flex-end; -`; - -export const TitleBlock = styled(H3)` - flex: 1 0 0; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; -`; - -const PageTitle: React.FC = ({ title, withLine, middleComponent, middleTitleBlock, endComponent }) => ( - - {title} - {middleTitleBlock ? ( - {middleTitleBlock} - ) : ( - {middleComponent} - )} - {endComponent} - -); - -export default PageTitle; diff --git a/airbyte-webapp/src/components/PageTitle/index.tsx b/airbyte-webapp/src/components/PageTitle/index.tsx deleted file mode 100644 index 9557b396dbbfc..0000000000000 --- a/airbyte-webapp/src/components/PageTitle/index.tsx +++ /dev/null @@ -1,4 +0,0 @@ -import PageTitle from "./PageTitle"; - -export default PageTitle; -export { PageTitle }; diff --git a/airbyte-webapp/src/components/ReleaseStageBadge/ReleaseStageBadge.tsx b/airbyte-webapp/src/components/ReleaseStageBadge/ReleaseStageBadge.tsx index 73a283442cb89..af9693a35b615 100644 --- a/airbyte-webapp/src/components/ReleaseStageBadge/ReleaseStageBadge.tsx +++ b/airbyte-webapp/src/components/ReleaseStageBadge/ReleaseStageBadge.tsx @@ -1,8 +1,8 @@ import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { Tooltip } from "components/base/Tooltip"; import { GAIcon } from "components/icons/GAIcon"; +import { Tooltip } from "components/ui/Tooltip"; import { ReleaseStage } from "core/request/AirbyteClient"; diff --git a/airbyte-webapp/src/components/base/Button/Button.tsx b/airbyte-webapp/src/components/base/Button/Button.tsx deleted file mode 100644 index 3e304cce814f4..0000000000000 --- a/airbyte-webapp/src/components/base/Button/Button.tsx +++ /dev/null @@ -1,134 +0,0 @@ -import styled from "styled-components"; -import { Theme } from "theme"; - -import { ButtonProps } from "./types"; - -type IStyleProps = ButtonProps & { theme: Theme }; - -const getBorderColor = (props: IStyleProps) => { - if ((props.secondary && props.wasActive) || props.iconOnly) { - return "transparent"; - } - - if (props.secondary) { - return props.theme.greyColor30; - } else if (props.danger) { - return props.theme.dangerColor; - } - - return props.theme.primaryColor; -}; - -const getBackgroundColor = (props: IStyleProps) => { - if (props.wasActive) { - if (props.secondary || props.iconOnly) { - return props.theme.primaryColor12; - } - return "transparent"; - } else if (props.secondary || props.iconOnly) { - return "transparent"; - } else if (props.danger) { - return props.theme.dangerColor; - } - - return props.theme.primaryColor; -}; - -const getTextColor = (props: IStyleProps) => { - if (props.wasActive) { - if (props.danger) { - return props.theme.dangerColor; - } - return props.theme.primaryColor; - } else if (props.secondary || props.iconOnly) { - return props.theme.darkGreyColor; - } - - return props.theme.whiteColor; -}; - -const getDisabledTextColor = (props: IStyleProps) => { - if (props.danger) { - return props.theme.dangerColor; - } else if (props.iconOnly) { - return props.theme.greyColor40; - } - - return getTextColor(props); -}; - -const getDisabledOpacity = (props: IStyleProps) => { - if (props.danger) { - return ".5"; - } else if (props.iconOnly) { - return "1"; - } - - return ".3"; -}; - -const getShadowOnHover = (props: IStyleProps) => { - if (props.secondary || props.iconOnly || (props.wasActive && !props.clickable)) { - return "none"; - } - - return "0 1px 3px rgba(53, 53, 66, .2), 0 1px 2px rgba(53, 53, 66, .12), 0 1px 1px rgba(53, 53, 66, .14)"; -}; - -const getFontSize = (props: IStyleProps) => { - if (props.size === "xl") { - return 16; - } - if (props.iconOnly) { - return 14; - } - return 12; -}; - -const getPadding = (props: IStyleProps) => { - if (props.size === "xl") { - return ".8em 2.5em"; - } - if (props.iconOnly) { - return "1.5px 3px"; - } - - return "5px 16px"; -}; - -const Button = styled.button` - width: ${(props) => (props.full ? "100%" : "auto")}; - display: ${(props) => (props.full ? "block" : "inline-block")}; - border: 1px solid ${(props) => getBorderColor(props)}; - outline: none; - border-radius: 4px; - padding: ${(props) => getPadding(props)}; - font-weight: ${(props) => (props.size === "xl" ? 600 : 500)}; - font-size: ${(props) => getFontSize(props)}px; - /* TODO: should try to get rid of line-height altogether */ - line-height: ${(props) => (props.size === "xl" ? "initial" : "15px")}; - text-align: center; - letter-spacing: 0.03em; - cursor: pointer; - pointer-events: ${(props) => (props.wasActive && !props.clickable ? "none" : "all")}; - color: ${(props) => getTextColor(props)}; - background: ${(props) => getBackgroundColor(props)}; - text-decoration: none; - - &:disabled { - opacity: ${(props) => getDisabledOpacity(props)}; - background: ${(props) => props.danger && "transparent"}; - border: ${(props) => props.danger && "none"}; - color: ${(props) => getDisabledTextColor(props)}; - pointer-events: none; - } - - &:hover { - box-shadow: ${(props) => getShadowOnHover(props)}; - border-color: ${(props) => - (props.secondary && props.theme.greyColor40) || (props.iconOnly && props.theme.greyColor20)}; - color: ${(props) => (props.secondary || props.iconOnly) && props.theme.textColor}; - } -`; - -export default Button; diff --git a/airbyte-webapp/src/components/base/Button/LoadingButton.tsx b/airbyte-webapp/src/components/base/Button/LoadingButton.tsx deleted file mode 100644 index 2432565072736..0000000000000 --- a/airbyte-webapp/src/components/base/Button/LoadingButton.tsx +++ /dev/null @@ -1,62 +0,0 @@ -import { faCircleNotch } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import React from "react"; -import styled, { keyframes } from "styled-components"; - -import Button from "./Button"; -import { ButtonProps } from "./types"; - -export const SpinAnimation = keyframes` - 0% { - transform: rotate(0deg); - } - 100% { - transform: rotate(360deg); - } -`; - -const SymbolSpinner = styled(FontAwesomeIcon)` - display: inline-block; - font-size: 18px; - position: absolute; - left: 50%; - animation: ${SpinAnimation} 1.5s linear 0s infinite; - color: ${({ theme, danger }) => (danger ? theme.dangerColor : theme.primaryColor)}; - margin: -1px 0 -3px -9px; -`; - -const ButtonView = styled(Button)` - pointer-events: none; - background: ${({ theme, danger }) => (danger ? theme.dangerColor25 : theme.primaryColor25)}; - border-color: transparent; - position: relative; -`; - -const Invisible = styled.div` - color: rgba(255, 255, 255, 0); -`; -/* - * TODO: this component need to be refactored - we need to have - * the only one + ); +}); + +Button.defaultProps = { + full: false, + size: "xs", + variant: "primary", + iconPosition: "left", +}; diff --git a/airbyte-webapp/src/components/ui/Button/index.stories.tsx b/airbyte-webapp/src/components/ui/Button/index.stories.tsx new file mode 100644 index 0000000000000..26fade189d628 --- /dev/null +++ b/airbyte-webapp/src/components/ui/Button/index.stories.tsx @@ -0,0 +1,77 @@ +import { faTimes } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { ComponentStory, ComponentMeta } from "@storybook/react"; + +import { Button } from "./Button"; + +export default { + title: "UI/Button", + component: Button, + argTypes: { + backgroundColor: { control: "color" }, + }, +} as ComponentMeta; + +const Template: ComponentStory = (args) => + icon={} + /> ) : null} ); diff --git a/airbyte-webapp/src/components/base/Input/index.stories.tsx b/airbyte-webapp/src/components/ui/Input/index.stories.tsx similarity index 100% rename from airbyte-webapp/src/components/base/Input/index.stories.tsx rename to airbyte-webapp/src/components/ui/Input/index.stories.tsx diff --git a/airbyte-webapp/src/components/base/Input/index.tsx b/airbyte-webapp/src/components/ui/Input/index.tsx similarity index 100% rename from airbyte-webapp/src/components/base/Input/index.tsx rename to airbyte-webapp/src/components/ui/Input/index.tsx diff --git a/airbyte-webapp/src/components/ui/Modal/Modal.tsx b/airbyte-webapp/src/components/ui/Modal/Modal.tsx index 880c41b5ca64c..857174bb281ac 100644 --- a/airbyte-webapp/src/components/ui/Modal/Modal.tsx +++ b/airbyte-webapp/src/components/ui/Modal/Modal.tsx @@ -2,7 +2,7 @@ import { Dialog } from "@headlessui/react"; import classNames from "classnames"; import React, { useState } from "react"; -import { Card } from "../../base/Card"; +import { Card } from "../Card"; import styles from "./Modal.module.scss"; export interface ModalProps { diff --git a/airbyte-webapp/src/components/ui/Modal/index.stories.tsx b/airbyte-webapp/src/components/ui/Modal/index.stories.tsx index c57c0e9300404..fce5759c3fae8 100644 --- a/airbyte-webapp/src/components/ui/Modal/index.stories.tsx +++ b/airbyte-webapp/src/components/ui/Modal/index.stories.tsx @@ -1,6 +1,6 @@ import { ComponentStory, ComponentMeta } from "@storybook/react"; -import { Button } from "components/base/Button"; +import { Button } from "components/ui/Button"; import { Modal, ModalBody, ModalFooter } from "."; diff --git a/airbyte-webapp/src/components/base/Multiselect/Multiselect.tsx b/airbyte-webapp/src/components/ui/Multiselect/Multiselect.tsx similarity index 100% rename from airbyte-webapp/src/components/base/Multiselect/Multiselect.tsx rename to airbyte-webapp/src/components/ui/Multiselect/Multiselect.tsx diff --git a/airbyte-webapp/src/components/base/Multiselect/index.tsx b/airbyte-webapp/src/components/ui/Multiselect/index.tsx similarity index 100% rename from airbyte-webapp/src/components/base/Multiselect/index.tsx rename to airbyte-webapp/src/components/ui/Multiselect/index.tsx diff --git a/airbyte-webapp/src/components/PageTitle/PageTitle.module.scss b/airbyte-webapp/src/components/ui/PageHeader/PageHeader.module.scss similarity index 90% rename from airbyte-webapp/src/components/PageTitle/PageTitle.module.scss rename to airbyte-webapp/src/components/ui/PageHeader/PageHeader.module.scss index 692b21ee1cb9a..6ad22a10a7f39 100644 --- a/airbyte-webapp/src/components/PageTitle/PageTitle.module.scss +++ b/airbyte-webapp/src/components/ui/PageHeader/PageHeader.module.scss @@ -1,5 +1,5 @@ -@use "../../scss/colors"; -@use "../../scss/variables"; +@use "../../../scss/colors"; +@use "../../../scss/variables"; .container { padding: variables.$spacing-xl 32px 0; @@ -20,7 +20,7 @@ white-space: nowrap; overflow: hidden; text-overflow: ellipsis; - + &.middle { justify-content: center; } @@ -40,4 +40,4 @@ flex: 1 0 0; display: flex; justify-content: flex-end; -} \ No newline at end of file +} diff --git a/airbyte-webapp/src/components/ui/PageHeader/PageHeader.tsx b/airbyte-webapp/src/components/ui/PageHeader/PageHeader.tsx new file mode 100644 index 0000000000000..91acb27afb4c4 --- /dev/null +++ b/airbyte-webapp/src/components/ui/PageHeader/PageHeader.tsx @@ -0,0 +1,48 @@ +import classNames from "classnames"; +import React from "react"; + +import { Text } from "components/ui/Text"; + +import styles from "./PageHeader.module.scss"; + +interface PageHeaderProps { + withLine?: boolean; + middleComponent?: React.ReactNode; + middleTitleBlock?: React.ReactNode; + endComponent?: React.ReactNode; + title: React.ReactNode; +} + +export const PageHeader: React.FC = ({ + title, + withLine, + middleComponent, + middleTitleBlock, + endComponent, +}) => ( +
+ + {title} + + {middleTitleBlock ? ( + + {middleTitleBlock} + + ) : ( +
{middleComponent}
+ )} +
{endComponent}
+
+); diff --git a/airbyte-webapp/src/components/ui/PageHeader/index.stories.tsx b/airbyte-webapp/src/components/ui/PageHeader/index.stories.tsx new file mode 100644 index 0000000000000..dfd9bfdac3b78 --- /dev/null +++ b/airbyte-webapp/src/components/ui/PageHeader/index.stories.tsx @@ -0,0 +1,28 @@ +import { faPlus } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { ComponentStory, ComponentMeta } from "@storybook/react"; + +import { Button } from "../Button"; +import { PageHeader } from "./PageHeader"; + +export default { + title: "UI/PageHeader", + component: PageHeader, + argTypes: {}, +} as ComponentMeta; + +const Template: ComponentStory = (args) => ; + +const title = "Connections"; + +const endComponent = ( + +); + +export const Primary = Template.bind({}); +Primary.args = { + title, + endComponent, +}; diff --git a/airbyte-webapp/src/components/ui/PageHeader/index.tsx b/airbyte-webapp/src/components/ui/PageHeader/index.tsx new file mode 100644 index 0000000000000..78b169af8a23a --- /dev/null +++ b/airbyte-webapp/src/components/ui/PageHeader/index.tsx @@ -0,0 +1 @@ +export * from "./PageHeader"; diff --git a/airbyte-webapp/src/components/base/Popout/Popout.tsx b/airbyte-webapp/src/components/ui/Popout/Popout.tsx similarity index 93% rename from airbyte-webapp/src/components/base/Popout/Popout.tsx rename to airbyte-webapp/src/components/ui/Popout/Popout.tsx index 86aecf027e575..65ece47a34bf9 100644 --- a/airbyte-webapp/src/components/base/Popout/Popout.tsx +++ b/airbyte-webapp/src/components/ui/Popout/Popout.tsx @@ -4,7 +4,7 @@ import { useToggle } from "react-use"; import styled from "styled-components"; import { DropDown } from "components"; -import { DropdownProps } from "components/base/DropDown"; +import { DropdownProps } from "components/ui/DropDown"; const OutsideClickListener = styled.div` bottom: 0; @@ -33,7 +33,7 @@ interface PopoutProps extends DropdownProps { title?: string; } -const Popout: React.FC = ({ onChange, targetComponent, ...props }) => { +export const Popout: React.FC = ({ onChange, targetComponent, ...props }) => { const [isOpen, toggleOpen] = useToggle(false); const onSelectChange = (value: Value, meta: ActionMeta) => { @@ -88,5 +88,3 @@ const Popout: React.FC = ({ onChange, targetComponent, ...props }) ); }; - -export { Popout }; diff --git a/airbyte-webapp/src/components/base/Popout/index.stories.tsx b/airbyte-webapp/src/components/ui/Popout/index.stories.tsx similarity index 96% rename from airbyte-webapp/src/components/base/Popout/index.stories.tsx rename to airbyte-webapp/src/components/ui/Popout/index.stories.tsx index f0921148bf9e7..4dfcf152d19cc 100644 --- a/airbyte-webapp/src/components/base/Popout/index.stories.tsx +++ b/airbyte-webapp/src/components/ui/Popout/index.stories.tsx @@ -1,7 +1,7 @@ import { ComponentStory, ComponentMeta } from "@storybook/react"; import React from "react"; -import Button from "../Button"; +import { Button } from "../Button"; import { Popout } from "./Popout"; export default { diff --git a/airbyte-webapp/src/components/base/Popout/index.tsx b/airbyte-webapp/src/components/ui/Popout/index.tsx similarity index 100% rename from airbyte-webapp/src/components/base/Popout/index.tsx rename to airbyte-webapp/src/components/ui/Popout/index.tsx diff --git a/airbyte-webapp/src/components/base/RadioButton/RadioButton.tsx b/airbyte-webapp/src/components/ui/RadioButton/RadioButton.tsx similarity index 100% rename from airbyte-webapp/src/components/base/RadioButton/RadioButton.tsx rename to airbyte-webapp/src/components/ui/RadioButton/RadioButton.tsx diff --git a/airbyte-webapp/src/components/base/RadioButton/index.tsx b/airbyte-webapp/src/components/ui/RadioButton/index.tsx similarity index 100% rename from airbyte-webapp/src/components/base/RadioButton/index.tsx rename to airbyte-webapp/src/components/ui/RadioButton/index.tsx diff --git a/airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.tsx b/airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.tsx index 22316f3e28741..ec31d79174e80 100644 --- a/airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.tsx +++ b/airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.tsx @@ -4,7 +4,7 @@ import classnames from "classnames"; import React, { useMemo } from "react"; import Slider, { CustomArrowProps, Settings as SliderProps } from "react-slick"; -import { Text } from "components/base/Text"; +import { Text } from "components/ui/Text"; import styles from "./SlickSlider.module.scss"; diff --git a/airbyte-webapp/src/components/base/Switch/Progress.svg b/airbyte-webapp/src/components/ui/Switch/Progress.svg similarity index 100% rename from airbyte-webapp/src/components/base/Switch/Progress.svg rename to airbyte-webapp/src/components/ui/Switch/Progress.svg diff --git a/airbyte-webapp/src/components/base/Switch/ProgressReverse.svg b/airbyte-webapp/src/components/ui/Switch/ProgressReverse.svg similarity index 100% rename from airbyte-webapp/src/components/base/Switch/ProgressReverse.svg rename to airbyte-webapp/src/components/ui/Switch/ProgressReverse.svg diff --git a/airbyte-webapp/src/components/base/Switch/Switch.module.scss b/airbyte-webapp/src/components/ui/Switch/Switch.module.scss similarity index 100% rename from airbyte-webapp/src/components/base/Switch/Switch.module.scss rename to airbyte-webapp/src/components/ui/Switch/Switch.module.scss diff --git a/airbyte-webapp/src/components/base/Switch/Switch.stories.tsx b/airbyte-webapp/src/components/ui/Switch/Switch.stories.tsx similarity index 100% rename from airbyte-webapp/src/components/base/Switch/Switch.stories.tsx rename to airbyte-webapp/src/components/ui/Switch/Switch.stories.tsx diff --git a/airbyte-webapp/src/components/base/Switch/Switch.tsx b/airbyte-webapp/src/components/ui/Switch/Switch.tsx similarity index 100% rename from airbyte-webapp/src/components/base/Switch/Switch.tsx rename to airbyte-webapp/src/components/ui/Switch/Switch.tsx diff --git a/airbyte-webapp/src/components/base/Switch/index.tsx b/airbyte-webapp/src/components/ui/Switch/index.tsx similarity index 100% rename from airbyte-webapp/src/components/base/Switch/index.tsx rename to airbyte-webapp/src/components/ui/Switch/index.tsx diff --git a/airbyte-webapp/src/components/ui/Table/Table.tsx b/airbyte-webapp/src/components/ui/Table/Table.tsx index b0380ce1bd9af..94b5805d5a6ea 100644 --- a/airbyte-webapp/src/components/ui/Table/Table.tsx +++ b/airbyte-webapp/src/components/ui/Table/Table.tsx @@ -2,7 +2,7 @@ import React, { memo, useMemo } from "react"; import { Cell, Column, ColumnInstance, SortingRule, useSortBy, useTable } from "react-table"; import styled from "styled-components"; -import { Card } from "components"; +import { Card } from "../Card"; interface PaddingProps { left?: number; diff --git a/airbyte-webapp/src/components/base/TagInput/TagInput.tsx b/airbyte-webapp/src/components/ui/TagInput/TagInput.tsx similarity index 100% rename from airbyte-webapp/src/components/base/TagInput/TagInput.tsx rename to airbyte-webapp/src/components/ui/TagInput/TagInput.tsx diff --git a/airbyte-webapp/src/components/base/TagInput/TagItem.tsx b/airbyte-webapp/src/components/ui/TagInput/TagItem.tsx similarity index 100% rename from airbyte-webapp/src/components/base/TagInput/TagItem.tsx rename to airbyte-webapp/src/components/ui/TagInput/TagItem.tsx diff --git a/airbyte-webapp/src/components/base/TagInput/index.tsx b/airbyte-webapp/src/components/ui/TagInput/index.tsx similarity index 100% rename from airbyte-webapp/src/components/base/TagInput/index.tsx rename to airbyte-webapp/src/components/ui/TagInput/index.tsx diff --git a/airbyte-webapp/src/components/base/Text/Text.tsx b/airbyte-webapp/src/components/ui/Text/Text.tsx similarity index 100% rename from airbyte-webapp/src/components/base/Text/Text.tsx rename to airbyte-webapp/src/components/ui/Text/Text.tsx diff --git a/airbyte-webapp/src/components/base/Text/heading.module.scss b/airbyte-webapp/src/components/ui/Text/heading.module.scss similarity index 100% rename from airbyte-webapp/src/components/base/Text/heading.module.scss rename to airbyte-webapp/src/components/ui/Text/heading.module.scss diff --git a/airbyte-webapp/src/components/base/Text/index.stories.tsx b/airbyte-webapp/src/components/ui/Text/index.stories.tsx similarity index 100% rename from airbyte-webapp/src/components/base/Text/index.stories.tsx rename to airbyte-webapp/src/components/ui/Text/index.stories.tsx diff --git a/airbyte-webapp/src/components/base/Text/index.ts b/airbyte-webapp/src/components/ui/Text/index.ts similarity index 100% rename from airbyte-webapp/src/components/base/Text/index.ts rename to airbyte-webapp/src/components/ui/Text/index.ts diff --git a/airbyte-webapp/src/components/base/Text/text.module.scss b/airbyte-webapp/src/components/ui/Text/text.module.scss similarity index 100% rename from airbyte-webapp/src/components/base/Text/text.module.scss rename to airbyte-webapp/src/components/ui/Text/text.module.scss diff --git a/airbyte-webapp/src/components/base/TextArea/TextArea.module.scss b/airbyte-webapp/src/components/ui/TextArea/TextArea.module.scss similarity index 100% rename from airbyte-webapp/src/components/base/TextArea/TextArea.module.scss rename to airbyte-webapp/src/components/ui/TextArea/TextArea.module.scss diff --git a/airbyte-webapp/src/components/base/TextArea/TextArea.tsx b/airbyte-webapp/src/components/ui/TextArea/TextArea.tsx similarity index 100% rename from airbyte-webapp/src/components/base/TextArea/TextArea.tsx rename to airbyte-webapp/src/components/ui/TextArea/TextArea.tsx diff --git a/airbyte-webapp/src/components/base/TextArea/index.stories.tsx b/airbyte-webapp/src/components/ui/TextArea/index.stories.tsx similarity index 100% rename from airbyte-webapp/src/components/base/TextArea/index.stories.tsx rename to airbyte-webapp/src/components/ui/TextArea/index.stories.tsx diff --git a/airbyte-webapp/src/components/base/TextArea/index.tsx b/airbyte-webapp/src/components/ui/TextArea/index.tsx similarity index 100% rename from airbyte-webapp/src/components/base/TextArea/index.tsx rename to airbyte-webapp/src/components/ui/TextArea/index.tsx diff --git a/airbyte-webapp/src/components/ui/Toast/SingletonCard.module.scss b/airbyte-webapp/src/components/ui/Toast/SingletonCard.module.scss new file mode 100644 index 0000000000000..0bf06de4cd58f --- /dev/null +++ b/airbyte-webapp/src/components/ui/Toast/SingletonCard.module.scss @@ -0,0 +1,3 @@ +.closeButton { + margin-left: 10px; +} diff --git a/airbyte-webapp/src/components/ui/Toast/Toast.tsx b/airbyte-webapp/src/components/ui/Toast/Toast.tsx index 376174dee483e..223ccd6ff66dd 100644 --- a/airbyte-webapp/src/components/ui/Toast/Toast.tsx +++ b/airbyte-webapp/src/components/ui/Toast/Toast.tsx @@ -3,9 +3,11 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; import styled, { keyframes } from "styled-components"; -import { Button, H5 } from "components"; +import { H5 } from "components/base/Titles"; +import { Button } from "../Button"; import ErrorSign from "./ErrorSign"; +import styles from "./SingletonCard.module.scss"; interface ToastProps { title: string | React.ReactNode; @@ -65,10 +67,6 @@ const Text = styled.div` margin-top: 5px; `; -const CloseButton = styled(Button)` - margin-left: 10px; -`; - export const Toast: React.FC = (props) => ( {props.hasError && } @@ -77,9 +75,12 @@ export const Toast: React.FC = (props) => ( {props.text && {props.text}} {props.onClose && ( - - - + )} diff --git a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.module.scss b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.module.scss index ff3dba6c7155f..ad435811580a9 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.module.scss +++ b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.module.scss @@ -1,11 +1,3 @@ -.logInBtn { - width: 90px; - height: 44px; - font-weight: 600 !important; // FIXME: solve style order issue - font-size: 16px !important; // FIXME: solve style order issue - line-height: 19px; -} - .forgotPassword { font-size: 12px; } diff --git a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx index 869fdfe6b277b..3ec8565d32c06 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx @@ -4,7 +4,7 @@ import { FormattedMessage, useIntl } from "react-intl"; import { NavigateOptions, To, useNavigate } from "react-router-dom"; import * as yup from "yup"; -import { LabeledInput, Link, LoadingButton } from "components"; +import { LabeledInput, Link, Button } from "components"; import HeadTitle from "components/HeadTitle"; import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; @@ -103,9 +103,9 @@ const LoginPage: React.FC = () => { > - + diff --git a/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx index c1bef301b8976..d2c1c6011170e 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx @@ -3,7 +3,7 @@ import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; import * as yup from "yup"; -import { LoadingButton, LabeledInput, Link } from "components"; +import { LabeledInput, Link, Button } from "components"; import HeadTitle from "components/HeadTitle"; import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; @@ -75,9 +75,9 @@ const ResetPasswordPage: React.FC = () => { - + )} diff --git a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/SignupPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/SignupPage.tsx index ac266480f2add..b3a24552754d7 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/SignupPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/SignupPage.tsx @@ -1,8 +1,8 @@ import React from "react"; import { FormattedMessage } from "react-intl"; -import { Text } from "components/base/Text"; import HeadTitle from "components/HeadTitle"; +import { Text } from "components/ui/Text"; import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; diff --git a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.module.scss b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.module.scss index a88c508c3586c..ce19205fa86b4 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.module.scss +++ b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.module.scss @@ -1,14 +1,6 @@ @use "../../../../../../scss/colors"; @use "../../../../../../scss/variables"; -.signUpButton { - width: 100% !important; - font-weight: 600 !important; - font-size: 16px !important; - line-height: 19px !important; - height: 44px !important; -} - .statusMessage { margin-top: variables.$spacing-md; color: colors.$red; diff --git a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx index 4bceec89a9b8c..3cfb47e738aa4 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx @@ -5,7 +5,7 @@ import { useSearchParams } from "react-router-dom"; import styled from "styled-components"; import * as yup from "yup"; -import { LabeledInput, Link, LoadingButton } from "components"; +import { LabeledInput, Link, Button } from "components"; import { useConfig } from "config"; import { useExperiment } from "hooks/services/Experiment"; @@ -165,9 +165,9 @@ export const SignupButton: React.FC = ({ disabled, buttonMessageId = "login.signup.submitButton", }) => ( - + ); export const SignupFormStatusMessage: React.FC> = ({ children }) => ( diff --git a/airbyte-webapp/src/packages/cloud/views/auth/components/CheckBoxControl.tsx b/airbyte-webapp/src/packages/cloud/views/auth/components/CheckBoxControl.tsx index ea17dcf9e4f3e..b9fec4ab9964e 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/components/CheckBoxControl.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/components/CheckBoxControl.tsx @@ -1,7 +1,7 @@ import React from "react"; import styled from "styled-components"; -import { CheckBox } from "components/base"; +import { CheckBox } from "components/ui/CheckBox"; type IProps = { message?: React.ReactNode; diff --git a/airbyte-webapp/src/packages/cloud/views/auth/components/FormTitle/FormTitle.tsx b/airbyte-webapp/src/packages/cloud/views/auth/components/FormTitle/FormTitle.tsx index 6750cc06307a1..110ed680e810b 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/components/FormTitle/FormTitle.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/components/FormTitle/FormTitle.tsx @@ -1,6 +1,6 @@ import React from "react"; -import { Text } from "components/base/Text"; +import { Text } from "components/ui/Text"; import styles from "./FormTitle.module.scss"; diff --git a/airbyte-webapp/src/packages/cloud/views/auth/components/Header/Header.tsx b/airbyte-webapp/src/packages/cloud/views/auth/components/Header/Header.tsx index e049db2c450e5..d738e81513986 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/components/Header/Header.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/components/Header/Header.tsx @@ -1,6 +1,6 @@ import React from "react"; import { FormattedMessage } from "react-intl"; -import { Link } from "react-router-dom"; +import { useNavigate } from "react-router-dom"; import { Button } from "components"; @@ -11,15 +11,18 @@ interface HeaderProps { toLogin?: boolean; } -export const Header: React.FC = ({ toLogin }) => ( -
-
-
- +export const Header: React.FC = ({ toLogin }) => { + const navigate = useNavigate(); + return ( +
+
+
+ +
+
-
-
-); + ); +}; diff --git a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/CreditsPage.tsx b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/CreditsPage.tsx index 00bcf92a30d4c..7d485ce749cbd 100644 --- a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/CreditsPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/CreditsPage.tsx @@ -2,9 +2,9 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { PageTitle } from "components"; import HeadTitle from "components/HeadTitle"; import MainPageWithScroll from "components/MainPageWithScroll"; +import { PageHeader } from "components/ui/PageHeader"; import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; import { useAuthService } from "packages/cloud/services/auth/AuthService"; @@ -28,7 +28,7 @@ const CreditsPage: React.FC = () => { return ( } - pageTitle={} />} + pageTitle={} />} > {!emailVerified && } diff --git a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/CreditsUsage.tsx b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/CreditsUsage.tsx index a3b20b81b5110..2d2db8aed60e1 100644 --- a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/CreditsUsage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/CreditsUsage.tsx @@ -2,8 +2,8 @@ import React, { useMemo } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; -import { Card } from "components/base/Card"; import { BarChart } from "components/ui/BarChart"; +import { Card } from "components/ui/Card"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; import { useGetCloudWorkspaceUsage } from "packages/cloud/services/workspaces/CloudWorkspacesService"; diff --git a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/RemainingCredits.tsx b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/RemainingCredits.tsx index d8af4b013c336..6a61d38c82488 100644 --- a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/RemainingCredits.tsx +++ b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/RemainingCredits.tsx @@ -1,10 +1,12 @@ +import { faPlus } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React, { useEffect, useRef, useState } from "react"; import { FormattedMessage, FormattedNumber } from "react-intl"; import { useSearchParams } from "react-router-dom"; import { useEffectOnce } from "react-use"; import styled from "styled-components"; -import { Button, LoadingButton } from "components"; +import { Button } from "components"; import { useConfig } from "config"; import { Action, Namespace } from "core/analytics"; @@ -30,7 +32,7 @@ const Block = styled.div` display: flex; justify-content: space-between; align-items: center; - margin: 10px 0px; + margin: 10px 0; `; const CreditView = styled.div` text-transform: uppercase; @@ -123,15 +125,17 @@ const RemainingCredits: React.FC = ({ selfServiceCheckoutEnabled }) => { - } > - - + diff --git a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/UsagePerConnectionTable.module.scss b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/UsagePerConnectionTable.module.scss new file mode 100644 index 0000000000000..2e032aaea4659 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/UsagePerConnectionTable.module.scss @@ -0,0 +1,7 @@ +.tableHeaderButton { + user-select: none; + + &:hover { + cursor: pointer; + } +} diff --git a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/UsagePerConnectionTable.tsx b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/UsagePerConnectionTable.tsx index 871668e583493..e552120503c2d 100644 --- a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/UsagePerConnectionTable.tsx +++ b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/UsagePerConnectionTable.tsx @@ -5,7 +5,7 @@ import { useNavigate } from "react-router-dom"; import { CellProps } from "react-table"; import styled from "styled-components"; -import SortButton from "components/EntityTable/components/SortButton"; +import SortIcon from "components/EntityTable/components/SortIcon"; import { SortOrderEnum } from "components/EntityTable/types"; import { Table } from "components/ui/Table"; @@ -16,6 +16,7 @@ import { useSourceDefinitionList } from "services/connector/SourceDefinitionServ import ConnectionCell from "./ConnectionCell"; import UsageCell from "./UsageCell"; +import styles from "./UsagePerConnectionTable.module.scss"; const Content = styled.div` padding: 0 60px 0 15px; @@ -111,14 +112,10 @@ const UsagePerConnectionTable: React.FC = ({ credi () => [ { Header: ( - <> + ), customWidth: 30, accessor: "sourceDefinitionName", @@ -133,14 +130,10 @@ const UsagePerConnectionTable: React.FC = ({ credi }, { Header: ( - <> + ), accessor: "creditsConsumed", collapse: true, diff --git a/airbyte-webapp/src/packages/cloud/views/layout/MainView/MainView.tsx b/airbyte-webapp/src/packages/cloud/views/layout/MainView/MainView.tsx index a4c3f2b666caf..460fe334b61be 100644 --- a/airbyte-webapp/src/packages/cloud/views/layout/MainView/MainView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/layout/MainView/MainView.tsx @@ -4,7 +4,7 @@ import { FormattedMessage, useIntl } from "react-intl"; import { Link, Outlet } from "react-router-dom"; import { LoadingPage } from "components"; -import { AlertBanner } from "components/base/Banner/AlertBanner"; +import { AlertBanner } from "components/ui/Banner/AlertBanner"; import { CloudRoutes } from "packages/cloud/cloudRoutes"; import { CreditStatus } from "packages/cloud/lib/domain/cloudWorkspaces/types"; diff --git a/airbyte-webapp/src/packages/cloud/views/layout/SideBar/SideBar.tsx b/airbyte-webapp/src/packages/cloud/views/layout/SideBar/SideBar.tsx index 04ae79a6e104d..00ff54deb12d5 100644 --- a/airbyte-webapp/src/packages/cloud/views/layout/SideBar/SideBar.tsx +++ b/airbyte-webapp/src/packages/cloud/views/layout/SideBar/SideBar.tsx @@ -7,8 +7,8 @@ import { FormattedMessage, FormattedNumber } from "react-intl"; import { NavLink } from "react-router-dom"; import { Link } from "components"; -import { Text } from "components/base/Text"; import { CreditsIcon } from "components/icons/CreditsIcon"; +import { Text } from "components/ui/Text"; import { useConfig } from "config"; import { FeatureItem, IfFeatureEnabled } from "hooks/services/Feature"; diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx index 92f95e51ed416..f6aaa699cc490 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx @@ -3,13 +3,13 @@ import { FormattedMessage } from "react-intl"; import { useMutation } from "react-query"; import styled from "styled-components"; -import { LoadingButton } from "components"; +import { Button } from "components"; import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; import { useAuthService } from "packages/cloud/services/auth/AuthService"; import { SettingsCard } from "pages/SettingsPage/pages/SettingsComponents"; -import { EmailSection, PasswordSection, NameSection } from "./components"; +import { EmailSection, NameSection, PasswordSection } from "./components"; const Header = styled.div` display: flex; @@ -31,9 +31,9 @@ const AccountSettingsView: React.FC = () => { title={
- logout()} isLoading={isLoggingOut} data-testid="button.signout"> +
} /> diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection/NameSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection/NameSection.tsx index 5d6c1c38401e7..3d52e8768b0e5 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection/NameSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection/NameSection.tsx @@ -3,7 +3,7 @@ import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; import * as yup from "yup"; -import { LoadingButton } from "components"; +import { Button } from "components"; import { LabeledInput } from "components/LabeledInput"; import { useCurrentUser } from "packages/cloud/services/auth/AuthService"; @@ -54,9 +54,9 @@ export const NameSection: React.FC = () => { )} - + )} diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection/PasswordSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection/PasswordSection.tsx index 9568d38eb73de..f3178b57b6480 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection/PasswordSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection/PasswordSection.tsx @@ -2,7 +2,7 @@ import { Field, FieldProps, Form, Formik } from "formik"; import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import { LoadingButton } from "components"; +import { Button } from "components"; import { LabeledInput } from "components/LabeledInput"; import { FieldItem } from "packages/cloud/views/auth/components/FormComponents"; @@ -74,9 +74,9 @@ const PasswordSection: React.FC = () => { )} - + )} diff --git a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.module.scss b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.module.scss new file mode 100644 index 0000000000000..ee9274edbfece --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.module.scss @@ -0,0 +1,8 @@ +.sendInvitationButton { + margin-left: 10px; +} + +.deleteButton { + width: 34px; + height: 34px; +} diff --git a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx index 7d528fbc52c67..c62ecb15b83a3 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx @@ -6,13 +6,16 @@ import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; import * as yup from "yup"; -import { Button, DropDown, H5, Input, LoadingButton } from "components"; +import { Button, DropDown, Input } from "components"; +import { H5 } from "components/base/Titles"; import { Cell, Header, Row } from "components/SimpleTableComponents"; import { Modal } from "components/ui/Modal"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; import { useUserHook } from "packages/cloud/services/users/UseUserHook"; +import styles from "./InviteUsersModal.module.scss"; + const requestConnectorValidationSchema = yup.object({ users: yup.array().of( yup.object().shape({ @@ -33,10 +36,6 @@ const Controls = styled.div` margin-top: 26px; `; -const SendInvitationButton = styled(LoadingButton)` - margin-left: 10px; -`; - const FormHeader = styled(Header)` margin-bottom: 14px; `; @@ -45,11 +44,6 @@ const FormRow = styled(Row)` margin-bottom: 8px; `; -const DeleteButton = styled(Button)` - width: 34px; - height: 34px; -`; - const ROLE_OPTIONS = [ { value: "admin", @@ -137,9 +131,9 @@ export const InviteUsersModal: React.FC<{ )} - { setFieldValue("users", [ @@ -147,10 +141,9 @@ export const InviteUsersModal: React.FC<{ ...values.users.slice(index + 1), ]); }} - secondary - > - - + variant="secondary" + icon={} + /> ))} @@ -171,17 +164,18 @@ export const InviteUsersModal: React.FC<{ /> - - - + diff --git a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx index 654a8b38cf649..15f9ab29027da 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx @@ -1,9 +1,12 @@ +import { faPlus } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; import { FormattedMessage } from "react-intl"; import { CellProps } from "react-table"; import { useToggle } from "react-use"; -import { Button, H5, LoadingButton } from "components"; +import { H5 } from "components/base/Titles"; +import { Button } from "components/ui/Button"; import { Table } from "components/ui/Table"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; @@ -35,9 +38,9 @@ const RemoveUserSection: React.FC<{ workspaceId: string; email: string }> = ({ w }; return ( - + ); }; @@ -87,7 +90,6 @@ export const UsersSettingsView: React.FC = () => { user?.userId !== row.original.userId ? ( ) : null, - // cell.value === "invited" && , ].filter(Boolean), }, ], @@ -100,8 +102,12 @@ export const UsersSettingsView: React.FC = () => {
-
diff --git a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/components/RoleToolTip.tsx b/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/components/RoleToolTip.tsx index d6bf28980d79e..1bcfe77f130e2 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/components/RoleToolTip.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/components/RoleToolTip.tsx @@ -2,7 +2,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { InfoTooltip } from "components/base/Tooltip"; +import { InfoTooltip } from "components/ui/Tooltip"; const LineBlock = styled.div` text-transform: none; diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacePopout/WorkspacePopout.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacePopout/WorkspacePopout.tsx index 87834316cc646..d8f4379a2508a 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacePopout/WorkspacePopout.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacePopout/WorkspacePopout.tsx @@ -4,7 +4,7 @@ import { components, MenuListProps } from "react-select"; import styled from "styled-components"; import { Popout } from "components"; -import { IDataItem } from "components/base/DropDown/components/Option"; +import { IDataItem } from "components/ui/DropDown/components/Option"; import { useListCloudWorkspacesAsync } from "packages/cloud/services/workspaces/CloudWorkspacesService"; import { useCurrentWorkspace, useWorkspaceService } from "services/workspaces/WorkspacesService"; diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx index 619a976c576f2..5982f9a047c2c 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx @@ -4,8 +4,9 @@ import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; import * as yup from "yup"; -import { Button, Label, LabeledInput, LabeledSwitch, LoadingButton } from "components"; -import { InfoTooltip } from "components/base/Tooltip"; +import { Label, LabeledInput, LabeledSwitch } from "components"; +import { Button } from "components/ui/Button"; +import { InfoTooltip } from "components/ui/Tooltip"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useAdvancedModeSetting } from "hooks/services/useAdvancedModeSetting"; @@ -102,12 +103,12 @@ export const WorkspaceSettingsView: React.FC = () => {
- - +
@@ -118,13 +119,13 @@ export const WorkspaceSettingsView: React.FC = () => { title={
- removeCloudWorkspace(workspace.workspaceId)} > - +
} /> diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/WorkspacesPage.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/WorkspacesPage.tsx index 03897e20093f8..ca6ef67ed3794 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/WorkspacesPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/WorkspacesPage.tsx @@ -1,7 +1,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; -import { Text } from "components/base/Text"; +import { Text } from "components/ui/Text"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/CreateWorkspaceForm.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/CreateWorkspaceForm.tsx index 0b2d17868ddca..1d674717c395a 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/CreateWorkspaceForm.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/CreateWorkspaceForm.tsx @@ -4,7 +4,7 @@ import { FormattedMessage } from "react-intl"; import styled from "styled-components"; import * as yup from "yup"; -import { LoadingButton, Input } from "components"; +import { Input, Button } from "components"; interface CreateWorkspaceFormProps { onSubmit: (values: { name: string }) => Promise; @@ -52,9 +52,9 @@ const CreateWorkspaceForm: React.FC = ({ onSubmit }) = )} - + )} diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspaceControl.module.scss b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspaceControl.module.scss new file mode 100644 index 0000000000000..a28d1aee6f7a5 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspaceControl.module.scss @@ -0,0 +1,3 @@ +.createButton { + margin-top: 25px; +} diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspaceItem.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspaceItem.tsx index 72fe32291e5ad..1c0bfe7e1cda0 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspaceItem.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspaceItem.tsx @@ -2,7 +2,7 @@ import { faChevronRight } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; -import { H5 } from "components"; +import { H5 } from "components/base/Titles"; import styles from "./WorkspaceItem.module.scss"; diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspacesControl.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspacesControl.tsx index dcb83a047e4b3..664c178ffb8c3 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspacesControl.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/WorkspacesControl.tsx @@ -6,10 +6,8 @@ import styled from "styled-components"; import { Button, Card } from "components"; import CreateWorkspaceForm from "./CreateWorkspaceForm"; +import styles from "./WorkspaceControl.module.scss"; -const CreateButton = styled(Button)` - margin-top: 25px; -`; const FormContent = styled(Card)` padding: 15px 20px 16px 20px; `; @@ -29,9 +27,9 @@ const WorkspacesControl: React.FC<{ ) : ( - + ); }; diff --git a/airbyte-webapp/src/pages/ConnectionPage/ConnectionPage.tsx b/airbyte-webapp/src/pages/ConnectionPage/ConnectionPage.tsx index 5dede84b3b1d0..828278e8557d1 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/ConnectionPage.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/ConnectionPage.tsx @@ -8,7 +8,7 @@ import { StartOverErrorView } from "views/common/StartOverErrorView"; import { RoutePaths } from "../routePaths"; import AllConnectionsPage from "./pages/AllConnectionsPage"; -import ConnectionItemPage from "./pages/ConnectionItemPage"; +import { ConnectionItemPage } from "./pages/ConnectionItemPage/ConnectionItemPage"; import { CreationFormPage } from "./pages/CreationFormPage/CreationFormPage"; export const ConnectionPage: React.FC = () => ( diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/AllConnectionsPage.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/AllConnectionsPage.tsx index ef09a7e47e6b9..d444bc23f3362 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/AllConnectionsPage.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/AllConnectionsPage.tsx @@ -1,10 +1,13 @@ +import { faPlus } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React, { Suspense } from "react"; import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; -import { Button, LoadingPage, MainPageWithScroll, PageTitle } from "components"; +import { Button, LoadingPage, MainPageWithScroll } from "components"; import { EmptyResourceListView } from "components/EmptyResourceListView"; import HeadTitle from "components/HeadTitle"; +import { PageHeader } from "components/ui/PageHeader"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useConnectionList } from "hooks/services/useConnectionHook"; @@ -26,10 +29,10 @@ const AllConnectionsPage: React.FC = () => { } pageTitle={ - } endComponent={ - } diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/components/ConnectionsTable.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/components/ConnectionsTable.tsx index 271c50608386e..faf9575893267 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/components/ConnectionsTable.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/components/ConnectionsTable.tsx @@ -6,24 +6,17 @@ import useSyncActions from "components/EntityTable/hooks"; import { ITableDataItem } from "components/EntityTable/types"; import { getConnectionTableData } from "components/EntityTable/utils"; -import { useDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; -import { useSourceDefinitionList } from "services/connector/SourceDefinitionService"; - -import { WebBackendConnectionRead } from "../../../../../core/request/AirbyteClient"; +import { WebBackendConnectionListItem } from "../../../../../core/request/AirbyteClient"; interface IProps { - connections: WebBackendConnectionRead[]; + connections: WebBackendConnectionListItem[]; } const ConnectionsTable: React.FC = ({ connections }) => { const navigate = useNavigate(); const { syncManualConnection } = useSyncActions(); - const { sourceDefinitions } = useSourceDefinitionList(); - - const { destinationDefinitions } = useDestinationDefinitionList(); - - const data = getConnectionTableData(connections, sourceDefinitions, destinationDefinitions, "connection"); + const data = getConnectionTableData(connections, "connection"); const onSync = useCallback( async (connectionId: string) => { diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx index 646253cc47a33..909d8499e9757 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx @@ -9,15 +9,15 @@ import { Action, Namespace } from "core/analytics"; import { ConnectionStatus } from "core/request/AirbyteClient"; import { useAnalyticsService, useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useGetConnection } from "hooks/services/useConnectionHook"; -import TransformationView from "pages/ConnectionPage/pages/ConnectionItemPage/components/TransformationView"; -import ConnectionPageTitle from "./components/ConnectionPageTitle"; -import { ReplicationView } from "./components/ReplicationView"; -import SettingsView from "./components/SettingsView"; -import StatusView from "./components/StatusView"; +import { ConnectionPageTitle } from "./ConnectionPageTitle"; +import { ConnectionReplicationTab } from "./ConnectionReplicationTab"; import { ConnectionSettingsRoutes } from "./ConnectionSettingsRoutes"; +import { ConnectionSettingsTab } from "./ConnectionSettingsTab"; +import { ConnectionStatusTab } from "./ConnectionStatusTab"; +import { ConnectionTransformationTab } from "./ConnectionTransformationTab"; -const ConnectionItemPage: React.FC = () => { +export const ConnectionItemPage: React.FC = () => { const params = useParams<{ connectionId: string; "*": ConnectionSettingsRoutes; @@ -74,19 +74,21 @@ const ConnectionItemPage: React.FC = () => { } + element={} /> } + element={} /> } + element={} /> : } + element={ + isConnectionDeleted ? : + } /> } /> @@ -94,5 +96,3 @@ const ConnectionItemPage: React.FC = () => { ); }; - -export default ConnectionItemPage; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.module.scss b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionName.module.scss similarity index 89% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.module.scss rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionName.module.scss index 674551b045daa..689a99b2c03a2 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.module.scss +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionName.module.scss @@ -1,5 +1,5 @@ -@use "../../../../../scss/colors"; -@use "../../../../../scss/variables"; +@use "../../../../scss/colors"; +@use "../../../../scss/variables"; .container { margin-top: variables.$spacing-md; @@ -20,6 +20,8 @@ display: none; position: absolute; right: variables.$spacing-xl; + + // used to control svg size, not text size font-size: 18px; color: colors.$blue; } @@ -35,11 +37,7 @@ width: 100%; h2 { - font-weight: 700; - font-size: 24px; - line-height: 29px; text-align: center; - color: colors.$dark-blue-900; margin: variables.$spacing-md; word-wrap: break-word; } diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionName.tsx similarity index 93% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionName.tsx index 0e56a206d7bb2..8bb38d8df64fb 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionName.tsx @@ -3,6 +3,7 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React, { ChangeEvent, useState } from "react"; import { Input } from "components"; +import { Text } from "components/ui/Text"; import { buildConnectionUpdate } from "core/domain/connection"; import { WebBackendConnectionRead } from "core/request/AirbyteClient"; @@ -17,7 +18,7 @@ interface ConnectionNameProps { const InputWithKeystroke = withKeystrokeHandler(Input); -const ConnectionName: React.FC = ({ connection }) => { +export const ConnectionName: React.FC = ({ connection }) => { const { name } = connection; const [editingState, setEditingState] = useState(false); const [loading, setLoading] = useState(false); @@ -85,7 +86,9 @@ const ConnectionName: React.FC = ({ connection }) => { ) : ( @@ -93,5 +96,3 @@ const ConnectionName: React.FC = ({ connection }) => { ); }; - -export default ConnectionName; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.module.scss b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionPageTitle.module.scss similarity index 56% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.module.scss rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionPageTitle.module.scss index 838da2b08ecc0..c281e8f8916dc 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.module.scss +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionPageTitle.module.scss @@ -1,9 +1,9 @@ -@use "../../../../../scss/variables" as vars; -@use "../../../../../scss/colors"; +@use "../../../../scss/colors"; +@use "../../../../scss/variables"; .container { text-align: center; - padding: vars.$spacing-lg vars.$spacing-xl; + padding: variables.$spacing-lg variables.$spacing-xl; } .connectionTitle { @@ -11,8 +11,8 @@ } .connectionDeleted { - max-width: vars.$width-modal-md; - margin: vars.$spacing-lg auto; + max-width: variables.$width-modal-md; + margin: variables.$spacing-lg auto; } .statusContainer { diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionPageTitle.tsx similarity index 90% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionPageTitle.tsx index 689151f327f54..ec474c2fa9f5a 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionPageTitle.tsx @@ -3,15 +3,15 @@ import React, { useCallback, useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; -import { Text } from "components/base/Text"; import { InfoBox } from "components/ui/InfoBox"; import { StepsMenu } from "components/ui/StepsMenu"; +import { Text } from "components/ui/Text"; import { ConnectionStatus, DestinationRead, SourceRead, WebBackendConnectionRead } from "core/request/AirbyteClient"; -import { ConnectionSettingsRoutes } from "../ConnectionSettingsRoutes"; -import ConnectionName from "./ConnectionName"; +import { ConnectionName } from "./ConnectionName"; import styles from "./ConnectionPageTitle.module.scss"; +import { ConnectionSettingsRoutes } from "./ConnectionSettingsRoutes"; import { StatusMainInfo } from "./StatusMainInfo"; interface ConnectionPageTitleProps { @@ -22,7 +22,7 @@ interface ConnectionPageTitleProps { onStatusUpdating?: (updating: boolean) => void; } -const ConnectionPageTitle: React.FC = ({ +export const ConnectionPageTitle: React.FC = ({ source, destination, connection, @@ -90,5 +90,3 @@ const ConnectionPageTitle: React.FC = ({ ); }; - -export default ConnectionPageTitle; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionReplicationTab.module.scss b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionReplicationTab.module.scss new file mode 100644 index 0000000000000..10360aaf922ec --- /dev/null +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionReplicationTab.module.scss @@ -0,0 +1,14 @@ +@use "../../../../scss/variables"; + +.tryArrow { + margin: 0 variables.$spacing-md -1px 0; + + // used to control svg size + font-size: 14px; +} + +.content { + max-width: 1279px; + margin: 0 auto; + padding-bottom: variables.$spacing-md; +} diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionReplicationTab.tsx similarity index 92% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionReplicationTab.tsx index 14a557a40bf47..97cb8b508f6cc 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionReplicationTab.tsx @@ -3,18 +3,17 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React, { useMemo, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useAsyncFn, useUnmount } from "react-use"; -import styled from "styled-components"; -import { Button } from "components/base/Button"; import { LabeledSwitch } from "components/LabeledSwitch"; import LoadingSchema from "components/LoadingSchema"; +import { Button } from "components/ui/Button"; import { ModalBody, ModalFooter } from "components/ui/Modal"; import { toWebBackendConnectionUpdate } from "core/domain/connection"; import { ConnectionStateType, ConnectionStatus } from "core/request/AirbyteClient"; import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; -import { ConnectionFormServiceProvider } from "hooks/services/Connection/ConnectionFormService"; +import { ConnectionFormServiceProvider } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useChangedFormsById, useUniqueFormId } from "hooks/services/FormChangeTracker"; import { ModalCancel, useModalService } from "hooks/services/Modal"; import { @@ -27,7 +26,9 @@ import { equal, naturalComparatorBy } from "utils/objects"; import { CatalogDiffModal } from "views/Connection/CatalogDiffModal/CatalogDiffModal"; import { ConnectionForm } from "views/Connection/ConnectionForm"; -interface ReplicationViewProps { +import styles from "./ConnectionReplicationTab.module.scss"; + +interface ConnectionReplicationTabProps { onAfterSaveSchema: () => void; connectionId: string; } @@ -63,7 +64,7 @@ const ResetWarningModal: React.FC = ({ onCancel, onClose

- } @@ -242,6 +239,6 @@ export const ReplicationView: React.FC = ({ onAfterSaveSch ) : ( )} - + ); }; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.module.scss b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionSettingsTab.module.scss similarity index 100% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.module.scss rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionSettingsTab.module.scss diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.test.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionSettingsTab.test.tsx similarity index 87% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.test.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionSettingsTab.test.tsx index e2635a4bdb05a..665b728cecff2 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.test.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionSettingsTab.test.tsx @@ -1,6 +1,6 @@ import { render, mockConnection } from "test-utils/testutils"; -import SettingsView from "./SettingsView"; +import { ConnectionSettingsTab } from "./ConnectionSettingsTab"; let mockIsAdvancedMode = false; const setMockIsAdvancedMode = (newSetting: boolean) => { @@ -40,11 +40,11 @@ describe("", () => { let container: HTMLElement; setMockIsAdvancedMode(false); - ({ container } = await render()); + ({ container } = await render()); expect(container.textContent).not.toContain("Connection State"); setMockIsAdvancedMode(true); - ({ container } = await render()); + ({ container } = await render()); expect(container.textContent).toContain("Connection State"); }); }); diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionSettingsTab.tsx similarity index 75% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionSettingsTab.tsx index 8b71da6e1fa4e..0fbe9c03c2a0c 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionSettingsTab.tsx @@ -2,19 +2,19 @@ import React from "react"; import DeleteBlock from "components/DeleteBlock"; +import { WebBackendConnectionRead } from "core/request/AirbyteClient"; import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; import { useAdvancedModeSetting } from "hooks/services/useAdvancedModeSetting"; import { useDeleteConnection } from "hooks/services/useConnectionHook"; -import { WebBackendConnectionRead } from "../../../../../core/request/AirbyteClient"; -import styles from "./SettingsView.module.scss"; +import styles from "./ConnectionSettingsTab.module.scss"; import { StateBlock } from "./StateBlock"; -interface SettingsViewProps { +interface ConnectionSettingsTabProps { connection: WebBackendConnectionRead; } -const SettingsView: React.FC = ({ connection }) => { +export const ConnectionSettingsTab: React.FC = ({ connection }) => { const { mutateAsync: deleteConnection } = useDeleteConnection(); const [isAdvancedMode] = useAdvancedModeSetting(); @@ -28,5 +28,3 @@ const SettingsView: React.FC = ({ connection }) => { ); }; - -export default SettingsView; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.module.scss b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionStatusTab.module.scss similarity index 96% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.module.scss rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionStatusTab.module.scss index 509eb61455252..f3add6f11095e 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.module.scss +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionStatusTab.module.scss @@ -1,4 +1,4 @@ -@use "../../../../../scss/colors"; +@use "../../../../scss/colors"; .statusView { margin: 0 10px; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionStatusTab.tsx similarity index 83% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionStatusTab.tsx index 74dc2483778ae..58241a3c6192a 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionStatusTab.tsx @@ -4,12 +4,12 @@ import React, { useEffect, useState } from "react"; import { FormattedMessage } from "react-intl"; import { Link, useLocation } from "react-router-dom"; -import { Button, LoadingButton } from "components"; -import { Card } from "components/base/Card"; -import { Tooltip } from "components/base/Tooltip"; import EmptyResource from "components/EmptyResourceBlock"; import { RotateIcon } from "components/icons/RotateIcon"; import { useAttemptLink } from "components/JobItem/attemptLinkUtils"; +import { Button } from "components/ui/Button"; +import { Card } from "components/ui/Card"; +import { Tooltip } from "components/ui/Tooltip"; import { getFrequencyType } from "config/utils"; import { Action, Namespace } from "core/analytics"; @@ -21,8 +21,8 @@ import { FeatureItem, useFeature } from "hooks/services/Feature"; import { useResetConnection, useSyncConnection } from "hooks/services/useConnectionHook"; import { useCancelJob, useListJobs } from "services/job/JobService"; +import styles from "./ConnectionStatusTab.module.scss"; import JobsList from "./JobsList"; -import styles from "./StatusView.module.scss"; const JOB_PAGE_SIZE_INCREMENT = 25; @@ -37,7 +37,7 @@ interface ActiveJob { isCanceling: boolean; } -interface StatusViewProps { +interface ConnectionStatusTabProps { connection: WebBackendConnectionRead; isStatusUpdating?: boolean; } @@ -49,7 +49,7 @@ const getJobRunningOrPending = (jobs: JobWithAttemptsRead[]) => { }); }; -const StatusView: React.FC = ({ connection }) => { +export const ConnectionStatusTab: React.FC = ({ connection }) => { useTrackPage(PageTrackingCodes.CONNECTIONS_ITEM_STATUS); const [activeJob, setActiveJob] = useState(); const [jobPageSize, setJobPageSize] = useState(JOB_PAGE_SIZE_INCREMENT); @@ -127,6 +127,12 @@ const StatusView: React.FC = ({ connection }) => { setActiveJob((state) => ({ ...state, isCanceling: true } as ActiveJob)); return cancelJob(activeJob.id); }; + let label = null; + if (activeJob?.action === ActionType.RESET) { + label = ; + } else if (activeJob?.action === ActionType.SYNC) { + label = ; + } const onLoadMoreJobs = () => { setJobPageSize((prevJobPageSize) => prevJobPageSize + JOB_PAGE_SIZE_INCREMENT); @@ -144,10 +150,13 @@ const StatusView: React.FC = ({ connection }) => { }; const cancelJobBtn = ( - ); @@ -162,13 +171,19 @@ const StatusView: React.FC = ({ connection }) => {
{!activeJob?.action && ( <> - -
+ } + > @@ -202,13 +217,11 @@ const StatusView: React.FC = ({ connection }) => { {(moreJobPagesAvailable || isJobPageLoading) && (
- +
)} ); }; - -export default StatusView; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/TransformationView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab.tsx similarity index 96% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/TransformationView.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab.tsx index 7392db6e03200..9e0790cb71cfb 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/TransformationView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab.tsx @@ -4,8 +4,8 @@ import { FormattedMessage } from "react-intl"; import { useToggle } from "react-use"; import styled from "styled-components"; -import { Card } from "components/base/Card"; -import { Text } from "components/base/Text"; +import { Card } from "components/ui/Card"; +import { Text } from "components/ui/Text"; import { buildConnectionUpdate, NormalizationType } from "core/domain/connection"; import { @@ -31,7 +31,7 @@ import { } from "views/Connection/ConnectionForm/formConfig"; import { FormCard } from "views/Connection/FormCard"; -interface TransformationViewProps { +interface ConnectionTransformationTabProps { connection: WebBackendConnectionRead; } @@ -118,7 +118,7 @@ const NormalizationCard: React.FC<{ ); }; -const TransformationView: React.FC = ({ connection }) => { +export const ConnectionTransformationTab: React.FC = ({ connection }) => { const definition = useGetDestinationDefinitionSpecification(connection.destination.destinationDefinitionId); const { mutateAsync: updateConnection } = useUpdateConnection(); const workspace = useCurrentWorkspace(); @@ -181,5 +181,3 @@ const TransformationView: React.FC = ({ connection }) = ); }; - -export default TransformationView; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/EnabledControl.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/EnabledControl.tsx similarity index 96% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/EnabledControl.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/EnabledControl.tsx index 2b16ead1997b7..595bcd2c05ec9 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/EnabledControl.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/EnabledControl.tsx @@ -7,11 +7,10 @@ import { Switch } from "components"; import { Action, Namespace } from "core/analytics"; import { buildConnectionUpdate } from "core/domain/connection"; +import { ConnectionStatus, WebBackendConnectionRead } from "core/request/AirbyteClient"; import { useAnalyticsService } from "hooks/services/Analytics"; import { useUpdateConnection } from "hooks/services/useConnectionHook"; -import { ConnectionStatus, WebBackendConnectionRead } from "../../../../../core/request/AirbyteClient"; - const ToggleLabel = styled.label` text-transform: uppercase; font-size: 14px; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/JobsList.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/JobsList.tsx similarity index 89% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/JobsList.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/JobsList.tsx index 2e9de41d68ef6..ac51d00349831 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/JobsList.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/JobsList.tsx @@ -2,7 +2,7 @@ import React, { useMemo } from "react"; import { JobItem } from "components/JobItem/JobItem"; -import { JobWithAttemptsRead } from "../../../../../core/request/AirbyteClient"; +import { JobWithAttemptsRead } from "core/request/AirbyteClient"; interface JobsListProps { jobs: JobWithAttemptsRead[]; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StateBlock.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/StateBlock.tsx similarity index 94% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StateBlock.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/StateBlock.tsx index b5bd764a0f0bf..ec2aae125e562 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StateBlock.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/StateBlock.tsx @@ -1,7 +1,8 @@ import React, { useMemo } from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import { H5, Card } from "components"; +import { H5 } from "components/base/Titles"; +import { Card } from "components/ui/Card"; import { ConnectionState } from "core/request/AirbyteClient"; import { useGetConnectionState } from "hooks/services/useConnectionHook"; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusMainInfo.module.scss b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/StatusMainInfo.module.scss similarity index 89% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusMainInfo.module.scss rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/StatusMainInfo.module.scss index eeaf0593c8e17..0600e7479c57e 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusMainInfo.module.scss +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/StatusMainInfo.module.scss @@ -1,5 +1,5 @@ -@use "../../../../../scss/colors"; -@use "../../../../../scss/variables"; +@use "../../../../scss/colors"; +@use "../../../../scss/variables"; .connectorLink { cursor: pointer; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusMainInfo.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/StatusMainInfo.tsx similarity index 100% rename from airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusMainInfo.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/StatusMainInfo.tsx diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/index.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/index.tsx deleted file mode 100644 index 69bbc3aa93cdf..0000000000000 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/index.tsx +++ /dev/null @@ -1,3 +0,0 @@ -import ConnectionItemPage from "./ConnectionItemPage"; - -export default ConnectionItemPage; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/CreationFormPage.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/CreationFormPage.tsx index 2666b15999e1d..69224657cb98d 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/CreationFormPage.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/CreationFormPage.tsx @@ -2,11 +2,12 @@ import React, { useState } from "react"; import { FormattedMessage } from "react-intl"; import { useLocation, useNavigate } from "react-router-dom"; -import { LoadingPage, PageTitle } from "components"; +import { LoadingPage } from "components"; import ConnectionBlock from "components/ConnectionBlock"; import { FormPageContent } from "components/ConnectorBlocks"; -import CreateConnectionContent from "components/CreateConnectionContent"; +import { CreateConnection } from "components/CreateConnection/CreateConnection"; import HeadTitle from "components/HeadTitle"; +import { PageHeader } from "components/ui/PageHeader"; import { StepsMenu } from "components/ui/StepsMenu"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; @@ -23,9 +24,9 @@ import { SourceDefinitionRead, SourceRead, } from "../../../../core/request/AirbyteClient"; -import { ConnectionCreateDestinationForm } from "./components/DestinationForm"; -import ExistingEntityForm from "./components/ExistingEntityForm"; -import { ConnectionCreateSourceForm } from "./components/SourceForm"; +import { ConnectionCreateDestinationForm } from "./DestinationForm"; +import ExistingEntityForm from "./ExistingEntityForm"; +import { ConnectionCreateSourceForm } from "./SourceForm"; export enum StepsTypes { CREATE_ENTITY = "createEntity", @@ -167,7 +168,7 @@ export const CreationFormPage: React.FC = () => { return ; } - return ; + return ; }; const steps = @@ -214,7 +215,7 @@ export const CreationFormPage: React.FC = () => { <> - } middleComponent={} /> diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/components/DestinationForm.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/DestinationForm.tsx similarity index 100% rename from airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/components/DestinationForm.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/DestinationForm.tsx diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/ExistingEntityForm.module.scss b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/ExistingEntityForm.module.scss new file mode 100644 index 0000000000000..e6049bbfa295c --- /dev/null +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/ExistingEntityForm.module.scss @@ -0,0 +1,3 @@ +.submitButton { + margin-left: auto; +} diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/components/ExistingEntityForm.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/ExistingEntityForm.tsx similarity index 91% rename from airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/components/ExistingEntityForm.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/ExistingEntityForm.tsx index e9bade5f89c31..2552b13403050 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/components/ExistingEntityForm.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/ExistingEntityForm.tsx @@ -5,14 +5,15 @@ import styled from "styled-components"; import * as yup from "yup"; import { Button, ControlLabels, DropDown } from "components"; -import { Card } from "components/base/Card"; import { ConnectorIcon } from "components/ConnectorIcon"; +import { Card } from "components/ui/Card"; +import { useDestinationList } from "hooks/services/useDestinationHook"; +import { useSourceList } from "hooks/services/useSourceHook"; import { useDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; import { useSourceDefinitionList } from "services/connector/SourceDefinitionService"; -import { useDestinationList } from "../../../../../hooks/services/useDestinationHook"; -import { useSourceList } from "../../../../../hooks/services/useSourceHook"; +import styles from "./ExistingEntityForm.module.scss"; interface IProps { type: "source" | "destination"; @@ -110,7 +111,7 @@ const ExistingEntityForm: React.FC = ({ type, onSubmit }) => { )} - diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/components/SourceForm.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/SourceForm.tsx similarity index 100% rename from airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/components/SourceForm.tsx rename to airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/SourceForm.tsx diff --git a/airbyte-webapp/src/pages/DestinationPage/pages/AllDestinationsPage/AllDestinationsPage.tsx b/airbyte-webapp/src/pages/DestinationPage/pages/AllDestinationsPage/AllDestinationsPage.tsx index 153ee827d63b9..0aa6aaac0b533 100644 --- a/airbyte-webapp/src/pages/DestinationPage/pages/AllDestinationsPage/AllDestinationsPage.tsx +++ b/airbyte-webapp/src/pages/DestinationPage/pages/AllDestinationsPage/AllDestinationsPage.tsx @@ -1,3 +1,5 @@ +import { faPlus } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; @@ -5,7 +7,7 @@ import { useNavigate } from "react-router-dom"; import { Button, MainPageWithScroll } from "components"; import { EmptyResourceListView } from "components/EmptyResourceListView"; import HeadTitle from "components/HeadTitle"; -import PageTitle from "components/PageTitle"; +import { PageHeader } from "components/ui/PageHeader"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useDestinationList } from "hooks/services/useDestinationHook"; @@ -24,10 +26,15 @@ const AllDestinationsPage: React.FC = () => { } pageTitle={ - } endComponent={ - } diff --git a/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/CreateDestinationPage.tsx b/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/CreateDestinationPage.tsx index cd1605a137682..e21148398bc83 100644 --- a/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/CreateDestinationPage.tsx +++ b/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/CreateDestinationPage.tsx @@ -4,7 +4,7 @@ import { useNavigate } from "react-router-dom"; import { FormPageContent } from "components/ConnectorBlocks"; import HeadTitle from "components/HeadTitle"; -import PageTitle from "components/PageTitle"; +import { PageHeader } from "components/ui/PageHeader"; import { ConnectionConfiguration } from "core/domain/connection"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; @@ -43,7 +43,7 @@ export const CreateDestinationPage: React.FC = () => { <> - } /> + } /> { - } middleComponent={} /> diff --git a/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationConnectionTable.tsx b/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationConnectionTable.tsx index ce9f783c44d89..1385ae462a673 100644 --- a/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationConnectionTable.tsx +++ b/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationConnectionTable.tsx @@ -7,23 +7,18 @@ import { ITableDataItem } from "components/EntityTable/types"; import { getConnectionTableData } from "components/EntityTable/utils"; import { RoutePaths } from "pages/routePaths"; -import { useDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; -import { useSourceDefinitionList } from "services/connector/SourceDefinitionService"; -import { WebBackendConnectionRead } from "../../../../../core/request/AirbyteClient"; +import { WebBackendConnectionListItem } from "../../../../../core/request/AirbyteClient"; interface IProps { - connections: WebBackendConnectionRead[]; + connections: WebBackendConnectionListItem[]; } const DestinationConnectionTable: React.FC = ({ connections }) => { const navigate = useNavigate(); const { syncManualConnection } = useSyncActions(); - const { sourceDefinitions } = useSourceDefinitionList(); - const { destinationDefinitions } = useDestinationDefinitionList(); - - const data = getConnectionTableData(connections, sourceDefinitions, destinationDefinitions, "destination"); + const data = getConnectionTableData(connections, "destination"); const onSync = useCallback( async (connectionId: string) => { diff --git a/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationSettings.tsx b/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationSettings.tsx index 39e2e9ac61feb..84140aad0f392 100644 --- a/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationSettings.tsx +++ b/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationSettings.tsx @@ -5,7 +5,7 @@ import DeleteBlock from "components/DeleteBlock"; import { ConnectionConfiguration } from "core/domain/connection"; import { Connector } from "core/domain/connector"; -import { DestinationRead, WebBackendConnectionRead } from "core/request/AirbyteClient"; +import { DestinationRead, WebBackendConnectionListItem } from "core/request/AirbyteClient"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; import { useDeleteDestination, useUpdateDestination } from "hooks/services/useDestinationHook"; @@ -17,7 +17,7 @@ import styles from "./DestinationSettings.module.scss"; interface DestinationsSettingsProps { currentDestination: DestinationRead; - connectionsWithDestination: WebBackendConnectionRead[]; + connectionsWithDestination: WebBackendConnectionListItem[]; } const DestinationsSettings: React.FC = ({ diff --git a/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx b/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx index c90a34c28bf31..1e0e83e5be7b6 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx @@ -151,12 +151,10 @@ const OnboardingPage: React.FC = () => {
-
diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/ConnectionStep.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/ConnectionStep.tsx index 95e860691a5a9..7826992684d85 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/components/ConnectionStep.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/components/ConnectionStep.tsx @@ -1,24 +1,20 @@ import React from "react"; -import CreateConnectionContent from "components/CreateConnectionContent"; +import { CreateConnection } from "components/CreateConnection/CreateConnection"; import { useDestinationList } from "hooks/services/useDestinationHook"; import { useSourceList } from "hooks/services/useSourceHook"; -interface IProps { +interface ConnectionStepProps { onNextStep: () => void; } -const ConnectionStep: React.FC = ({ onNextStep: afterSubmitConnection }) => { +const ConnectionStep: React.FC = ({ onNextStep: afterSubmitConnection }) => { const { sources } = useSourceList(); const { destinations } = useDestinationList(); return ( - + ); }; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/FinalStep.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/FinalStep.tsx index 718285cefe63e..a03f22cfb6037 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/components/FinalStep.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/components/FinalStep.tsx @@ -1,7 +1,7 @@ import React, { useEffect, useState } from "react"; import { FormattedMessage } from "react-intl"; -import { Text } from "components/base/Text"; +import { Text } from "components/ui/Text"; import { useConfig } from "config"; import Status from "core/statuses"; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/FirstSuccessfulSync/FirstSuccessfulSync.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/FirstSuccessfulSync/FirstSuccessfulSync.tsx index 2de6f671f31ed..1fb464151b639 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/components/FirstSuccessfulSync/FirstSuccessfulSync.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/components/FirstSuccessfulSync/FirstSuccessfulSync.tsx @@ -1,7 +1,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; -import { Text } from "components/base/Text"; +import { Text } from "components/ui/Text"; import styles from "./FirstSuccessfulSync.module.scss"; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.module.scss b/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.module.scss new file mode 100644 index 0000000000000..a80281793cee3 --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.module.scss @@ -0,0 +1,3 @@ +.paddedButton { + margin-left: 10px; +} diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx index b4cc1e0517240..492f5c9b2fc2c 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx @@ -4,14 +4,15 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import styled, { keyframes } from "styled-components"; -import { Button } from "components/base"; -import { Text } from "components/base/Text"; import Link from "components/Link"; +import { Button } from "components/ui"; +import { Text } from "components/ui/Text"; import Status from "core/statuses"; import { JobStatus, WebBackendConnectionRead } from "../../../core/request/AirbyteClient"; import { RoutePaths } from "../../routePaths"; +import styles from "./ProgressBlock.module.scss"; const run = keyframes` from { @@ -54,9 +55,6 @@ const ControlBlock = styled.div` justify-content: center; align-items: center; `; -const PaddedButton = styled(Button)` - margin-left: 10px; -`; interface ProgressBlockProps { connection: WebBackendConnectionRead; @@ -88,9 +86,9 @@ const ProgressBlock: React.FC = ({ connection, onSync }) => {showMessage(connection.latestSyncJobStatus)} - + ); } diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/TitlesBlock.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/TitlesBlock.tsx index 7563394fe7c1c..86a176600c611 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/components/TitlesBlock.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/components/TitlesBlock.tsx @@ -1,6 +1,6 @@ import React from "react"; -import { Text } from "components/base/Text"; +import { Text } from "components/ui/Text"; import styles from "./TitlesBlock.module.scss"; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/UseCaseBlock.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/UseCaseBlock.tsx index 3626dcf709118..3e2b9c14f99f7 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/components/UseCaseBlock.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/components/UseCaseBlock.tsx @@ -2,7 +2,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; import styled from "styled-components"; -import { Card } from "components/base/Card"; +import { Card } from "components/ui/Card"; interface UseCaseBlockProps { count: number; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/components/ShowVideo.module.scss b/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/components/ShowVideo.module.scss new file mode 100644 index 0000000000000..a9165858eaa95 --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/components/ShowVideo.module.scss @@ -0,0 +1,13 @@ +@use "../../../../../scss/colors"; + +.closeButton { + position: absolute; + top: 30px; + right: 30px; + color: colors.$white; + font-size: 20px; + + &:hover { + border: none; + } +} diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/components/ShowVideo.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/components/ShowVideo.tsx index 21ed9d47fcb6a..3a594e3d7bbe4 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/components/ShowVideo.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/components/ShowVideo.tsx @@ -1,34 +1,21 @@ import { faTimes } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; -import styled from "styled-components"; -import { Button } from "components/base"; +import { Button } from "components/ui/Button"; import { Modal } from "components/ui/Modal"; +import styles from "./ShowVideo.module.scss"; + interface ShowVideoProps { videoId?: string; onClose: () => void; } -const CloseButton = styled(Button)` - position: absolute; - top: 30px; - right: 30px; - color: ${({ theme }) => theme.whiteColor}; - font-size: 20px; - - &:hover { - border: none; - } -`; - const ShowVideo: React.FC = ({ videoId, onClose }) => { return ( - - - +