forked from airbytehq/airbyte
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
✨Source Apify Dataset: Migrate Python CDK to Low-code CDK (airbytehq#…
…29859) Co-authored-by: Marcos Marx <marcosmarxm@users.noreply.github.com> Co-authored-by: marcosmarxm <marcosmarxm@gmail.com>
- Loading branch information
Showing
28 changed files
with
488 additions
and
338 deletions.
There are no files selected for viewing
1 change: 0 additions & 1 deletion
1
airbyte-integrations/connectors/source-apify-dataset/.dockerignore
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,5 @@ | ||
* | ||
!Dockerfile | ||
!Dockerfile.test | ||
!main.py | ||
!source_apify_dataset | ||
!setup.py | ||
|
36 changes: 29 additions & 7 deletions
36
airbyte-integrations/connectors/source-apify-dataset/Dockerfile
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,16 +1,38 @@ | ||
FROM python:3.9-slim | ||
FROM python:3.9.11-alpine3.15 as base | ||
|
||
# build and load all requirements | ||
FROM base as builder | ||
WORKDIR /airbyte/integration_code | ||
|
||
# upgrade pip to the latest version | ||
RUN apk --no-cache upgrade \ | ||
&& pip install --upgrade pip \ | ||
&& apk --no-cache add tzdata build-base | ||
|
||
# Bash is installed for more convenient debugging. | ||
RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* | ||
|
||
COPY setup.py ./ | ||
# install necessary packages to a temporary folder | ||
RUN pip install --prefix=/install . | ||
|
||
# build a clean environment | ||
FROM base | ||
WORKDIR /airbyte/integration_code | ||
COPY source_apify_dataset ./source_apify_dataset | ||
|
||
# copy all loaded and built libraries to a pure basic image | ||
COPY --from=builder /install /usr/local | ||
# add default timezone settings | ||
COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime | ||
RUN echo "Etc/UTC" > /etc/timezone | ||
|
||
# bash is installed for more convenient debugging. | ||
RUN apk --no-cache add bash | ||
|
||
# copy payload code only | ||
COPY main.py ./ | ||
COPY setup.py ./ | ||
RUN pip install . | ||
COPY source_apify_dataset ./source_apify_dataset | ||
|
||
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" | ||
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] | ||
|
||
LABEL io.airbyte.version=0.2.0 | ||
LABEL io.airbyte.version=1.0.0 | ||
LABEL io.airbyte.name=airbyte/source-apify-dataset |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
4 changes: 0 additions & 4 deletions
4
...rce-apify-dataset/unit_tests/unit_test.py → ...nnectors/source-apify-dataset/__init__.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,3 @@ | ||
# | ||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
# | ||
|
||
|
||
def test_example_method(): | ||
assert True |
51 changes: 38 additions & 13 deletions
51
airbyte-integrations/connectors/source-apify-dataset/acceptance-test-config.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,19 +1,44 @@ | ||
# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) | ||
# for more information about how to configure these tests | ||
connector_image: airbyte/source-apify-dataset:dev | ||
tests: | ||
acceptance_tests: | ||
spec: | ||
- spec_path: "source_apify_dataset/spec.json" | ||
tests: | ||
- spec_path: "source_apify_dataset/spec.yaml" | ||
backward_compatibility_tests_config: | ||
disable_for_version: 0.2.0 | ||
connection: | ||
- config_path: "secrets/config.json" | ||
status: "succeed" | ||
- config_path: "integration_tests/invalid_config.json" | ||
status: "failed" | ||
tests: | ||
- config_path: "secrets/config.json" | ||
status: "succeed" | ||
- config_path: "integration_tests/invalid_config.json" | ||
status: "failed" | ||
discovery: | ||
- config_path: "secrets/config.json" | ||
tests: | ||
- config_path: "secrets/config.json" | ||
backward_compatibility_tests_config: | ||
disable_for_version: 0.2.0 | ||
basic_read: | ||
- config_path: "secrets/config.json" | ||
configured_catalog_path: "integration_tests/configured_catalog.json" | ||
tests: | ||
- config_path: "secrets/config.json" | ||
configured_catalog_path: "integration_tests/configured_catalog.json" | ||
incremental: | ||
bypass_reason: Connector doesn't use incremental sync | ||
# tests: | ||
# - config_path: "secrets/config.json" | ||
# configured_catalog_path: "integration_tests/configured_catalog.json" | ||
# future_state: | ||
# future_state_path: "integration_tests/abnormal_state.json" | ||
full_refresh: | ||
- config_path: "secrets/config.json" | ||
configured_catalog_path: "integration_tests/configured_catalog.json" | ||
tests: | ||
- config_path: "secrets/config.json" | ||
configured_catalog_path: "integration_tests/configured_catalog.json" | ||
ignored_fields: | ||
datasets: | ||
- name: "accessedAt" | ||
bypass_reason: "Change everytime" | ||
- name: "stats/readCount" | ||
bypass_reason: "Change everytime" | ||
dataset: | ||
- name: "accessedAt" | ||
bypass_reason: "Change everytime" | ||
- name: "stats/readCount" | ||
bypass_reason: "Change everytime" |
1 change: 1 addition & 0 deletions
1
airbyte-integrations/connectors/source-apify-dataset/acceptance-test-docker.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,2 +1,3 @@ | ||
#!/usr/bin/env sh | ||
|
||
source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# | ||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
# |
16 changes: 16 additions & 0 deletions
16
airbyte-integrations/connectors/source-apify-dataset/integration_tests/abnormal_state.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
[ | ||
{ | ||
"type": "STREAM", | ||
"stream": { | ||
"stream_state": { "modifiedAt": "3021-09-08T07:04:28.000Z" }, | ||
"stream_descriptor": { "name": "dataset" } | ||
} | ||
}, | ||
{ | ||
"type": "STREAM", | ||
"stream": { | ||
"stream_state": { "modifiedAt": "3021-09-08T07:04:28.000Z" }, | ||
"stream_descriptor": { "name": "datasets" } | ||
} | ||
} | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
20 changes: 0 additions & 20 deletions
20
airbyte-integrations/connectors/source-apify-dataset/integration_tests/catalog.json
This file was deleted.
Oops, something went wrong.
39 changes: 23 additions & 16 deletions
39
...te-integrations/connectors/source-apify-dataset/integration_tests/configured_catalog.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,24 +1,31 @@ | ||
{ | ||
"streams": [ | ||
{ | ||
"stream": { | ||
"name": "datasets", | ||
"json_schema": {}, | ||
"supported_sync_modes": ["full_refresh"] | ||
}, | ||
"sync_mode": "full_refresh", | ||
"destination_sync_mode": "overwrite" | ||
}, | ||
{ | ||
"stream": { | ||
"name": "dataset", | ||
"json_schema": {}, | ||
"supported_sync_modes": ["full_refresh"] | ||
}, | ||
"sync_mode": "full_refresh", | ||
"destination_sync_mode": "overwrite", | ||
"destination_sync_mode": "overwrite" | ||
}, | ||
{ | ||
"stream": { | ||
"name": "DatasetItems", | ||
"supported_sync_modes": ["full_refresh"], | ||
"destination_sync_mode": "overwrite", | ||
"json_schema": { | ||
"$schema": "http://json-schema.org/draft-07/schema#", | ||
"type": "object", | ||
"properties": { | ||
"data": { | ||
"type": "object", | ||
"additionalProperties": true | ||
} | ||
}, | ||
"additionalProperties": true | ||
} | ||
} | ||
"name": "item_collection", | ||
"json_schema": {}, | ||
"supported_sync_modes": ["full_refresh"] | ||
}, | ||
"sync_mode": "full_refresh", | ||
"destination_sync_mode": "overwrite" | ||
} | ||
] | ||
} |
2 changes: 2 additions & 0 deletions
2
...yte-integrations/connectors/source-apify-dataset/integration_tests/expected_records.jsonl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
{"stream": "datasets", "data": {"id":"Mxnvcv4Rspg9P9aP0","name":"my-dataset-name","userId":"YnGtyk7naKpwpousW","username":"encouraging_cliff","createdAt":"2023-08-25T19:19:33.588Z","modifiedAt":"2023-08-25T19:19:33.588Z","accessedAt":"2023-08-25T19:19:43.646Z","itemCount":0,"cleanItemCount":0,"actId":null,"actRunId":null,"schema":null,"stats":{"inflatedBytes":0,"readCount":0,"writeCount":0}}, "emitted_at": 1692990238010} | ||
{"stream": "dataset", "data": {"id":"Mxnvcv4Rspg9P9aP0","name":"my-dataset-name","userId":"YnGtyk7naKpwpousW","createdAt":"2023-08-25T19:19:33.588Z","modifiedAt":"2023-08-25T19:19:33.588Z","accessedAt":"2023-08-25T19:19:43.646Z","itemCount":0,"cleanItemCount":0,"actId":null,"actRunId":null,"schema":null,"stats":{"readCount":0,"writeCount":0,"storageBytes":0},"fields":[]}, "emitted_at": 1692990238010} |
4 changes: 2 additions & 2 deletions
4
airbyte-integrations/connectors/source-apify-dataset/integration_tests/invalid_config.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
{ | ||
"datasetId": "non_existent_dataset_id", | ||
"clean": false | ||
"token": "abc", | ||
"start_date": "2099-08-25T00:00:59.244Z" | ||
} |
4 changes: 4 additions & 0 deletions
4
airbyte-integrations/connectors/source-apify-dataset/integration_tests/sample_config.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
{ | ||
"token": "apify_api_XXXXXXXXXXXXXXXXXXXX", | ||
"start_date": "2023-08-25T00:00:59.244Z" | ||
} |
9 changes: 9 additions & 0 deletions
9
airbyte-integrations/connectors/source-apify-dataset/integration_tests/sample_state.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
[ | ||
{ | ||
"type": "STREAM", | ||
"stream": { | ||
"stream_state": { "modifiedAt": "3021-09-08T07:04:28.000Z" }, | ||
"stream_descriptor": { "name": "example" } | ||
} | ||
} | ||
] |
30 changes: 18 additions & 12 deletions
30
airbyte-integrations/connectors/source-apify-dataset/metadata.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,24 +1,30 @@ | ||
data: | ||
allowedHosts: | ||
hosts: | ||
- api.apify.com | ||
registries: | ||
oss: | ||
enabled: true | ||
cloud: | ||
enabled: true | ||
connectorSubtype: api | ||
connectorType: source | ||
definitionId: 47f17145-fe20-4ef5-a548-e29b048adf84 | ||
dockerImageTag: 0.2.0 | ||
dockerImageTag: 1.0.0 | ||
dockerRepository: airbyte/source-apify-dataset | ||
githubIssueLabel: source-apify-dataset | ||
icon: apify.svg | ||
icon: apify-dataset.svg | ||
license: MIT | ||
name: Apify Dataset | ||
registries: | ||
cloud: | ||
enabled: true | ||
oss: | ||
enabled: true | ||
releaseDate: 2023-08-25 | ||
releaseStage: alpha | ||
releases: | ||
breakingChanges: | ||
1.0.0: | ||
upgradeDeadline: 2023-08-30 | ||
message: "Update spec to use token and ingest all 3 streams correctly" | ||
supportLevel: community | ||
documentationUrl: https://docs.airbyte.com/integrations/sources/apify-dataset | ||
tags: | ||
- language:python | ||
ab_internal: | ||
sl: 100 | ||
ql: 100 | ||
supportLevel: community | ||
- language:lowcode | ||
metadataSpecVersion: "1.0" |
Oops, something went wrong.