Skip to content

Commit

Permalink
Merge branch 'master' into koconder-connector-genesys
Browse files Browse the repository at this point in the history
  • Loading branch information
koconder committed Oct 5, 2022
2 parents d6bbb62 + 515f1e5 commit ced87f0
Show file tree
Hide file tree
Showing 220 changed files with 1,489 additions and 796 deletions.
87 changes: 87 additions & 0 deletions .github/workflows/commands-for-testing-tool.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
name: Run Testing Tool Commands

on:
issue_comment:
types: [created]

jobs:
set-params:
# Only allow slash commands on pull request (not on issues)
if: ${{ github.event.issue.pull_request }}
runs-on: ubuntu-latest
outputs:
repo: ${{ steps.getref.outputs.repo }}
ref: ${{ steps.getref.outputs.ref }}
comment-id: ${{ steps.comment-info.outputs.comment-id }}
command: ${{ steps.regex.outputs.first_match }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
- name: Check PAT rate limits
run: |
./tools/bin/find_non_rate_limited_PAT \
${{ secrets.AIRBYTEIO_PAT }} \
${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
${{ secrets.SUPERTOPHER_PAT }} \
${{ secrets.DAVINCHIA_PAT }}
- name: Get PR repo and ref
id: getref
run: |
pr_info="$(curl ${{ github.event.issue.pull_request.url }})"
echo ::set-output name=ref::"$(echo $pr_info | jq -r '.head.ref')"
echo ::set-output name=repo::"$(echo $pr_info | jq -r '.head.repo.full_name')"
- name: Get comment id
id: comment-info
run: |
echo ::set-output name=comment-id::"${{ github.event.comment.id }}"
- name: Get command
id: regex
uses: AsasInnab/regex-action@v1
with:
regex_pattern: "^/[a-zA-Z0-9_/-]+"
regex_flags: "i"
search_string: ${{ github.event.comment.body }}
helps-run:
runs-on: ubuntu-latest
if: |
needs.set-params.outputs.command == '/help-full' ||
needs.set-params.outputs.command == '/help'
needs: set-params
steps:
- name: Update comment for processing
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ needs.set-params.outputs.comment-id }}
reactions: eyes, rocket
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Pull Testing Tool docker image
run: docker pull airbyte/airbyte-e2e-testing-tool:latest
- name: Create input and output folders
run: |
mkdir secrets
mkdir result
- name: Run docker container with params
run: docker run -v $(pwd)/secrets:/secrets -v $(pwd)/result:/result airbyte/airbyte-e2e-testing-tool:latest ${{ github.event.comment.body }}
- name: Read file with results
id: read_file
uses: andstor/file-reader-action@v1
with:
path: "result/log"
- name: Add Success Comment
if: needs.set-params.outputs.comment-id && success()
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ needs.set-params.outputs.comment-id }}
body: |
> :white_check_mark: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
${{ steps.read_file.outputs.contents }}
reactions: +1
- name: Add Failure Comment
if: needs.set-params.outputs.comment-id && failure()
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ needs.set-params.outputs.comment-id }}
body: |
> :x: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
reactions: -1
4 changes: 4 additions & 0 deletions airbyte-cdk/python/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# Changelog

## 0.1.92
- Low-code: Properly propagate $options to array items
- Low-code: Log request and response when running check operation in debug mode

## 0.1.91
- Low-code: Rename LimitPaginator to DefaultPaginator and move page_size field to PaginationStrategy

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,13 +138,11 @@ def build(self, class_or_class_name: Union[str, Type], config, instantiate: bool
class_ = self._get_class_from_fully_qualified_class_name(class_or_class_name)
else:
class_ = class_or_class_name

# create components in options before propagating them
if OPTIONS_STR in kwargs:
kwargs[OPTIONS_STR] = {
k: self._create_subcomponent(k, v, kwargs, config, class_, instantiate) for k, v in kwargs[OPTIONS_STR].items()
}

updated_kwargs = {k: self._create_subcomponent(k, v, kwargs, config, class_, instantiate) for k, v in kwargs.items()}

if instantiate:
Expand Down Expand Up @@ -216,7 +214,7 @@ def _create_subcomponent(self, key, definition, kwargs, config, parent_class, in
self._create_subcomponent(
key,
sub,
self._merge_dicts(kwargs.get(OPTIONS_STR, dict()), self._get_subcomponent_options(sub)),
kwargs,
config,
parent_class,
instantiate,
Expand Down
2 changes: 1 addition & 1 deletion airbyte-cdk/python/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

setup(
name="airbyte-cdk",
version="0.1.91",
version="0.1.92",
description="A framework for writing Airbyte Connectors.",
long_description=README,
long_description_content_type="text/markdown",
Expand Down
60 changes: 54 additions & 6 deletions airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ def test_full_config():
primary_key: "id"
extractor:
$ref: "*ref(extractor)"
field_pointer: ["result"]
field_pointer: ["{{ options['name'] }}"]
retriever:
$ref: "*ref(retriever)"
requester:
Expand Down Expand Up @@ -365,7 +365,7 @@ def test_full_config():
assert type(stream.retriever.record_selector) == RecordSelector
assert type(stream.retriever.record_selector.extractor.decoder) == JsonDecoder

assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_pointer] == ["result"]
assert [fp.eval(input_config) for fp in stream.retriever.record_selector.extractor.field_pointer] == ["lists"]
assert type(stream.retriever.record_selector.record_filter) == RecordFilter
assert stream.retriever.record_selector.record_filter._filter_interpolator.condition == "{{ record['id'] > stream_state['id'] }}"
assert stream.schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.json"
Expand All @@ -378,18 +378,24 @@ def test_full_config():
assert stream.retriever.requester.path.default == "marketing/lists"


def test_create_record_selector():
content = """
@pytest.mark.parametrize(
"test_name, record_selector, expected_runtime_selector",
[("test_static_record_selector", "result", "result"), ("test_options_record_selector", "{{ options['name'] }}", "lists")],
)
def test_create_record_selector(test_name, record_selector, expected_runtime_selector):
content = f"""
extractor:
type: DpathExtractor
selector:
$options:
name: "lists"
class_name: airbyte_cdk.sources.declarative.extractors.record_selector.RecordSelector
record_filter:
class_name: airbyte_cdk.sources.declarative.extractors.record_filter.RecordFilter
condition: "{{ record['id'] > stream_state['id'] }}"
extractor:
$ref: "*ref(extractor)"
field_pointer: ["result"]
field_pointer: ["{record_selector}"]
"""
config = parser.parse(content)

Expand All @@ -398,10 +404,52 @@ def test_create_record_selector():
selector = factory.create_component(config["selector"], input_config)()
assert isinstance(selector, RecordSelector)
assert isinstance(selector.extractor, DpathExtractor)
assert [fp.eval(input_config) for fp in selector.extractor.field_pointer] == ["result"]
assert [fp.eval(input_config) for fp in selector.extractor.field_pointer] == [expected_runtime_selector]
assert isinstance(selector.record_filter, RecordFilter)


@pytest.mark.parametrize(
"test_name, content, expected_field_pointer_value",
[
(
"test_option_in_selector",
"""
extractor:
type: DpathExtractor
field_pointer: ["{{ options['name'] }}"]
selector:
class_name: airbyte_cdk.sources.declarative.extractors.record_selector.RecordSelector
$options:
name: "selector"
extractor: "*ref(extractor)"
""",
"selector",
),
(
"test_option_in_extractor",
"""
extractor:
type: DpathExtractor
$options:
name: "extractor"
field_pointer: ["{{ options['name'] }}"]
selector:
class_name: airbyte_cdk.sources.declarative.extractors.record_selector.RecordSelector
$options:
name: "selector"
extractor: "*ref(extractor)"
""",
"extractor",
),
],
)
def test_options_propagation(test_name, content, expected_field_pointer_value):
config = parser.parse(content)

selector = factory.create_component(config["selector"], input_config, True)()
assert selector.extractor.field_pointer[0].eval(input_config) == expected_field_pointer_value


def test_create_requester():
content = """
requester:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ public void waitForTemporalNamespace() {
// cache before continuing on with any additional configuration/bean creation.
temporalService.blockingStub().describeNamespace(DescribeNamespaceRequest.newBuilder().setNamespace(temporalNamespace).build());
namespaceExists = true;
} catch (final StatusRuntimeException e) {
// This is to allow the configured namespace to be available in the Temporal
// cache before continuing on with any additional configuration/bean creation.
Thread.sleep(TimeUnit.SECONDS.toMillis(5));
} catch (final InterruptedException | StatusRuntimeException e) {
log.debug("Namespace '{}' does not exist yet. Re-checking...", temporalNamespace);
try {
Thread.sleep(TimeUnit.SECONDS.toMillis(5));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
- name: Amplitude
sourceDefinitionId: fa9f58c6-2d03-4237-aaa4-07d75e0c1396
dockerRepository: airbyte/source-amplitude
dockerImageTag: 0.1.14
dockerImageTag: 0.1.15
documentationUrl: https://docs.airbyte.io/integrations/sources/amplitude
icon: amplitude.svg
sourceType: api
Expand Down Expand Up @@ -279,7 +279,7 @@
- name: Facebook Marketing
sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
dockerRepository: airbyte/source-facebook-marketing
dockerImageTag: 0.2.66
dockerImageTag: 0.2.67
documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing
icon: facebook.svg
sourceType: api
Expand Down Expand Up @@ -354,7 +354,7 @@
- name: GitHub
sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
dockerRepository: airbyte/source-github
dockerImageTag: 0.3.3
dockerImageTag: 0.3.4
documentationUrl: https://docs.airbyte.io/integrations/sources/github
icon: github.svg
sourceType: api
Expand Down Expand Up @@ -472,7 +472,7 @@
- name: HubSpot
sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
dockerRepository: airbyte/source-hubspot
dockerImageTag: 0.2.1
dockerImageTag: 0.2.2
documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot
icon: hubspot.svg
sourceType: api
Expand Down Expand Up @@ -504,7 +504,7 @@
- name: Iterable
sourceDefinitionId: 2e875208-0c0b-4ee4-9e92-1cb3156ea799
dockerRepository: airbyte/source-iterable
dockerImageTag: 0.1.17
dockerImageTag: 0.1.18
documentationUrl: https://docs.airbyte.io/integrations/sources/iterable
icon: iterable.svg
sourceType: api
Expand Down Expand Up @@ -765,11 +765,11 @@
- name: Paypal Transaction
sourceDefinitionId: d913b0f2-cc51-4e55-a44c-8ba1697b9239
dockerRepository: airbyte/source-paypal-transaction
dockerImageTag: 0.1.9
dockerImageTag: 0.1.10
documentationUrl: https://docs.airbyte.io/integrations/sources/paypal-transaction
icon: paypal.svg
sourceType: api
releaseStage: beta
releaseStage: generally_available
- name: Paystack
sourceDefinitionId: 193bdcb8-1dd9-48d1-aade-91cadfd74f9b
dockerRepository: airbyte/source-paystack
Expand Down Expand Up @@ -836,7 +836,7 @@
- name: Postgres
sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
dockerRepository: airbyte/source-postgres
dockerImageTag: 1.0.13
dockerImageTag: 1.0.14
documentationUrl: https://docs.airbyte.io/integrations/sources/postgres
icon: postgresql.svg
sourceType: database
Expand Down Expand Up @@ -1059,7 +1059,7 @@
- name: TikTok Marketing
sourceDefinitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35
dockerRepository: airbyte/source-tiktok-marketing
dockerImageTag: 0.1.16
dockerImageTag: 0.1.17
documentationUrl: https://docs.airbyte.io/integrations/sources/tiktok-marketing
icon: tiktok.svg
sourceType: api
Expand Down
16 changes: 8 additions & 8 deletions airbyte-config/init/src/main/resources/seed/source_specs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1105,7 +1105,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-amplitude:0.1.14"
- dockerImage: "airbyte/source-amplitude:0.1.15"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/amplitude"
connectionSpecification:
Expand Down Expand Up @@ -2561,7 +2561,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-facebook-marketing:0.2.66"
- dockerImage: "airbyte/source-facebook-marketing:0.2.67"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
Expand Down Expand Up @@ -3503,7 +3503,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-github:0.3.3"
- dockerImage: "airbyte/source-github:0.3.4"
spec:
documentationUrl: "https://docs.airbyte.com/integrations/sources/github"
connectionSpecification:
Expand Down Expand Up @@ -4715,7 +4715,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-hubspot:0.2.1"
- dockerImage: "airbyte/source-hubspot:0.2.2"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot"
connectionSpecification:
Expand Down Expand Up @@ -5002,7 +5002,7 @@
oauthFlowInitParameters: []
oauthFlowOutputParameters:
- - "access_token"
- dockerImage: "airbyte/source-iterable:0.1.17"
- dockerImage: "airbyte/source-iterable:0.1.18"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/iterable"
connectionSpecification:
Expand Down Expand Up @@ -8179,7 +8179,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-paypal-transaction:0.1.9"
- dockerImage: "airbyte/source-paypal-transaction:0.1.10"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/paypal-transactions"
connectionSpecification:
Expand Down Expand Up @@ -8590,7 +8590,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-postgres:1.0.13"
- dockerImage: "airbyte/source-postgres:1.0.14"
spec:
documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
connectionSpecification:
Expand Down Expand Up @@ -11181,7 +11181,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-tiktok-marketing:0.1.16"
- dockerImage: "airbyte/source-tiktok-marketing:0.1.17"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/tiktok-marketing"
changelogUrl: "https://docs.airbyte.io/integrations/sources/tiktok-marketing"
Expand Down
Loading

0 comments on commit ced87f0

Please sign in to comment.