diff --git a/.gitignore b/.gitignore index 82fcdcf890c651..1876fcbfcf486c 100644 --- a/.gitignore +++ b/.gitignore @@ -77,5 +77,8 @@ airbyte-config/**/resources/seed/oss_catalog.json # Helm charts .tgz dependencies charts/**/charts +# Snyk +.dccache + # Datadog dd-java-agent.jar diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index cf605d97da516d..791de533546585 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -2935,6 +2935,8 @@ components: $ref: "#/components/schemas/CatalogDiff" breakingChange: type: boolean + connectionStatus: + $ref: "#/components/schemas/ConnectionStatus" SourceSearch: type: object properties: diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java index 3db73387fec0c7..b0c987151527a6 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java @@ -194,10 +194,20 @@ public void close() throws Exception { return; } - LOGGER.debug("Closing normalization process"); + LOGGER.info("Terminating normalization process {}...", process.pid()); WorkerUtils.gentleClose(process, 1, TimeUnit.MINUTES); - if (process.isAlive() || process.exitValue() != 0) { - throw new WorkerException("Normalization process wasn't successful"); + + /* + * After attempting to close the process check the following: + * + * Did the process actually terminate? If "yes", did it do so nominally? + */ + if (process.isAlive()) { + throw new WorkerException("Normalization process " + process.pid() + " did not terminate after 1 minute."); + } else if (process.exitValue() != 0) { + throw new WorkerException("Normalization process " + process.pid() + " did not terminate normally (exit code: " + process.exitValue() + ")"); + } else { + LOGGER.info("Normalization process {} successfully terminated.", process.pid()); } } diff --git a/airbyte-config/init/src/main/resources/icons/senseforce.svg b/airbyte-config/init/src/main/resources/icons/senseforce.svg new file mode 100644 index 00000000000000..ad02b1c4a7d9e8 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/senseforce.svg @@ -0,0 +1,10 @@ + + + + SF Icon Copy + Created with Sketch. + + + + + \ No newline at end of file diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 0ac2e4e4694c80..e0b55dcc3b1e26 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -86,6 +86,7 @@ dockerRepository: airbyte/destination-clickhouse dockerImageTag: 0.2.0 documentationUrl: https://docs.airbyte.com/integrations/destinations/clickhouse + icon: clickhouse.svg releaseStage: alpha normalizationRepository: airbyte/normalization-clickhouse normalizationTag: 0.2.24 diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index d970fd3f4ff0bd..c0d08bd614a9e4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -228,7 +228,7 @@ dockerRepository: airbyte/source-clickhouse dockerImageTag: 0.1.14 documentationUrl: https://docs.airbyte.com/integrations/sources/clickhouse - icon: cliskhouse.svg + icon: clickhouse.svg sourceType: database releaseStage: alpha - name: ClickUp @@ -1148,6 +1148,13 @@ icon: persistiq.svg sourceType: api releaseStage: alpha +- name: Pexels API + sourceDefinitionId: 69d9eb65-8026-47dc-baf1-e4bf67901fd6 + dockerRepository: airbyte/source-pexels-api + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/pexels-api + sourceType: api + releaseStage: alpha - name: Pinterest sourceDefinitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003 dockerRepository: airbyte/source-pinterest @@ -1434,6 +1441,7 @@ dockerRepository: airbyte/source-senseforce dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.com/integrations/sources/senseforce + icon: senseforce.svg sourceType: api releaseStage: alpha - name: Sendinblue diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index a22df670b01d06..ae696de453cf01 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -10614,6 +10614,72 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-pexels-api:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/pexels-api" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Pexel API Spec" + type: "object" + required: + - "api_key" + - "query" + additionalProperties: true + properties: + api_key: + title: "API Key from the pexels website" + type: "string" + description: "API key is required to access pexels api, For getting your's\ + \ goto https://www.pexels.com/api/documentation and create account for\ + \ free." + airbyte_secret: true + query: + title: "Specific query for the search" + type: "string" + description: "Optional, the search query, Example Ocean, Tigers, Pears,\ + \ etc." + examples: + - "people" + - "oceans" + orientation: + title: "Specific orientation for the search" + type: "string" + description: "Optional, Desired photo orientation. The current supported\ + \ orientations are landscape, portrait or square" + examples: + - "square" + - "landscape" + size: + title: "Specific size for the search" + type: "string" + description: "Optional, Minimum photo size. The current supported sizes\ + \ are large(24MP), medium(12MP) or small(4MP)." + examples: + - "large" + - "small" + color: + title: "Specific color for the search" + type: "string" + description: "Optional, Desired photo color. Supported colors red, orange,\ + \ yellow, green, turquoise, blue, violet, pink, brown, black, gray, white\ + \ or any hexidecimal color code." + examples: + - "red" + - "orange" + locale: + title: "Specific locale for the search" + type: "string" + description: "Optional, The locale of the search you are performing. The\ + \ current supported locales are 'en-US' 'pt-BR' 'es-ES' 'ca-ES' 'de-DE'\ + \ 'it-IT' 'fr-FR' 'sv-SE' 'id-ID' 'pl-PL' 'ja-JP' 'zh-TW' 'zh-CN' 'ko-KR'\ + \ 'th-TH' 'nl-NL' 'hu-HU' 'vi-VN' 'cs-CZ' 'da-DK' 'fi-FI' 'uk-UA' 'el-GR'\ + \ 'ro-RO' 'nb-NO' 'sk-SK' 'tr-TR' 'ru-RU'." + examples: + - "en-US" + - "pt-BR" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-pinterest:0.1.9" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/pinterest" diff --git a/airbyte-connector-builder-server/dist/connector_builder_server-0.40.21-py3.10.egg b/airbyte-connector-builder-server/dist/connector_builder_server-0.40.21-py3.10.egg new file mode 100644 index 00000000000000..acb8ae91ba50e7 Binary files /dev/null and b/airbyte-connector-builder-server/dist/connector_builder_server-0.40.21-py3.10.egg differ diff --git a/airbyte-integrations/connectors/source-pexels-api/.dockerignore b/airbyte-integrations/connectors/source-pexels-api/.dockerignore new file mode 100644 index 00000000000000..49145deae91d61 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_pexels_api +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-pexels-api/Dockerfile b/airbyte-integrations/connectors/source-pexels-api/Dockerfile new file mode 100644 index 00000000000000..f2c50f9de98258 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_pexels_api ./source_pexels_api + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-pexels-api diff --git a/airbyte-integrations/connectors/source-pexels-api/README.md b/airbyte-integrations/connectors/source-pexels-api/README.md new file mode 100644 index 00000000000000..9185269d731ee2 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/README.md @@ -0,0 +1,103 @@ +# Pexels Api Source + +This is the repository for the Pexels Api configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/pexels-api). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-pexels-api:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pexels-api) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pexels_api/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source pexels-api test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-pexels-api:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-pexels-api:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-pexels-api:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pexels-api:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pexels-api:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pexels-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-pexels-api:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-pexels-api:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-pexels-api/__init__.py b/airbyte-integrations/connectors/source-pexels-api/__init__.py new file mode 100644 index 00000000000000..1100c1c58cf510 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-pexels-api/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pexels-api/acceptance-test-config.yml new file mode 100644 index 00000000000000..fac407da2ceb18 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/acceptance-test-config.yml @@ -0,0 +1,25 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-pexels-api:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_pexels_api/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-pexels-api/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-pexels-api/acceptance-test-docker.sh new file mode 100644 index 00000000000000..c51577d10690c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-pexels-api/bootstrap.md b/airbyte-integrations/connectors/source-pexels-api/bootstrap.md new file mode 100644 index 00000000000000..ef9f1c3f6da6fb --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/bootstrap.md @@ -0,0 +1,40 @@ +# Pexels-API + +The connector uses the v1 API documented here: https://www.pexels.com/api/documentation . It is +straightforward HTTP REST API with API based authentication. + +## API key + +Api key is mandate for this connector to work, It could be generated by a free account at https://www.pexels.com/api/new/. +Just pass the generated API key and optional parameters for establishing the connection. + +## Implementation details + +## Setup guide + +### Step 1: Set up Pexels-API connection + +- Generate an API key (Example: 12345) +- Params (If specific info is needed) +- Available params + - query: Ocean, Tigers, Pears, etc. Default is people + - orientation: landscape, portrait or square. Default is landscape + - size: large, medium, small. Default is large + - color: red, orange, yellow, green, turquoise, blue, violet, pink, brown, black, gray, white or any hexidecimal color code. + - locale: en-US, pt-BR, es-ES, ca-ES, de-DE, it-IT, fr-FR, sv-SE, id-ID, pl-PL, ja-JP, zh-TW, zh-CN, ko-KR, th-TH, nl-NL, hu-HU, vi-VN,
cs-CZ, da-DK, fi-FI, uk-UA, el-GR, ro-RO, nb-NO, sk-SK, tr-TR, ru-RU. Default is en-US + +## Step 2: Generate schema for the endpoint + +### Custom schema is generated and tested with different IDs + +## Step 3: Spec, Secrets, and connector yaml files are configured with reference to the Airbyte documentation. + +## In a nutshell: + +1. Navigate to the Airbyte Open Source dashboard. +2. Set the name for your source. +3. Enter your `api_key`. +5. Enter your config params if needed. (Optional) +6. Click **Set up source**. + + * We use only GET methods, towards the API endpoints which is straightforward \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pexels-api/build.gradle b/airbyte-integrations/connectors/source-pexels-api/build.gradle new file mode 100644 index 00000000000000..3309355431d7cf --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_pexels_api' +} diff --git a/airbyte-integrations/connectors/source-pexels-api/integration_tests/__init__.py b/airbyte-integrations/connectors/source-pexels-api/integration_tests/__init__.py new file mode 100644 index 00000000000000..1100c1c58cf510 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-pexels-api/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-pexels-api/integration_tests/abnormal_state.json new file mode 100644 index 00000000000000..52b0f2c2118f45 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-pexels-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pexels-api/integration_tests/acceptance.py new file mode 100644 index 00000000000000..1302b2f57e10ec --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-pexels-api/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-pexels-api/integration_tests/configured_catalog.json new file mode 100644 index 00000000000000..4cc129d87333c0 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/integration_tests/configured_catalog.json @@ -0,0 +1,49 @@ +{ + "streams": [ + { + "stream": { + "name": "photos_search", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "photos_curated", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "videos_search", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "videos_popular", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "collection_featured", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-pexels-api/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-pexels-api/integration_tests/invalid_config.json new file mode 100644 index 00000000000000..7720051d1530df --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/integration_tests/invalid_config.json @@ -0,0 +1,8 @@ +{ + "api_key": "", + "query":"", + "orientation": "", + "size": "", + "color": "", + "locale": "" +} diff --git a/airbyte-integrations/connectors/source-pexels-api/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-pexels-api/integration_tests/sample_config.json new file mode 100644 index 00000000000000..4b22317519972a --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/integration_tests/sample_config.json @@ -0,0 +1,8 @@ +{ + "api_key": "", + "query":"", + "orientation": "", + "size": "", + "color": "", + "locale": "" +} diff --git a/airbyte-integrations/connectors/source-pexels-api/main.py b/airbyte-integrations/connectors/source-pexels-api/main.py new file mode 100644 index 00000000000000..49c78e42ca7283 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pexels_api import SourcePexelsApi + +if __name__ == "__main__": + source = SourcePexelsApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pexels-api/requirements.txt b/airbyte-integrations/connectors/source-pexels-api/requirements.txt new file mode 100644 index 00000000000000..0411042aa0911f --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-pexels-api/setup.py b/airbyte-integrations/connectors/source-pexels-api/setup.py new file mode 100644 index 00000000000000..595906638a3b2c --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_pexels_api", + description="Source implementation for Pexels Api.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/__init__.py b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/__init__.py new file mode 100644 index 00000000000000..0a2b7916714d32 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourcePexelsApi + +__all__ = ["SourcePexelsApi"] diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/pexels_api.yaml b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/pexels_api.yaml new file mode 100644 index 00000000000000..0f561d6caa28ef --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/pexels_api.yaml @@ -0,0 +1,135 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: [] + + requester_stream: + request_parameters: + query: | + {{ + config['query'] + if options['name'] == 'photos_search' or options['name'] == 'videos_search' + else '' + }} + orientation: | + {{ + config['orientation'] + if options['name'] == 'photos_search' or options['name'] == 'videos_search' + else '' + }} + size: | + {{ + config['size'] + if options['name'] == 'photos_search' or options['name'] == 'videos_search' + else '' + }} + locale: | + {{ + config['locale'] + if options['name'] == 'photos_search' or options['name'] == 'videos_search' + else '' + }} + color: | + {{ + config['color'] + if options['name'] == 'photos_search' + else '' + }} + + requester: + url_base: "https://api.pexels.com" + http_method: "GET" + request_options_provider: + request_parameters: + $ref: "*ref(definitions.requester_stream.request_parameters)" + authenticator: + type: ApiKeyAuthenticator + header: "Authorization" + api_token: "{{ config['api_key'] }}" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [500] + action: FAIL + + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + + base_stream: + schema_loader: + type: JsonSchema + file_path: "./source_pexels_api/schemas/{{ options['name'] }}.json" + retriever: + $ref: "*ref(definitions.retriever)" + + page_stream: + schema_loader: + type: JsonSchema + file_path: "./source_pexels_api/schemas/{{ options['name'] }}.json" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: "DefaultPaginator" + url_base: "*ref(definitions.requester.url_base)" + pagination_strategy: + type: "PageIncrement" + page_size: 1000 + page_token_option: + inject_into: "request_parameter" + field_name: "page" + page_size_option: + inject_into: "request_parameter" + field_name: "per_page" + requester: + $ref: "*ref(definitions.requester)" + + photos_search_stream: + $ref: "*ref(definitions.page_stream)" + $options: + name: "photos_search" + path: "/v1/search" + + photos_curated_stream: + $ref: "*ref(definitions.page_stream)" + $options: + name: "photos_curated" + path: "/v1/curated" + + videos_search_stream: + $ref: "*ref(definitions.page_stream)" + $options: + name: "videos_search" + path: "/videos/search" + + videos_popular_stream: + $ref: "*ref(definitions.page_stream)" + $options: + name: "videos_popular" + path: "/videos/popular" + + collection_featured_stream: + $ref: "*ref(definitions.page_stream)" + $options: + name: "collection_featured" + path: "/v1/collections/featured" + +streams: + - "*ref(definitions.photos_search_stream)" + - "*ref(definitions.photos_curated_stream)" + - "*ref(definitions.collection_featured_stream)" + - "*ref(definitions.videos_search_stream)" + - "*ref(definitions.videos_popular_stream)" + +check: + stream_names: + - "videos_popular" diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/collection_featured.json b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/collection_featured.json new file mode 100644 index 00000000000000..c3d248b7e5de43 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/collection_featured.json @@ -0,0 +1,98 @@ +{ + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/object1667390893.json", + "title": "Root", + "type": "object", + "properties": { + "collections": { + "$id": "#root/collections", + "title": "Collections", + "type": "array", + "default": [], + "items": { + "$id": "#root/collections/items", + "title": "Items", + "type": "object", + "properties": { + "id": { + "$id": "#root/collections/items/id", + "title": "Id", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "title": { + "$id": "#root/collections/items/title", + "title": "Title", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "description": { + "$id": "#root/collections/items/description", + "title": "Description", + "type": ["string","null"], + "default": null + }, + "private": { + "$id": "#root/collections/items/private", + "title": "Private", + "type": "boolean", + "default": true + }, + "media_count": { + "$id": "#root/collections/items/media_count", + "title": "Media_count", + "type": "integer", + "default": 0 + }, + "photos_count": { + "$id": "#root/collections/items/photos_count", + "title": "Photos_count", + "type": "integer", + "default": 0 + }, + "videos_count": { + "$id": "#root/collections/items/videos_count", + "title": "Videos_count", + "type": "integer", + "default": 0 + } + } + } + }, + "page": { + "$id": "#root/page", + "title": "Page", + "type": "integer", + "default": 0 + }, + "per_page": { + "$id": "#root/per_page", + "title": "Per_page", + "type": "integer", + "default": 0 + }, + "total_results": { + "$id": "#root/total_results", + "title": "Total_results", + "type": "integer", + "default": 0 + }, + "next_page": { + "$id": "#root/next_page", + "title": "Next_page", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "prev_page": { + "$id": "#root/prev_page", + "title": "Prev_page", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } +} diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/collection_get_by_id.json b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/collection_get_by_id.json new file mode 100644 index 00000000000000..0f528423c24283 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/collection_get_by_id.json @@ -0,0 +1,189 @@ +{ + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/object1667390919.json", + "title": "Root", + "type": "object", + "properties": { + "id": { + "$id": "#root/id", + "title": "Id", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "media": { + "$id": "#root/media", + "title": "Media", + "type": "array", + "default": [], + "items": { + "$id": "#root/media/items", + "title": "Items", + "type": "object", + "properties": { + "type": { + "$id": "#root/media/items/type", + "title": "Type", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "id": { + "$id": "#root/media/items/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "width": { + "$id": "#root/media/items/width", + "title": "Width", + "type": "integer", + "default": 0 + }, + "height": { + "$id": "#root/media/items/height", + "title": "Height", + "type": "integer", + "default": 0 + }, + "url": { + "$id": "#root/media/items/url", + "title": "Url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "photographer": { + "$id": "#root/media/items/photographer", + "title": "Photographer", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "photographer_url": { + "$id": "#root/media/items/photographer_url", + "title": "Photographer_url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "photographer_id": { + "$id": "#root/media/items/photographer_id", + "title": "Photographer_id", + "type": "integer", + "default": 0 + }, + "avg_color": { + "$id": "#root/media/items/avg_color", + "title": "Avg_color", + "type": ["null", "string"], + "default": "", + "pattern": "^.*$" + }, + "src": { + "$id": "#root/media/items/src", + "title": "Src", + "type": "object", + "properties": { + "original": { + "$id": "#root/media/items/src/original", + "title": "Original", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "large2x": { + "$id": "#root/media/items/src/large2x", + "title": "Large2x", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "large": { + "$id": "#root/media/items/src/large", + "title": "Large", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "medium": { + "$id": "#root/media/items/src/medium", + "title": "Medium", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "small": { + "$id": "#root/media/items/src/small", + "title": "Small", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "portrait": { + "$id": "#root/media/items/src/portrait", + "title": "Portrait", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "landscape": { + "$id": "#root/media/items/src/landscape", + "title": "Landscape", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "tiny": { + "$id": "#root/media/items/src/tiny", + "title": "Tiny", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + }, + "liked": { + "$id": "#root/media/items/liked", + "title": "Liked", + "type": "boolean", + "default": true + } + } + } + }, + "page": { + "$id": "#root/page", + "title": "Page", + "type": "integer", + "default": 0 + }, + "per_page": { + "$id": "#root/per_page", + "title": "Per_page", + "type": "integer", + "default": 0 + }, + "total_results": { + "$id": "#root/total_results", + "title": "Total_results", + "type": "integer", + "default": 0 + }, + "next_page": { + "$id": "#root/next_page", + "title": "Next_page", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "prev_page": { + "$id": "#root/prev_page", + "title": "Prev_page", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } +} diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/photos_curated.json b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/photos_curated.json new file mode 100644 index 00000000000000..376010a78ae48d --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/photos_curated.json @@ -0,0 +1,175 @@ +{ + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/object1667389456.json", + "title": "Root", + "type": "object", + "properties": { + "page": { + "$id": "#root/page", + "title": "Page", + "type": "integer", + "default": 0 + }, + "per_page": { + "$id": "#root/per_page", + "title": "Per_page", + "type": "integer", + "default": 0 + }, + "photos": { + "$id": "#root/photos", + "title": "Photos", + "type": "array", + "default": [], + "items": { + "$id": "#root/photos/items", + "title": "Items", + "type": "object", + "properties": { + "id": { + "$id": "#root/photos/items/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "width": { + "$id": "#root/photos/items/width", + "title": "Width", + "type": "integer", + "default": 0 + }, + "height": { + "$id": "#root/photos/items/height", + "title": "Height", + "type": "integer", + "default": 0 + }, + "url": { + "$id": "#root/photos/items/url", + "title": "Url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "photographer": { + "$id": "#root/photos/items/photographer", + "title": "Photographer", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "photographer_url": { + "$id": "#root/photos/items/photographer_url", + "title": "Photographer_url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "photographer_id": { + "$id": "#root/photos/items/photographer_id", + "title": "Photographer_id", + "type": "integer", + "default": 0 + }, + "avg_color": { + "$id": "#root/photos/items/avg_color", + "title": "Avg_color", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "src": { + "$id": "#root/photos/items/src", + "title": "Src", + "type": "object", + "properties": { + "original": { + "$id": "#root/photos/items/src/original", + "title": "Original", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "large2x": { + "$id": "#root/photos/items/src/large2x", + "title": "Large2x", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "large": { + "$id": "#root/photos/items/src/large", + "title": "Large", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "medium": { + "$id": "#root/photos/items/src/medium", + "title": "Medium", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "small": { + "$id": "#root/photos/items/src/small", + "title": "Small", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "portrait": { + "$id": "#root/photos/items/src/portrait", + "title": "Portrait", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "landscape": { + "$id": "#root/photos/items/src/landscape", + "title": "Landscape", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "tiny": { + "$id": "#root/photos/items/src/tiny", + "title": "Tiny", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + }, + "liked": { + "$id": "#root/photos/items/liked", + "title": "Liked", + "type": "boolean", + "default": true + }, + "alt": { + "$id": "#root/photos/items/alt", + "title": "Alt", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + } + }, + "total_results": { + "$id": "#root/total_results", + "title": "Total_results", + "type": "integer", + "default": 0 + }, + "next_page": { + "$id": "#root/next_page", + "title": "Next_page", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } +} diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/photos_get_by_id.json b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/photos_get_by_id.json new file mode 100644 index 00000000000000..21ece3efb890af --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/photos_get_by_id.json @@ -0,0 +1,137 @@ +{ + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/object1667389413.json", + "title": "Root", + "type": "object", + "properties": { + "id": { + "$id": "#root/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "width": { + "$id": "#root/width", + "title": "Width", + "type": "integer", + "default": 0 + }, + "height": { + "$id": "#root/height", + "title": "Height", + "type": "integer", + "default": 0 + }, + "url": { + "$id": "#root/url", + "title": "Url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "photographer": { + "$id": "#root/photographer", + "title": "Photographer", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "photographer_url": { + "$id": "#root/photographer_url", + "title": "Photographer_url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "photographer_id": { + "$id": "#root/photographer_id", + "title": "Photographer_id", + "type": "integer", + "default": 0 + }, + "avg_color": { + "$id": "#root/avg_color", + "title": "Avg_color", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "src": { + "$id": "#root/src", + "title": "Src", + "type": "object", + "properties": { + "original": { + "$id": "#root/src/original", + "title": "Original", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "large2x": { + "$id": "#root/src/large2x", + "title": "Large2x", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "large": { + "$id": "#root/src/large", + "title": "Large", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "medium": { + "$id": "#root/src/medium", + "title": "Medium", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "small": { + "$id": "#root/src/small", + "title": "Small", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "portrait": { + "$id": "#root/src/portrait", + "title": "Portrait", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "landscape": { + "$id": "#root/src/landscape", + "title": "Landscape", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "tiny": { + "$id": "#root/src/tiny", + "title": "Tiny", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + }, + "liked": { + "$id": "#root/liked", + "title": "Liked", + "type": "boolean", + "default": true + }, + "alt": { + "$id": "#root/alt", + "title": "Alt", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } +} diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/photos_search.json b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/photos_search.json new file mode 100644 index 00000000000000..efddf13582852c --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/photos_search.json @@ -0,0 +1,177 @@ +{ + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/object1667402697.json", + "title": "Root", + "type": "object", + "properties": { + "page": { + "$id": "#root/page", + "title": "Page", + "type": "integer", + "default": 0 + }, + "per_page": { + "$id": "#root/per_page", + "title": "Per_page", + "type": "integer", + "default": 0 + }, + "photos": { + "$id": "#root/photos", + "title": "Photos", + "type": "array", + "default": [], + "items":{ + "$id": "#root/photos/items", + "title": "Items", + "type": "object", + "properties": { + "id": { + "$id": "#root/photos/items/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "width": { + "$id": "#root/photos/items/width", + "title": "Width", + "type": "integer", + "default": 0 + }, + "height": { + "$id": "#root/photos/items/height", + "title": "Height", + "type": "integer", + "default": 0 + }, + "url": { + "$id": "#root/photos/items/url", + "title": "Url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "photographer": { + "$id": "#root/photos/items/photographer", + "title": "Photographer", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "photographer_url": { + "$id": "#root/photos/items/photographer_url", + "title": "Photographer_url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "photographer_id": { + "$id": "#root/photos/items/photographer_id", + "title": "Photographer_id", + "type": "integer", + "default": 0 + }, + "avg_color": { + "$id": "#root/photos/items/avg_color", + "title": "Avg_color", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "src": { + "$id": "#root/photos/items/src", + "title": "Src", + "type": "object", + "properties": { + "original": { + "$id": "#root/photos/items/src/original", + "title": "Original", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "large2x": { + "$id": "#root/photos/items/src/large2x", + "title": "Large2x", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "large": { + "$id": "#root/photos/items/src/large", + "title": "Large", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "medium": { + "$id": "#root/photos/items/src/medium", + "title": "Medium", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "small": { + "$id": "#root/photos/items/src/small", + "title": "Small", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "portrait": { + "$id": "#root/photos/items/src/portrait", + "title": "Portrait", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "landscape": { + "$id": "#root/photos/items/src/landscape", + "title": "Landscape", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "tiny": { + "$id": "#root/photos/items/src/tiny", + "title": "Tiny", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + } +, + "liked": { + "$id": "#root/photos/items/liked", + "title": "Liked", + "type": "boolean", + "default": true + }, + "alt": { + "$id": "#root/photos/items/alt", + "title": "Alt", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + } + + }, + "total_results": { + "$id": "#root/total_results", + "title": "Total_results", + "type": "integer", + "default": 0 + }, + "next_page": { + "$id": "#root/next_page", + "title": "Next_page", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } +} diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/videos_get_by_id.json b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/videos_get_by_id.json new file mode 100644 index 00000000000000..9ad4ab06780605 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/videos_get_by_id.json @@ -0,0 +1,196 @@ +{ + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/object1667389811.json", + "title": "Root", + "type": "object", + "properties": { + "page": { + "$id": "#root/page", + "title": "Page", + "type": "integer", + "default": 0 + }, + "per_page": { + "$id": "#root/per_page", + "title": "Per_page", + "type": "integer", + "default": 0 + }, + "total_results": { + "$id": "#root/total_results", + "title": "Total_results", + "type": "integer", + "default": 0 + }, + "url": { + "$id": "#root/url", + "title": "Url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "videos": { + "$id": "#root/videos", + "title": "Videos", + "type": "array", + "default": [], + "items": { + "$id": "#root/videos/items", + "title": "Items", + "type": "object", + "properties": { + "id": { + "$id": "#root/videos/items/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "width": { + "$id": "#root/videos/items/width", + "title": "Width", + "type": ["number", "integer", "null"], + "default": 0 + }, + "height": { + "$id": "#root/videos/items/height", + "title": "Height", + "type": ["number", "integer", "null"], + "default": 0 + }, + "url": { + "$id": "#root/videos/items/url", + "title": "Url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "image": { + "$id": "#root/videos/items/image", + "title": "Image", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "duration": { + "$id": "#root/videos/items/duration", + "title": "Duration", + "type": "integer", + "default": 0 + }, + "user": { + "$id": "#root/videos/items/user", + "title": "User", + "type": "object", + "properties": { + "id": { + "$id": "#root/videos/items/user/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "name": { + "$id": "#root/videos/items/user/name", + "title": "Name", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "url": { + "$id": "#root/videos/items/user/url", + "title": "Url", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + }, + "video_files": { + "$id": "#root/videos/items/video_files", + "title": "Video_files", + "type": "array", + "default": [], + "items": { + "$id": "#root/videos/items/video_files/items", + "title": "Items", + "type": "object", + "properties": { + "id": { + "$id": "#root/videos/items/video_files/items/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "quality": { + "$id": "#root/videos/items/video_files/items/quality", + "title": "Quality", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "file_type": { + "$id": "#root/videos/items/video_files/items/file_type", + "title": "File_type", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "width": { + "$id": "#root/videos/items/video_files/items/width", + "title": "Width", + "type": ["number", "integer", "null"], + "default": 0 + }, + "height": { + "$id": "#root/videos/items/video_files/items/height", + "title": "Height", + "type": ["number", "integer", "null"], + "default": 0 + }, + "link": { + "$id": "#root/videos/items/video_files/items/link", + "title": "Link", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + } + }, + "video_pictures": { + "$id": "#root/videos/items/video_pictures", + "title": "Video_pictures", + "type": "array", + "default": [], + "items": { + "$id": "#root/videos/items/video_pictures/items", + "title": "Items", + "type": "object", + "properties": { + "id": { + "$id": "#root/videos/items/video_pictures/items/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "picture": { + "$id": "#root/videos/items/video_pictures/items/picture", + "title": "Picture", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "nr": { + "$id": "#root/videos/items/video_pictures/items/nr", + "title": "Nr", + "type": "integer", + "default": 0 + } + } + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/videos_popular.json b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/videos_popular.json new file mode 100644 index 00000000000000..9ad4ab06780605 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/videos_popular.json @@ -0,0 +1,196 @@ +{ + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/object1667389811.json", + "title": "Root", + "type": "object", + "properties": { + "page": { + "$id": "#root/page", + "title": "Page", + "type": "integer", + "default": 0 + }, + "per_page": { + "$id": "#root/per_page", + "title": "Per_page", + "type": "integer", + "default": 0 + }, + "total_results": { + "$id": "#root/total_results", + "title": "Total_results", + "type": "integer", + "default": 0 + }, + "url": { + "$id": "#root/url", + "title": "Url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "videos": { + "$id": "#root/videos", + "title": "Videos", + "type": "array", + "default": [], + "items": { + "$id": "#root/videos/items", + "title": "Items", + "type": "object", + "properties": { + "id": { + "$id": "#root/videos/items/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "width": { + "$id": "#root/videos/items/width", + "title": "Width", + "type": ["number", "integer", "null"], + "default": 0 + }, + "height": { + "$id": "#root/videos/items/height", + "title": "Height", + "type": ["number", "integer", "null"], + "default": 0 + }, + "url": { + "$id": "#root/videos/items/url", + "title": "Url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "image": { + "$id": "#root/videos/items/image", + "title": "Image", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "duration": { + "$id": "#root/videos/items/duration", + "title": "Duration", + "type": "integer", + "default": 0 + }, + "user": { + "$id": "#root/videos/items/user", + "title": "User", + "type": "object", + "properties": { + "id": { + "$id": "#root/videos/items/user/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "name": { + "$id": "#root/videos/items/user/name", + "title": "Name", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "url": { + "$id": "#root/videos/items/user/url", + "title": "Url", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + }, + "video_files": { + "$id": "#root/videos/items/video_files", + "title": "Video_files", + "type": "array", + "default": [], + "items": { + "$id": "#root/videos/items/video_files/items", + "title": "Items", + "type": "object", + "properties": { + "id": { + "$id": "#root/videos/items/video_files/items/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "quality": { + "$id": "#root/videos/items/video_files/items/quality", + "title": "Quality", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "file_type": { + "$id": "#root/videos/items/video_files/items/file_type", + "title": "File_type", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "width": { + "$id": "#root/videos/items/video_files/items/width", + "title": "Width", + "type": ["number", "integer", "null"], + "default": 0 + }, + "height": { + "$id": "#root/videos/items/video_files/items/height", + "title": "Height", + "type": ["number", "integer", "null"], + "default": 0 + }, + "link": { + "$id": "#root/videos/items/video_files/items/link", + "title": "Link", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + } + }, + "video_pictures": { + "$id": "#root/videos/items/video_pictures", + "title": "Video_pictures", + "type": "array", + "default": [], + "items": { + "$id": "#root/videos/items/video_pictures/items", + "title": "Items", + "type": "object", + "properties": { + "id": { + "$id": "#root/videos/items/video_pictures/items/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "picture": { + "$id": "#root/videos/items/video_pictures/items/picture", + "title": "Picture", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "nr": { + "$id": "#root/videos/items/video_pictures/items/nr", + "title": "Nr", + "type": "integer", + "default": 0 + } + } + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/videos_search.json b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/videos_search.json new file mode 100644 index 00000000000000..d4def3f2a10f37 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/schemas/videos_search.json @@ -0,0 +1,196 @@ +{ + "definitions": {}, + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/object1667389704.json", + "title": "Root", + "type": "object", + "properties": { + "page": { + "$id": "#root/page", + "title": "Page", + "type": "integer", + "default": 0 + }, + "per_page": { + "$id": "#root/per_page", + "title": "Per_page", + "type": "integer", + "default": 0 + }, + "total_results": { + "$id": "#root/total_results", + "title": "Total_results", + "type": "integer", + "default": 0 + }, + "url": { + "$id": "#root/url", + "title": "Url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "videos": { + "$id": "#root/videos", + "title": "Videos", + "type": "array", + "default": [], + "items": { + "$id": "#root/videos/items", + "title": "Items", + "type": "object", + "properties": { + "id": { + "$id": "#root/videos/items/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "width": { + "$id": "#root/videos/items/width", + "title": "Width", + "type": ["null", "number", "integer"], + "default": 0 + }, + "height": { + "$id": "#root/videos/items/height", + "title": "Height", + "type": ["null", "number", "integer"], + "default": 0 + }, + "url": { + "$id": "#root/videos/items/url", + "title": "Url", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "image": { + "$id": "#root/videos/items/image", + "title": "Image", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "duration": { + "$id": "#root/videos/items/duration", + "title": "Duration", + "type": "integer", + "default": 0 + }, + "user": { + "$id": "#root/videos/items/user", + "title": "User", + "type": "object", + "properties": { + "id": { + "$id": "#root/videos/items/user/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "name": { + "$id": "#root/videos/items/user/name", + "title": "Name", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "url": { + "$id": "#root/videos/items/user/url", + "title": "Url", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + }, + "video_files": { + "$id": "#root/videos/items/video_files", + "title": "Video_files", + "type": "array", + "default": [], + "items": { + "$id": "#root/videos/items/video_files/items", + "title": "Items", + "type": "object", + "properties": { + "id": { + "$id": "#root/videos/items/video_files/items/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "quality": { + "$id": "#root/videos/items/video_files/items/quality", + "title": "Quality", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "file_type": { + "$id": "#root/videos/items/video_files/items/file_type", + "title": "File_type", + "type": "string", + "default": "", + "pattern": "^.*$" + }, + "width": { + "$id": "#root/videos/items/video_files/items/width", + "title": "Width", + "type": ["null", "number", "integer"], + "default": 0 + }, + "height": { + "$id": "#root/videos/items/video_files/items/height", + "title": "Height", + "type": ["null", "number", "integer"], + "default": 0 + }, + "link": { + "$id": "#root/videos/items/video_files/items/link", + "title": "Link", + "type": "string", + "default": "", + "pattern": "^.*$" + } + } + } + }, + "video_pictures": { + "$id": "#root/videos/items/video_pictures", + "title": "Video_pictures", + "type": "array", + "default": [], + "items": { + "$id": "#root/videos/items/video_pictures/items", + "title": "Items", + "type": "object", + "properties": { + "id": { + "$id": "#root/videos/items/video_pictures/items/id", + "title": "Id", + "type": "integer", + "default": 0 + }, + "picture": { + "$id": "#root/videos/items/video_pictures/items/picture", + "title": "Picture", + "type": ["null","string"], + "default": "", + "pattern": "^.*$" + }, + "nr": { + "$id": "#root/videos/items/video_pictures/items/nr", + "title": "Nr", + "type": "integer", + "default": 0 + } + } + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/source.py b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/source.py new file mode 100644 index 00000000000000..f7a5cf4eee2d87 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourcePexelsApi(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "pexels_api.yaml"}) diff --git a/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/spec.yaml b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/spec.yaml new file mode 100644 index 00000000000000..870d3822995421 --- /dev/null +++ b/airbyte-integrations/connectors/source-pexels-api/source_pexels_api/spec.yaml @@ -0,0 +1,50 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/pexels-api +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Pexel API Spec + type: object + required: + - api_key + - query + additionalProperties: true + properties: + api_key: + title: API Key from the pexels website + type: string + description: API key is required to access pexels api, For getting your's goto https://www.pexels.com/api/documentation and create account for free. + airbyte_secret: true + query: + title: Specific query for the search + type: string + description: Optional, the search query, Example Ocean, Tigers, Pears, etc. + examples: + - people + - oceans + orientation: + title: Specific orientation for the search + type: string + description: Optional, Desired photo orientation. The current supported orientations are landscape, portrait or square + examples: + - square + - landscape + size: + title: Specific size for the search + type: string + description: Optional, Minimum photo size. The current supported sizes are large(24MP), medium(12MP) or small(4MP). + examples: + - large + - small + color: + title: Specific color for the search + type: string + description: Optional, Desired photo color. Supported colors red, orange, yellow, green, turquoise, blue, violet, pink, brown, black, gray, white or any hexidecimal color code. + examples: + - red + - orange + locale: + title: Specific locale for the search + type: string + description: Optional, The locale of the search you are performing. The current supported locales are 'en-US' 'pt-BR' 'es-ES' 'ca-ES' 'de-DE' 'it-IT' 'fr-FR' 'sv-SE' 'id-ID' 'pl-PL' 'ja-JP' 'zh-TW' 'zh-CN' 'ko-KR' 'th-TH' 'nl-NL' 'hu-HU' 'vi-VN' 'cs-CZ' 'da-DK' 'fi-FI' 'uk-UA' 'el-GR' 'ro-RO' 'nb-NO' 'sk-SK' 'tr-TR' 'ru-RU'. + examples: + - en-US + - pt-BR \ No newline at end of file diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index e74c9109441834..f4f3fd9eaf31c5 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -7,6 +7,7 @@ import io.airbyte.analytics.Deployment; import io.airbyte.analytics.TrackingClient; import io.airbyte.analytics.TrackingClientSingleton; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.lang.CloseableShutdownHook; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.temporal.ConnectionManagerUtils; @@ -213,6 +214,8 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, final TrackingClient trackingClient = TrackingClientSingleton.get(); final JobTracker jobTracker = new JobTracker(configRepository, jobPersistence, trackingClient); + final EnvVariableFeatureFlags envVariableFeatureFlags = new EnvVariableFeatureFlags(); + final WebUrlHelper webUrlHelper = new WebUrlHelper(configs.getWebappUrl()); final JobErrorReportingClient jobErrorReportingClient = JobErrorReportingClientFactory.getClient(configs.getJobErrorReportingStrategy(), configs); final JobErrorReporter jobErrorReporter = @@ -286,7 +289,8 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, configs.getWorkerEnvironment(), configs.getLogConfigs(), eventRunner, - connectionsHandler); + connectionsHandler, + envVariableFeatureFlags); final DbMigrationHandler dbMigrationHandler = new DbMigrationHandler(configsDatabase, configsFlyway, jobsDatabase, jobsFlyway); diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java index 49a4a75940e91f..7621cb4baf936a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java @@ -15,6 +15,8 @@ import io.airbyte.api.model.generated.CheckConnectionRead; import io.airbyte.api.model.generated.CheckConnectionRead.StatusEnum; import io.airbyte.api.model.generated.ConnectionIdRequestBody; +import io.airbyte.api.model.generated.ConnectionRead; +import io.airbyte.api.model.generated.ConnectionStatus; import io.airbyte.api.model.generated.ConnectionUpdate; import io.airbyte.api.model.generated.DestinationCoreConfig; import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; @@ -27,6 +29,7 @@ import io.airbyte.api.model.generated.JobIdRequestBody; import io.airbyte.api.model.generated.JobInfoRead; import io.airbyte.api.model.generated.LogRead; +import io.airbyte.api.model.generated.NonBreakingChangesPreference; import io.airbyte.api.model.generated.SourceCoreConfig; import io.airbyte.api.model.generated.SourceDefinitionIdWithWorkspaceId; import io.airbyte.api.model.generated.SourceDefinitionSpecificationRead; @@ -39,6 +42,7 @@ import io.airbyte.api.model.generated.SynchronousJobRead; import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.enums.Enums; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.ErrorCode; import io.airbyte.commons.temporal.TemporalClient.ManualOperationResult; @@ -95,6 +99,7 @@ public class SchedulerHandler { private final JobPersistence jobPersistence; private final JobConverter jobConverter; private final EventRunner eventRunner; + private final EnvVariableFeatureFlags envVariableFeatureFlags; public SchedulerHandler(final ConfigRepository configRepository, final SecretsRepositoryReader secretsRepositoryReader, @@ -104,7 +109,8 @@ public SchedulerHandler(final ConfigRepository configRepository, final WorkerEnvironment workerEnvironment, final LogConfigs logConfigs, final EventRunner eventRunner, - final ConnectionsHandler connectionsHandler) { + final ConnectionsHandler connectionsHandler, + final EnvVariableFeatureFlags envVariableFeatureFlags) { this( configRepository, secretsRepositoryWriter, @@ -114,7 +120,8 @@ public SchedulerHandler(final ConfigRepository configRepository, jobPersistence, eventRunner, new JobConverter(workerEnvironment, logConfigs), - connectionsHandler); + connectionsHandler, + envVariableFeatureFlags); } @VisibleForTesting @@ -126,7 +133,8 @@ public SchedulerHandler(final ConfigRepository configRepository, final JobPersistence jobPersistence, final EventRunner eventRunner, final JobConverter jobConverter, - final ConnectionsHandler connectionsHandler) { + final ConnectionsHandler connectionsHandler, + final EnvVariableFeatureFlags envVariableFeatureFlags) { this.configRepository = configRepository; this.secretsRepositoryWriter = secretsRepositoryWriter; this.synchronousSchedulerClient = synchronousSchedulerClient; @@ -136,6 +144,7 @@ public SchedulerHandler(final ConfigRepository configRepository, this.eventRunner = eventRunner; this.jobConverter = jobConverter; this.connectionsHandler = connectionsHandler; + this.envVariableFeatureFlags = envVariableFeatureFlags; } public CheckConnectionRead checkSourceConnectionFromSourceId(final SourceIdRequestBody sourceIdRequestBody) @@ -360,16 +369,32 @@ private void discoveredSchemaWithCatalogDiff(SourceDiscoverSchemaRead discovered throws JsonValidationException, ConfigNotFoundException, IOException { final Optional catalogUsedToMakeConfiguredCatalog = connectionsHandler .getConnectionAirbyteCatalog(discoverSchemaRequestBody.getConnectionId()); + final ConnectionRead connectionRead = connectionsHandler.getConnection(discoverSchemaRequestBody.getConnectionId()); final io.airbyte.api.model.generated.@NotNull AirbyteCatalog currentAirbyteCatalog = - connectionsHandler.getConnection(discoverSchemaRequestBody.getConnectionId()).getSyncCatalog(); + connectionRead.getSyncCatalog(); CatalogDiff diff = connectionsHandler.getDiff(catalogUsedToMakeConfiguredCatalog.orElse(currentAirbyteCatalog), discoveredSchema.getCatalog(), CatalogConverter.toProtocol(currentAirbyteCatalog)); boolean containsBreakingChange = containsBreakingChange(diff); ConnectionUpdate updateObject = new ConnectionUpdate().breakingChange(containsBreakingChange).connectionId(discoverSchemaRequestBody.getConnectionId()); + ConnectionStatus connectionStatus; + if (shouldDisableConnection(containsBreakingChange, connectionRead.getNonBreakingChangesPreference(), diff)) { + connectionStatus = ConnectionStatus.INACTIVE; + } else { + connectionStatus = ConnectionStatus.ACTIVE; + } + updateObject.status(connectionStatus); connectionsHandler.updateConnection(updateObject); - discoveredSchema.catalogDiff(diff).breakingChange(containsBreakingChange); + discoveredSchema.catalogDiff(diff).breakingChange(containsBreakingChange).connectionStatus(connectionStatus); + + } + + private boolean shouldDisableConnection(boolean containsBreakingChange, NonBreakingChangesPreference preference, CatalogDiff diff) { + if (!envVariableFeatureFlags.autoDetectSchema()) { + return false; + } + return containsBreakingChange || (preference == NonBreakingChangesPreference.DISABLE && !diff.getTransforms().isEmpty()); } private CheckConnectionRead reportConnectionStatus(final SynchronousResponse response) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java index a76d0e6bf61afa..fa4822a6da6765 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @@ -353,6 +353,7 @@ public WebBackendConnectionRead webBackendGetConnection(final WebBackendConnecti */ diff = refreshedCatalog.get().getCatalogDiff(); connection.setBreakingChange(refreshedCatalog.get().getBreakingChange()); + connection.setStatus(refreshedCatalog.get().getConnectionStatus()); } else if (catalogUsedToMakeConfiguredCatalog.isPresent()) { // reconstructs a full picture of the full schema at the time the catalog was configured. syncCatalog = updateSchemaWithDiscovery(configuredCatalog, catalogUsedToMakeConfiguredCatalog.get()); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java index 2eb3e9be56d7cb..f06b4d6a8d695f 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java @@ -25,6 +25,7 @@ import io.airbyte.api.model.generated.CheckConnectionRead; import io.airbyte.api.model.generated.ConnectionIdRequestBody; import io.airbyte.api.model.generated.ConnectionRead; +import io.airbyte.api.model.generated.ConnectionStatus; import io.airbyte.api.model.generated.ConnectionUpdate; import io.airbyte.api.model.generated.DestinationCoreConfig; import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; @@ -34,6 +35,7 @@ import io.airbyte.api.model.generated.FieldTransform; import io.airbyte.api.model.generated.JobIdRequestBody; import io.airbyte.api.model.generated.JobInfoRead; +import io.airbyte.api.model.generated.NonBreakingChangesPreference; import io.airbyte.api.model.generated.SourceCoreConfig; import io.airbyte.api.model.generated.SourceDefinitionIdWithWorkspaceId; import io.airbyte.api.model.generated.SourceDefinitionSpecificationRead; @@ -45,6 +47,7 @@ import io.airbyte.api.model.generated.StreamTransform.TransformTypeEnum; import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.enums.Enums; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.Exceptions; import io.airbyte.commons.temporal.ErrorCode; @@ -106,9 +109,11 @@ class SchedulerHandlerTest { private static final String DESTINATION_PROTOCOL_VERSION = "0.7.9"; private static final String NAME = "name"; private static final String DOGS = "dogs"; + private static final String SHOES = "shoes"; + private static final String SKU = "sku"; - private static final AirbyteCatalog airbyteCatalog = CatalogHelpers.createAirbyteCatalog("shoes", - Field.of("sku", JsonSchemaType.STRING)); + private static final AirbyteCatalog airbyteCatalog = CatalogHelpers.createAirbyteCatalog(SHOES, + Field.of(SKU, JsonSchemaType.STRING)); private static final SourceConnection SOURCE = new SourceConnection() .withName("my postgres db") @@ -145,6 +150,7 @@ class SchedulerHandlerTest { private EventRunner eventRunner; private JobConverter jobConverter; private ConnectionsHandler connectionsHandler; + private EnvVariableFeatureFlags envVariableFeatureFlags; @BeforeEach void setup() { @@ -162,6 +168,7 @@ void setup() { jobPersistence = mock(JobPersistence.class); eventRunner = mock(EventRunner.class); connectionsHandler = mock(ConnectionsHandler.class); + envVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); jobConverter = spy(new JobConverter(WorkerEnvironment.DOCKER, LogConfigs.EMPTY)); @@ -174,7 +181,8 @@ void setup() { jobPersistence, eventRunner, jobConverter, - connectionsHandler); + connectionsHandler, + envVariableFeatureFlags); } @Test @@ -555,7 +563,7 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreaking() throws IOExcept when(discoverResponse.getOutput()).thenReturn(discoveredCatalogId); final AirbyteCatalog airbyteCatalogCurrent = new AirbyteCatalog().withStreams(Lists.newArrayList( - CatalogHelpers.createAirbyteStream("shoes", Field.of("sku", JsonSchemaType.STRING)), + CatalogHelpers.createAirbyteStream(SHOES, Field.of(SKU, JsonSchemaType.STRING)), CatalogHelpers.createAirbyteStream(DOGS, Field.of(NAME, JsonSchemaType.STRING)))); final ConnectionRead connectionRead = new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent)); @@ -577,6 +585,110 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreaking() throws IOExcept assertEquals(actual.getCatalog(), expectedActorCatalog); } + @Test + void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreakingDisableConnectionPreferenceNoFeatureFlag() + throws IOException, JsonValidationException, ConfigNotFoundException { + final SourceConnection source = SourceHelpers.generateSource(UUID.randomUUID()); + final UUID connectionId = UUID.randomUUID(); + final UUID discoveredCatalogId = UUID.randomUUID(); + final SynchronousResponse discoverResponse = (SynchronousResponse) jobResponse; + final SourceDiscoverSchemaRequestBody request = + new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true); + final StreamTransform streamTransform = new StreamTransform().transformType(TransformTypeEnum.REMOVE_STREAM) + .streamDescriptor(new io.airbyte.api.model.generated.StreamDescriptor().name(DOGS)); + final CatalogDiff catalogDiff = new CatalogDiff().addTransformsItem(streamTransform); + when(envVariableFeatureFlags.autoDetectSchema()).thenReturn(false); + when(configRepository.getStandardSourceDefinition(source.getSourceDefinitionId())) + .thenReturn(new StandardSourceDefinition() + .withDockerRepository(SOURCE_DOCKER_REPO) + .withDockerImageTag(SOURCE_DOCKER_TAG) + .withProtocolVersion(SOURCE_PROTOCOL_VERSION) + .withSourceDefinitionId(source.getSourceDefinitionId())); + when(configRepository.getSourceConnection(source.getSourceId())).thenReturn(source); + when(synchronousSchedulerClient.createDiscoverSchemaJob(source, SOURCE_DOCKER_IMAGE, SOURCE_DOCKER_TAG, new Version(SOURCE_PROTOCOL_VERSION))) + .thenReturn(discoverResponse); + + when(discoverResponse.isSuccess()).thenReturn(true); + when(discoverResponse.getOutput()).thenReturn(discoveredCatalogId); + + final AirbyteCatalog airbyteCatalogCurrent = new AirbyteCatalog().withStreams(Lists.newArrayList( + CatalogHelpers.createAirbyteStream(SHOES, Field.of(SKU, JsonSchemaType.STRING)), + CatalogHelpers.createAirbyteStream(DOGS, Field.of(NAME, JsonSchemaType.STRING)))); + + final ConnectionRead connectionRead = + new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent)).nonBreakingChangesPreference( + NonBreakingChangesPreference.DISABLE); + when(connectionsHandler.getConnection(request.getConnectionId())).thenReturn(connectionRead); + when(connectionsHandler.getDiff(any(), any(), any())).thenReturn(catalogDiff); + + final ActorCatalog actorCatalog = new ActorCatalog() + .withCatalog(Jsons.jsonNode(airbyteCatalog)) + .withCatalogHash("") + .withId(discoveredCatalogId); + when(configRepository.getActorCatalogById(discoveredCatalogId)).thenReturn(actorCatalog); + + final AirbyteCatalog persistenceCatalog = Jsons.object(actorCatalog.getCatalog(), + io.airbyte.protocol.models.AirbyteCatalog.class); + final io.airbyte.api.model.generated.AirbyteCatalog expectedActorCatalog = CatalogConverter.toApi(persistenceCatalog); + + final SourceDiscoverSchemaRead actual = schedulerHandler.discoverSchemaForSourceFromSourceId(request); + assertEquals(actual.getCatalogDiff(), catalogDiff); + assertEquals(actual.getCatalog(), expectedActorCatalog); + assertEquals(actual.getConnectionStatus(), ConnectionStatus.ACTIVE); + } + + @Test + void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreakingDisableConnectionPreferenceFeatureFlag() + throws IOException, JsonValidationException, ConfigNotFoundException { + final SourceConnection source = SourceHelpers.generateSource(UUID.randomUUID()); + final UUID connectionId = UUID.randomUUID(); + final UUID discoveredCatalogId = UUID.randomUUID(); + final SynchronousResponse discoverResponse = (SynchronousResponse) jobResponse; + final SourceDiscoverSchemaRequestBody request = + new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true); + final StreamTransform streamTransform = new StreamTransform().transformType(TransformTypeEnum.REMOVE_STREAM) + .streamDescriptor(new io.airbyte.api.model.generated.StreamDescriptor().name(DOGS)); + final CatalogDiff catalogDiff = new CatalogDiff().addTransformsItem(streamTransform); + when(envVariableFeatureFlags.autoDetectSchema()).thenReturn(true); + when(configRepository.getStandardSourceDefinition(source.getSourceDefinitionId())) + .thenReturn(new StandardSourceDefinition() + .withDockerRepository(SOURCE_DOCKER_REPO) + .withDockerImageTag(SOURCE_DOCKER_TAG) + .withProtocolVersion(SOURCE_PROTOCOL_VERSION) + .withSourceDefinitionId(source.getSourceDefinitionId())); + when(configRepository.getSourceConnection(source.getSourceId())).thenReturn(source); + when(synchronousSchedulerClient.createDiscoverSchemaJob(source, SOURCE_DOCKER_IMAGE, SOURCE_DOCKER_TAG, new Version(SOURCE_PROTOCOL_VERSION))) + .thenReturn(discoverResponse); + + when(discoverResponse.isSuccess()).thenReturn(true); + when(discoverResponse.getOutput()).thenReturn(discoveredCatalogId); + + final AirbyteCatalog airbyteCatalogCurrent = new AirbyteCatalog().withStreams(Lists.newArrayList( + CatalogHelpers.createAirbyteStream(SHOES, Field.of(SKU, JsonSchemaType.STRING)), + CatalogHelpers.createAirbyteStream(DOGS, Field.of(NAME, JsonSchemaType.STRING)))); + + final ConnectionRead connectionRead = + new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent)).nonBreakingChangesPreference( + NonBreakingChangesPreference.DISABLE); + when(connectionsHandler.getConnection(request.getConnectionId())).thenReturn(connectionRead); + when(connectionsHandler.getDiff(any(), any(), any())).thenReturn(catalogDiff); + + final ActorCatalog actorCatalog = new ActorCatalog() + .withCatalog(Jsons.jsonNode(airbyteCatalog)) + .withCatalogHash("") + .withId(discoveredCatalogId); + when(configRepository.getActorCatalogById(discoveredCatalogId)).thenReturn(actorCatalog); + + final AirbyteCatalog persistenceCatalog = Jsons.object(actorCatalog.getCatalog(), + io.airbyte.protocol.models.AirbyteCatalog.class); + final io.airbyte.api.model.generated.AirbyteCatalog expectedActorCatalog = CatalogConverter.toApi(persistenceCatalog); + + final SourceDiscoverSchemaRead actual = schedulerHandler.discoverSchemaForSourceFromSourceId(request); + assertEquals(actual.getCatalogDiff(), catalogDiff); + assertEquals(actual.getCatalog(), expectedActorCatalog); + assertEquals(actual.getConnectionStatus(), ConnectionStatus.INACTIVE); + } + @Test void testDiscoverSchemaFromSourceIdWithConnectionIdBreaking() throws IOException, JsonValidationException, ConfigNotFoundException { final SourceConnection source = SourceHelpers.generateSource(UUID.randomUUID()); @@ -603,7 +715,7 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdBreaking() throws IOException when(discoverResponse.getOutput()).thenReturn(discoveredCatalogId); final AirbyteCatalog airbyteCatalogCurrent = new AirbyteCatalog().withStreams(Lists.newArrayList( - CatalogHelpers.createAirbyteStream("shoes", Field.of("sku", JsonSchemaType.STRING)), + CatalogHelpers.createAirbyteStream(SHOES, Field.of(SKU, JsonSchemaType.STRING)), CatalogHelpers.createAirbyteStream(DOGS, Field.of(NAME, JsonSchemaType.STRING)))); final ConnectionRead connectionRead = new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent)); @@ -619,14 +731,119 @@ void testDiscoverSchemaFromSourceIdWithConnectionIdBreaking() throws IOException final AirbyteCatalog persistenceCatalog = Jsons.object(actorCatalog.getCatalog(), io.airbyte.protocol.models.AirbyteCatalog.class); final io.airbyte.api.model.generated.AirbyteCatalog expectedActorCatalog = CatalogConverter.toApi(persistenceCatalog); - final ConnectionUpdate expectedConnectionUpdate = new ConnectionUpdate().connectionId(connectionId).breakingChange(true); + final ConnectionUpdate expectedConnectionUpdate = + new ConnectionUpdate().connectionId(connectionId).breakingChange(true).status(ConnectionStatus.ACTIVE); final SourceDiscoverSchemaRead actual = schedulerHandler.discoverSchemaForSourceFromSourceId(request); assertEquals(actual.getCatalogDiff(), catalogDiff); assertEquals(actual.getCatalog(), expectedActorCatalog); + assertEquals(actual.getConnectionStatus(), ConnectionStatus.ACTIVE); verify(connectionsHandler).updateConnection(expectedConnectionUpdate); } + @Test + void testDiscoverSchemaFromSourceIdWithConnectionIdBreakingFeatureFlagOn() throws IOException, JsonValidationException, ConfigNotFoundException { + final SourceConnection source = SourceHelpers.generateSource(UUID.randomUUID()); + final UUID connectionId = UUID.randomUUID(); + final UUID discoveredCatalogId = UUID.randomUUID(); + final SynchronousResponse discoverResponse = (SynchronousResponse) jobResponse; + final SourceDiscoverSchemaRequestBody request = + new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true); + final StreamTransform streamTransform = new StreamTransform().transformType(TransformTypeEnum.UPDATE_STREAM) + .streamDescriptor(new io.airbyte.api.model.generated.StreamDescriptor().name(DOGS)).addUpdateStreamItem(new FieldTransform().transformType( + FieldTransform.TransformTypeEnum.REMOVE_FIELD).breaking(true)); + final CatalogDiff catalogDiff = new CatalogDiff().addTransformsItem(streamTransform); + when(envVariableFeatureFlags.autoDetectSchema()).thenReturn(true); + when(configRepository.getStandardSourceDefinition(source.getSourceDefinitionId())) + .thenReturn(new StandardSourceDefinition() + .withDockerRepository(SOURCE_DOCKER_REPO) + .withDockerImageTag(SOURCE_DOCKER_TAG) + .withProtocolVersion(SOURCE_PROTOCOL_VERSION) + .withSourceDefinitionId(source.getSourceDefinitionId())); + when(configRepository.getSourceConnection(source.getSourceId())).thenReturn(source); + when(synchronousSchedulerClient.createDiscoverSchemaJob(source, SOURCE_DOCKER_IMAGE, SOURCE_DOCKER_TAG, new Version(SOURCE_PROTOCOL_VERSION))) + .thenReturn(discoverResponse); + + when(discoverResponse.isSuccess()).thenReturn(true); + when(discoverResponse.getOutput()).thenReturn(discoveredCatalogId); + + final AirbyteCatalog airbyteCatalogCurrent = new AirbyteCatalog().withStreams(Lists.newArrayList( + CatalogHelpers.createAirbyteStream(SHOES, Field.of(SKU, JsonSchemaType.STRING)), + CatalogHelpers.createAirbyteStream(DOGS, Field.of(NAME, JsonSchemaType.STRING)))); + + final ConnectionRead connectionRead = new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent)); + when(connectionsHandler.getConnection(request.getConnectionId())).thenReturn(connectionRead); + when(connectionsHandler.getDiff(any(), any(), any())).thenReturn(catalogDiff); + + final ActorCatalog actorCatalog = new ActorCatalog() + .withCatalog(Jsons.jsonNode(airbyteCatalog)) + .withCatalogHash("") + .withId(discoveredCatalogId); + when(configRepository.getActorCatalogById(discoveredCatalogId)).thenReturn(actorCatalog); + + final AirbyteCatalog persistenceCatalog = Jsons.object(actorCatalog.getCatalog(), + io.airbyte.protocol.models.AirbyteCatalog.class); + final io.airbyte.api.model.generated.AirbyteCatalog expectedActorCatalog = CatalogConverter.toApi(persistenceCatalog); + final ConnectionUpdate expectedConnectionUpdate = + new ConnectionUpdate().connectionId(connectionId).breakingChange(true).status(ConnectionStatus.INACTIVE); + + final SourceDiscoverSchemaRead actual = schedulerHandler.discoverSchemaForSourceFromSourceId(request); + assertEquals(actual.getCatalogDiff(), catalogDiff); + assertEquals(actual.getCatalog(), expectedActorCatalog); + assertEquals(actual.getConnectionStatus(), ConnectionStatus.INACTIVE); + verify(connectionsHandler).updateConnection(expectedConnectionUpdate); + } + + @Test + void testDiscoverSchemaFromSourceIdWithConnectionIdNonBreakingDisableConnectionPreferenceFeatureFlagNoDiff() + throws IOException, JsonValidationException, ConfigNotFoundException { + final SourceConnection source = SourceHelpers.generateSource(UUID.randomUUID()); + final UUID connectionId = UUID.randomUUID(); + final UUID discoveredCatalogId = UUID.randomUUID(); + final SynchronousResponse discoverResponse = (SynchronousResponse) jobResponse; + final SourceDiscoverSchemaRequestBody request = + new SourceDiscoverSchemaRequestBody().sourceId(source.getSourceId()).connectionId(connectionId).disableCache(true); + final CatalogDiff catalogDiff = new CatalogDiff(); + when(envVariableFeatureFlags.autoDetectSchema()).thenReturn(true); + when(configRepository.getStandardSourceDefinition(source.getSourceDefinitionId())) + .thenReturn(new StandardSourceDefinition() + .withDockerRepository(SOURCE_DOCKER_REPO) + .withDockerImageTag(SOURCE_DOCKER_TAG) + .withProtocolVersion(SOURCE_PROTOCOL_VERSION) + .withSourceDefinitionId(source.getSourceDefinitionId())); + when(configRepository.getSourceConnection(source.getSourceId())).thenReturn(source); + when(synchronousSchedulerClient.createDiscoverSchemaJob(source, SOURCE_DOCKER_IMAGE, SOURCE_DOCKER_TAG, new Version(SOURCE_PROTOCOL_VERSION))) + .thenReturn(discoverResponse); + + when(discoverResponse.isSuccess()).thenReturn(true); + when(discoverResponse.getOutput()).thenReturn(discoveredCatalogId); + + final AirbyteCatalog airbyteCatalogCurrent = new AirbyteCatalog().withStreams(Lists.newArrayList( + CatalogHelpers.createAirbyteStream(SHOES, Field.of(SKU, JsonSchemaType.STRING)), + CatalogHelpers.createAirbyteStream(DOGS, Field.of(NAME, JsonSchemaType.STRING)))); + + final ConnectionRead connectionRead = + new ConnectionRead().syncCatalog(CatalogConverter.toApi(airbyteCatalogCurrent)).nonBreakingChangesPreference( + NonBreakingChangesPreference.DISABLE); + when(connectionsHandler.getConnection(request.getConnectionId())).thenReturn(connectionRead); + when(connectionsHandler.getDiff(any(), any(), any())).thenReturn(catalogDiff); + + final ActorCatalog actorCatalog = new ActorCatalog() + .withCatalog(Jsons.jsonNode(airbyteCatalog)) + .withCatalogHash("") + .withId(discoveredCatalogId); + when(configRepository.getActorCatalogById(discoveredCatalogId)).thenReturn(actorCatalog); + + final AirbyteCatalog persistenceCatalog = Jsons.object(actorCatalog.getCatalog(), + io.airbyte.protocol.models.AirbyteCatalog.class); + final io.airbyte.api.model.generated.AirbyteCatalog expectedActorCatalog = CatalogConverter.toApi(persistenceCatalog); + + final SourceDiscoverSchemaRead actual = schedulerHandler.discoverSchemaForSourceFromSourceId(request); + assertEquals(actual.getCatalogDiff(), catalogDiff); + assertEquals(actual.getCatalog(), expectedActorCatalog); + assertEquals(actual.getConnectionStatus(), ConnectionStatus.ACTIVE); + } + @Test void testDiscoverSchemaForSourceFromSourceCreate() throws JsonValidationException, IOException, ConfigNotFoundException { final SourceConnection source = new SourceConnection() diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java index 78b9211352987b..b390d7a4c3fc67 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java @@ -76,6 +76,7 @@ import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.Status; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.ConfigRepository.DestinationAndDefinition; @@ -173,9 +174,10 @@ void setup() throws IOException, JsonValidationException, ConfigNotFoundExceptio final DestinationRead destinationRead = DestinationHelpers.getDestinationRead(destination, destinationDefinition); final StandardSync standardSync = - ConnectionHelpers.generateSyncWithSourceAndDestinationId(source.getSourceId(), destination.getDestinationId(), false); + ConnectionHelpers.generateSyncWithSourceAndDestinationId(source.getSourceId(), destination.getDestinationId(), false, Status.ACTIVE); final StandardSync brokenStandardSync = - ConnectionHelpers.generateSyncWithSourceAndDestinationId(source.getSourceId(), destination.getDestinationId(), true); + ConnectionHelpers.generateSyncWithSourceAndDestinationId(source.getSourceId(), destination.getDestinationId(), true, Status.INACTIVE); + when(configRepository.listWorkspaceStandardSyncs(sourceRead.getWorkspaceId(), false)) .thenReturn(Collections.singletonList(standardSync)); when(configRepository.getSourceAndDefinitionsFromSourceIds(Collections.singletonList(source.getSourceId()))) @@ -277,7 +279,7 @@ void setup() throws IOException, JsonValidationException, ConfigNotFoundExceptio .streamDescriptor(new io.airbyte.api.model.generated.StreamDescriptor().name("users-data1")) .updateStream(null)))); - expectedWithNewSchemaAndBreakingChange = expectedWebBackendConnectionReadObject(connectionRead, sourceRead, destinationRead, + expectedWithNewSchemaAndBreakingChange = expectedWebBackendConnectionReadObject(brokenConnectionRead, sourceRead, destinationRead, new OperationReadList().operations(expected.getOperations()), SchemaChange.BREAKING, now, modifiedCatalog, null) .catalogDiff(new CatalogDiff().transforms(List.of( new StreamTransform().transformType(TransformTypeEnum.ADD_STREAM) @@ -418,7 +420,7 @@ void testWebBackendGetConnectionWithDiscoveryAndNewSchema() throws ConfigNotFoun when(configRepository.getActorCatalogById(any())).thenReturn(new ActorCatalog().withId(UUID.randomUUID())); SourceDiscoverSchemaRead schemaRead = new SourceDiscoverSchemaRead().catalogDiff(expectedWithNewSchema.getCatalogDiff()).catalog(expectedWithNewSchema.getSyncCatalog()) - .breakingChange(false); + .breakingChange(false).connectionStatus(ConnectionStatus.ACTIVE); when(schedulerHandler.discoverSchemaForSourceFromSourceId(any())).thenReturn(schemaRead); final WebBackendConnectionRead result = testWebBackendGetConnection(true, connectionRead, @@ -435,10 +437,10 @@ void testWebBackendGetConnectionWithDiscoveryAndNewSchemaBreakingChange() throws when(configRepository.getActorCatalogById(any())).thenReturn(new ActorCatalog().withId(UUID.randomUUID())); SourceDiscoverSchemaRead schemaRead = new SourceDiscoverSchemaRead().catalogDiff(expectedWithNewSchema.getCatalogDiff()).catalog(expectedWithNewSchema.getSyncCatalog()) - .breakingChange(true); + .breakingChange(true).connectionStatus(ConnectionStatus.INACTIVE); when(schedulerHandler.discoverSchemaForSourceFromSourceId(any())).thenReturn(schemaRead); - final WebBackendConnectionRead result = testWebBackendGetConnection(true, connectionRead, + final WebBackendConnectionRead result = testWebBackendGetConnection(true, brokenConnectionRead, operationReadList); assertEquals(expectedWithNewSchemaAndBreakingChange, result); } diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java index ab742a079b3676..908a5749e72fdc 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java @@ -33,6 +33,7 @@ import io.airbyte.config.Schedule.TimeUnit; import io.airbyte.config.ScheduleData; import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.Status; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; @@ -107,7 +108,10 @@ public static StandardSync generateSyncWithDestinationId(final UUID destinationI .withManual(true); } - public static StandardSync generateSyncWithSourceAndDestinationId(final UUID sourceId, final UUID destinationId, final boolean isBroken) { + public static StandardSync generateSyncWithSourceAndDestinationId(final UUID sourceId, + final UUID destinationId, + final boolean isBroken, + final Status status) { final UUID connectionId = UUID.randomUUID(); return new StandardSync() @@ -116,7 +120,7 @@ public static StandardSync generateSyncWithSourceAndDestinationId(final UUID sou .withNamespaceDefinition(NamespaceDefinitionType.SOURCE) .withNamespaceFormat(null) .withPrefix(STANDARD_SYNC_PREFIX) - .withStatus(StandardSync.Status.ACTIVE) + .withStatus(status) .withCatalog(generateBasicConfiguredAirbyteCatalog()) .withSourceCatalogId(UUID.randomUUID()) .withSourceId(sourceId) @@ -166,7 +170,6 @@ public static ConnectionRead generateExpectedConnectionRead(final UUID connectio .namespaceDefinition(io.airbyte.api.model.generated.NamespaceDefinitionType.SOURCE) .namespaceFormat(null) .prefix("presto_to_hudi") - .status(ConnectionStatus.ACTIVE) .schedule(generateBasicConnectionSchedule()) .scheduleType(ConnectionScheduleType.BASIC) .scheduleData(generateBasicConnectionScheduleData()) @@ -199,6 +202,14 @@ public static ConnectionRead generateExpectedConnectionRead(final StandardSync s .units(standardSync.getSchedule().getUnits())); } + if (standardSync.getStatus() == Status.INACTIVE) { + connectionRead.setStatus(ConnectionStatus.INACTIVE); + } else if (standardSync.getStatus() == Status.ACTIVE) { + connectionRead.setStatus(ConnectionStatus.ACTIVE); + } else if (standardSync.getStatus() == Status.DEPRECATED) { + connectionRead.setStatus(ConnectionStatus.DEPRECATED); + } + return connectionRead; } diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 2371fde0d81b16..e392b5bf0a2eb9 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -10,7 +10,7 @@ "prestart": "npm run generate-client", "start": "craco start", "prestart:cloud": "npm run generate-client", - "start:cloud": "AB_ENV=frontend-dev node -r ./scripts/environment.js ./node_modules/.bin/craco start", + "start:cloud": "AB_ENV=${AB_ENV-frontend-dev} node -r ./scripts/environment.js ./node_modules/.bin/craco start", "prebuild": "npm run generate-client", "build": "BUILD_PATH='./build/app' craco build", "pretest": "npm run generate-client", diff --git a/airbyte-webapp/src/packages/cloud/views/layout/SideBar/SideBar.tsx b/airbyte-webapp/src/packages/cloud/views/layout/SideBar/SideBar.tsx index c5c5091c62c119..8db1d3bdd1dc53 100644 --- a/airbyte-webapp/src/packages/cloud/views/layout/SideBar/SideBar.tsx +++ b/airbyte-webapp/src/packages/cloud/views/layout/SideBar/SideBar.tsx @@ -10,7 +10,7 @@ import { NavLink } from "react-router-dom"; import { Link } from "components"; import { CreditsIcon } from "components/icons/CreditsIcon"; import { DocsIcon } from "components/icons/DocsIcon"; -import { DropdownMenu } from "components/ui/DropdownMenu"; +import { DropdownMenu, DropdownMenuOptionType } from "components/ui/DropdownMenu"; import { Text } from "components/ui/Text"; import { FeatureItem, IfFeatureEnabled } from "hooks/services/Feature"; @@ -39,7 +39,7 @@ const SideBar: React.FC = () => { const cloudWorkspace = useGetCloudWorkspace(workspace.workspaceId); const { show } = useIntercom(); const { formatMessage } = useIntl(); - const handleChatUs = () => show(); + const handleChatUs = (data: DropdownMenuOptionType) => data.value === "chatUs" && show(); return (