Skip to content

Commit

Permalink
Standard tests: update template (#2937)
Browse files Browse the repository at this point in the history
* update template for python sources
* update templates and their docs
* update docs, rename folders, add acceptance script
* adding hooks for custom setup and teardown
+ improve plugin behaviour, enable it explicitly
+ add acceptance.py to run acceptance tests together with integration tests
+ remove helper acceptance-test-python.sh
+ fix gradle command for acceptance tests
+ update docs

Co-authored-by: Eugene Kulak <kulak.eugene@gmail.com>
Co-authored-by: Sherif A. Nada <snadalive@gmail.com>
  • Loading branch information
3 people committed Apr 28, 2021
1 parent 94d5b27 commit 88b77aa
Show file tree
Hide file tree
Showing 50 changed files with 579 additions and 310 deletions.
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
*
!Dockerfile
!standard_test
!source_acceptance_test
!setup.py
6 changes: 3 additions & 3 deletions airbyte-integrations/bases/source-acceptance-test/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
FROM airbyte/integration-base-python:dev

FROM airbyte/integration-base-python:0.1.4
RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
ENV CODE_PATH="source_acceptance_test"

WORKDIR /airbyte/source_acceptance_test
Expand All @@ -10,4 +10,4 @@ RUN pip install .
LABEL io.airbyte.version=0.1.0
LABEL io.airbyte.name=airbyte/source-acceptance-test

ENTRYPOINT ["python", "-m", "pytest"]
ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin"]
3 changes: 1 addition & 2 deletions airbyte-integrations/bases/source-acceptance-test/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,10 @@

setuptools.setup(
name="source-acceptance-test",
description="Contains classes for running integration tests.",
description="Contains acceptance tests for source connectors.",
author="Airbyte",
author_email="contact@airbyte.io",
url="https://github.com/airbytehq/airbyte",
packages=setuptools.find_packages(),
install_requires=MAIN_REQUIREMENTS,
entry_points={"pytest11": ["pytest-airbyte = source_acceptance_test.plugin"]},
)
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,17 @@


@pytest.fixture(name="base_path")
def base_path_fixture(pytestconfig, standard_test_config) -> Path:
def base_path_fixture(pytestconfig, acceptance_test_config) -> Path:
"""Fixture to define base path for every path-like fixture"""
if standard_test_config.base_path:
return Path(standard_test_config.base_path).absolute()
if acceptance_test_config.base_path:
return Path(acceptance_test_config.base_path).absolute()
return Path(pytestconfig.getoption("--acceptance-test-config")).absolute()


@pytest.fixture(name="standard_test_config", scope="session")
def standard_test_config_fixture(pytestconfig) -> Config:
@pytest.fixture(name="acceptance_test_config", scope="session")
def acceptance_test_config_fixture(pytestconfig) -> Config:
"""Fixture with test's config"""
return load_config(pytestconfig.getoption("--acceptance-test-config"))
return load_config(pytestconfig.getoption("--acceptance-test-config", skip=True))


@pytest.fixture(name="connector_config_path")
Expand Down Expand Up @@ -89,8 +89,8 @@ def catalog_fixture(configured_catalog: ConfiguredAirbyteCatalog) -> Optional[Ai


@pytest.fixture(name="image_tag")
def image_tag_fixture(standard_test_config) -> str:
return standard_test_config.connector_image
def image_tag_fixture(acceptance_test_config) -> str:
return acceptance_test_config.connector_image


@pytest.fixture(name="connector_config")
Expand Down Expand Up @@ -126,10 +126,10 @@ def docker_runner_fixture(image_tag, tmp_path) -> ConnectorRunner:


@pytest.fixture(scope="session", autouse=True)
def pull_docker_image(standard_test_config) -> None:
def pull_docker_image(acceptance_test_config) -> None:
"""Startup fixture to pull docker image"""
print("Pulling docker image", standard_test_config.connector_image)
ConnectorRunner(image_name=standard_test_config.connector_image, volume=Path("."))
print("Pulling docker image", acceptance_test_config.connector_image)
ConnectorRunner(image_name=acceptance_test_config.connector_image, volume=Path("."))
print("Pulling completed")


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,25 @@



from pathlib import Path
from typing import List

import pytest
from _pytest.config import Config
from _pytest.config.argparsing import Parser
from source_acceptance_test.utils import diff_dicts, load_config


HERE = Path(__file__).parent.absolute()


def pytest_load_initial_conftests(early_config: Config, parser: Parser, args: List[str]):
"""Hook function to add acceptance tests to args"""
args.append(str(HERE / "tests"))


def pytest_addoption(parser):
"""Hook function to add CLI option `standard_test_config`"""
"""Hook function to add CLI option `acceptance-test-config`"""
parser.addoption(
"--acceptance-test-config", action="store", default=".", help="Folder with standard test config - acceptance_test_config.yml"
)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# {{titleCase name}} Source
# {{titleCase name}} Source

This is the repository for the {{titleCase name}} source connector, written in Python.
This is the repository for the {{titleCase name}} source connector, written in Python.
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/{{dashCase name}}).

## Local development
Expand Down Expand Up @@ -42,7 +42,6 @@ See `sample_files/sample_config.json` for a sample config file.
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source {{dashCase name}} test creds`
and place them into `secrets/config.json`.


### Locally running the connector
```
python main_dev.py spec
Expand All @@ -51,12 +50,6 @@ python main_dev.py discover --config secrets/config.json
python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
```

### Unit Tests
To run unit tests locally, from the connector directory run:
```
python -m pytest unit_tests
```

### Locally running the connector docker image

#### Build
Expand All @@ -80,19 +73,55 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-{{dashCase name}}:dev
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-{{dashCase name}}:dev discover --config /secrets/config.json
docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-{{dashCase name}}:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json
```
## Testing
Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
First install test dependencies into your virtual environment:
```
pip install .[test]
```
### Unit Tests
To run unit tests locally, from the connector directory run:
```
python -m pytest unit_tests
```

### Integration Tests
1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:integrationTest` to run the standard integration test suite.
1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`.
Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector).
#### Custom Integration tests
Place custom tests inside `integration_tests/` folder, then, from the connector root, run
```
python -m pytest integration_tests
```
#### Acceptance Tests
Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information.
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
To run your integration tests with acceptance tests, from the connector root, run
```
python -m pytest integration_tests -p integration_tests.acceptance
```
To run your integration tests with docker

### Using gradle to run tests
All commands should be run from airbyte project root.
To run unittest run:
```
./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:unitTest
```
To run acceptance and custom integration tests run:
```
./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:IntegrationTest
```

## Dependency Management
All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
We split dependencies between two groups, dependencies that are:
* required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
* required for the testing need to go to `TEST_REQUIREMENTS` list

### Publishing a new version of the connector
You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
1. Make sure your changes are passing unit and integration tests
1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer).
1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)).
1. Create a Pull Request
1. Pat yourself on the back for being an awesome contributor
1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# See [Source Acceptance Tests](https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/source-acceptance-tests.md)
# for more information about how to configure these tests
connector_image: airbyte/{{dashCase name}}
tests:
spec:
- spec_path: "source_{{snakeCase name}}/spec.json"
connection:
- config_path: "secrets/config.json"
status: "succeed"
- config_path: "integration_tests/invalid_config.json"
status: "exception"
discovery:
- config_path: "secrets/config.json"
basic_read:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
validate_output_from_all_streams: yes
incremental: # TODO if your connector does not implement incremental sync, remove this block
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
state_path: "integration_tests/abnormal_state.json"
full_refresh:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/usr/bin/env sh
docker run --rm -i airbyte/source-acceptance-test \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /tmp:/tmp \
-v ./:/test_input \
-w /test_input \
-p integration_tests.acceptance
Original file line number Diff line number Diff line change
@@ -1,32 +1,14 @@
plugins {
id 'airbyte-python'
id 'airbyte-docker'
id 'airbyte-standard-source-test-file'
id 'airbyte-source-acceptance-test'
}

airbytePython {
moduleDirectory 'source_{{snakeCase name}}'
moduleDirectory 'source_{{snakeCase name}}_singer'
}

airbyteStandardSourceTestFile {
// For more information on standard source tests, see https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/testing-connectors

// All these input paths must live inside this connector's directory (or subdirectories)
// TODO update the spec JSON file
specPath = "source_{{snakeCase name}}/spec.json"

// configPath points to a config file which matches the spec.json supplied above. secrets/ is gitignored by default, so place your config file
// there (in case it contains any credentials)
// TODO update the config file to contain actual credentials
configPath = "secrets/config.json"
// TODO update the sample configured_catalog JSON for use in testing
// Note: If your source supports incremental syncing, then make sure that the catalog that is returned in the get_catalog method is configured
// for incremental syncing (e.g. include cursor fields, etc).
configuredCatalogPath = "sample_files/configured_catalog.json"
}


dependencies {
implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs)
implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs)
implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
}
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"todo-stream-name": {
"todo-field-name": "todo-abnormal-value"
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,32 +19,17 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#


import pytest
from source_hubspot.client import Client
from source_hubspot.errors import HubspotInvalidAuth


@pytest.fixture(name="wrong_credentials")
def wrong_credentials_fixture():
return {"api_key": "wrongkey-key1-key2-key3-wrongkey1234"}


def test__health_check_with_wrong_token(wrong_credentials):
client = Client(start_date="2021-02-01T00:00:00Z", credentials=wrong_credentials)
alive, error = client.health_check()

assert not alive
assert (
error
== "HubspotInvalidAuth('The API key provided is invalid. View or manage your API key here: https://app.hubspot.com/l/api-key/')"
)
pytest_plugins = ("source_acceptance_test.plugin",)


def test__stream_iterator_with_wrong_token(wrong_credentials):
client = Client(start_date="2021-02-01T00:00:00Z", credentials=wrong_credentials)
with pytest.raises(
HubspotInvalidAuth, match="The API key provided is invalid. View or manage your API key here: https://app.hubspot.com/l/api-key/"
):
_ = list(client.streams)
@pytest.fixture(scope="session", autouse=True)
def connector_setup():
""" This fixture is a placeholder for external resources that acceptance test might require."""
# TODO: setup test dependencies
yield
# TODO: clean up test dependencies
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"todo-wrong-field": "this should be an incomplete config file, used in standard tests"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"fix-me": "TODO"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"todo-stream-name": {
"todo-field-name": "value"
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies.
-e ../../bases/airbyte-protocol
-e ../../bases/base-python
-e ../../bases/source-acceptance-test
-e .
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,25 @@

from setuptools import find_packages, setup

MAIN_REQUIREMENTS = [
"airbyte-protocol",
"base-python",
]

TEST_REQUIREMENTS = [
"pytest==6.1.2",
"source-acceptance-test",
]

setup(
name="source_{{snakeCase name}}",
description="Source implementation for {{titleCase name}}.",
author="Airbyte",
author_email="contact@airbyte.io",
packages=find_packages(),
install_requires=["airbyte-protocol", "base-python", "pytest==6.1.2"],
package_data={"": ["*.json"]}
install_requires=MAIN_REQUIREMENTS,
package_data={"": ["*.json"]},
extras_require={
"tests": TEST_REQUIREMENTS,
},
)
Loading

0 comments on commit 88b77aa

Please sign in to comment.