From f76a0d34cc5490fb370d50984a38649fa9f21213 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 13 Dec 2023 10:32:31 +0100 Subject: [PATCH 01/22] base implementation --- .../connectors/source-spacex-api/setup.py | 5 + .../source_spacex_api/run.py | 13 + .../connectors/source-stripe/main.py | 1 + .../connectors/source-stripe/setup.py | 5 + .../source-stripe/source_stripe/run.py | 45 + airbyte-lib/.gitignore | 1 + airbyte-lib/README.md | 2 + airbyte-lib/airbyte_lib/__init__.py | 6 + airbyte-lib/airbyte_lib/cache.py | 19 + airbyte-lib/airbyte_lib/registry.py | 31 + airbyte-lib/airbyte_lib/source.py | 218 +++++ airbyte-lib/airbyte_lib/sync.py | 5 + airbyte-lib/examples/run_spacex.py | 20 + airbyte-lib/examples/run_stripe.py | 13 + airbyte-lib/poetry.lock | 871 ++++++++++++++++++ airbyte-lib/pyproject.toml | 17 + airbyte-lib/tests/__init__.py | 0 17 files changed, 1272 insertions(+) create mode 100644 airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py create mode 100644 airbyte-integrations/connectors/source-stripe/source_stripe/run.py create mode 100644 airbyte-lib/.gitignore create mode 100644 airbyte-lib/README.md create mode 100644 airbyte-lib/airbyte_lib/__init__.py create mode 100644 airbyte-lib/airbyte_lib/cache.py create mode 100644 airbyte-lib/airbyte_lib/registry.py create mode 100644 airbyte-lib/airbyte_lib/source.py create mode 100644 airbyte-lib/airbyte_lib/sync.py create mode 100644 airbyte-lib/examples/run_spacex.py create mode 100644 airbyte-lib/examples/run_stripe.py create mode 100644 airbyte-lib/poetry.lock create mode 100644 airbyte-lib/pyproject.toml create mode 100644 airbyte-lib/tests/__init__.py diff --git a/airbyte-integrations/connectors/source-spacex-api/setup.py b/airbyte-integrations/connectors/source-spacex-api/setup.py index 2d202b30ec9a25..3da038cc7e4966 100644 --- a/airbyte-integrations/connectors/source-spacex-api/setup.py +++ b/airbyte-integrations/connectors/source-spacex-api/setup.py @@ -26,4 +26,9 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + 'console_scripts': [ + 'source-spacex-api=source_spacex_api.run:run', + ], + }, ) diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py new file mode 100644 index 00000000000000..4ebe799e3470c0 --- /dev/null +++ b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from .source import SourceSpacexApi + +def run(): + source = SourceSpacexApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-stripe/main.py b/airbyte-integrations/connectors/source-stripe/main.py index a8ed671a829223..9abfa6ba03e439 100644 --- a/airbyte-integrations/connectors/source-stripe/main.py +++ b/airbyte-integrations/connectors/source-stripe/main.py @@ -39,6 +39,7 @@ def _get_source(args: List[str]): if __name__ == "__main__": + print(sys.argv) _args = sys.argv[1:] source = _get_source(_args) if source: diff --git a/airbyte-integrations/connectors/source-stripe/setup.py b/airbyte-integrations/connectors/source-stripe/setup.py index aab9a737d197e7..04024433371180 100644 --- a/airbyte-integrations/connectors/source-stripe/setup.py +++ b/airbyte-integrations/connectors/source-stripe/setup.py @@ -20,4 +20,9 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + 'console_scripts': [ + 'source-stripe=source_stripe.run:run', + ], + }, ) diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/run.py b/airbyte-integrations/connectors/source-stripe/source_stripe/run.py new file mode 100644 index 00000000000000..d5d3f613ee9404 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/run.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys +import traceback +from datetime import datetime +from typing import List + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type +from .source import SourceStripe + + +def _get_source(args: List[str]): + catalog_path = AirbyteEntrypoint.extract_catalog(args) + config_path = AirbyteEntrypoint.extract_config(args) + try: + return SourceStripe( + SourceStripe.read_catalog(catalog_path) if catalog_path else None, + SourceStripe.read_config(config_path) if config_path else None, + ) + except Exception as error: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + return None + + +def run(): + _args = sys.argv[1:] + source = _get_source(_args) + if source: + launch(source, _args) diff --git a/airbyte-lib/.gitignore b/airbyte-lib/.gitignore new file mode 100644 index 00000000000000..3eb9954af8d1c8 --- /dev/null +++ b/airbyte-lib/.gitignore @@ -0,0 +1 @@ +.venv* \ No newline at end of file diff --git a/airbyte-lib/README.md b/airbyte-lib/README.md new file mode 100644 index 00000000000000..a6a0156502fe11 --- /dev/null +++ b/airbyte-lib/README.md @@ -0,0 +1,2 @@ +# airbyte-lib + diff --git a/airbyte-lib/airbyte_lib/__init__.py b/airbyte-lib/airbyte_lib/__init__.py new file mode 100644 index 00000000000000..def4bec05e64ce --- /dev/null +++ b/airbyte-lib/airbyte_lib/__init__.py @@ -0,0 +1,6 @@ + +from .source import get_connector +from .cache import get_in_memory_cache +from .sync import sync + +__all__ = ["get_connector", "get_in_memory_cache", "sync"] diff --git a/airbyte-lib/airbyte_lib/cache.py b/airbyte-lib/airbyte_lib/cache.py new file mode 100644 index 00000000000000..584ab8baa59adc --- /dev/null +++ b/airbyte-lib/airbyte_lib/cache.py @@ -0,0 +1,19 @@ + + +from typing import Iterable +from airbyte_cdk.models import AirbyteRecordMessage + +class InMemoryCache: + """The in-memory cache is accepting airbyte messages and stores them in a dictionary for streams (one list of dicts per stream).""" + + def __init__(self): + self.streams = {} + + def write(self, messages: Iterable[AirbyteRecordMessage]): + for message in messages: + if message.stream not in self.streams: + self.streams[message.stream] = [] + self.streams[message.stream].append(message.data) + +def get_in_memory_cache(): + return InMemoryCache() \ No newline at end of file diff --git a/airbyte-lib/airbyte_lib/registry.py b/airbyte-lib/airbyte_lib/registry.py new file mode 100644 index 00000000000000..8312e6512b253b --- /dev/null +++ b/airbyte-lib/airbyte_lib/registry.py @@ -0,0 +1,31 @@ +import requests +from dataclasses import dataclass + +_cache = None + +REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" + +@dataclass +class ConnectorMetadata: + name: str + version: str + + +def _update_cache(): + global _cache + print("Updating cache") + response = requests.get(REGISTRY_URL) + response.raise_for_status() + data = response.json() + _cache = {} + for connector in data["sources"]: + name = connector["dockerRepository"].replace("airbyte/", "") + _cache[name] = ConnectorMetadata(name, connector["dockerImageTag"]) + +def get_connector_metadata(name: str): + """ + check the cache for the connector. If the cache is empty, populate by calling update_cache + """ + if not _cache: + _update_cache() + return _cache[name] \ No newline at end of file diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py new file mode 100644 index 00000000000000..f57ffd37740925 --- /dev/null +++ b/airbyte-lib/airbyte_lib/source.py @@ -0,0 +1,218 @@ + + +from airbyte_lib.registry import ConnectorMetadata, get_connector_metadata +import jsonschema +from functools import lru_cache +import json +from pathlib import Path +import subprocess +from typing import Any, Dict, Iterable, List, Optional, IO +from airbyte_cdk.models import AirbyteMessage, AirbyteCatalog, Type, AirbyteRecordMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, SyncMode, DestinationSyncMode, Status, ConnectorSpecification +import tempfile +from abc import ABC, abstractmethod +from contextlib import contextmanager + + +@contextmanager +def as_temp_files(files: List[Any]): + temp_files: List[Any] = [] + try: + for content in files: + temp_file = tempfile.NamedTemporaryFile(mode='w+t', delete=True) + temp_file.write(json.dumps(content) if isinstance(content, dict) else content) + temp_file.flush() + temp_files.append(temp_file) + yield [file.name for file in temp_files] + finally: + for temp_file in temp_files: + try: + temp_file.close() + except: + pass + + +from abc import ABC, abstractmethod +from contextlib import contextmanager +from pathlib import Path +import subprocess +from typing import List, IO + + +class Executor(ABC): + def __init__(self, metadata: ConnectorMetadata): + self.metadata = metadata + + @abstractmethod + @contextmanager + def execute(self, args: List[str]) -> IO[str]: + pass + +class VenvExecutor(Executor): + def __init__(self, metadata: ConnectorMetadata): + super().__init__(metadata) + + @contextmanager + def execute(self, args: List[str]) -> IO[str]: + venv_name = f".venv-{self.metadata.name}" + venv_path = Path(venv_name) + if not venv_path.exists(): + raise Exception(f"Could not find venv {venv_name}") + + connector_path = Path(venv_path, "bin", self.metadata.name) + if not connector_path.exists(): + raise Exception(f"Could not find connector {self.metadata.name} in venv {venv_name}") + + process = subprocess.Popen( + [str(connector_path)] + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + try: + yield process.stdout + finally: + exit_code = process.wait() + if exit_code != 0: + raise Exception(f"Process exited with code {exit_code}") + + +class Source: + """This class is representing a source that can be called""" + + def __init__(self, executor: Executor, name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None, streams: Optional[List[str]] = None): + self.executor = executor + self.name = name + self.version = version + if not config is None: + self.set_config(config) + if not streams is None: + self.set_streams(streams) + + def set_streams(self, streams: List[str]): + available_streams = self.get_available_streams() + for stream in streams: + if stream not in available_streams: + raise Exception(f"Stream {stream} is not available for connector {self.name}, choose from {available_streams}") + self.streams = streams + + def set_config(self, config: Dict[str, Any]): + self._validate_config(config) + self.config = config + + def discover(self) -> AirbyteCatalog: + """ + Call discover on the connector. + + This involves the following steps: + * Write the config to a temporary file + * execute the connector with discover --config + * Listen to the messages and return the first AirbyteCatalog that comes along. + * Make sure the subprocess is killed when the function returns. + """ + with as_temp_files([self.config]) as [config_file]: + for msg in self._execute(["discover", "--config", config_file]): + if msg.type == Type.CATALOG and msg.catalog: + return msg.catalog + raise Exception("Connector did not return a catalog") + + def _validate_config(self, config: Dict[str, any]) -> None: + """ + Validate the config against the spec. + """ + spec = self._spec() + jsonschema.validate(config, spec.connectionSpecification) + + def get_available_streams(self) -> List[str]: + """ + Get the available streams from the spec. + """ + return [s.name for s in self.discover().streams] + + @lru_cache(maxsize=1) + def _spec(self) -> ConnectorSpecification: + """ + Call spec on the connector. + + This involves the following steps: + * execute the connector with spec + * Listen to the messages and return the first AirbyteCatalog that comes along. + * Make sure the subprocess is killed when the function returns. + """ + for msg in self._execute(["spec"]): + if msg.type == Type.SPEC and msg.spec: + return msg.spec + raise Exception("Connector did not return a spec") + + def check(self): + """ + Call check on the connector. + + This involves the following steps: + * Write the config to a temporary file + * execute the connector with check --config + * Listen to the messages and return the first AirbyteCatalog that comes along. + * Make sure the subprocess is killed when the function returns. + """ + with as_temp_files([self.config]) as [config_file]: + for msg in self._execute(["check", "--config", config_file]): + if msg.type == Type.CONNECTION_STATUS and msg.connectionStatus: + if msg.connectionStatus.status == Status.FAILED: + raise Exception(f"Connector returned failed status: {msg.connectionStatus.message}") + else: + return + raise Exception("Connector did not return check status") + + def read(self) -> Iterable[AirbyteRecordMessage]: + """ + Call read on the connector. + + This involves the following steps: + * Call discover to get the catalog + * Generate a configured catalog that syncs all streams in full_refresh mode + * Write the configured catalog and the config to a temporary file + * execute the connector with read --config --catalog + * Listen to the messages and return the AirbyteRecordMessages that come along. + """ + catalog = self.discover() + configured_catalog = ConfiguredAirbyteCatalog(streams=[ConfiguredAirbyteStream(stream=s, sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite) for s in catalog.streams if s.name in self.streams or self.streams is None]) + with as_temp_files([self.config, configured_catalog.json()]) as [config_file, catalog_file]: + for msg in self._execute(["read", "--config", config_file, "--catalog", catalog_file]): + if msg.type == Type.RECORD: + yield msg.record + + + def _execute(self, args: List[str]) -> Iterable[AirbyteMessage]: + """ + Execute the connector with the given arguments. + + This involves the following steps: + * Locate the right venv. It is called ".venv-" + * Spawn a subprocess with .venv-/bin/ + * Read the output line by line of the subprocess and serialize them AirbyteMessage objects. Drop if not valid. + """ + + last_log_messages = [] + try: + with self.executor.execute(args) as output: + for line in output: + try: + message = AirbyteMessage.parse_raw(line) + yield message + if message.type == Type.LOG: + last_log_messages.append(message.log.message) + if len(last_log_messages) > 10: + last_log_messages.pop(0) + except Exception: + # line is probably a log message, add it to the last log messages + last_log_messages.append(line) + if len(last_log_messages) > 10: + last_log_messages.pop(0) + except Exception as e: + raise Exception(f"{str(e)}. Last logs: {last_log_messages}") + + +def get_connector(name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None): + """Get a connector by name and version.""" + metadata = get_connector_metadata(name) + return Source(VenvExecutor(metadata), name, version, config) \ No newline at end of file diff --git a/airbyte-lib/airbyte_lib/sync.py b/airbyte-lib/airbyte_lib/sync.py new file mode 100644 index 00000000000000..b3709322d40eef --- /dev/null +++ b/airbyte-lib/airbyte_lib/sync.py @@ -0,0 +1,5 @@ +from .source import Source +from .cache import InMemoryCache + +def sync(connector: Source, store: InMemoryCache): + store.write(connector.read()) \ No newline at end of file diff --git a/airbyte-lib/examples/run_spacex.py b/airbyte-lib/examples/run_spacex.py new file mode 100644 index 00000000000000..9da553fb97f05e --- /dev/null +++ b/airbyte-lib/examples/run_spacex.py @@ -0,0 +1,20 @@ +import airbyte_lib as ab + +# preparation (from airbyte-lib main folder): +# python -m venv .venv-source-spacex-api +# source .venv-source-spacex-api/bin/activate +# pip install -e ../airbyte-integrations/connectors/source-spacex-api +# In separate terminal: +# poetry run python examples/run_spacex.py + +source = ab.get_connector("source-spacex-api", config={"id": "605b4b6aaa5433645e37d03f"}) +cache = ab.get_in_memory_cache() + +source.check() + +source.set_streams(["launches", "rockets"]) + +ab.sync(source, cache) + +for name, records in cache.streams.items(): + print(f"Stream {name}: {len(records)} records") \ No newline at end of file diff --git a/airbyte-lib/examples/run_stripe.py b/airbyte-lib/examples/run_stripe.py new file mode 100644 index 00000000000000..679cdfe70de08f --- /dev/null +++ b/airbyte-lib/examples/run_stripe.py @@ -0,0 +1,13 @@ +from airbyte_lib import get_connector + +# preparation (from airbyte-lib main folder): +# python -m venv .venv-source-stripe +# source .venv-source-stripe/bin/activate +# pip install -e ../airbyte-integrations/connectors/source-stripe +# In separate terminal: +# poetry run python examples/run_stripe.py + +con = get_connector("source-stripe") + +for msg in con._execute(["spec"]): + print(msg) \ No newline at end of file diff --git a/airbyte-lib/poetry.lock b/airbyte-lib/poetry.lock new file mode 100644 index 00000000000000..e2f9cbbf0021cd --- /dev/null +++ b/airbyte-lib/poetry.lock @@ -0,0 +1,871 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.57.1" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.57.1.tar.gz", hash = "sha256:ef2067c5921cd0a6e784ed091ec63b5b94583810490026498ae1b403f9d40e39"}, + {file = "airbyte_cdk-0.57.1-py3-none-any.whl", hash = "sha256:824d34518af049a911036e57600ea84216c9f8176e61ac183ba646f8e29d7336"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.2" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.2" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, + {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "attrs" +version = "23.1.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, + {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2023.11.17" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.3" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, + {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.1.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.1.0-py3-none-any.whl", hash = "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380"}, + {file = "platformdirs-4.1.0.tar.gz", hash = "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "pydantic" +version = "1.10.13" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, + {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, + {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, + {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, + {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, + {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, + {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, + {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, + {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, + {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.0" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.0-py3-none-any.whl", hash = "sha256:a4b21e7a0e99bee328d8f01b23addf664931e693b83212b0fc0059d66ed143b4"}, + {file = "pyrate_limiter-3.1.0.tar.gz", hash = "sha256:6e9ebc756cf6c7afdacb666c182990fb4499d71a542f19b90a05347904b5f8b0"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=4.0.0,<5.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "setuptools" +version = "69.0.2" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.2-py3-none-any.whl", hash = "sha256:1e8fdff6797d3865f37397be788a4e3cba233608e9b509382a2777d25ebde7f2"}, + {file = "setuptools-69.0.2.tar.gz", hash = "sha256:735896e78a4742605974de002ac60562d286fa8051a7e2299445e8e8fbb01aa6"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.1.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.1.0-py3-none-any.whl", hash = "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3"}, + {file = "urllib3-2.1.0.tar.gz", hash = "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "23d733e34a9f9291dc36d2c13090736350672274e541207b0bb9fc5da9b8fe18" diff --git a/airbyte-lib/pyproject.toml b/airbyte-lib/pyproject.toml new file mode 100644 index 00000000000000..c745a236a40e9b --- /dev/null +++ b/airbyte-lib/pyproject.toml @@ -0,0 +1,17 @@ +[tool.poetry] +name = "airbyte-lib" +version = "0.1.0" +description = "" +authors = ["Joe Reuter "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.10" +airbyte-cdk = ">=0.57.1" +jsonschema = "3.2.0" +requests = "^2.31.0" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/airbyte-lib/tests/__init__.py b/airbyte-lib/tests/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 From a960851bdb4995b20023a7d795d2a27108525f5a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 13 Dec 2023 15:01:05 +0100 Subject: [PATCH 02/22] implement peek --- airbyte-lib/airbyte_lib/registry.py | 1 - airbyte-lib/airbyte_lib/source.py | 155 +++++++++++++++++++++++----- airbyte-lib/examples/run_spacex.py | 6 +- 3 files changed, 134 insertions(+), 28 deletions(-) diff --git a/airbyte-lib/airbyte_lib/registry.py b/airbyte-lib/airbyte_lib/registry.py index 8312e6512b253b..65a6f6866b6173 100644 --- a/airbyte-lib/airbyte_lib/registry.py +++ b/airbyte-lib/airbyte_lib/registry.py @@ -13,7 +13,6 @@ class ConnectorMetadata: def _update_cache(): global _cache - print("Updating cache") response = requests.get(REGISTRY_URL) response.raise_for_status() data = response.json() diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index f57ffd37740925..05bef14d6b4288 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -1,5 +1,3 @@ - - from airbyte_lib.registry import ConnectorMetadata, get_connector_metadata import jsonschema from functools import lru_cache @@ -7,10 +5,22 @@ from pathlib import Path import subprocess from typing import Any, Dict, Iterable, List, Optional, IO -from airbyte_cdk.models import AirbyteMessage, AirbyteCatalog, Type, AirbyteRecordMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, SyncMode, DestinationSyncMode, Status, ConnectorSpecification +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteCatalog, + Type, + AirbyteRecordMessage, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + SyncMode, + DestinationSyncMode, + Status, + ConnectorSpecification, +) import tempfile from abc import ABC, abstractmethod from contextlib import contextmanager +from itertools import islice @contextmanager @@ -18,8 +28,10 @@ def as_temp_files(files: List[Any]): temp_files: List[Any] = [] try: for content in files: - temp_file = tempfile.NamedTemporaryFile(mode='w+t', delete=True) - temp_file.write(json.dumps(content) if isinstance(content, dict) else content) + temp_file = tempfile.NamedTemporaryFile(mode="w+t", delete=True) + temp_file.write( + json.dumps(content) if isinstance(content, dict) else content + ) temp_file.flush() temp_files.append(temp_file) yield [file.name for file in temp_files] @@ -47,6 +59,7 @@ def __init__(self, metadata: ConnectorMetadata): def execute(self, args: List[str]) -> IO[str]: pass + class VenvExecutor(Executor): def __init__(self, metadata: ConnectorMetadata): super().__init__(metadata) @@ -60,46 +73,76 @@ def execute(self, args: List[str]) -> IO[str]: connector_path = Path(venv_path, "bin", self.metadata.name) if not connector_path.exists(): - raise Exception(f"Could not find connector {self.metadata.name} in venv {venv_name}") + raise Exception( + f"Could not find connector {self.metadata.name} in venv {venv_name}" + ) process = subprocess.Popen( [str(connector_path)] + args, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stderr=subprocess.STDOUT, universal_newlines=True, ) try: yield process.stdout finally: + # Close the stdout stream + if process.stdout: + process.stdout.close() + + # Terminate the process if it is still running + if process.poll() is None: # Check if the process is still running + process.terminate() + try: + # Wait for a short period to allow process to terminate gracefully + process.wait(timeout=10) + except subprocess.TimeoutExpired: + # If the process does not terminate within the timeout, force kill it + process.kill() + + # Now, the process is either terminated or killed. Check the exit code. exit_code = process.wait() - if exit_code != 0: + + # If the exit code is not 0 or -15 (SIGTERM), raise an exception + if exit_code != 0 and exit_code != -15: raise Exception(f"Process exited with code {exit_code}") class Source: """This class is representing a source that can be called""" - def __init__(self, executor: Executor, name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None, streams: Optional[List[str]] = None): + def __init__( + self, + executor: Executor, + name: str, + version: str = "latest", + config: Optional[Dict[str, Any]] = None, + streams: Optional[List[str]] = None, + ): self.executor = executor self.name = name self.version = version + self.streams = None + self.config = None if not config is None: self.set_config(config) if not streams is None: self.set_streams(streams) - + def set_streams(self, streams: List[str]): available_streams = self.get_available_streams() for stream in streams: if stream not in available_streams: - raise Exception(f"Stream {stream} is not available for connector {self.name}, choose from {available_streams}") + raise Exception( + f"Stream {stream} is not available for connector {self.name}, choose from {available_streams}" + ) self.streams = streams - + def set_config(self, config: Dict[str, Any]): self._validate_config(config) self.config = config - + def discover(self) -> AirbyteCatalog: """ Call discover on the connector. @@ -115,20 +158,20 @@ def discover(self) -> AirbyteCatalog: if msg.type == Type.CATALOG and msg.catalog: return msg.catalog raise Exception("Connector did not return a catalog") - + def _validate_config(self, config: Dict[str, any]) -> None: """ Validate the config against the spec. """ spec = self._spec() jsonschema.validate(config, spec.connectionSpecification) - + def get_available_streams(self) -> List[str]: """ Get the available streams from the spec. """ return [s.name for s in self.discover().streams] - + @lru_cache(maxsize=1) def _spec(self) -> ConnectorSpecification: """ @@ -144,6 +187,37 @@ def _spec(self) -> ConnectorSpecification: return msg.spec raise Exception("Connector did not return a spec") + def peek(self, stream: str, max_n: int = 10) -> List[Dict[str, Any]]: + """ + Peek at a stream. + + This involves the following steps: + * Call discover to get the catalog + * Generate a configured catalog that syncs the given stream in full_refresh mode + * Write the configured catalog and the config to a temporary file + * execute the connector with read --config --catalog + * Listen to the messages and return the first AirbyteRecordMessages that come along. + * Make sure the subprocess is killed when the function returns. + """ + catalog = self.discover() + configured_catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=s, + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + for s in catalog.streams + if s.name == stream + ] + ) + if len(configured_catalog.streams) == 0: + raise Exception( + f"Stream {stream} is not available for connector {self.name}, choose from {self.get_available_streams()}" + ) + messages = islice(self._read(configured_catalog), max_n) + return [m.data for m in messages] + def check(self): """ Call check on the connector. @@ -158,11 +232,13 @@ def check(self): for msg in self._execute(["check", "--config", config_file]): if msg.type == Type.CONNECTION_STATUS and msg.connectionStatus: if msg.connectionStatus.status == Status.FAILED: - raise Exception(f"Connector returned failed status: {msg.connectionStatus.message}") + raise Exception( + f"Connector returned failed status: {msg.connectionStatus.message}" + ) else: return raise Exception("Connector did not return check status") - + def read(self) -> Iterable[AirbyteRecordMessage]: """ Call read on the connector. @@ -175,13 +251,40 @@ def read(self) -> Iterable[AirbyteRecordMessage]: * Listen to the messages and return the AirbyteRecordMessages that come along. """ catalog = self.discover() - configured_catalog = ConfiguredAirbyteCatalog(streams=[ConfiguredAirbyteStream(stream=s, sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite) for s in catalog.streams if s.name in self.streams or self.streams is None]) - with as_temp_files([self.config, configured_catalog.json()]) as [config_file, catalog_file]: - for msg in self._execute(["read", "--config", config_file, "--catalog", catalog_file]): + configured_catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=s, + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + for s in catalog.streams + if self.streams is None or s.name in self.streams + ] + ) + yield from self._read(configured_catalog) + + def _read( + self, catalog: ConfiguredAirbyteCatalog + ) -> Iterable[AirbyteRecordMessage]: + """ + Call read on the connector. + + This involves the following steps: + * Write the config to a temporary file + * execute the connector with read --config --catalog + * Listen to the messages and return the AirbyteRecordMessages that come along. + """ + with as_temp_files([self.config, catalog.json()]) as [ + config_file, + catalog_file, + ]: + for msg in self._execute( + ["read", "--config", config_file, "--catalog", catalog_file] + ): if msg.type == Type.RECORD: yield msg.record - def _execute(self, args: List[str]) -> Iterable[AirbyteMessage]: """ Execute the connector with the given arguments. @@ -210,9 +313,11 @@ def _execute(self, args: List[str]) -> Iterable[AirbyteMessage]: last_log_messages.pop(0) except Exception as e: raise Exception(f"{str(e)}. Last logs: {last_log_messages}") - -def get_connector(name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None): + +def get_connector( + name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None +): """Get a connector by name and version.""" metadata = get_connector_metadata(name) - return Source(VenvExecutor(metadata), name, version, config) \ No newline at end of file + return Source(VenvExecutor(metadata), name, version, config) diff --git a/airbyte-lib/examples/run_spacex.py b/airbyte-lib/examples/run_spacex.py index 9da553fb97f05e..5cd2c430eb3c18 100644 --- a/airbyte-lib/examples/run_spacex.py +++ b/airbyte-lib/examples/run_spacex.py @@ -12,9 +12,11 @@ source.check() -source.set_streams(["launches", "rockets"]) +source.set_streams(["launches", "rockets", "capsules"]) ab.sync(source, cache) for name, records in cache.streams.items(): - print(f"Stream {name}: {len(records)} records") \ No newline at end of file + print(f"Stream {name}: {len(records)} records") + +print(source.peek("capsules")) \ No newline at end of file From 17c17b95d1c1804c0c796f0d93e0c24d928e41e7 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 13 Dec 2023 15:09:34 +0100 Subject: [PATCH 03/22] prepare --- airbyte-lib/README.md | 8 ++++++++ airbyte-lib/examples/run_stripe.py | 13 ------------- airbyte-lib/tests/test_source.py | 3 +++ 3 files changed, 11 insertions(+), 13 deletions(-) delete mode 100644 airbyte-lib/examples/run_stripe.py create mode 100644 airbyte-lib/tests/test_source.py diff --git a/airbyte-lib/README.md b/airbyte-lib/README.md index a6a0156502fe11..f70528eb3d233d 100644 --- a/airbyte-lib/README.md +++ b/airbyte-lib/README.md @@ -1,2 +1,10 @@ # airbyte-lib +airbyte-lib is a library that allows to run Airbyte syncs embedded into any Python application, without the need to run Airbyte server. + +## Development + +* Make sure [Poetry is installed](https://python-poetry.org/docs/#). +* Run `poetry install` +* For examples, check out the `examples` folder. They can be run via `poetry run python examples/` +* Unit tests can be run via `poetry run pytest` \ No newline at end of file diff --git a/airbyte-lib/examples/run_stripe.py b/airbyte-lib/examples/run_stripe.py deleted file mode 100644 index 679cdfe70de08f..00000000000000 --- a/airbyte-lib/examples/run_stripe.py +++ /dev/null @@ -1,13 +0,0 @@ -from airbyte_lib import get_connector - -# preparation (from airbyte-lib main folder): -# python -m venv .venv-source-stripe -# source .venv-source-stripe/bin/activate -# pip install -e ../airbyte-integrations/connectors/source-stripe -# In separate terminal: -# poetry run python examples/run_stripe.py - -con = get_connector("source-stripe") - -for msg in con._execute(["spec"]): - print(msg) \ No newline at end of file diff --git a/airbyte-lib/tests/test_source.py b/airbyte-lib/tests/test_source.py new file mode 100644 index 00000000000000..e3127a16e94d9d --- /dev/null +++ b/airbyte-lib/tests/test_source.py @@ -0,0 +1,3 @@ + +def test_my_test(): + pass \ No newline at end of file From 6dd881543ce1f05f668173701300924729036669 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 13 Dec 2023 16:13:04 +0100 Subject: [PATCH 04/22] various things --- airbyte-lib/airbyte_lib/__init__.py | 11 +- airbyte-lib/airbyte_lib/cache.py | 20 +- airbyte-lib/airbyte_lib/connection.py | 37 ++ airbyte-lib/airbyte_lib/executor.py | 72 +++ airbyte-lib/airbyte_lib/factories.py | 27 ++ airbyte-lib/airbyte_lib/source.py | 82 +--- airbyte-lib/airbyte_lib/sync.py | 5 - airbyte-lib/examples/run_spacex.py | 8 +- airbyte-lib/poetry.lock | 614 ++------------------------ airbyte-lib/pyproject.toml | 2 +- 10 files changed, 209 insertions(+), 669 deletions(-) create mode 100644 airbyte-lib/airbyte_lib/connection.py create mode 100644 airbyte-lib/airbyte_lib/executor.py create mode 100644 airbyte-lib/airbyte_lib/factories.py delete mode 100644 airbyte-lib/airbyte_lib/sync.py diff --git a/airbyte-lib/airbyte_lib/__init__.py b/airbyte-lib/airbyte_lib/__init__.py index def4bec05e64ce..40ba3fbd2c203c 100644 --- a/airbyte-lib/airbyte_lib/__init__.py +++ b/airbyte-lib/airbyte_lib/__init__.py @@ -1,6 +1,9 @@ -from .source import get_connector -from .cache import get_in_memory_cache -from .sync import sync +from .factories import (create_connection, get_connector, get_in_memory_cache, sync) -__all__ = ["get_connector", "get_in_memory_cache", "sync"] +__all__ = [ + "create_connection", + "get_connector", + "get_in_memory_cache", + "sync", +] diff --git a/airbyte-lib/airbyte_lib/cache.py b/airbyte-lib/airbyte_lib/cache.py index 584ab8baa59adc..524c2d27eddc6c 100644 --- a/airbyte-lib/airbyte_lib/cache.py +++ b/airbyte-lib/airbyte_lib/cache.py @@ -1,19 +1,23 @@ -from typing import Iterable -from airbyte_cdk.models import AirbyteRecordMessage +from typing import Any, Dict, Iterable, List +from abc import ABC, abstractmethod +from airbyte_protocol.models import AirbyteRecordMessage -class InMemoryCache: +class Cache(ABC): + @abstractmethod + def write(self, messages: Iterable[AirbyteRecordMessage]): + pass + + +class InMemoryCache(Cache): """The in-memory cache is accepting airbyte messages and stores them in a dictionary for streams (one list of dicts per stream).""" def __init__(self): - self.streams = {} + self.streams: Dict[str, List[Dict[str, Any]]] = {} def write(self, messages: Iterable[AirbyteRecordMessage]): for message in messages: if message.stream not in self.streams: self.streams[message.stream] = [] - self.streams[message.stream].append(message.data) - -def get_in_memory_cache(): - return InMemoryCache() \ No newline at end of file + self.streams[message.stream].append(message.data) \ No newline at end of file diff --git a/airbyte-lib/airbyte_lib/connection.py b/airbyte-lib/airbyte_lib/connection.py new file mode 100644 index 00000000000000..cb6af8f0bf2949 --- /dev/null +++ b/airbyte-lib/airbyte_lib/connection.py @@ -0,0 +1,37 @@ +from typing import Iterable, TypeVar, Generic +from airbyte_lib.cache import Cache +from airbyte_lib.source import Source +from airbyte_protocol.models import AirbyteRecordMessage, Type +from dataclasses import dataclass + +TCache = TypeVar('TCache', bound=Cache) + +@dataclass +class SyncResult(Generic[TCache]): + processed_records: int + cache: TCache + +class Connection(Generic[TCache]): + """This class is representing a source that can be called""" + + def __init__( + self, + source: Source, + cache: TCache, + ): + self.source = source + self.cache = cache + + def _process(self, messages: Iterable[AirbyteRecordMessage]): + self._processed_records = 0 + for message in messages: + self._processed_records += 1 + yield message + + def sync(self) -> SyncResult[TCache]: + self.cache.write(self._process(self.source.read())) + + return SyncResult( + processed_records=self._processed_records, + cache=self.cache, + ) diff --git a/airbyte-lib/airbyte_lib/executor.py b/airbyte-lib/airbyte_lib/executor.py new file mode 100644 index 00000000000000..1625dda33b00e1 --- /dev/null +++ b/airbyte-lib/airbyte_lib/executor.py @@ -0,0 +1,72 @@ +from abc import ABC, abstractmethod +from contextlib import contextmanager +from pathlib import Path +import subprocess +from typing import List, IO +from airbyte_lib.registry import ConnectorMetadata +from pathlib import Path +import subprocess +from typing import List, IO +from abc import ABC, abstractmethod +from contextlib import contextmanager + + + +class Executor(ABC): + def __init__(self, metadata: ConnectorMetadata): + self.metadata = metadata + + @abstractmethod + @contextmanager + def execute(self, args: List[str]) -> IO[str]: + pass + + +class VenvExecutor(Executor): + def __init__(self, metadata: ConnectorMetadata): + super().__init__(metadata) + + @contextmanager + def execute(self, args: List[str]) -> IO[str]: + venv_name = f".venv-{self.metadata.name}" + venv_path = Path(venv_name) + if not venv_path.exists(): + raise Exception(f"Could not find venv {venv_name}") + + connector_path = Path(venv_path, "bin", self.metadata.name) + if not connector_path.exists(): + raise Exception( + f"Could not find connector {self.metadata.name} in venv {venv_name}" + ) + + process = subprocess.Popen( + [str(connector_path)] + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + + try: + yield process.stdout + finally: + # Close the stdout stream + if process.stdout: + process.stdout.close() + + # Terminate the process if it is still running + if process.poll() is None: # Check if the process is still running + process.terminate() + try: + # Wait for a short period to allow process to terminate gracefully + process.wait(timeout=10) + except subprocess.TimeoutExpired: + # If the process does not terminate within the timeout, force kill it + process.kill() + + # Now, the process is either terminated or killed. Check the exit code. + exit_code = process.wait() + + # If the exit code is not 0 or -15 (SIGTERM), raise an exception + if exit_code != 0 and exit_code != -15: + raise Exception(f"Process exited with code {exit_code}") + diff --git a/airbyte-lib/airbyte_lib/factories.py b/airbyte-lib/airbyte_lib/factories.py new file mode 100644 index 00000000000000..7ebbd76de7e36f --- /dev/null +++ b/airbyte-lib/airbyte_lib/factories.py @@ -0,0 +1,27 @@ + + +from typing import Any, Dict, Optional, TypeVar, Generic +from airbyte_lib.cache import InMemoryCache, Cache +from airbyte_lib.connection import Connection, SyncResult +from airbyte_lib.executor import VenvExecutor +from airbyte_lib.registry import get_connector_metadata +from airbyte_lib.source import Source + +TCache = TypeVar('TCache', bound=Cache) + +def get_in_memory_cache(): + return InMemoryCache() + +def get_connector( + name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None +): + metadata = get_connector_metadata(name) + return Source(VenvExecutor(metadata), name, version, config) + +def sync(connector: Source, store: TCache) -> SyncResult[TCache]: + return create_connection(connector, store).sync() + +def create_connection( + source: Source, cache: TCache +) -> Connection[TCache]: + return Connection(source, cache) \ No newline at end of file diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index 05bef14d6b4288..56a29039a13fb8 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -1,11 +1,10 @@ -from airbyte_lib.registry import ConnectorMetadata, get_connector_metadata +from airbyte_lib.executor import Executor, VenvExecutor +from airbyte_lib.registry import get_connector_metadata import jsonschema from functools import lru_cache import json -from pathlib import Path -import subprocess -from typing import Any, Dict, Iterable, List, Optional, IO -from airbyte_cdk.models import ( +from typing import Any, Dict, Iterable, List, Optional +from airbyte_protocol.models import ( AirbyteMessage, AirbyteCatalog, Type, @@ -18,7 +17,6 @@ ConnectorSpecification, ) import tempfile -from abc import ABC, abstractmethod from contextlib import contextmanager from itertools import islice @@ -43,71 +41,6 @@ def as_temp_files(files: List[Any]): pass -from abc import ABC, abstractmethod -from contextlib import contextmanager -from pathlib import Path -import subprocess -from typing import List, IO - - -class Executor(ABC): - def __init__(self, metadata: ConnectorMetadata): - self.metadata = metadata - - @abstractmethod - @contextmanager - def execute(self, args: List[str]) -> IO[str]: - pass - - -class VenvExecutor(Executor): - def __init__(self, metadata: ConnectorMetadata): - super().__init__(metadata) - - @contextmanager - def execute(self, args: List[str]) -> IO[str]: - venv_name = f".venv-{self.metadata.name}" - venv_path = Path(venv_name) - if not venv_path.exists(): - raise Exception(f"Could not find venv {venv_name}") - - connector_path = Path(venv_path, "bin", self.metadata.name) - if not connector_path.exists(): - raise Exception( - f"Could not find connector {self.metadata.name} in venv {venv_name}" - ) - - process = subprocess.Popen( - [str(connector_path)] + args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - ) - - try: - yield process.stdout - finally: - # Close the stdout stream - if process.stdout: - process.stdout.close() - - # Terminate the process if it is still running - if process.poll() is None: # Check if the process is still running - process.terminate() - try: - # Wait for a short period to allow process to terminate gracefully - process.wait(timeout=10) - except subprocess.TimeoutExpired: - # If the process does not terminate within the timeout, force kill it - process.kill() - - # Now, the process is either terminated or killed. Check the exit code. - exit_code = process.wait() - - # If the exit code is not 0 or -15 (SIGTERM), raise an exception - if exit_code != 0 and exit_code != -15: - raise Exception(f"Process exited with code {exit_code}") - class Source: """This class is representing a source that can be called""" @@ -314,10 +247,3 @@ def _execute(self, args: List[str]) -> Iterable[AirbyteMessage]: except Exception as e: raise Exception(f"{str(e)}. Last logs: {last_log_messages}") - -def get_connector( - name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None -): - """Get a connector by name and version.""" - metadata = get_connector_metadata(name) - return Source(VenvExecutor(metadata), name, version, config) diff --git a/airbyte-lib/airbyte_lib/sync.py b/airbyte-lib/airbyte_lib/sync.py deleted file mode 100644 index b3709322d40eef..00000000000000 --- a/airbyte-lib/airbyte_lib/sync.py +++ /dev/null @@ -1,5 +0,0 @@ -from .source import Source -from .cache import InMemoryCache - -def sync(connector: Source, store: InMemoryCache): - store.write(connector.read()) \ No newline at end of file diff --git a/airbyte-lib/examples/run_spacex.py b/airbyte-lib/examples/run_spacex.py index 5cd2c430eb3c18..eaae9313b20ae2 100644 --- a/airbyte-lib/examples/run_spacex.py +++ b/airbyte-lib/examples/run_spacex.py @@ -14,9 +14,9 @@ source.set_streams(["launches", "rockets", "capsules"]) -ab.sync(source, cache) +result = ab.sync(source, cache) -for name, records in cache.streams.items(): - print(f"Stream {name}: {len(records)} records") +print(source.peek("capsules")) -print(source.peek("capsules")) \ No newline at end of file +for name, records in result.cache.streams.items(): + print(f"Stream {name}: {len(records)} records") diff --git a/airbyte-lib/poetry.lock b/airbyte-lib/poetry.lock index e2f9cbbf0021cd..3eacea93a77863 100644 --- a/airbyte-lib/poetry.lock +++ b/airbyte-lib/poetry.lock @@ -1,55 +1,18 @@ # This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. -[[package]] -name = "airbyte-cdk" -version = "0.57.1" -description = "A framework for writing Airbyte Connectors." -optional = false -python-versions = ">=3.8" -files = [ - {file = "airbyte-cdk-0.57.1.tar.gz", hash = "sha256:ef2067c5921cd0a6e784ed091ec63b5b94583810490026498ae1b403f9d40e39"}, - {file = "airbyte_cdk-0.57.1-py3-none-any.whl", hash = "sha256:824d34518af049a911036e57600ea84216c9f8176e61ac183ba646f8e29d7336"}, -] - -[package.dependencies] -airbyte-protocol-models = "0.4.2" -backoff = "*" -cachetools = "*" -Deprecated = ">=1.2,<2.0" -dpath = ">=2.0.1,<2.1.0" -genson = "1.2.2" -isodate = ">=0.6.1,<0.7.0" -Jinja2 = ">=3.1.2,<3.2.0" -jsonref = ">=0.2,<1.0" -jsonschema = ">=3.2.0,<3.3.0" -pendulum = "*" -pydantic = ">=1.10.8,<2.0.0" -pyrate-limiter = ">=3.1.0,<3.2.0" -python-dateutil = "*" -PyYAML = ">=6.0.1" -requests = "*" -requests-cache = "*" -wcmatch = "8.4" - -[package.extras] -dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] -file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] -sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] -vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] - [[package]] name = "airbyte-protocol-models" -version = "0.4.2" +version = "1.0.1" description = "Declares the Airbyte Protocol." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, - {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, + {file = "airbyte_protocol_models-1.0.1-py3-none-any.whl", hash = "sha256:2c214fb8cb42b74aa6408beeea2cd52f094bc8a3ba0e78af20bb358e5404f4a8"}, + {file = "airbyte_protocol_models-1.0.1.tar.gz", hash = "sha256:caa860d15c9c9073df4b221f58280b9855d36de07519e010d1e610546458d0a7"}, ] [package.dependencies] -pydantic = ">=1.9.2,<2.0.0" +pydantic = ">=1.9.2,<1.10.0" [[package]] name = "attrs" @@ -69,64 +32,6 @@ docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib- tests = ["attrs[tests-no-zope]", "zope-interface"] tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -[[package]] -name = "backoff" -version = "2.2.1" -description = "Function decoration for backoff and retry" -optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, - {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, -] - -[[package]] -name = "bracex" -version = "2.4" -description = "Bash style brace expander." -optional = false -python-versions = ">=3.8" -files = [ - {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, - {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, -] - -[[package]] -name = "cachetools" -version = "5.3.2" -description = "Extensible memoizing collections and decorators" -optional = false -python-versions = ">=3.7" -files = [ - {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, - {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, -] - -[[package]] -name = "cattrs" -version = "23.2.3" -description = "Composable complex class support for attrs and dataclasses." -optional = false -python-versions = ">=3.8" -files = [ - {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, - {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, -] - -[package.dependencies] -attrs = ">=23.1.0" -exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} - -[package.extras] -bson = ["pymongo (>=4.4.0)"] -cbor2 = ["cbor2 (>=5.4.6)"] -msgpack = ["msgpack (>=1.0.5)"] -orjson = ["orjson (>=3.9.2)"] -pyyaml = ["pyyaml (>=6.0)"] -tomlkit = ["tomlkit (>=0.11.8)"] -ujson = ["ujson (>=5.7.0)"] - [[package]] name = "certifi" version = "2023.11.17" @@ -237,58 +142,6 @@ files = [ {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] -[[package]] -name = "deprecated" -version = "1.2.14" -description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, - {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, -] - -[package.dependencies] -wrapt = ">=1.10,<2" - -[package.extras] -dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] - -[[package]] -name = "dpath" -version = "2.0.8" -description = "Filesystem-like pathing and searching for dictionaries" -optional = false -python-versions = ">=3.7" -files = [ - {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, - {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, -] - -[[package]] -name = "exceptiongroup" -version = "1.2.0" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, - {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, -] - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "genson" -version = "1.2.2" -description = "GenSON is a powerful, user-friendly JSON Schema generator." -optional = false -python-versions = "*" -files = [ - {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, -] - [[package]] name = "idna" version = "3.6" @@ -300,48 +153,6 @@ files = [ {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] -[[package]] -name = "isodate" -version = "0.6.1" -description = "An ISO 8601 date/time/duration parser and formatter" -optional = false -python-versions = "*" -files = [ - {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, - {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, -] - -[package.dependencies] -six = "*" - -[[package]] -name = "jinja2" -version = "3.1.2" -description = "A very fast and expressive template engine." -optional = false -python-versions = ">=3.7" -files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, -] - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "jsonref" -version = "0.3.0" -description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." -optional = false -python-versions = ">=3.3,<4.0" -files = [ - {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, - {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, -] - [[package]] name = "jsonschema" version = "3.2.0" @@ -363,181 +174,57 @@ six = ">=1.11.0" format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] -[[package]] -name = "markupsafe" -version = "2.1.3" -description = "Safely add untrusted strings to HTML/XML markup." -optional = false -python-versions = ">=3.7" -files = [ - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, - {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, -] - -[[package]] -name = "pendulum" -version = "2.1.2" -description = "Python datetimes made easy" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, - {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, - {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, - {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, - {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, - {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, - {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, - {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, - {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, - {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, - {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, - {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, - {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, - {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, - {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, - {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, - {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, - {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, - {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, - {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, - {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, -] - -[package.dependencies] -python-dateutil = ">=2.6,<3.0" -pytzdata = ">=2020.1" - -[[package]] -name = "platformdirs" -version = "4.1.0" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -optional = false -python-versions = ">=3.8" -files = [ - {file = "platformdirs-4.1.0-py3-none-any.whl", hash = "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380"}, - {file = "platformdirs-4.1.0.tar.gz", hash = "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"}, -] - -[package.extras] -docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] - [[package]] name = "pydantic" -version = "1.10.13" +version = "1.9.2" description = "Data validation and settings management using python type hints" optional = false -python-versions = ">=3.7" -files = [ - {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, - {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, - {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, - {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, - {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, - {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, - {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, - {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, - {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, - {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, - {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, - {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, - {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, - {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, - {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, - {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, - {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, - {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, - {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, - {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, - {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, - {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, - {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, - {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, - {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, - {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, - {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, - {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, - {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, - {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, - {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, - {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, - {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, - {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, - {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, - {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, +python-versions = ">=3.6.1" +files = [ + {file = "pydantic-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c9e04a6cdb7a363d7cb3ccf0efea51e0abb48e180c0d31dca8d247967d85c6e"}, + {file = "pydantic-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fafe841be1103f340a24977f61dee76172e4ae5f647ab9e7fd1e1fca51524f08"}, + {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afacf6d2a41ed91fc631bade88b1d319c51ab5418870802cedb590b709c5ae3c"}, + {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ee0d69b2a5b341fc7927e92cae7ddcfd95e624dfc4870b32a85568bd65e6131"}, + {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ff68fc85355532ea77559ede81f35fff79a6a5543477e168ab3a381887caea76"}, + {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0f5e142ef8217019e3eef6ae1b6b55f09a7a15972958d44fbd228214cede567"}, + {file = "pydantic-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:615661bfc37e82ac677543704437ff737418e4ea04bef9cf11c6d27346606044"}, + {file = "pydantic-1.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:328558c9f2eed77bd8fffad3cef39dbbe3edc7044517f4625a769d45d4cf7555"}, + {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd446bdb7755c3a94e56d7bdfd3ee92396070efa8ef3a34fab9579fe6aa1d84"}, + {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0b214e57623a535936005797567231a12d0da0c29711eb3514bc2b3cd008d0f"}, + {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d8ce3fb0841763a89322ea0432f1f59a2d3feae07a63ea2c958b2315e1ae8adb"}, + {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b34ba24f3e2d0b39b43f0ca62008f7ba962cff51efa56e64ee25c4af6eed987b"}, + {file = "pydantic-1.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:84d76ecc908d917f4684b354a39fd885d69dd0491be175f3465fe4b59811c001"}, + {file = "pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56"}, + {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4"}, + {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f"}, + {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979"}, + {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d"}, + {file = "pydantic-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e0b4fb13ad4db4058a7c3c80e2569adbd810c25e6ca3bbd8b2a9cc2cc871d7"}, + {file = "pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3"}, + {file = "pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa"}, + {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3"}, + {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0"}, + {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8"}, + {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8"}, + {file = "pydantic-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:d70916235d478404a3fa8c997b003b5f33aeac4686ac1baa767234a0f8ac2326"}, + {file = "pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801"}, + {file = "pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44"}, + {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747"}, + {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453"}, + {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb"}, + {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15"}, + {file = "pydantic-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1a68f4f65a9ee64b6ccccb5bf7e17db07caebd2730109cb8a95863cfa9c4e55"}, + {file = "pydantic-1.9.2-py3-none-any.whl", hash = "sha256:78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e"}, + {file = "pydantic-1.9.2.tar.gz", hash = "sha256:8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d"}, ] [package.dependencies] -typing-extensions = ">=4.2.0" +typing-extensions = ">=3.7.4.3" [package.extras] dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] -[[package]] -name = "pyrate-limiter" -version = "3.1.0" -description = "Python Rate-Limiter using Leaky-Bucket Algorithm" -optional = false -python-versions = ">=3.8,<4.0" -files = [ - {file = "pyrate_limiter-3.1.0-py3-none-any.whl", hash = "sha256:a4b21e7a0e99bee328d8f01b23addf664931e693b83212b0fc0059d66ed143b4"}, - {file = "pyrate_limiter-3.1.0.tar.gz", hash = "sha256:6e9ebc756cf6c7afdacb666c182990fb4499d71a542f19b90a05347904b5f8b0"}, -] - -[package.extras] -all = ["filelock (>=3.0)", "redis (>=4.0.0,<5.0.0)"] -docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] - [[package]] name = "pyrsistent" version = "0.20.0" @@ -579,80 +266,6 @@ files = [ {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, ] -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytzdata" -version = "2020.1" -description = "The Olson timezone database for Python." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, - {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, -] - -[[package]] -name = "pyyaml" -version = "6.0.1" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.6" -files = [ - {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, - {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, - {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, - {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, - {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, - {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, - {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, - {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, -] - [[package]] name = "requests" version = "2.31.0" @@ -674,36 +287,6 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] -[[package]] -name = "requests-cache" -version = "1.1.1" -description = "A persistent cache for python requests" -optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, - {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, -] - -[package.dependencies] -attrs = ">=21.2" -cattrs = ">=22.2" -platformdirs = ">=2.5" -requests = ">=2.22" -url-normalize = ">=1.4" -urllib3 = ">=1.25.5" - -[package.extras] -all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] -bson = ["bson (>=0.5)"] -docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] -dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] -json = ["ujson (>=5.4)"] -mongodb = ["pymongo (>=3)"] -redis = ["redis (>=3)"] -security = ["itsdangerous (>=2.0)"] -yaml = ["pyyaml (>=5.4)"] - [[package]] name = "setuptools" version = "69.0.2" @@ -742,20 +325,6 @@ files = [ {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] -[[package]] -name = "url-normalize" -version = "1.4.3" -description = "URL normalization for Python" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" -files = [ - {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, - {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, -] - -[package.dependencies] -six = "*" - [[package]] name = "urllib3" version = "2.1.0" @@ -772,100 +341,7 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] -[[package]] -name = "wcmatch" -version = "8.4" -description = "Wildcard/glob file name matcher." -optional = false -python-versions = ">=3.7" -files = [ - {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, - {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, -] - -[package.dependencies] -bracex = ">=2.1.1" - -[[package]] -name = "wrapt" -version = "1.16.0" -description = "Module for decorators, wrappers and monkey patching." -optional = false -python-versions = ">=3.6" -files = [ - {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, - {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, - {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, - {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, - {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, - {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, - {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, - {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, - {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, - {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, - {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, - {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, - {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, - {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, - {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, - {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, - {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, - {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, - {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, - {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, - {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, - {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, - {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, - {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, - {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, - {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, - {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, - {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, - {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, - {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, - {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, - {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, - {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, - {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, - {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, - {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, - {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, - {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, - {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, - {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, - {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, - {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, - {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, - {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, - {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, - {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, - {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, - {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, - {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, - {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, - {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, - {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, - {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, - {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, - {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, - {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, - {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, - {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, - {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, - {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, - {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, - {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, - {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, - {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, - {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, - {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, - {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, - {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, - {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, - {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, -] - [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "23d733e34a9f9291dc36d2c13090736350672274e541207b0bb9fc5da9b8fe18" +content-hash = "c9c9ef5965cefe590d38e25b1cd11bcac2feed3f01c51115691174742bca52e4" diff --git a/airbyte-lib/pyproject.toml b/airbyte-lib/pyproject.toml index c745a236a40e9b..7967b1815fdb46 100644 --- a/airbyte-lib/pyproject.toml +++ b/airbyte-lib/pyproject.toml @@ -7,9 +7,9 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.10" -airbyte-cdk = ">=0.57.1" jsonschema = "3.2.0" requests = "^2.31.0" +airbyte-protocol-models = "^1.0.1" [build-system] From 81cc2141763ccda29f65ef5a31ed20d91d2e8ac9 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 13 Dec 2023 18:50:29 +0100 Subject: [PATCH 05/22] tests and stuff --- airbyte-lib/airbyte_lib/executor.py | 66 +++++++-- airbyte-lib/airbyte_lib/factories.py | 2 +- airbyte-lib/airbyte_lib/registry.py | 14 +- airbyte-lib/airbyte_lib/source.py | 9 +- airbyte-lib/examples/run_test_source.py | 22 +++ airbyte-lib/poetry.lock | 97 ++++++++++++- airbyte-lib/pyproject.toml | 2 + airbyte-lib/tests/fixtures/registry.json | 42 ++++++ .../tests/fixtures/source-test/setup.py | 20 +++ .../fixtures/source-test/source_test/run.py | 129 ++++++++++++++++++ airbyte-lib/tests/test_integration.py | 90 ++++++++++++ airbyte-lib/tests/test_source.py | 3 - 12 files changed, 476 insertions(+), 20 deletions(-) create mode 100644 airbyte-lib/examples/run_test_source.py create mode 100644 airbyte-lib/tests/fixtures/registry.json create mode 100644 airbyte-lib/tests/fixtures/source-test/setup.py create mode 100644 airbyte-lib/tests/fixtures/source-test/source_test/run.py create mode 100644 airbyte-lib/tests/test_integration.py delete mode 100644 airbyte-lib/tests/test_source.py diff --git a/airbyte-lib/airbyte_lib/executor.py b/airbyte-lib/airbyte_lib/executor.py index 1625dda33b00e1..1e1b94305bcb07 100644 --- a/airbyte-lib/airbyte_lib/executor.py +++ b/airbyte-lib/airbyte_lib/executor.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod from contextlib import contextmanager +import os from pathlib import Path import subprocess from typing import List, IO @@ -9,36 +10,85 @@ from typing import List, IO from abc import ABC, abstractmethod from contextlib import contextmanager - +import sys class Executor(ABC): - def __init__(self, metadata: ConnectorMetadata): + def __init__(self, metadata: ConnectorMetadata, target_version: str = "latest"): self.metadata = metadata + self.target_version = target_version if target_version != "latest" else metadata.latest_available_version @abstractmethod @contextmanager def execute(self, args: List[str]) -> IO[str]: pass + @abstractmethod + def ensure_installation(self): + pass + class VenvExecutor(Executor): - def __init__(self, metadata: ConnectorMetadata): - super().__init__(metadata) + def _get_venv_name(self): + return f".venv-{self.metadata.name}" + + def _get_connector_path(self): + return Path(self._get_venv_name(), "bin", self.metadata.name) + + def _run_subprocess_and_raise_on_failure(self, args: List[str]): + result = subprocess.run(args) + if result.returncode != 0: + raise Exception(f"Install process exited with code {result.returncode}") + + def _install(self): + venv_name = self._get_venv_name() + self._run_subprocess_and_raise_on_failure([sys.executable, "-m", "venv", venv_name]) - @contextmanager - def execute(self, args: List[str]) -> IO[str]: + pip_path = os.path.join(venv_name, "bin", "pip") + + # TODO this is a temporary install path that will be replaced with a proper package name once they are published. At this point we are also using the version + package_to_install = f"../airbyte-integrations/connectors/{self.metadata.name}" + self._run_subprocess_and_raise_on_failure([pip_path, "install", "-e", package_to_install]) + + def _get_installed_version(self): + """ + In the venv, run the following: python -c "from importlib.metadata import version; print(version(''))" + """ + venv_name = self._get_venv_name() + connector_name = self.metadata.name + return subprocess.check_output( + [os.path.join(venv_name, "bin", "python"), "-c", f"from importlib.metadata import version; print(version('{connector_name}'))"], + universal_newlines=True + ).strip() + + def ensure_installation(self): venv_name = f".venv-{self.metadata.name}" venv_path = Path(venv_name) if not venv_path.exists(): - raise Exception(f"Could not find venv {venv_name}") + self._install() - connector_path = Path(venv_path, "bin", self.metadata.name) + connector_path = self._get_connector_path() if not connector_path.exists(): raise Exception( f"Could not find connector {self.metadata.name} in venv {venv_name}" ) + installed_version = self._get_installed_version() + if installed_version != self.target_version: + # If the version doesn't match, reinstall + self._install() + + # Check the version again + version_after_install = self._get_installed_version() + if version_after_install != self.target_version: + raise Exception( + f"Failed to install connector {self.metadata.name} version {self.target_version}. Installed version is {version_after_install}" + ) + + @contextmanager + def execute(self, args: List[str]) -> IO[str]: + connector_path = self._get_connector_path() + process = subprocess.Popen( [str(connector_path)] + args, stdout=subprocess.PIPE, diff --git a/airbyte-lib/airbyte_lib/factories.py b/airbyte-lib/airbyte_lib/factories.py index 7ebbd76de7e36f..6ac536c47ff44c 100644 --- a/airbyte-lib/airbyte_lib/factories.py +++ b/airbyte-lib/airbyte_lib/factories.py @@ -16,7 +16,7 @@ def get_connector( name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None ): metadata = get_connector_metadata(name) - return Source(VenvExecutor(metadata), name, version, config) + return Source(VenvExecutor(metadata, version), name, config) def sync(connector: Source, store: TCache) -> SyncResult[TCache]: return create_connection(connector, store).sync() diff --git a/airbyte-lib/airbyte_lib/registry.py b/airbyte-lib/airbyte_lib/registry.py index 65a6f6866b6173..b25a5ef69cf942 100644 --- a/airbyte-lib/airbyte_lib/registry.py +++ b/airbyte-lib/airbyte_lib/registry.py @@ -1,4 +1,6 @@ +import json import requests +import os from dataclasses import dataclass _cache = None @@ -8,14 +10,18 @@ @dataclass class ConnectorMetadata: name: str - version: str + latest_available_version: str def _update_cache(): global _cache - response = requests.get(REGISTRY_URL) - response.raise_for_status() - data = response.json() + if os.environ.get("AIRBYTE_LOCAL_REGISTRY"): + with open(os.environ.get("AIRBYTE_LOCAL_REGISTRY"), "r") as f: + data = json.load(f) + else: + response = requests.get(REGISTRY_URL) + response.raise_for_status() + data = response.json() _cache = {} for connector in data["sources"]: name = connector["dockerRepository"].replace("airbyte/", "") diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index 56a29039a13fb8..256c3a2c4bd436 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -49,13 +49,11 @@ def __init__( self, executor: Executor, name: str, - version: str = "latest", config: Optional[Dict[str, Any]] = None, streams: Optional[List[str]] = None, ): self.executor = executor self.name = name - self.version = version self.streams = None self.config = None if not config is None: @@ -172,6 +170,9 @@ def check(self): return raise Exception("Connector did not return check status") + def install(self): + self.executor.ensure_installation() + def read(self) -> Iterable[AirbyteRecordMessage]: """ Call read on the connector. @@ -228,6 +229,8 @@ def _execute(self, args: List[str]) -> Iterable[AirbyteMessage]: * Read the output line by line of the subprocess and serialize them AirbyteMessage objects. Drop if not valid. """ + self.executor.ensure_installation() + last_log_messages = [] try: with self.executor.execute(args) as output: @@ -239,7 +242,7 @@ def _execute(self, args: List[str]) -> Iterable[AirbyteMessage]: last_log_messages.append(message.log.message) if len(last_log_messages) > 10: last_log_messages.pop(0) - except Exception: + except Exception as e: # line is probably a log message, add it to the last log messages last_log_messages.append(line) if len(last_log_messages) > 10: diff --git a/airbyte-lib/examples/run_test_source.py b/airbyte-lib/examples/run_test_source.py new file mode 100644 index 00000000000000..a5238395d2c21e --- /dev/null +++ b/airbyte-lib/examples/run_test_source.py @@ -0,0 +1,22 @@ +import os +import airbyte_lib as ab + +# preparation (from airbyte-lib main folder): +# python -m venv .venv-source-test +# source .venv-source-test/bin/activate +# pip install -e ./tests/fixtures/source-test +# In separate terminal: +# poetry run python examples/run_test_source.py + +os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./tests/fixtures/registry.json" + +source = ab.get_connector("source-test", config={"apiKey": "test"}) +cache = ab.get_in_memory_cache() + +source.check() + +print(source.get_available_streams()) + +result = ab.sync(source, cache) + +print(result.cache.streams) \ No newline at end of file diff --git a/airbyte-lib/poetry.lock b/airbyte-lib/poetry.lock index 3eacea93a77863..29329679291912 100644 --- a/airbyte-lib/poetry.lock +++ b/airbyte-lib/poetry.lock @@ -142,6 +142,31 @@ files = [ {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "idna" version = "3.6" @@ -153,6 +178,17 @@ files = [ {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "jsonschema" version = "3.2.0" @@ -174,6 +210,32 @@ six = ">=1.11.0" format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pluggy" +version = "1.3.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + [[package]] name = "pydantic" version = "1.9.2" @@ -266,6 +328,28 @@ files = [ {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, ] +[[package]] +name = "pytest" +version = "7.4.3" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, + {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + [[package]] name = "requests" version = "2.31.0" @@ -314,6 +398,17 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + [[package]] name = "typing-extensions" version = "4.9.0" @@ -344,4 +439,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "c9c9ef5965cefe590d38e25b1cd11bcac2feed3f01c51115691174742bca52e4" +content-hash = "cc3bec51c4678167ddfacd5a75f8c2216ae8c50d6d8ee85218d73f3045e84e0e" diff --git a/airbyte-lib/pyproject.toml b/airbyte-lib/pyproject.toml index 7967b1815fdb46..df48d682c3cbb1 100644 --- a/airbyte-lib/pyproject.toml +++ b/airbyte-lib/pyproject.toml @@ -11,6 +11,8 @@ jsonschema = "3.2.0" requests = "^2.31.0" airbyte-protocol-models = "^1.0.1" +[tool.poetry.group.dev.dependencies] +pytest = "^7.4.3" [build-system] requires = ["poetry-core"] diff --git a/airbyte-lib/tests/fixtures/registry.json b/airbyte-lib/tests/fixtures/registry.json new file mode 100644 index 00000000000000..aa8bcbe89ea0b8 --- /dev/null +++ b/airbyte-lib/tests/fixtures/registry.json @@ -0,0 +1,42 @@ +{ + "sources": [ + { + "sourceDefinitionId": "9f32dab3-77cb-45a1-9d33-347aa5fbe363", + "name": "Test Source", + "dockerRepository": "airbyte/source-test", + "dockerImageTag": "0.0.1", + "documentationUrl": "https://docs.airbyte.com/integrations/sources/test", + "icon": "test.svg", + "iconUrl": "https://connectors.airbyte.com/files/metadata/airbyte/source-test/latest/icon.svg", + "sourceType": "api", + "spec": { + "documentationUrl": "https://docs.airbyte.com/integrations/sources/test", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "apiKey": { + "type": "string", + "title": "API Key", + "description": "The API key for the service" + } + } + } + }, + "tombstone": false, + "public": true, + "custom": false, + "releaseStage": "alpha", + "supportLevel": "community", + "ab_internal": { + "sl": 100, + "ql": 200 + }, + "tags": [ + "language:python" + ], + "githubIssueLabel": "source-test", + "license": "MIT" + } + ] +} \ No newline at end of file diff --git a/airbyte-lib/tests/fixtures/source-test/setup.py b/airbyte-lib/tests/fixtures/source-test/setup.py new file mode 100644 index 00000000000000..0744427547bf4d --- /dev/null +++ b/airbyte-lib/tests/fixtures/source-test/setup.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +setup( + name="source_test", + version="0.0.1", + description="Test Soutce", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + entry_points={ + 'console_scripts': [ + 'source-test=source_test.run:run', + ], + }, +) diff --git a/airbyte-lib/tests/fixtures/source-test/source_test/run.py b/airbyte-lib/tests/fixtures/source-test/source_test/run.py new file mode 100644 index 00000000000000..b5c4f8a4893608 --- /dev/null +++ b/airbyte-lib/tests/fixtures/source-test/source_test/run.py @@ -0,0 +1,129 @@ +import sys +import json + + +sample_catalog = { + "type": "CATALOG", + "catalog": { + "streams": [ + { + "name": "stream1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["column1"], + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": {"type": "string"}, + "column2": {"type": "number"}, + }, + }, + }, + { + "name": "stream2", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": {"type": "string"}, + "column2": {"type": "number"}, + }, + }, + }, + ] + }, +} + +sample_connection_specification = { + "type": "SPEC", + "spec": { + "documentationUrl": "https://example.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "apiKey": { + "type": "string", + "title": "API Key", + "description": "The API key for the service", + } + }, + }, + }, +} + +sample_connection_check_success = { + "type": "CONNECTION_STATUS", + "connectionStatus": {"status": "SUCCEEDED"}, +} + +sample_connection_check_failure = { + "type": "CONNECTION_STATUS", + "connectionStatus": {"status": "FAILED", "message": "An error"}, +} + +sample_record1_stream1 = { + "type": "RECORD", + "record": { + "data": {"column1": "value1", "column2": 1}, + "stream": "stream1", + "emitted_at": 123456789, + }, +} +sample_record2_stream1 = { + "type": "RECORD", + "record": { + "data": {"column1": "value2", "column2": 2}, + "stream": "stream1", + "emitted_at": 123456789, + }, +} +sample_record_stream2 = { + "type": "RECORD", + "record": { + "data": {"column1": "value1", "column2": 1}, + "stream": "stream2", + "emitted_at": 123456789, + }, +} + + +def parse_args(): + arg_dict = {} + args = sys.argv[2:] + for i in range(0, len(args), 2): + arg_dict[args[i]] = args[i + 1] + + return arg_dict + + +def get_json_file(path): + with open(path, "r") as f: + return json.load(f) + + +def run(): + args = sys.argv[1:] + if args[0] == "spec": + print(json.dumps(sample_connection_specification)) + elif args[0] == "discover": + print(json.dumps(sample_catalog)) + elif args[0] == "check": + args = parse_args() + config = get_json_file(args["--config"]) + if config.get("apiKey") == "test": + print(json.dumps(sample_connection_check_success)) + else: + print(json.dumps(sample_connection_check_failure)) + elif args[0] == "read": + args = parse_args() + catalog = get_json_file(args["--catalog"]) + for stream in catalog["streams"]: + if stream["stream"]["name"] == "stream1": + print(json.dumps(sample_record1_stream1)) + print(json.dumps(sample_record2_stream1)) + elif stream["stream"]["name"] == "stream2": + print(json.dumps(sample_record_stream2)) diff --git a/airbyte-lib/tests/test_integration.py b/airbyte-lib/tests/test_integration.py new file mode 100644 index 00000000000000..51e17b3ae9af49 --- /dev/null +++ b/airbyte-lib/tests/test_integration.py @@ -0,0 +1,90 @@ +import pytest +import os +import shutil +import airbyte_lib as ab + + +@pytest.fixture(scope="module", autouse=True) +def prepare_test_env(): + """ + Prepare test environment. This will pre-install the test source from the fixtures array and set the environment variable to use the local json file as registry. + """ + if os.path.exists(".venv-source-test"): + shutil.rmtree(".venv-source-test") + + os.system("python -m venv .venv-source-test") + os.system( + "source .venv-source-test/bin/activate && pip install -e ./tests/fixtures/source-test" + ) + os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./tests/fixtures/registry.json" + + yield + + shutil.rmtree(".venv-source-test") + + +def test_list_streams(): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + + assert source.get_available_streams() == ["stream1", "stream2"] + +def test_invalid_config(): + with pytest.raises(Exception): + ab.get_connector("source-test", config={"apiKey": 1234}) + +def test_non_existing_connector(): + with pytest.raises(Exception): + ab.get_connector("source-not-existing", config={"apiKey": "abc"}) + +def test_wrong_version(): + with pytest.raises(Exception): + ab.get_connector("source-test", version="1.2.3", config={"apiKey": "abc"}) + +def test_check(): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + + source.check() + + +def test_check_fail(): + source = ab.get_connector("source-test", config={"apiKey": "wrong"}) + + with pytest.raises(Exception): + source.check() + + +def test_sync(): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + cache = ab.get_in_memory_cache() + + result = ab.sync(source, cache) + + assert result.processed_records == 3 + assert result.cache.streams == { + "stream1": [{"column1": "value1", "column2": 1}, {"column1": "value2", "column2": 2}], + "stream2": [{"column1": "value1", "column2": 1}], + } + +def test_sync_limited_streams(): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + cache = ab.get_in_memory_cache() + + source.set_streams(["stream2"]) + + result = ab.sync(source, cache) + + assert result.processed_records == 1 + assert result.cache.streams == { + "stream2": [{"column1": "value1", "column2": 1}], + } + +def test_peek(): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + + assert source.peek("stream1", 1) == [{"column1": "value1", "column2": 1}] + +def test_peek_nonexisting_stream(): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + + with pytest.raises(Exception): + source.peek("non-existing") \ No newline at end of file diff --git a/airbyte-lib/tests/test_source.py b/airbyte-lib/tests/test_source.py deleted file mode 100644 index e3127a16e94d9d..00000000000000 --- a/airbyte-lib/tests/test_source.py +++ /dev/null @@ -1,3 +0,0 @@ - -def test_my_test(): - pass \ No newline at end of file From 585631c66d7a3680e177bed70e1fa71d279d0d12 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 13 Dec 2023 18:52:03 +0100 Subject: [PATCH 06/22] format --- .../connectors/source-spacex-api/setup.py | 5 +- .../source_spacex_api/run.py | 2 + .../connectors/source-stripe/setup.py | 4 +- .../source-stripe/source_stripe/run.py | 1 + airbyte-lib/airbyte_lib/cache.py | 9 ++- airbyte-lib/airbyte_lib/connection.py | 13 +++- airbyte-lib/airbyte_lib/executor.py | 35 ++++----- airbyte-lib/airbyte_lib/factories.py | 22 +++--- airbyte-lib/airbyte_lib/registry.py | 9 ++- airbyte-lib/airbyte_lib/source.py | 51 +++++------- airbyte-lib/examples/run_spacex.py | 2 + airbyte-lib/examples/run_test_source.py | 5 +- airbyte-lib/tests/fixtures/registry.json | 78 +++++++++---------- .../tests/fixtures/source-test/setup.py | 4 +- .../fixtures/source-test/source_test/run.py | 5 +- airbyte-lib/tests/test_integration.py | 18 +++-- 16 files changed, 139 insertions(+), 124 deletions(-) diff --git a/airbyte-integrations/connectors/source-spacex-api/setup.py b/airbyte-integrations/connectors/source-spacex-api/setup.py index 3da038cc7e4966..eb1d484b04f68c 100644 --- a/airbyte-integrations/connectors/source-spacex-api/setup.py +++ b/airbyte-integrations/connectors/source-spacex-api/setup.py @@ -17,6 +17,7 @@ setup( name="source_spacex_api", + version="0.1.1", description="Source implementation for Spacex Api.", author="Airbyte", author_email="contact@airbyte.io", @@ -27,8 +28,8 @@ "tests": TEST_REQUIREMENTS, }, entry_points={ - 'console_scripts': [ - 'source-spacex-api=source_spacex_api.run:run', + "console_scripts": [ + "source-spacex-api=source_spacex_api.run:run", ], }, ) diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py index 4ebe799e3470c0..fe9b66c956c565 100644 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py +++ b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py @@ -6,8 +6,10 @@ import sys from airbyte_cdk.entrypoint import launch + from .source import SourceSpacexApi + def run(): source = SourceSpacexApi() launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-stripe/setup.py b/airbyte-integrations/connectors/source-stripe/setup.py index 04024433371180..2d05b4aec1c85b 100644 --- a/airbyte-integrations/connectors/source-stripe/setup.py +++ b/airbyte-integrations/connectors/source-stripe/setup.py @@ -21,8 +21,8 @@ "tests": TEST_REQUIREMENTS, }, entry_points={ - 'console_scripts': [ - 'source-stripe=source_stripe.run:run', + "console_scripts": [ + "source-stripe=source_stripe.run:run", ], }, ) diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/run.py b/airbyte-integrations/connectors/source-stripe/source_stripe/run.py index d5d3f613ee9404..60537a985f1aa2 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/run.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/run.py @@ -10,6 +10,7 @@ from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type + from .source import SourceStripe diff --git a/airbyte-lib/airbyte_lib/cache.py b/airbyte-lib/airbyte_lib/cache.py index 524c2d27eddc6c..8801375b0cb7f2 100644 --- a/airbyte-lib/airbyte_lib/cache.py +++ b/airbyte-lib/airbyte_lib/cache.py @@ -1,9 +1,12 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -from typing import Any, Dict, Iterable, List from abc import ABC, abstractmethod +from typing import Any, Dict, Iterable, List + from airbyte_protocol.models import AirbyteRecordMessage + class Cache(ABC): @abstractmethod def write(self, messages: Iterable[AirbyteRecordMessage]): @@ -15,9 +18,9 @@ class InMemoryCache(Cache): def __init__(self): self.streams: Dict[str, List[Dict[str, Any]]] = {} - + def write(self, messages: Iterable[AirbyteRecordMessage]): for message in messages: if message.stream not in self.streams: self.streams[message.stream] = [] - self.streams[message.stream].append(message.data) \ No newline at end of file + self.streams[message.stream].append(message.data) diff --git a/airbyte-lib/airbyte_lib/connection.py b/airbyte-lib/airbyte_lib/connection.py index cb6af8f0bf2949..8556dd8d70b834 100644 --- a/airbyte-lib/airbyte_lib/connection.py +++ b/airbyte-lib/airbyte_lib/connection.py @@ -1,16 +1,21 @@ -from typing import Iterable, TypeVar, Generic +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from dataclasses import dataclass +from typing import Generic, Iterable, TypeVar + from airbyte_lib.cache import Cache from airbyte_lib.source import Source from airbyte_protocol.models import AirbyteRecordMessage, Type -from dataclasses import dataclass -TCache = TypeVar('TCache', bound=Cache) +TCache = TypeVar("TCache", bound=Cache) + @dataclass class SyncResult(Generic[TCache]): processed_records: int cache: TCache + class Connection(Generic[TCache]): """This class is representing a source that can be called""" @@ -21,7 +26,7 @@ def __init__( ): self.source = source self.cache = cache - + def _process(self, messages: Iterable[AirbyteRecordMessage]): self._processed_records = 0 for message in messages: diff --git a/airbyte-lib/airbyte_lib/executor.py b/airbyte-lib/airbyte_lib/executor.py index 1e1b94305bcb07..c6282cc19fde9c 100644 --- a/airbyte-lib/airbyte_lib/executor.py +++ b/airbyte-lib/airbyte_lib/executor.py @@ -1,16 +1,14 @@ -from abc import ABC, abstractmethod -from contextlib import contextmanager +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os -from pathlib import Path -import subprocess -from typing import List, IO -from airbyte_lib.registry import ConnectorMetadata -from pathlib import Path import subprocess -from typing import List, IO +import sys from abc import ABC, abstractmethod from contextlib import contextmanager -import sys +from pathlib import Path +from typing import IO, List + +from airbyte_lib.registry import ConnectorMetadata class Executor(ABC): @@ -31,15 +29,15 @@ def ensure_installation(self): class VenvExecutor(Executor): def _get_venv_name(self): return f".venv-{self.metadata.name}" - + def _get_connector_path(self): return Path(self._get_venv_name(), "bin", self.metadata.name) - + def _run_subprocess_and_raise_on_failure(self, args: List[str]): result = subprocess.run(args) if result.returncode != 0: raise Exception(f"Install process exited with code {result.returncode}") - + def _install(self): venv_name = self._get_venv_name() self._run_subprocess_and_raise_on_failure([sys.executable, "-m", "venv", venv_name]) @@ -49,7 +47,7 @@ def _install(self): # TODO this is a temporary install path that will be replaced with a proper package name once they are published. At this point we are also using the version package_to_install = f"../airbyte-integrations/connectors/{self.metadata.name}" self._run_subprocess_and_raise_on_failure([pip_path, "install", "-e", package_to_install]) - + def _get_installed_version(self): """ In the venv, run the following: python -c "from importlib.metadata import version; print(version(''))" @@ -58,7 +56,7 @@ def _get_installed_version(self): connector_name = self.metadata.name return subprocess.check_output( [os.path.join(venv_name, "bin", "python"), "-c", f"from importlib.metadata import version; print(version('{connector_name}'))"], - universal_newlines=True + universal_newlines=True, ).strip() def ensure_installation(self): @@ -69,15 +67,13 @@ def ensure_installation(self): connector_path = self._get_connector_path() if not connector_path.exists(): - raise Exception( - f"Could not find connector {self.metadata.name} in venv {venv_name}" - ) + raise Exception(f"Could not find connector {self.metadata.name} in venv {venv_name}") installed_version = self._get_installed_version() if installed_version != self.target_version: # If the version doesn't match, reinstall self._install() - + # Check the version again version_after_install = self._get_installed_version() if version_after_install != self.target_version: @@ -102,7 +98,7 @@ def execute(self, args: List[str]) -> IO[str]: # Close the stdout stream if process.stdout: process.stdout.close() - + # Terminate the process if it is still running if process.poll() is None: # Check if the process is still running process.terminate() @@ -119,4 +115,3 @@ def execute(self, args: List[str]) -> IO[str]: # If the exit code is not 0 or -15 (SIGTERM), raise an exception if exit_code != 0 and exit_code != -15: raise Exception(f"Process exited with code {exit_code}") - diff --git a/airbyte-lib/airbyte_lib/factories.py b/airbyte-lib/airbyte_lib/factories.py index 6ac536c47ff44c..e2b6c003461a2b 100644 --- a/airbyte-lib/airbyte_lib/factories.py +++ b/airbyte-lib/airbyte_lib/factories.py @@ -1,27 +1,29 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -from typing import Any, Dict, Optional, TypeVar, Generic -from airbyte_lib.cache import InMemoryCache, Cache +from typing import Any, Dict, Generic, Optional, TypeVar + +from airbyte_lib.cache import Cache, InMemoryCache from airbyte_lib.connection import Connection, SyncResult from airbyte_lib.executor import VenvExecutor from airbyte_lib.registry import get_connector_metadata from airbyte_lib.source import Source -TCache = TypeVar('TCache', bound=Cache) +TCache = TypeVar("TCache", bound=Cache) + def get_in_memory_cache(): return InMemoryCache() -def get_connector( - name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None -): + +def get_connector(name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None): metadata = get_connector_metadata(name) return Source(VenvExecutor(metadata, version), name, config) + def sync(connector: Source, store: TCache) -> SyncResult[TCache]: return create_connection(connector, store).sync() -def create_connection( - source: Source, cache: TCache -) -> Connection[TCache]: - return Connection(source, cache) \ No newline at end of file + +def create_connection(source: Source, cache: TCache) -> Connection[TCache]: + return Connection(source, cache) diff --git a/airbyte-lib/airbyte_lib/registry.py b/airbyte-lib/airbyte_lib/registry.py index b25a5ef69cf942..a8911e81894c56 100644 --- a/airbyte-lib/airbyte_lib/registry.py +++ b/airbyte-lib/airbyte_lib/registry.py @@ -1,12 +1,16 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import json -import requests import os from dataclasses import dataclass +import requests + _cache = None REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" + @dataclass class ConnectorMetadata: name: str @@ -27,10 +31,11 @@ def _update_cache(): name = connector["dockerRepository"].replace("airbyte/", "") _cache[name] = ConnectorMetadata(name, connector["dockerImageTag"]) + def get_connector_metadata(name: str): """ check the cache for the connector. If the cache is empty, populate by calling update_cache """ if not _cache: _update_cache() - return _cache[name] \ No newline at end of file + return _cache[name] diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index 256c3a2c4bd436..fd15e676bba4b1 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -1,24 +1,27 @@ -from airbyte_lib.executor import Executor, VenvExecutor -from airbyte_lib.registry import get_connector_metadata -import jsonschema -from functools import lru_cache +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import json +import tempfile +from contextlib import contextmanager +from functools import lru_cache +from itertools import islice from typing import Any, Dict, Iterable, List, Optional + +import jsonschema +from airbyte_lib.executor import Executor, VenvExecutor +from airbyte_lib.registry import get_connector_metadata from airbyte_protocol.models import ( - AirbyteMessage, AirbyteCatalog, - Type, + AirbyteMessage, AirbyteRecordMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, - SyncMode, + ConnectorSpecification, DestinationSyncMode, Status, - ConnectorSpecification, + SyncMode, + Type, ) -import tempfile -from contextlib import contextmanager -from itertools import islice @contextmanager @@ -27,9 +30,7 @@ def as_temp_files(files: List[Any]): try: for content in files: temp_file = tempfile.NamedTemporaryFile(mode="w+t", delete=True) - temp_file.write( - json.dumps(content) if isinstance(content, dict) else content - ) + temp_file.write(json.dumps(content) if isinstance(content, dict) else content) temp_file.flush() temp_files.append(temp_file) yield [file.name for file in temp_files] @@ -41,7 +42,6 @@ def as_temp_files(files: List[Any]): pass - class Source: """This class is representing a source that can be called""" @@ -65,9 +65,7 @@ def set_streams(self, streams: List[str]): available_streams = self.get_available_streams() for stream in streams: if stream not in available_streams: - raise Exception( - f"Stream {stream} is not available for connector {self.name}, choose from {available_streams}" - ) + raise Exception(f"Stream {stream} is not available for connector {self.name}, choose from {available_streams}") self.streams = streams def set_config(self, config: Dict[str, Any]): @@ -143,9 +141,7 @@ def peek(self, stream: str, max_n: int = 10) -> List[Dict[str, Any]]: ] ) if len(configured_catalog.streams) == 0: - raise Exception( - f"Stream {stream} is not available for connector {self.name}, choose from {self.get_available_streams()}" - ) + raise Exception(f"Stream {stream} is not available for connector {self.name}, choose from {self.get_available_streams()}") messages = islice(self._read(configured_catalog), max_n) return [m.data for m in messages] @@ -163,9 +159,7 @@ def check(self): for msg in self._execute(["check", "--config", config_file]): if msg.type == Type.CONNECTION_STATUS and msg.connectionStatus: if msg.connectionStatus.status == Status.FAILED: - raise Exception( - f"Connector returned failed status: {msg.connectionStatus.message}" - ) + raise Exception(f"Connector returned failed status: {msg.connectionStatus.message}") else: return raise Exception("Connector did not return check status") @@ -198,9 +192,7 @@ def read(self) -> Iterable[AirbyteRecordMessage]: ) yield from self._read(configured_catalog) - def _read( - self, catalog: ConfiguredAirbyteCatalog - ) -> Iterable[AirbyteRecordMessage]: + def _read(self, catalog: ConfiguredAirbyteCatalog) -> Iterable[AirbyteRecordMessage]: """ Call read on the connector. @@ -213,9 +205,7 @@ def _read( config_file, catalog_file, ]: - for msg in self._execute( - ["read", "--config", config_file, "--catalog", catalog_file] - ): + for msg in self._execute(["read", "--config", config_file, "--catalog", catalog_file]): if msg.type == Type.RECORD: yield msg.record @@ -249,4 +239,3 @@ def _execute(self, args: List[str]) -> Iterable[AirbyteMessage]: last_log_messages.pop(0) except Exception as e: raise Exception(f"{str(e)}. Last logs: {last_log_messages}") - diff --git a/airbyte-lib/examples/run_spacex.py b/airbyte-lib/examples/run_spacex.py index eaae9313b20ae2..7269eff11afde5 100644 --- a/airbyte-lib/examples/run_spacex.py +++ b/airbyte-lib/examples/run_spacex.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import airbyte_lib as ab # preparation (from airbyte-lib main folder): diff --git a/airbyte-lib/examples/run_test_source.py b/airbyte-lib/examples/run_test_source.py index a5238395d2c21e..a2c4d967c6eb95 100644 --- a/airbyte-lib/examples/run_test_source.py +++ b/airbyte-lib/examples/run_test_source.py @@ -1,4 +1,7 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os + import airbyte_lib as ab # preparation (from airbyte-lib main folder): @@ -19,4 +22,4 @@ result = ab.sync(source, cache) -print(result.cache.streams) \ No newline at end of file +print(result.cache.streams) diff --git a/airbyte-lib/tests/fixtures/registry.json b/airbyte-lib/tests/fixtures/registry.json index aa8bcbe89ea0b8..f668681a11a98b 100644 --- a/airbyte-lib/tests/fixtures/registry.json +++ b/airbyte-lib/tests/fixtures/registry.json @@ -1,42 +1,40 @@ { - "sources": [ - { - "sourceDefinitionId": "9f32dab3-77cb-45a1-9d33-347aa5fbe363", - "name": "Test Source", - "dockerRepository": "airbyte/source-test", - "dockerImageTag": "0.0.1", - "documentationUrl": "https://docs.airbyte.com/integrations/sources/test", - "icon": "test.svg", - "iconUrl": "https://connectors.airbyte.com/files/metadata/airbyte/source-test/latest/icon.svg", - "sourceType": "api", - "spec": { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/test", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "apiKey": { - "type": "string", - "title": "API Key", - "description": "The API key for the service" - } - } - } - }, - "tombstone": false, - "public": true, - "custom": false, - "releaseStage": "alpha", - "supportLevel": "community", - "ab_internal": { - "sl": 100, - "ql": 200 - }, - "tags": [ - "language:python" - ], - "githubIssueLabel": "source-test", - "license": "MIT" + "sources": [ + { + "sourceDefinitionId": "9f32dab3-77cb-45a1-9d33-347aa5fbe363", + "name": "Test Source", + "dockerRepository": "airbyte/source-test", + "dockerImageTag": "0.0.1", + "documentationUrl": "https://docs.airbyte.com/integrations/sources/test", + "icon": "test.svg", + "iconUrl": "https://connectors.airbyte.com/files/metadata/airbyte/source-test/latest/icon.svg", + "sourceType": "api", + "spec": { + "documentationUrl": "https://docs.airbyte.com/integrations/sources/test", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "apiKey": { + "type": "string", + "title": "API Key", + "description": "The API key for the service" + } + } } - ] -} \ No newline at end of file + }, + "tombstone": false, + "public": true, + "custom": false, + "releaseStage": "alpha", + "supportLevel": "community", + "ab_internal": { + "sl": 100, + "ql": 200 + }, + "tags": ["language:python"], + "githubIssueLabel": "source-test", + "license": "MIT" + } + ] +} diff --git a/airbyte-lib/tests/fixtures/source-test/setup.py b/airbyte-lib/tests/fixtures/source-test/setup.py index 0744427547bf4d..b59aca2ec5c3ac 100644 --- a/airbyte-lib/tests/fixtures/source-test/setup.py +++ b/airbyte-lib/tests/fixtures/source-test/setup.py @@ -13,8 +13,8 @@ author_email="contact@airbyte.io", packages=find_packages(), entry_points={ - 'console_scripts': [ - 'source-test=source_test.run:run', + "console_scripts": [ + "source-test=source_test.run:run", ], }, ) diff --git a/airbyte-lib/tests/fixtures/source-test/source_test/run.py b/airbyte-lib/tests/fixtures/source-test/source_test/run.py index b5c4f8a4893608..518d35e81576aa 100644 --- a/airbyte-lib/tests/fixtures/source-test/source_test/run.py +++ b/airbyte-lib/tests/fixtures/source-test/source_test/run.py @@ -1,6 +1,7 @@ -import sys -import json +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +import json +import sys sample_catalog = { "type": "CATALOG", diff --git a/airbyte-lib/tests/test_integration.py b/airbyte-lib/tests/test_integration.py index 51e17b3ae9af49..1928811310945c 100644 --- a/airbyte-lib/tests/test_integration.py +++ b/airbyte-lib/tests/test_integration.py @@ -1,7 +1,10 @@ -import pytest +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os import shutil + import airbyte_lib as ab +import pytest @pytest.fixture(scope="module", autouse=True) @@ -13,9 +16,7 @@ def prepare_test_env(): shutil.rmtree(".venv-source-test") os.system("python -m venv .venv-source-test") - os.system( - "source .venv-source-test/bin/activate && pip install -e ./tests/fixtures/source-test" - ) + os.system("source .venv-source-test/bin/activate && pip install -e ./tests/fixtures/source-test") os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./tests/fixtures/registry.json" yield @@ -28,18 +29,22 @@ def test_list_streams(): assert source.get_available_streams() == ["stream1", "stream2"] + def test_invalid_config(): with pytest.raises(Exception): ab.get_connector("source-test", config={"apiKey": 1234}) + def test_non_existing_connector(): with pytest.raises(Exception): ab.get_connector("source-not-existing", config={"apiKey": "abc"}) + def test_wrong_version(): with pytest.raises(Exception): ab.get_connector("source-test", version="1.2.3", config={"apiKey": "abc"}) + def test_check(): source = ab.get_connector("source-test", config={"apiKey": "test"}) @@ -65,6 +70,7 @@ def test_sync(): "stream2": [{"column1": "value1", "column2": 1}], } + def test_sync_limited_streams(): source = ab.get_connector("source-test", config={"apiKey": "test"}) cache = ab.get_in_memory_cache() @@ -78,13 +84,15 @@ def test_sync_limited_streams(): "stream2": [{"column1": "value1", "column2": 1}], } + def test_peek(): source = ab.get_connector("source-test", config={"apiKey": "test"}) assert source.peek("stream1", 1) == [{"column1": "value1", "column2": 1}] + def test_peek_nonexisting_stream(): source = ab.get_connector("source-test", config={"apiKey": "test"}) with pytest.raises(Exception): - source.peek("non-existing") \ No newline at end of file + source.peek("non-existing") From 7a96d614973e550004243dffaa30d9a28fa004b4 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 14 Dec 2023 11:47:47 +0100 Subject: [PATCH 07/22] hook into CI --- airbyte-lib/airbyte_lib/connection.py | 2 +- airbyte-lib/airbyte_lib/factories.py | 2 +- airbyte-lib/airbyte_lib/source.py | 19 +++++++---------- airbyte-lib/build.gradle | 3 +++ .../{tests => integration_tests}/__init__.py | 0 .../fixtures/registry.json | 0 .../fixtures/source-test/setup.py | 0 .../fixtures/source-test/source_test/run.py | 0 .../test_integration.py | 4 ++-- .../src/main/groovy/airbyte-python.gradle | 21 +++++++++++++++++-- settings.gradle | 2 ++ 11 files changed, 36 insertions(+), 17 deletions(-) create mode 100644 airbyte-lib/build.gradle rename airbyte-lib/{tests => integration_tests}/__init__.py (100%) rename airbyte-lib/{tests => integration_tests}/fixtures/registry.json (100%) rename airbyte-lib/{tests => integration_tests}/fixtures/source-test/setup.py (100%) rename airbyte-lib/{tests => integration_tests}/fixtures/source-test/source_test/run.py (100%) rename airbyte-lib/{tests => integration_tests}/test_integration.py (94%) diff --git a/airbyte-lib/airbyte_lib/connection.py b/airbyte-lib/airbyte_lib/connection.py index 8556dd8d70b834..3ea0558a536290 100644 --- a/airbyte-lib/airbyte_lib/connection.py +++ b/airbyte-lib/airbyte_lib/connection.py @@ -5,7 +5,7 @@ from airbyte_lib.cache import Cache from airbyte_lib.source import Source -from airbyte_protocol.models import AirbyteRecordMessage, Type +from airbyte_protocol.models import AirbyteRecordMessage TCache = TypeVar("TCache", bound=Cache) diff --git a/airbyte-lib/airbyte_lib/factories.py b/airbyte-lib/airbyte_lib/factories.py index e2b6c003461a2b..c5376db84d05cf 100644 --- a/airbyte-lib/airbyte_lib/factories.py +++ b/airbyte-lib/airbyte_lib/factories.py @@ -1,7 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -from typing import Any, Dict, Generic, Optional, TypeVar +from typing import Any, Dict, Optional, TypeVar from airbyte_lib.cache import Cache, InMemoryCache from airbyte_lib.connection import Connection, SyncResult diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index fd15e676bba4b1..f732573166b422 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -8,8 +8,7 @@ from typing import Any, Dict, Iterable, List, Optional import jsonschema -from airbyte_lib.executor import Executor, VenvExecutor -from airbyte_lib.registry import get_connector_metadata +from airbyte_lib.executor import Executor from airbyte_protocol.models import ( AirbyteCatalog, AirbyteMessage, @@ -38,7 +37,7 @@ def as_temp_files(files: List[Any]): for temp_file in temp_files: try: temp_file.close() - except: + except Exception: pass @@ -56,9 +55,9 @@ def __init__( self.name = name self.streams = None self.config = None - if not config is None: + if config is not None: self.set_config(config) - if not streams is None: + if streams is not None: self.set_streams(streams) def set_streams(self, streams: List[str]): @@ -224,18 +223,16 @@ def _execute(self, args: List[str]) -> Iterable[AirbyteMessage]: last_log_messages = [] try: with self.executor.execute(args) as output: + last_log_messages = [] for line in output: try: message = AirbyteMessage.parse_raw(line) yield message if message.type == Type.LOG: last_log_messages.append(message.log.message) - if len(last_log_messages) > 10: - last_log_messages.pop(0) - except Exception as e: - # line is probably a log message, add it to the last log messages + last_log_messages = last_log_messages[-10:] + except Exception: last_log_messages.append(line) - if len(last_log_messages) > 10: - last_log_messages.pop(0) + last_log_messages = last_log_messages[-10:] except Exception as e: raise Exception(f"{str(e)}. Last logs: {last_log_messages}") diff --git a/airbyte-lib/build.gradle b/airbyte-lib/build.gradle new file mode 100644 index 00000000000000..8aabcbc11d3ae0 --- /dev/null +++ b/airbyte-lib/build.gradle @@ -0,0 +1,3 @@ +plugins { + id 'airbyte-python' +} \ No newline at end of file diff --git a/airbyte-lib/tests/__init__.py b/airbyte-lib/integration_tests/__init__.py similarity index 100% rename from airbyte-lib/tests/__init__.py rename to airbyte-lib/integration_tests/__init__.py diff --git a/airbyte-lib/tests/fixtures/registry.json b/airbyte-lib/integration_tests/fixtures/registry.json similarity index 100% rename from airbyte-lib/tests/fixtures/registry.json rename to airbyte-lib/integration_tests/fixtures/registry.json diff --git a/airbyte-lib/tests/fixtures/source-test/setup.py b/airbyte-lib/integration_tests/fixtures/source-test/setup.py similarity index 100% rename from airbyte-lib/tests/fixtures/source-test/setup.py rename to airbyte-lib/integration_tests/fixtures/source-test/setup.py diff --git a/airbyte-lib/tests/fixtures/source-test/source_test/run.py b/airbyte-lib/integration_tests/fixtures/source-test/source_test/run.py similarity index 100% rename from airbyte-lib/tests/fixtures/source-test/source_test/run.py rename to airbyte-lib/integration_tests/fixtures/source-test/source_test/run.py diff --git a/airbyte-lib/tests/test_integration.py b/airbyte-lib/integration_tests/test_integration.py similarity index 94% rename from airbyte-lib/tests/test_integration.py rename to airbyte-lib/integration_tests/test_integration.py index 1928811310945c..045a4650940259 100644 --- a/airbyte-lib/tests/test_integration.py +++ b/airbyte-lib/integration_tests/test_integration.py @@ -16,8 +16,8 @@ def prepare_test_env(): shutil.rmtree(".venv-source-test") os.system("python -m venv .venv-source-test") - os.system("source .venv-source-test/bin/activate && pip install -e ./tests/fixtures/source-test") - os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./tests/fixtures/registry.json" + os.system("source .venv-source-test/bin/activate && pip install -e ./integration_tests/fixtures/source-test") + os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./integration_tests/fixtures/registry.json" yield diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle index 59f14890c75b20..4cfe4db8b3e001 100644 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -131,6 +131,15 @@ class AirbytePythonPlugin implements Plugin { inputs.file('setup.py') outputs.file('build/installedlocalreqs.txt') } + } else if (project.file('pyproject.toml').exists()) { + // If requirements.txt does not exists, install from setup.py instead, assume a dev or "tests" profile exists. + // In this case, there is no need to depend on the base python modules since everything should be contained in the setup.py. + project.tasks.register('installLocalReqs', PythonTask) { + module = "pip" + command = "install .[dev,tests]" + inputs.file('pyproject.toml') + outputs.file('build/installedlocalreqs.txt') + } } else { return } @@ -145,7 +154,11 @@ class AirbytePythonPlugin implements Plugin { def installReqs = project.tasks.register('installReqs', PythonTask) { module = "pip" command = "install .[main]" - inputs.file('setup.py') + if (project.file('setup.py').exists()) { + inputs.file('setup.py') + } else { + inputs.file('pyproject.toml') + } outputs.file('build/installedreqs.txt') } installReqs.configure { @@ -160,7 +173,11 @@ class AirbytePythonPlugin implements Plugin { def installTestReqs = project.tasks.register('installTestReqs', PythonTask) { module = "pip" command = "install .[tests]" - inputs.file('setup.py') + if (project.file('setup.py').exists()) { + inputs.file('setup.py') + } else { + inputs.file('pyproject.toml') + } outputs.file('build/installedtestreqs.txt') } installTestReqs.configure { diff --git a/settings.gradle b/settings.gradle index 91cb8e6871f129..aa690209830fce 100644 --- a/settings.gradle +++ b/settings.gradle @@ -161,6 +161,8 @@ rootProject.name = 'airbyte' include ':tools:code-generator' +include ':airbyte-lib' + include ':airbyte-cdk:python' include ':airbyte-cdk:java:airbyte-cdk' include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons' From 796ca18058e2de011cfac779a51c04050c43794d Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 14 Dec 2023 12:10:08 +0100 Subject: [PATCH 08/22] format --- airbyte-lib/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-lib/build.gradle b/airbyte-lib/build.gradle index 8aabcbc11d3ae0..6b4f54758c13d1 100644 --- a/airbyte-lib/build.gradle +++ b/airbyte-lib/build.gradle @@ -1,3 +1,3 @@ plugins { id 'airbyte-python' -} \ No newline at end of file +} From 24b7df61c19ae023a62be710173a6239e99a2e8e Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 14 Dec 2023 13:20:07 +0100 Subject: [PATCH 09/22] run integration_tests via airbyte-ci --- .github/workflows/airbyte-ci-tests.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml index ed450ee04b955e..2697fac12de05a 100644 --- a/.github/workflows/airbyte-ci-tests.yml +++ b/.github/workflows/airbyte-ci-tests.yml @@ -61,6 +61,9 @@ jobs: - 'airbyte-ci/connectors/metadata_service/lib/**' - 'airbyte-ci/connectors/metadata_service/orchestrator/**' - '!**/*.md' + airbyte_lib: + - 'airbyte_lib/**' + - '!**/*.md' - name: Run airbyte-ci/connectors/connector_ops tests if: steps.changes.outputs.ops_any_changed == 'true' @@ -132,3 +135,18 @@ jobs: docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + + - name: Run airbyte-lib tests + if: steps.changes.outputs.airbyte_lib_any_changed == 'true' + id: run-airbyte-lib-tests + uses: ./.github/actions/run-dagger-pipeline + with: + context: "pull_request" + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + subcommand: "test --test-directory=integration_tests airbyte-lib" + airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} From 4f349cc7254001e335e0125ababc2f80ac8a8f1f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 14 Dec 2023 14:37:13 +0100 Subject: [PATCH 10/22] fix CI invocation --- .github/workflows/airbyte-ci-tests.yml | 2 +- airbyte-lib/airbyte_lib/registry.py | 2 ++ airbyte-lib/build.gradle | 3 --- .../{ => tests}/integration_tests/__init__.py | 0 .../integration_tests/fixtures/registry.json | 0 .../fixtures/source-test/setup.py | 0 .../fixtures/source-test/source_test/run.py | 0 .../integration_tests/test_integration.py | 4 ++-- .../src/main/groovy/airbyte-python.gradle | 21 ++----------------- settings.gradle | 2 -- 10 files changed, 7 insertions(+), 27 deletions(-) delete mode 100644 airbyte-lib/build.gradle rename airbyte-lib/{ => tests}/integration_tests/__init__.py (100%) rename airbyte-lib/{ => tests}/integration_tests/fixtures/registry.json (100%) rename airbyte-lib/{ => tests}/integration_tests/fixtures/source-test/setup.py (100%) rename airbyte-lib/{ => tests}/integration_tests/fixtures/source-test/source_test/run.py (100%) rename airbyte-lib/{ => tests}/integration_tests/test_integration.py (94%) diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml index 2697fac12de05a..5a8b90291e3711 100644 --- a/.github/workflows/airbyte-ci-tests.yml +++ b/.github/workflows/airbyte-ci-tests.yml @@ -147,6 +147,6 @@ jobs: gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "test --test-directory=integration_tests airbyte-lib" + subcommand: "test airbyte-lib" airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }} tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} diff --git a/airbyte-lib/airbyte_lib/registry.py b/airbyte-lib/airbyte_lib/registry.py index a8911e81894c56..829a59d032f3b4 100644 --- a/airbyte-lib/airbyte_lib/registry.py +++ b/airbyte-lib/airbyte_lib/registry.py @@ -38,4 +38,6 @@ def get_connector_metadata(name: str): """ if not _cache: _update_cache() + if name not in _cache: + raise Exception(f"Connector {name} not found") return _cache[name] diff --git a/airbyte-lib/build.gradle b/airbyte-lib/build.gradle deleted file mode 100644 index 6b4f54758c13d1..00000000000000 --- a/airbyte-lib/build.gradle +++ /dev/null @@ -1,3 +0,0 @@ -plugins { - id 'airbyte-python' -} diff --git a/airbyte-lib/integration_tests/__init__.py b/airbyte-lib/tests/integration_tests/__init__.py similarity index 100% rename from airbyte-lib/integration_tests/__init__.py rename to airbyte-lib/tests/integration_tests/__init__.py diff --git a/airbyte-lib/integration_tests/fixtures/registry.json b/airbyte-lib/tests/integration_tests/fixtures/registry.json similarity index 100% rename from airbyte-lib/integration_tests/fixtures/registry.json rename to airbyte-lib/tests/integration_tests/fixtures/registry.json diff --git a/airbyte-lib/integration_tests/fixtures/source-test/setup.py b/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py similarity index 100% rename from airbyte-lib/integration_tests/fixtures/source-test/setup.py rename to airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py diff --git a/airbyte-lib/integration_tests/fixtures/source-test/source_test/run.py b/airbyte-lib/tests/integration_tests/fixtures/source-test/source_test/run.py similarity index 100% rename from airbyte-lib/integration_tests/fixtures/source-test/source_test/run.py rename to airbyte-lib/tests/integration_tests/fixtures/source-test/source_test/run.py diff --git a/airbyte-lib/integration_tests/test_integration.py b/airbyte-lib/tests/integration_tests/test_integration.py similarity index 94% rename from airbyte-lib/integration_tests/test_integration.py rename to airbyte-lib/tests/integration_tests/test_integration.py index 045a4650940259..6bb9424a6b54b7 100644 --- a/airbyte-lib/integration_tests/test_integration.py +++ b/airbyte-lib/tests/integration_tests/test_integration.py @@ -16,8 +16,8 @@ def prepare_test_env(): shutil.rmtree(".venv-source-test") os.system("python -m venv .venv-source-test") - os.system("source .venv-source-test/bin/activate && pip install -e ./integration_tests/fixtures/source-test") - os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./integration_tests/fixtures/registry.json" + os.system("source .venv-source-test/bin/activate && pip install -e ./tests/integration_tests/fixtures/source-test") + os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./tests/integration_tests/fixtures/registry.json" yield diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle index 4cfe4db8b3e001..59f14890c75b20 100644 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -131,15 +131,6 @@ class AirbytePythonPlugin implements Plugin { inputs.file('setup.py') outputs.file('build/installedlocalreqs.txt') } - } else if (project.file('pyproject.toml').exists()) { - // If requirements.txt does not exists, install from setup.py instead, assume a dev or "tests" profile exists. - // In this case, there is no need to depend on the base python modules since everything should be contained in the setup.py. - project.tasks.register('installLocalReqs', PythonTask) { - module = "pip" - command = "install .[dev,tests]" - inputs.file('pyproject.toml') - outputs.file('build/installedlocalreqs.txt') - } } else { return } @@ -154,11 +145,7 @@ class AirbytePythonPlugin implements Plugin { def installReqs = project.tasks.register('installReqs', PythonTask) { module = "pip" command = "install .[main]" - if (project.file('setup.py').exists()) { - inputs.file('setup.py') - } else { - inputs.file('pyproject.toml') - } + inputs.file('setup.py') outputs.file('build/installedreqs.txt') } installReqs.configure { @@ -173,11 +160,7 @@ class AirbytePythonPlugin implements Plugin { def installTestReqs = project.tasks.register('installTestReqs', PythonTask) { module = "pip" command = "install .[tests]" - if (project.file('setup.py').exists()) { - inputs.file('setup.py') - } else { - inputs.file('pyproject.toml') - } + inputs.file('setup.py') outputs.file('build/installedtestreqs.txt') } installTestReqs.configure { diff --git a/settings.gradle b/settings.gradle index aa690209830fce..91cb8e6871f129 100644 --- a/settings.gradle +++ b/settings.gradle @@ -161,8 +161,6 @@ rootProject.name = 'airbyte' include ':tools:code-generator' -include ':airbyte-lib' - include ':airbyte-cdk:python' include ':airbyte-cdk:java:airbyte-cdk' include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons' From aae69e52b9b629183a26a749ac3029f4cb4dc132 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 14 Dec 2023 18:06:40 +0100 Subject: [PATCH 11/22] clean up --- .../connectors/source-spacex-api/setup.py | 6 --- .../source_spacex_api/run.py | 15 ------ .../connectors/source-stripe/main.py | 1 - .../connectors/source-stripe/setup.py | 5 -- .../source-stripe/source_stripe/run.py | 46 ------------------- 5 files changed, 73 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py delete mode 100644 airbyte-integrations/connectors/source-stripe/source_stripe/run.py diff --git a/airbyte-integrations/connectors/source-spacex-api/setup.py b/airbyte-integrations/connectors/source-spacex-api/setup.py index eb1d484b04f68c..2d202b30ec9a25 100644 --- a/airbyte-integrations/connectors/source-spacex-api/setup.py +++ b/airbyte-integrations/connectors/source-spacex-api/setup.py @@ -17,7 +17,6 @@ setup( name="source_spacex_api", - version="0.1.1", description="Source implementation for Spacex Api.", author="Airbyte", author_email="contact@airbyte.io", @@ -27,9 +26,4 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, - entry_points={ - "console_scripts": [ - "source-spacex-api=source_spacex_api.run:run", - ], - }, ) diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py deleted file mode 100644 index fe9b66c956c565..00000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/run.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import sys - -from airbyte_cdk.entrypoint import launch - -from .source import SourceSpacexApi - - -def run(): - source = SourceSpacexApi() - launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-stripe/main.py b/airbyte-integrations/connectors/source-stripe/main.py index 9abfa6ba03e439..a8ed671a829223 100644 --- a/airbyte-integrations/connectors/source-stripe/main.py +++ b/airbyte-integrations/connectors/source-stripe/main.py @@ -39,7 +39,6 @@ def _get_source(args: List[str]): if __name__ == "__main__": - print(sys.argv) _args = sys.argv[1:] source = _get_source(_args) if source: diff --git a/airbyte-integrations/connectors/source-stripe/setup.py b/airbyte-integrations/connectors/source-stripe/setup.py index 2d05b4aec1c85b..aab9a737d197e7 100644 --- a/airbyte-integrations/connectors/source-stripe/setup.py +++ b/airbyte-integrations/connectors/source-stripe/setup.py @@ -20,9 +20,4 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, - entry_points={ - "console_scripts": [ - "source-stripe=source_stripe.run:run", - ], - }, ) diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/run.py b/airbyte-integrations/connectors/source-stripe/source_stripe/run.py deleted file mode 100644 index 60537a985f1aa2..00000000000000 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/run.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import sys -import traceback -from datetime import datetime -from typing import List - -from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch -from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type - -from .source import SourceStripe - - -def _get_source(args: List[str]): - catalog_path = AirbyteEntrypoint.extract_catalog(args) - config_path = AirbyteEntrypoint.extract_config(args) - try: - return SourceStripe( - SourceStripe.read_catalog(catalog_path) if catalog_path else None, - SourceStripe.read_config(config_path) if config_path else None, - ) - except Exception as error: - print( - AirbyteMessage( - type=Type.TRACE, - trace=AirbyteTraceMessage( - type=TraceType.ERROR, - emitted_at=int(datetime.now().timestamp() * 1000), - error=AirbyteErrorTraceMessage( - message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}", - stack_trace=traceback.format_exc(), - ), - ), - ).json() - ) - return None - - -def run(): - _args = sys.argv[1:] - source = _get_source(_args) - if source: - launch(source, _args) From 9e003a911a861dc657885c79c366e36c92bc16be Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 19 Dec 2023 11:51:23 +0100 Subject: [PATCH 12/22] airbyte-lib: Add path executor (#33600) Co-authored-by: Aaron ("AJ") Steers --- airbyte-lib/airbyte_lib/executor.py | 73 ++++++++++++------- airbyte-lib/airbyte_lib/factories.py | 13 +++- .../integration_tests/test_integration.py | 14 ++++ 3 files changed, 69 insertions(+), 31 deletions(-) diff --git a/airbyte-lib/airbyte_lib/executor.py b/airbyte-lib/airbyte_lib/executor.py index c6282cc19fde9c..457d847b7a2925 100644 --- a/airbyte-lib/airbyte_lib/executor.py +++ b/airbyte-lib/airbyte_lib/executor.py @@ -26,6 +26,39 @@ def ensure_installation(self): pass +def _stream_from_subprocess(args: List[str]) -> IO[str]: + process = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + + try: + yield process.stdout + finally: + # Close the stdout stream + if process.stdout: + process.stdout.close() + + # Terminate the process if it is still running + if process.poll() is None: # Check if the process is still running + process.terminate() + try: + # Wait for a short period to allow process to terminate gracefully + process.wait(timeout=10) + except subprocess.TimeoutExpired: + # If the process does not terminate within the timeout, force kill it + process.kill() + + # Now, the process is either terminated or killed. Check the exit code. + exit_code = process.wait() + + # If the exit code is not 0 or -15 (SIGTERM), raise an exception + if exit_code != 0 and exit_code != -15: + raise Exception(f"Process exited with code {exit_code}") + + class VenvExecutor(Executor): def _get_venv_name(self): return f".venv-{self.metadata.name}" @@ -85,33 +118,17 @@ def ensure_installation(self): def execute(self, args: List[str]) -> IO[str]: connector_path = self._get_connector_path() - process = subprocess.Popen( - [str(connector_path)] + args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - ) + return _stream_from_subprocess([str(connector_path)] + args) + +class PathExecutor(Executor): + def ensure_installation(self): try: - yield process.stdout - finally: - # Close the stdout stream - if process.stdout: - process.stdout.close() - - # Terminate the process if it is still running - if process.poll() is None: # Check if the process is still running - process.terminate() - try: - # Wait for a short period to allow process to terminate gracefully - process.wait(timeout=10) - except subprocess.TimeoutExpired: - # If the process does not terminate within the timeout, force kill it - process.kill() - - # Now, the process is either terminated or killed. Check the exit code. - exit_code = process.wait() - - # If the exit code is not 0 or -15 (SIGTERM), raise an exception - if exit_code != 0 and exit_code != -15: - raise Exception(f"Process exited with code {exit_code}") + self.execute(["spec"]) + except Exception as e: + raise Exception(f"Connector {self.metadata.name} is not available - executing it failed: {e}") + + @contextmanager + def execute(self, args: List[str]) -> IO[str]: + return _stream_from_subprocess([self.metadata.name] + args) + diff --git a/airbyte-lib/airbyte_lib/factories.py b/airbyte-lib/airbyte_lib/factories.py index c5376db84d05cf..05ee0011c6ee2f 100644 --- a/airbyte-lib/airbyte_lib/factories.py +++ b/airbyte-lib/airbyte_lib/factories.py @@ -5,7 +5,7 @@ from airbyte_lib.cache import Cache, InMemoryCache from airbyte_lib.connection import Connection, SyncResult -from airbyte_lib.executor import VenvExecutor +from airbyte_lib.executor import PathExecutor, VenvExecutor from airbyte_lib.registry import get_connector_metadata from airbyte_lib.source import Source @@ -16,9 +16,16 @@ def get_in_memory_cache(): return InMemoryCache() -def get_connector(name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None): +def get_connector(name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None, use_venv: bool = True): + """ + Get a connector by name and version. + :param name: connector name + :param version: connector version - if not provided, the most recent version will be used + :param config: connector config - if not provided, you need to set it later via the set_config method + :param use_venv: whether to use a virtual environment to run the connector. If False, the connector is expected to be available on the path (e.g. installed via pip). If True, the connector will be installed in a virtual environment. + """ metadata = get_connector_metadata(name) - return Source(VenvExecutor(metadata, version), name, config) + return Source(VenvExecutor(metadata, version) if use_venv else PathExecutor(metadata, version), name, config) def sync(connector: Source, store: TCache) -> SyncResult[TCache]: diff --git a/airbyte-lib/tests/integration_tests/test_integration.py b/airbyte-lib/tests/integration_tests/test_integration.py index 6bb9424a6b54b7..107d2b9e625e78 100644 --- a/airbyte-lib/tests/integration_tests/test_integration.py +++ b/airbyte-lib/tests/integration_tests/test_integration.py @@ -96,3 +96,17 @@ def test_peek_nonexisting_stream(): with pytest.raises(Exception): source.peek("non-existing") + +def test_failing_path_connector(): + with pytest.raises(Exception): + ab.get_connector("source-test", config={"apiKey": "test"}, use_venv=False) + +def test_succeeding_path_connector(): + old_path = os.environ["PATH"] + + # set path to include the test venv bin folder + os.environ["PATH"] = f"{os.path.abspath('.venv-source-test/bin')}:{os.environ['PATH']}" + source = ab.get_connector("source-test", config={"apiKey": "test"}, use_venv=False) + source.check() + + os.environ["PATH"] = old_path From 8418dca9fd8c79020120c257fe65c67d9325c445 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 19 Dec 2023 11:55:36 +0100 Subject: [PATCH 13/22] code format --- airbyte-lib/airbyte_lib/executor.py | 1 - airbyte-lib/airbyte_lib/factories.py | 6 +++--- airbyte-lib/tests/integration_tests/test_integration.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/airbyte-lib/airbyte_lib/executor.py b/airbyte-lib/airbyte_lib/executor.py index 457d847b7a2925..dc124c2179c5c7 100644 --- a/airbyte-lib/airbyte_lib/executor.py +++ b/airbyte-lib/airbyte_lib/executor.py @@ -131,4 +131,3 @@ def ensure_installation(self): @contextmanager def execute(self, args: List[str]) -> IO[str]: return _stream_from_subprocess([self.metadata.name] + args) - diff --git a/airbyte-lib/airbyte_lib/factories.py b/airbyte-lib/airbyte_lib/factories.py index 05ee0011c6ee2f..6fbb89e46f1dfc 100644 --- a/airbyte-lib/airbyte_lib/factories.py +++ b/airbyte-lib/airbyte_lib/factories.py @@ -16,16 +16,16 @@ def get_in_memory_cache(): return InMemoryCache() -def get_connector(name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None, use_venv: bool = True): +def get_connector(name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None, auto_install: bool = True): """ Get a connector by name and version. :param name: connector name :param version: connector version - if not provided, the most recent version will be used :param config: connector config - if not provided, you need to set it later via the set_config method - :param use_venv: whether to use a virtual environment to run the connector. If False, the connector is expected to be available on the path (e.g. installed via pip). If True, the connector will be installed in a virtual environment. + :param auto_install: whether to use a virtual environment to run the connector. If False, the connector is expected to be available on the path (e.g. installed via pip). If True, the connector will be installed automatically in a virtual environment. """ metadata = get_connector_metadata(name) - return Source(VenvExecutor(metadata, version) if use_venv else PathExecutor(metadata, version), name, config) + return Source(VenvExecutor(metadata, version) if auto_install else PathExecutor(metadata, version), name, config) def sync(connector: Source, store: TCache) -> SyncResult[TCache]: diff --git a/airbyte-lib/tests/integration_tests/test_integration.py b/airbyte-lib/tests/integration_tests/test_integration.py index 107d2b9e625e78..a43611ca6f6265 100644 --- a/airbyte-lib/tests/integration_tests/test_integration.py +++ b/airbyte-lib/tests/integration_tests/test_integration.py @@ -99,14 +99,14 @@ def test_peek_nonexisting_stream(): def test_failing_path_connector(): with pytest.raises(Exception): - ab.get_connector("source-test", config={"apiKey": "test"}, use_venv=False) + ab.get_connector("source-test", config={"apiKey": "test"}, auto_install=False) def test_succeeding_path_connector(): old_path = os.environ["PATH"] # set path to include the test venv bin folder os.environ["PATH"] = f"{os.path.abspath('.venv-source-test/bin')}:{os.environ['PATH']}" - source = ab.get_connector("source-test", config={"apiKey": "test"}, use_venv=False) + source = ab.get_connector("source-test", config={"apiKey": "test"}, auto_install=False) source.check() os.environ["PATH"] = old_path From babf99996436723e2901143ca25dd99dfee80b74 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 19 Dec 2023 15:40:15 +0100 Subject: [PATCH 14/22] mypy --- airbyte-lib/README.md | 3 +- airbyte-lib/airbyte_lib/cache.py | 4 +- airbyte-lib/airbyte_lib/executor.py | 31 ++-- airbyte-lib/airbyte_lib/registry.py | 16 ++- airbyte-lib/airbyte_lib/source.py | 29 ++-- airbyte-lib/poetry.lock | 211 +++++++++++++++++++++++++++- airbyte-lib/pyproject.toml | 6 + 7 files changed, 263 insertions(+), 37 deletions(-) diff --git a/airbyte-lib/README.md b/airbyte-lib/README.md index f70528eb3d233d..fe8cabeca7f8cb 100644 --- a/airbyte-lib/README.md +++ b/airbyte-lib/README.md @@ -7,4 +7,5 @@ airbyte-lib is a library that allows to run Airbyte syncs embedded into any Pyth * Make sure [Poetry is installed](https://python-poetry.org/docs/#). * Run `poetry install` * For examples, check out the `examples` folder. They can be run via `poetry run python examples/` -* Unit tests can be run via `poetry run pytest` \ No newline at end of file +* Unit tests can be run via `poetry run pytest` +* Mypy type checks can be run via `poetry run mypy .` \ No newline at end of file diff --git a/airbyte-lib/airbyte_lib/cache.py b/airbyte-lib/airbyte_lib/cache.py index 8801375b0cb7f2..461dc0efab2f4b 100644 --- a/airbyte-lib/airbyte_lib/cache.py +++ b/airbyte-lib/airbyte_lib/cache.py @@ -16,10 +16,10 @@ def write(self, messages: Iterable[AirbyteRecordMessage]): class InMemoryCache(Cache): """The in-memory cache is accepting airbyte messages and stores them in a dictionary for streams (one list of dicts per stream).""" - def __init__(self): + def __init__(self) -> None: self.streams: Dict[str, List[Dict[str, Any]]] = {} - def write(self, messages: Iterable[AirbyteRecordMessage]): + def write(self, messages: Iterable[AirbyteRecordMessage]) -> None: for message in messages: if message.stream not in self.streams: self.streams[message.stream] = [] diff --git a/airbyte-lib/airbyte_lib/executor.py b/airbyte-lib/airbyte_lib/executor.py index dc124c2179c5c7..f0b19028af95ef 100644 --- a/airbyte-lib/airbyte_lib/executor.py +++ b/airbyte-lib/airbyte_lib/executor.py @@ -6,7 +6,7 @@ from abc import ABC, abstractmethod from contextlib import contextmanager from pathlib import Path -from typing import IO, List +from typing import IO, Generator, Iterable, List from airbyte_lib.registry import ConnectorMetadata @@ -17,8 +17,7 @@ def __init__(self, metadata: ConnectorMetadata, target_version: str = "latest"): self.target_version = target_version if target_version != "latest" else metadata.latest_available_version @abstractmethod - @contextmanager - def execute(self, args: List[str]) -> IO[str]: + def execute(self, args: List[str]) -> Iterable[str]: pass @abstractmethod @@ -26,7 +25,8 @@ def ensure_installation(self): pass -def _stream_from_subprocess(args: List[str]) -> IO[str]: +@contextmanager +def _stream_from_subprocess(args: List[str]) -> Generator[Iterable[str], None, None]: process = subprocess.Popen( args, stdout=subprocess.PIPE, @@ -34,8 +34,17 @@ def _stream_from_subprocess(args: List[str]) -> IO[str]: universal_newlines=True, ) + def _stream_from_file(file: IO[str]): + while True: + line = file.readline() + if not line: + break + yield line + + if process.stdout is None: + raise Exception("Failed to start subprocess") try: - yield process.stdout + yield _stream_from_file(process.stdout) finally: # Close the stdout stream if process.stdout: @@ -114,11 +123,11 @@ def ensure_installation(self): f"Failed to install connector {self.metadata.name} version {self.target_version}. Installed version is {version_after_install}" ) - @contextmanager - def execute(self, args: List[str]) -> IO[str]: + def execute(self, args: List[str]) -> Iterable[str]: connector_path = self._get_connector_path() - return _stream_from_subprocess([str(connector_path)] + args) + with _stream_from_subprocess([str(connector_path)] + args) as stream: + yield from stream class PathExecutor(Executor): @@ -128,6 +137,6 @@ def ensure_installation(self): except Exception as e: raise Exception(f"Connector {self.metadata.name} is not available - executing it failed: {e}") - @contextmanager - def execute(self, args: List[str]) -> IO[str]: - return _stream_from_subprocess([self.metadata.name] + args) + def execute(self, args: List[str]) -> Iterable[str]: + with _stream_from_subprocess([self.metadata.name] + args) as stream: + yield from stream diff --git a/airbyte-lib/airbyte_lib/registry.py b/airbyte-lib/airbyte_lib/registry.py index 829a59d032f3b4..7ce3efb93682b3 100644 --- a/airbyte-lib/airbyte_lib/registry.py +++ b/airbyte-lib/airbyte_lib/registry.py @@ -3,13 +3,10 @@ import json import os from dataclasses import dataclass +from typing import Dict, Optional import requests -_cache = None - -REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" - @dataclass class ConnectorMetadata: @@ -17,10 +14,15 @@ class ConnectorMetadata: latest_available_version: str -def _update_cache(): +_cache: Optional[Dict[str, ConnectorMetadata]] = None + +REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" + + +def _update_cache() -> None: global _cache if os.environ.get("AIRBYTE_LOCAL_REGISTRY"): - with open(os.environ.get("AIRBYTE_LOCAL_REGISTRY"), "r") as f: + with open(str(os.environ.get("AIRBYTE_LOCAL_REGISTRY")), "r") as f: data = json.load(f) else: response = requests.get(REGISTRY_URL) @@ -38,6 +40,6 @@ def get_connector_metadata(name: str): """ if not _cache: _update_cache() - if name not in _cache: + if not _cache or name not in _cache: raise Exception(f"Connector {name} not found") return _cache[name] diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index f732573166b422..a9a7a560f694f0 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -53,8 +53,8 @@ def __init__( ): self.executor = executor self.name = name - self.streams = None - self.config = None + self.streams: Optional[List[str]] = None + self.config: Optional[Dict[str, Any]] = None if config is not None: self.set_config(config) if streams is not None: @@ -87,7 +87,7 @@ def discover(self) -> AirbyteCatalog: return msg.catalog raise Exception("Connector did not return a catalog") - def _validate_config(self, config: Dict[str, any]) -> None: + def _validate_config(self, config: Dict[str, Any]) -> None: """ Validate the config against the spec. """ @@ -220,19 +220,18 @@ def _execute(self, args: List[str]) -> Iterable[AirbyteMessage]: self.executor.ensure_installation() - last_log_messages = [] + last_log_messages: List[str] = [] try: - with self.executor.execute(args) as output: - last_log_messages = [] - for line in output: - try: - message = AirbyteMessage.parse_raw(line) - yield message - if message.type == Type.LOG: - last_log_messages.append(message.log.message) - last_log_messages = last_log_messages[-10:] - except Exception: - last_log_messages.append(line) + last_log_messages = [] + for line in self.executor.execute(args): + try: + message = AirbyteMessage.parse_raw(line) + yield message + if message.type == Type.LOG: + last_log_messages.append(message.log.message) last_log_messages = last_log_messages[-10:] + except Exception: + last_log_messages.append(line) + last_log_messages = last_log_messages[-10:] except Exception as e: raise Exception(f"{str(e)}. Last logs: {last_log_messages}") diff --git a/airbyte-lib/poetry.lock b/airbyte-lib/poetry.lock index 29329679291912..2323d479f3d5a0 100644 --- a/airbyte-lib/poetry.lock +++ b/airbyte-lib/poetry.lock @@ -210,6 +210,64 @@ six = ">=1.11.0" format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] +[[package]] +name = "mypy" +version = "1.7.1" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:12cce78e329838d70a204293e7b29af9faa3ab14899aec397798a4b41be7f340"}, + {file = "mypy-1.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1484b8fa2c10adf4474f016e09d7a159602f3239075c7bf9f1627f5acf40ad49"}, + {file = "mypy-1.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31902408f4bf54108bbfb2e35369877c01c95adc6192958684473658c322c8a5"}, + {file = "mypy-1.7.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f2c2521a8e4d6d769e3234350ba7b65ff5d527137cdcde13ff4d99114b0c8e7d"}, + {file = "mypy-1.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:fcd2572dd4519e8a6642b733cd3a8cfc1ef94bafd0c1ceed9c94fe736cb65b6a"}, + {file = "mypy-1.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4b901927f16224d0d143b925ce9a4e6b3a758010673eeded9b748f250cf4e8f7"}, + {file = "mypy-1.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2f7f6985d05a4e3ce8255396df363046c28bea790e40617654e91ed580ca7c51"}, + {file = "mypy-1.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:944bdc21ebd620eafefc090cdf83158393ec2b1391578359776c00de00e8907a"}, + {file = "mypy-1.7.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9c7ac372232c928fff0645d85f273a726970c014749b924ce5710d7d89763a28"}, + {file = "mypy-1.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:f6efc9bd72258f89a3816e3a98c09d36f079c223aa345c659622f056b760ab42"}, + {file = "mypy-1.7.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6dbdec441c60699288adf051f51a5d512b0d818526d1dcfff5a41f8cd8b4aaf1"}, + {file = "mypy-1.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4fc3d14ee80cd22367caaaf6e014494415bf440980a3045bf5045b525680ac33"}, + {file = "mypy-1.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c6e4464ed5f01dc44dc9821caf67b60a4e5c3b04278286a85c067010653a0eb"}, + {file = "mypy-1.7.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:d9b338c19fa2412f76e17525c1b4f2c687a55b156320acb588df79f2e6fa9fea"}, + {file = "mypy-1.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:204e0d6de5fd2317394a4eff62065614c4892d5a4d1a7ee55b765d7a3d9e3f82"}, + {file = "mypy-1.7.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:84860e06ba363d9c0eeabd45ac0fde4b903ad7aa4f93cd8b648385a888e23200"}, + {file = "mypy-1.7.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8c5091ebd294f7628eb25ea554852a52058ac81472c921150e3a61cdd68f75a7"}, + {file = "mypy-1.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40716d1f821b89838589e5b3106ebbc23636ffdef5abc31f7cd0266db936067e"}, + {file = "mypy-1.7.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5cf3f0c5ac72139797953bd50bc6c95ac13075e62dbfcc923571180bebb662e9"}, + {file = "mypy-1.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:78e25b2fd6cbb55ddfb8058417df193f0129cad5f4ee75d1502248e588d9e0d7"}, + {file = "mypy-1.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:75c4d2a6effd015786c87774e04331b6da863fc3fc4e8adfc3b40aa55ab516fe"}, + {file = "mypy-1.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2643d145af5292ee956aa0a83c2ce1038a3bdb26e033dadeb2f7066fb0c9abce"}, + {file = "mypy-1.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75aa828610b67462ffe3057d4d8a4112105ed211596b750b53cbfe182f44777a"}, + {file = "mypy-1.7.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ee5d62d28b854eb61889cde4e1dbc10fbaa5560cb39780c3995f6737f7e82120"}, + {file = "mypy-1.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:72cf32ce7dd3562373f78bd751f73c96cfb441de147cc2448a92c1a308bd0ca6"}, + {file = "mypy-1.7.1-py3-none-any.whl", hash = "sha256:f7c5d642db47376a0cc130f0de6d055056e010debdaf0707cd2b0fc7e7ef30ea"}, + {file = "mypy-1.7.1.tar.gz", hash = "sha256:fcb6d9afb1b6208b4c712af0dafdc650f518836065df0d4fb1d800f5d6773db2"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.1.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + [[package]] name = "packaging" version = "23.2" @@ -350,6 +408,21 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "referencing" +version = "0.32.0" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "referencing-0.32.0-py3-none-any.whl", hash = "sha256:bdcd3efb936f82ff86f993093f6da7435c7de69a3b3a5a06678a6050184bee99"}, + {file = "referencing-0.32.0.tar.gz", hash = "sha256:689e64fe121843dcfd57b71933318ef1f91188ffb45367332700a86ac8fd6161"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" + [[package]] name = "requests" version = "2.31.0" @@ -371,6 +444,114 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "rpds-py" +version = "0.15.2" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "rpds_py-0.15.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:337a8653fb11d2fbe7157c961cc78cb3c161d98cf44410ace9a3dc2db4fad882"}, + {file = "rpds_py-0.15.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:813a65f95bfcb7c8f2a70dd6add9b51e9accc3bdb3e03d0ff7a9e6a2d3e174bf"}, + {file = "rpds_py-0.15.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:082e0e55d73690ffb4da4352d1b5bbe1b5c6034eb9dc8c91aa2a3ee15f70d3e2"}, + {file = "rpds_py-0.15.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5595c80dd03d7e6c6afb73f3594bf3379a7d79fa57164b591d012d4b71d6ac4c"}, + {file = "rpds_py-0.15.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb10bb720348fe1647a94eb605accb9ef6a9b1875d8845f9e763d9d71a706387"}, + {file = "rpds_py-0.15.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:53304cc14b1d94487d70086e1cb0cb4c29ec6da994d58ae84a4d7e78c6a6d04d"}, + {file = "rpds_py-0.15.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d64a657de7aae8db2da60dc0c9e4638a0c3893b4d60101fd564a3362b2bfeb34"}, + {file = "rpds_py-0.15.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee40206d1d6e95eaa2b7b919195e3689a5cf6ded730632de7f187f35a1b6052c"}, + {file = "rpds_py-0.15.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1607cda6129f815493a3c184492acb5ae4aa6ed61d3a1b3663aa9824ed26f7ac"}, + {file = "rpds_py-0.15.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f3e6e2e502c4043c52a99316d89dc49f416acda5b0c6886e0dd8ea7bb35859e8"}, + {file = "rpds_py-0.15.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:044f6f46d62444800402851afa3c3ae50141f12013060c1a3a0677e013310d6d"}, + {file = "rpds_py-0.15.2-cp310-none-win32.whl", hash = "sha256:c827a931c6b57f50f1bb5de400dcfb00bad8117e3753e80b96adb72d9d811514"}, + {file = "rpds_py-0.15.2-cp310-none-win_amd64.whl", hash = "sha256:3bbc89ce2a219662ea142f0abcf8d43f04a41d5b1880be17a794c39f0d609cb0"}, + {file = "rpds_py-0.15.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:1fd0f0b1ccd7d537b858a56355a250108df692102e08aa2036e1a094fd78b2dc"}, + {file = "rpds_py-0.15.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b414ef79f1f06fb90b5165db8aef77512c1a5e3ed1b4807da8476b7e2c853283"}, + {file = "rpds_py-0.15.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c31272c674f725dfe0f343d73b0abe8c878c646967ec1c6106122faae1efc15b"}, + {file = "rpds_py-0.15.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a6945c2d61c42bb7e818677f43638675b8c1c43e858b67a96df3eb2426a86c9d"}, + {file = "rpds_py-0.15.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02744236ac1895d7be837878e707a5c35fb8edc5137602f253b63623d7ad5c8c"}, + {file = "rpds_py-0.15.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2181e86d4e1cdf49a7320cb72a36c45efcb7670d0a88f09fd2d3a7967c0540fd"}, + {file = "rpds_py-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a8ff8e809da81363bffca2b965cb6e4bf6056b495fc3f078467d1f8266fe27f"}, + {file = "rpds_py-0.15.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:97532802f14d383f37d603a56e226909f825a83ff298dc1b6697de00d2243999"}, + {file = "rpds_py-0.15.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:13716e53627ad97babf72ac9e01cf9a7d4af2f75dd5ed7b323a7a9520e948282"}, + {file = "rpds_py-0.15.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:2f1f295a5c28cfa74a7d48c95acc1c8a7acd49d7d9072040d4b694fe11cd7166"}, + {file = "rpds_py-0.15.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8ec464f20fe803ae00419bd1610934e3bda963aeba1e6181dfc9033dc7e8940c"}, + {file = "rpds_py-0.15.2-cp311-none-win32.whl", hash = "sha256:b61d5096e75fd71018b25da50b82dd70ec39b5e15bb2134daf7eb7bbbc103644"}, + {file = "rpds_py-0.15.2-cp311-none-win_amd64.whl", hash = "sha256:9d41ebb471a6f064c0d1c873c4f7dded733d16ca5db7d551fb04ff3805d87802"}, + {file = "rpds_py-0.15.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:13ff62d3561a23c17341b4afc78e8fcfd799ab67c0b1ca32091d71383a98ba4b"}, + {file = "rpds_py-0.15.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b70b45a40ad0798b69748b34d508259ef2bdc84fb2aad4048bc7c9cafb68ddb3"}, + {file = "rpds_py-0.15.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4ecbba7efd82bd2a4bb88aab7f984eb5470991c1347bdd1f35fb34ea28dba6e"}, + {file = "rpds_py-0.15.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9d38494a8d21c246c535b41ecdb2d562c4b933cf3d68de03e8bc43a0d41be652"}, + {file = "rpds_py-0.15.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:13152dfe7d7c27c40df8b99ac6aab12b978b546716e99f67e8a67a1d441acbc3"}, + {file = "rpds_py-0.15.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:164fcee32f15d04d61568c9cb0d919e37ff3195919cd604039ff3053ada0461b"}, + {file = "rpds_py-0.15.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a5122b17a4faf5d7a6d91fa67b479736c0cacc7afe791ddebb7163a8550b799"}, + {file = "rpds_py-0.15.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:46b4f3d47d1033db569173be62365fbf7808c2bd3fb742314d251f130d90d44c"}, + {file = "rpds_py-0.15.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c61e42b4ceb9759727045765e87d51c1bb9f89987aca1fcc8a040232138cad1c"}, + {file = "rpds_py-0.15.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d2aa3ca9552f83b0b4fa6ca8c6ce08da6580f37e3e0ab7afac73a1cfdc230c0e"}, + {file = "rpds_py-0.15.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ec19e823b4ccd87bd69e990879acbce9e961fc7aebe150156b8f4418d4b27b7f"}, + {file = "rpds_py-0.15.2-cp312-none-win32.whl", hash = "sha256:afeabb382c1256a7477b739820bce7fe782bb807d82927102cee73e79b41b38b"}, + {file = "rpds_py-0.15.2-cp312-none-win_amd64.whl", hash = "sha256:422b0901878a31ef167435c5ad46560362891816a76cc0d150683f3868a6f0d1"}, + {file = "rpds_py-0.15.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:baf744e5f9d5ee6531deea443be78b36ed1cd36c65a0b95ea4e8d69fa0102268"}, + {file = "rpds_py-0.15.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7e072f5da38d6428ba1fc1115d3cc0dae895df671cb04c70c019985e8c7606be"}, + {file = "rpds_py-0.15.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f138f550b83554f5b344d6be35d3ed59348510edc3cb96f75309db6e9bfe8210"}, + {file = "rpds_py-0.15.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b2a4cd924d0e2f4b1a68034abe4cadc73d69ad5f4cf02db6481c0d4d749f548f"}, + {file = "rpds_py-0.15.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5eb05b654a41e0f81ab27a7c3e88b6590425eb3e934e1d533ecec5dc88a6ffff"}, + {file = "rpds_py-0.15.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2ee066a64f0d2ba45391cac15b3a70dcb549e968a117bd0500634754cfe0e5fc"}, + {file = "rpds_py-0.15.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c51a899792ee2c696072791e56b2020caff58b275abecbc9ae0cb71af0645c95"}, + {file = "rpds_py-0.15.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ac2ac84a4950d627d84b61f082eba61314373cfab4b3c264b62efab02ababe83"}, + {file = "rpds_py-0.15.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:62b292fff4739c6be89e6a0240c02bda5a9066a339d90ab191cf66e9fdbdc193"}, + {file = "rpds_py-0.15.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:98ee201a52a7f65608e5494518932e1473fd43535f12cade0a1b4ab32737fe28"}, + {file = "rpds_py-0.15.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3d40fb3ca22e3d40f494d577441b263026a3bd8c97ae6ce89b2d3c4b39ac9581"}, + {file = "rpds_py-0.15.2-cp38-none-win32.whl", hash = "sha256:30479a9f1fce47df56b07460b520f49fa2115ec2926d3b1303c85c81f8401ed1"}, + {file = "rpds_py-0.15.2-cp38-none-win_amd64.whl", hash = "sha256:2df3d07a16a3bef0917b28cd564778fbb31f3ffa5b5e33584470e2d1b0f248f0"}, + {file = "rpds_py-0.15.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:56b51ba29a18e5f5810224bcf00747ad931c0716e3c09a76b4a1edd3d4aba71f"}, + {file = "rpds_py-0.15.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3c11bc5814554b018f6c5d6ae0969e43766f81e995000b53a5d8c8057055e886"}, + {file = "rpds_py-0.15.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2faa97212b0dc465afeedf49045cdd077f97be1188285e646a9f689cb5dfff9e"}, + {file = "rpds_py-0.15.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:86c01299942b0f4b5b5f28c8701689181ad2eab852e65417172dbdd6c5b3ccc8"}, + {file = "rpds_py-0.15.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd7d3608589072f63078b4063a6c536af832e76b0b3885f1bfe9e892abe6c207"}, + {file = "rpds_py-0.15.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:938518a11780b39998179d07f31a4a468888123f9b00463842cd40f98191f4d3"}, + {file = "rpds_py-0.15.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dccc623725d0b298f557d869a68496a2fd2a9e9c41107f234fa5f7a37d278ac"}, + {file = "rpds_py-0.15.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d46ee458452727a147d7897bb33886981ae1235775e05decae5d5d07f537695a"}, + {file = "rpds_py-0.15.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d9d7ebcd11ea76ba0feaae98485cd8e31467c3d7985210fab46983278214736b"}, + {file = "rpds_py-0.15.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8a5f574b92b3ee7d254e56d56e37ec0e1416acb1ae357c4956d76a1788dc58fb"}, + {file = "rpds_py-0.15.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3db0c998c92b909d7c90b66c965590d4f3cd86157176a6cf14aa1f867b77b889"}, + {file = "rpds_py-0.15.2-cp39-none-win32.whl", hash = "sha256:bbc7421cbd28b4316d1d017db338039a7943f945c6f2bb15e1439b14b5682d28"}, + {file = "rpds_py-0.15.2-cp39-none-win_amd64.whl", hash = "sha256:1c24e30d720c0009b6fb2e1905b025da56103c70a8b31b99138e4ed1c2a6c5b0"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1e6fcd0a0f62f2997107f758bb372397b8d5fd5f39cc6dcb86f7cb98a2172d6c"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d800a8e2ac62db1b9ea5d6d1724f1a93c53907ca061de4d05ed94e8dfa79050c"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e09d017e3f4d9bd7d17a30d3f59e4d6d9ba2d2ced280eec2425e84112cf623f"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b88c3ab98556bc351b36d6208a6089de8c8db14a7f6e1f57f82a334bd2c18f0b"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f333bfe782a2d05a67cfaa0cc9cd68b36b39ee6acfe099f980541ed973a7093"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b629db53fe17e6ce478a969d30bd1d0e8b53238c46e3a9c9db39e8b65a9ef973"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:485fbdd23becb822804ed05622907ee5c8e8a5f43f6f43894a45f463b2217045"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:893e38d0f4319dfa70c0f36381a37cc418985c87b11d9784365b1fff4fa6973b"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:8ffdeb7dbd0160d4e391e1f857477e4762d00aa2199c294eb95dfb9451aa1d9f"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:fc33267d58dfbb2361baed52668c5d8c15d24bc0372cecbb79fed77339b55e0d"}, + {file = "rpds_py-0.15.2-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:2e7e5633577b3bd56bf3af2ef6ae3778bbafb83743989d57f0e7edbf6c0980e4"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:8b9650f92251fdef843e74fc252cdfd6e3c700157ad686eeb0c6d7fdb2d11652"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:07a2e1d78d382f7181789713cdf0c16edbad4fe14fe1d115526cb6f0eef0daa3"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03f9c5875515820633bd7709a25c3e60c1ea9ad1c5d4030ce8a8c203309c36fd"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:580182fa5b269c2981e9ce9764367cb4edc81982ce289208d4607c203f44ffde"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa1e626c524d2c7972c0f3a8a575d654a3a9c008370dc2a97e46abd0eaa749b9"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae9d83a81b09ce3a817e2cbb23aabc07f86a3abc664c613cd283ce7a03541e95"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9235be95662559141934fced8197de6fee8c58870f36756b0584424b6d708393"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a72e00826a2b032dda3eb25aa3e3579c6d6773d22d8446089a57a123481cc46c"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:ab095edf1d840a6a6a4307e1a5b907a299a94e7b90e75436ee770b8c35d22a25"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:3b79c63d29101cbaa53a517683557bb550462394fb91044cc5998dd2acff7340"}, + {file = "rpds_py-0.15.2-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:911e600e798374c0d86235e7ef19109cf865d1336942d398ff313375a25a93ba"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3cd61e759c4075510052d1eca5cddbd297fe1164efec14ef1fce3f09b974dfe4"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:9d2ae79f31da5143e020a8d4fc74e1f0cbcb8011bdf97453c140aa616db51406"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e99d6510c8557510c220b865d966b105464740dcbebf9b79ecd4fbab30a13d9"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c43e1b89099279cc03eb1c725c5de12af6edcd2f78e2f8a022569efa639ada3"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac7187bee72384b9cfedf09a29a3b2b6e8815cc64c095cdc8b5e6aec81e9fd5f"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3423007fc0661827e06f8a185a3792c73dda41f30f3421562f210cf0c9e49569"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2974e6dff38afafd5ccf8f41cb8fc94600b3f4fd9b0a98f6ece6e2219e3158d5"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:93c18a1696a8e0388ed84b024fe1a188a26ba999b61d1d9a371318cb89885a8c"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:c7cd0841a586b7105513a7c8c3d5c276f3adc762a072d81ef7fae80632afad1e"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:709dc11af2f74ba89c68b1592368c6edcbccdb0a06ba77eb28c8fe08bb6997da"}, + {file = "rpds_py-0.15.2-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:fc066395e6332da1e7525d605b4c96055669f8336600bef8ac569d5226a7c76f"}, + {file = "rpds_py-0.15.2.tar.gz", hash = "sha256:373b76eeb79e8c14f6d82cb1d4d5293f9e4059baec6c1b16dca7ad13b6131b39"}, +] + [[package]] name = "setuptools" version = "69.0.2" @@ -409,6 +590,34 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "types-jsonschema" +version = "4.20.0.0" +description = "Typing stubs for jsonschema" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-jsonschema-4.20.0.0.tar.gz", hash = "sha256:0de1032d243f1d3dba8b745ad84efe8c1af71665a9deb1827636ac535dcb79c1"}, + {file = "types_jsonschema-4.20.0.0-py3-none-any.whl", hash = "sha256:e6d5df18aaca4412f0aae246a294761a92040e93d7bc840f002b7329a8b72d26"}, +] + +[package.dependencies] +referencing = "*" + +[[package]] +name = "types-requests" +version = "2.31.0.10" +description = "Typing stubs for requests" +optional = false +python-versions = ">=3.7" +files = [ + {file = "types-requests-2.31.0.10.tar.gz", hash = "sha256:dc5852a76f1eaf60eafa81a2e50aefa3d1f015c34cf0cba130930866b1b22a92"}, + {file = "types_requests-2.31.0.10-py3-none-any.whl", hash = "sha256:b32b9a86beffa876c0c3ac99a4cd3b8b51e973fb8e3bd4e0a6bb32c7efad80fc"}, +] + +[package.dependencies] +urllib3 = ">=2" + [[package]] name = "typing-extensions" version = "4.9.0" @@ -439,4 +648,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "cc3bec51c4678167ddfacd5a75f8c2216ae8c50d6d8ee85218d73f3045e84e0e" +content-hash = "d983f1b01018b56db7c106f87146337055cf33ba3cd558f7f9afcb60bd576140" diff --git a/airbyte-lib/pyproject.toml b/airbyte-lib/pyproject.toml index df48d682c3cbb1..0a06e405fb5b0f 100644 --- a/airbyte-lib/pyproject.toml +++ b/airbyte-lib/pyproject.toml @@ -13,7 +13,13 @@ airbyte-protocol-models = "^1.0.1" [tool.poetry.group.dev.dependencies] pytest = "^7.4.3" +mypy = "^1.7.1" +types-requests = "^2.31.0.10" +types-jsonschema = "^4.20.0.0" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.mypy] +ignore_missing_imports = true \ No newline at end of file From d2e1e46d3886739438e8cf384832a075dcd9c04e Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 19 Dec 2023 15:59:23 +0100 Subject: [PATCH 15/22] always show last log messages --- airbyte-lib/airbyte_lib/source.py | 22 ++++++++++++---------- airbyte-lib/examples/run_test_source.py | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index a9a7a560f694f0..b827d0289ab037 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -55,6 +55,7 @@ def __init__( self.name = name self.streams: Optional[List[str]] = None self.config: Optional[Dict[str, Any]] = None + self._last_log_messages: List[str] = [] if config is not None: self.set_config(config) if streams is not None: @@ -85,7 +86,7 @@ def discover(self) -> AirbyteCatalog: for msg in self._execute(["discover", "--config", config_file]): if msg.type == Type.CATALOG and msg.catalog: return msg.catalog - raise Exception("Connector did not return a catalog") + raise Exception(f"Connector did not return a catalog. Last logs: {self._last_log_messages}") def _validate_config(self, config: Dict[str, Any]) -> None: """ @@ -113,7 +114,7 @@ def _spec(self) -> ConnectorSpecification: for msg in self._execute(["spec"]): if msg.type == Type.SPEC and msg.spec: return msg.spec - raise Exception("Connector did not return a spec") + raise Exception(f"Connector did not return a spec. Last logs: {self._last_log_messages}") def peek(self, stream: str, max_n: int = 10) -> List[Dict[str, Any]]: """ @@ -161,7 +162,7 @@ def check(self): raise Exception(f"Connector returned failed status: {msg.connectionStatus.message}") else: return - raise Exception("Connector did not return check status") + raise Exception(f"Connector did not return check status. Last logs: {self._last_log_messages}") def install(self): self.executor.ensure_installation() @@ -208,6 +209,10 @@ def _read(self, catalog: ConfiguredAirbyteCatalog) -> Iterable[AirbyteRecordMess if msg.type == Type.RECORD: yield msg.record + def _add_to_logs(self, message: str): + self._last_log_messages.append(message) + self._last_log_messages = self._last_log_messages[-10:] + def _execute(self, args: List[str]) -> Iterable[AirbyteMessage]: """ Execute the connector with the given arguments. @@ -220,18 +225,15 @@ def _execute(self, args: List[str]) -> Iterable[AirbyteMessage]: self.executor.ensure_installation() - last_log_messages: List[str] = [] try: - last_log_messages = [] + self._last_log_messages = [] for line in self.executor.execute(args): try: message = AirbyteMessage.parse_raw(line) yield message if message.type == Type.LOG: - last_log_messages.append(message.log.message) - last_log_messages = last_log_messages[-10:] + self._add_to_logs(message.log.message) except Exception: - last_log_messages.append(line) - last_log_messages = last_log_messages[-10:] + self._add_to_logs(line) except Exception as e: - raise Exception(f"{str(e)}. Last logs: {last_log_messages}") + raise Exception(f"{str(e)}. Last logs: {self._last_log_messages}") diff --git a/airbyte-lib/examples/run_test_source.py b/airbyte-lib/examples/run_test_source.py index a2c4d967c6eb95..f6df9df7aa1f45 100644 --- a/airbyte-lib/examples/run_test_source.py +++ b/airbyte-lib/examples/run_test_source.py @@ -7,11 +7,11 @@ # preparation (from airbyte-lib main folder): # python -m venv .venv-source-test # source .venv-source-test/bin/activate -# pip install -e ./tests/fixtures/source-test +# pip install -e ./tests/integration_tests/fixtures/source-test # In separate terminal: # poetry run python examples/run_test_source.py -os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./tests/fixtures/registry.json" +os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./tests/integration_tests/fixtures/registry.json" source = ab.get_connector("source-test", config={"apiKey": "test"}) cache = ab.get_in_memory_cache() From e0904908170a57f3e30cc531423d37c2d1ad23c1 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 20 Dec 2023 14:29:02 +0100 Subject: [PATCH 16/22] add py.typed --- airbyte-lib/airbyte_lib/py.typed | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 airbyte-lib/airbyte_lib/py.typed diff --git a/airbyte-lib/airbyte_lib/py.typed b/airbyte-lib/airbyte_lib/py.typed new file mode 100644 index 00000000000000..e69de29bb2d1d6 From 4ca3a6ba4dae7583edc2b2154e2e61a3cb76da34 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 20 Dec 2023 14:50:52 +0100 Subject: [PATCH 17/22] add mypy checks --- airbyte-lib/README.md | 3 +-- airbyte-lib/poetry.lock | 38 +++++++++++++++++++++++++++++++++++++- airbyte-lib/pyproject.toml | 6 +++++- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/airbyte-lib/README.md b/airbyte-lib/README.md index fe8cabeca7f8cb..addfb2ed49dcc2 100644 --- a/airbyte-lib/README.md +++ b/airbyte-lib/README.md @@ -7,5 +7,4 @@ airbyte-lib is a library that allows to run Airbyte syncs embedded into any Pyth * Make sure [Poetry is installed](https://python-poetry.org/docs/#). * Run `poetry install` * For examples, check out the `examples` folder. They can be run via `poetry run python examples/` -* Unit tests can be run via `poetry run pytest` -* Mypy type checks can be run via `poetry run mypy .` \ No newline at end of file +* Unit tests and type checks can be run via `poetry run pytest` \ No newline at end of file diff --git a/airbyte-lib/poetry.lock b/airbyte-lib/poetry.lock index 2323d479f3d5a0..8261c5142dfedd 100644 --- a/airbyte-lib/poetry.lock +++ b/airbyte-lib/poetry.lock @@ -167,6 +167,22 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "filelock" +version = "3.13.1" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.13.1-py3-none-any.whl", hash = "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"}, + {file = "filelock-3.13.1.tar.gz", hash = "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +typing = ["typing-extensions (>=4.8)"] + [[package]] name = "idna" version = "3.6" @@ -408,6 +424,26 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-mypy" +version = "0.10.3" +description = "Mypy static type checker plugin for Pytest" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-mypy-0.10.3.tar.gz", hash = "sha256:f8458f642323f13a2ca3e2e61509f7767966b527b4d8adccd5032c3e7b4fd3db"}, + {file = "pytest_mypy-0.10.3-py3-none-any.whl", hash = "sha256:7638d0d3906848fc1810cb2f5cc7fceb4cc5c98524aafcac58f28620e3102053"}, +] + +[package.dependencies] +attrs = ">=19.0" +filelock = ">=3.0" +mypy = [ + {version = ">=0.900", markers = "python_version >= \"3.11\""}, + {version = ">=0.780", markers = "python_version >= \"3.9\" and python_version < \"3.11\""}, +] +pytest = {version = ">=6.2", markers = "python_version >= \"3.10\""} + [[package]] name = "referencing" version = "0.32.0" @@ -648,4 +684,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "d983f1b01018b56db7c106f87146337055cf33ba3cd558f7f9afcb60bd576140" +content-hash = "945526bad52aa7cbbcac56c9b34d024b3b03fd29c7e40bd4c22dce96f8e414d1" diff --git a/airbyte-lib/pyproject.toml b/airbyte-lib/pyproject.toml index 0a06e405fb5b0f..5d1b88ecdc7b01 100644 --- a/airbyte-lib/pyproject.toml +++ b/airbyte-lib/pyproject.toml @@ -16,10 +16,14 @@ pytest = "^7.4.3" mypy = "^1.7.1" types-requests = "^2.31.0.10" types-jsonschema = "^4.20.0.0" +pytest-mypy = "^0.10.3" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.mypy] -ignore_missing_imports = true \ No newline at end of file +ignore_missing_imports = true + +[tool.pytest.ini_options] +addopts = "--mypy" \ No newline at end of file From c6ec8039448c7d08c354fd593c571cec92a3bdd3 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 20 Dec 2023 15:13:26 +0100 Subject: [PATCH 18/22] rename peek to read_stream --- airbyte-lib/airbyte_lib/source.py | 9 ++++----- airbyte-lib/examples/run_spacex.py | 3 ++- airbyte-lib/tests/integration_tests/test_integration.py | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index b827d0289ab037..2faacbc319a629 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -4,7 +4,6 @@ import tempfile from contextlib import contextmanager from functools import lru_cache -from itertools import islice from typing import Any, Dict, Iterable, List, Optional import jsonschema @@ -116,9 +115,9 @@ def _spec(self) -> ConnectorSpecification: return msg.spec raise Exception(f"Connector did not return a spec. Last logs: {self._last_log_messages}") - def peek(self, stream: str, max_n: int = 10) -> List[Dict[str, Any]]: + def read_stream(self, stream: str) -> Iterable[Dict[str, Any]]: """ - Peek at a stream. + Read a stream from the connector. This involves the following steps: * Call discover to get the catalog @@ -142,8 +141,8 @@ def peek(self, stream: str, max_n: int = 10) -> List[Dict[str, Any]]: ) if len(configured_catalog.streams) == 0: raise Exception(f"Stream {stream} is not available for connector {self.name}, choose from {self.get_available_streams()}") - messages = islice(self._read(configured_catalog), max_n) - return [m.data for m in messages] + for message in self._read(configured_catalog): + yield message.data def check(self): """ diff --git a/airbyte-lib/examples/run_spacex.py b/airbyte-lib/examples/run_spacex.py index 7269eff11afde5..9b2ee701c876fd 100644 --- a/airbyte-lib/examples/run_spacex.py +++ b/airbyte-lib/examples/run_spacex.py @@ -1,6 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import airbyte_lib as ab +from itertools import islice # preparation (from airbyte-lib main folder): # python -m venv .venv-source-spacex-api @@ -18,7 +19,7 @@ result = ab.sync(source, cache) -print(source.peek("capsules")) +print(islice(source.read_stream("capsules"), 10)) for name, records in result.cache.streams.items(): print(f"Stream {name}: {len(records)} records") diff --git a/airbyte-lib/tests/integration_tests/test_integration.py b/airbyte-lib/tests/integration_tests/test_integration.py index a43611ca6f6265..0d604865596446 100644 --- a/airbyte-lib/tests/integration_tests/test_integration.py +++ b/airbyte-lib/tests/integration_tests/test_integration.py @@ -85,17 +85,17 @@ def test_sync_limited_streams(): } -def test_peek(): +def test_read_stream(): source = ab.get_connector("source-test", config={"apiKey": "test"}) - assert source.peek("stream1", 1) == [{"column1": "value1", "column2": 1}] + assert list(source.read_stream("stream1")) == [{"column1": "value1", "column2": 1}, {"column1": "value2", "column2": 2}] -def test_peek_nonexisting_stream(): +def test_read_stream_nonexisting(): source = ab.get_connector("source-test", config={"apiKey": "test"}) with pytest.raises(Exception): - source.peek("non-existing") + list(source.read_stream("non-existing")) def test_failing_path_connector(): with pytest.raises(Exception): From e69c568b41159788e6f7812bc7868ee16c1a8c4c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 20 Dec 2023 15:43:29 +0100 Subject: [PATCH 19/22] refactor to match new API --- airbyte-lib/airbyte_lib/__init__.py | 9 ++-- airbyte-lib/airbyte_lib/cache.py | 28 +++++++++++++ airbyte-lib/airbyte_lib/connection.py | 42 ------------------- airbyte-lib/airbyte_lib/factories.py | 15 +------ airbyte-lib/airbyte_lib/source.py | 34 +++++++++++---- airbyte-lib/airbyte_lib/sync_result.py | 27 ++++++++++++ airbyte-lib/examples/run_spacex.py | 5 ++- airbyte-lib/examples/run_test_source.py | 2 +- .../integration_tests/test_integration.py | 14 +++---- 9 files changed, 98 insertions(+), 78 deletions(-) delete mode 100644 airbyte-lib/airbyte_lib/connection.py create mode 100644 airbyte-lib/airbyte_lib/sync_result.py diff --git a/airbyte-lib/airbyte_lib/__init__.py b/airbyte-lib/airbyte_lib/__init__.py index 40ba3fbd2c203c..a0c1b81906c14a 100644 --- a/airbyte-lib/airbyte_lib/__init__.py +++ b/airbyte-lib/airbyte_lib/__init__.py @@ -1,9 +1,12 @@ -from .factories import (create_connection, get_connector, get_in_memory_cache, sync) +from .factories import (get_connector, get_in_memory_cache) +from .sync_result import (Dataset, SyncResult) +from .source import (Source) __all__ = [ - "create_connection", "get_connector", "get_in_memory_cache", - "sync", + "Dataset", + "SyncResult", + "Source", ] diff --git a/airbyte-lib/airbyte_lib/cache.py b/airbyte-lib/airbyte_lib/cache.py index 461dc0efab2f4b..9115442205f162 100644 --- a/airbyte-lib/airbyte_lib/cache.py +++ b/airbyte-lib/airbyte_lib/cache.py @@ -12,6 +12,22 @@ class Cache(ABC): def write(self, messages: Iterable[AirbyteRecordMessage]): pass + @abstractmethod + def get_iterable(self, stream: str) -> Iterable[Dict[str, Any]]: + pass + + @abstractmethod + def get_pandas(self, stream: str) -> Any: + pass + + @abstractmethod + def get_sql_table(self, stream: str) -> Any: + pass + + @abstractmethod + def get_sql_engine(self, stream: str) -> Any: + pass + class InMemoryCache(Cache): """The in-memory cache is accepting airbyte messages and stores them in a dictionary for streams (one list of dicts per stream).""" @@ -24,3 +40,15 @@ def write(self, messages: Iterable[AirbyteRecordMessage]) -> None: if message.stream not in self.streams: self.streams[message.stream] = [] self.streams[message.stream].append(message.data) + + def get_iterable(self, stream: str) -> Iterable[Dict[str, Any]]: + return iter(self.streams[stream]) + + def get_pandas(self, stream: str) -> Any: + raise NotImplementedError() + + def get_sql_table(self, stream: str) -> Any: + raise NotImplementedError() + + def get_sql_engine(self, stream: str) -> Any: + raise NotImplementedError() diff --git a/airbyte-lib/airbyte_lib/connection.py b/airbyte-lib/airbyte_lib/connection.py deleted file mode 100644 index 3ea0558a536290..00000000000000 --- a/airbyte-lib/airbyte_lib/connection.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -from dataclasses import dataclass -from typing import Generic, Iterable, TypeVar - -from airbyte_lib.cache import Cache -from airbyte_lib.source import Source -from airbyte_protocol.models import AirbyteRecordMessage - -TCache = TypeVar("TCache", bound=Cache) - - -@dataclass -class SyncResult(Generic[TCache]): - processed_records: int - cache: TCache - - -class Connection(Generic[TCache]): - """This class is representing a source that can be called""" - - def __init__( - self, - source: Source, - cache: TCache, - ): - self.source = source - self.cache = cache - - def _process(self, messages: Iterable[AirbyteRecordMessage]): - self._processed_records = 0 - for message in messages: - self._processed_records += 1 - yield message - - def sync(self) -> SyncResult[TCache]: - self.cache.write(self._process(self.source.read())) - - return SyncResult( - processed_records=self._processed_records, - cache=self.cache, - ) diff --git a/airbyte-lib/airbyte_lib/factories.py b/airbyte-lib/airbyte_lib/factories.py index 6fbb89e46f1dfc..6bd2ecd62426e7 100644 --- a/airbyte-lib/airbyte_lib/factories.py +++ b/airbyte-lib/airbyte_lib/factories.py @@ -1,16 +1,13 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -from typing import Any, Dict, Optional, TypeVar +from typing import Any, Dict, Optional -from airbyte_lib.cache import Cache, InMemoryCache -from airbyte_lib.connection import Connection, SyncResult +from airbyte_lib.cache import InMemoryCache from airbyte_lib.executor import PathExecutor, VenvExecutor from airbyte_lib.registry import get_connector_metadata from airbyte_lib.source import Source -TCache = TypeVar("TCache", bound=Cache) - def get_in_memory_cache(): return InMemoryCache() @@ -26,11 +23,3 @@ def get_connector(name: str, version: str = "latest", config: Optional[Dict[str, """ metadata = get_connector_metadata(name) return Source(VenvExecutor(metadata, version) if auto_install else PathExecutor(metadata, version), name, config) - - -def sync(connector: Source, store: TCache) -> SyncResult[TCache]: - return create_connection(connector, store).sync() - - -def create_connection(source: Source, cache: TCache) -> Connection[TCache]: - return Connection(source, cache) diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index 2faacbc319a629..465c96385aea20 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -7,7 +7,9 @@ from typing import Any, Dict, Iterable, List, Optional import jsonschema +from airbyte_lib.cache import Cache, InMemoryCache from airbyte_lib.executor import Executor +from airbyte_lib.sync_result import SyncResult from airbyte_protocol.models import ( AirbyteCatalog, AirbyteMessage, @@ -71,7 +73,7 @@ def set_config(self, config: Dict[str, Any]): self._validate_config(config) self.config = config - def discover(self) -> AirbyteCatalog: + def _discover(self) -> AirbyteCatalog: """ Call discover on the connector. @@ -98,7 +100,7 @@ def get_available_streams(self) -> List[str]: """ Get the available streams from the spec. """ - return [s.name for s in self.discover().streams] + return [s.name for s in self._discover().streams] @lru_cache(maxsize=1) def _spec(self) -> ConnectorSpecification: @@ -127,7 +129,7 @@ def read_stream(self, stream: str) -> Iterable[Dict[str, Any]]: * Listen to the messages and return the first AirbyteRecordMessages that come along. * Make sure the subprocess is killed when the function returns. """ - catalog = self.discover() + catalog = self._discover() configured_catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( @@ -141,7 +143,7 @@ def read_stream(self, stream: str) -> Iterable[Dict[str, Any]]: ) if len(configured_catalog.streams) == 0: raise Exception(f"Stream {stream} is not available for connector {self.name}, choose from {self.get_available_streams()}") - for message in self._read(configured_catalog): + for message in self._read_catalog(configured_catalog): yield message.data def check(self): @@ -166,7 +168,7 @@ def check(self): def install(self): self.executor.ensure_installation() - def read(self) -> Iterable[AirbyteRecordMessage]: + def _read(self) -> Iterable[AirbyteRecordMessage]: """ Call read on the connector. @@ -177,7 +179,7 @@ def read(self) -> Iterable[AirbyteRecordMessage]: * execute the connector with read --config --catalog * Listen to the messages and return the AirbyteRecordMessages that come along. """ - catalog = self.discover() + catalog = self._discover() configured_catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( @@ -189,9 +191,9 @@ def read(self) -> Iterable[AirbyteRecordMessage]: if self.streams is None or s.name in self.streams ] ) - yield from self._read(configured_catalog) + yield from self._read_catalog(configured_catalog) - def _read(self, catalog: ConfiguredAirbyteCatalog) -> Iterable[AirbyteRecordMessage]: + def _read_catalog(self, catalog: ConfiguredAirbyteCatalog) -> Iterable[AirbyteRecordMessage]: """ Call read on the connector. @@ -236,3 +238,19 @@ def _execute(self, args: List[str]) -> Iterable[AirbyteMessage]: self._add_to_logs(line) except Exception as e: raise Exception(f"{str(e)}. Last logs: {self._last_log_messages}") + + def _process(self, messages: Iterable[AirbyteRecordMessage]): + self._processed_records = 0 + for message in messages: + self._processed_records += 1 + yield message + + def read_all(self, cache: Optional[Cache]) -> SyncResult: + if cache is None: + cache = InMemoryCache() + cache.write(self._process(self._read())) + + return SyncResult( + processed_records=self._processed_records, + cache=cache, + ) diff --git a/airbyte-lib/airbyte_lib/sync_result.py b/airbyte-lib/airbyte_lib/sync_result.py new file mode 100644 index 00000000000000..2bfa72d2020217 --- /dev/null +++ b/airbyte-lib/airbyte_lib/sync_result.py @@ -0,0 +1,27 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_lib.cache import Cache + + +class Dataset: + def __init__(self, cache: Cache, stream: str) -> None: + self._cache = cache + self._stream = stream + + def __iter__(self): + return self._cache.get_iterable(self._stream) + + def to_pandas(self): + return self._cache.get_pandas(self._stream) + + def to_sql_table(self): + return self._cache.get_sql_table(self._stream) + + +class SyncResult: + def __init__(self, processed_records: int, cache: Cache) -> None: + self.processed_records = processed_records + self._cache = cache + + def __getitem__(self, stream: str) -> Dataset: + return Dataset(self._cache, stream) diff --git a/airbyte-lib/examples/run_spacex.py b/airbyte-lib/examples/run_spacex.py index 9b2ee701c876fd..2cfd46ab70f889 100644 --- a/airbyte-lib/examples/run_spacex.py +++ b/airbyte-lib/examples/run_spacex.py @@ -1,8 +1,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -import airbyte_lib as ab from itertools import islice +import airbyte_lib as ab + # preparation (from airbyte-lib main folder): # python -m venv .venv-source-spacex-api # source .venv-source-spacex-api/bin/activate @@ -17,7 +18,7 @@ source.set_streams(["launches", "rockets", "capsules"]) -result = ab.sync(source, cache) +result = source.read_all(cache) print(islice(source.read_stream("capsules"), 10)) diff --git a/airbyte-lib/examples/run_test_source.py b/airbyte-lib/examples/run_test_source.py index f6df9df7aa1f45..bd6ba23d0c3aeb 100644 --- a/airbyte-lib/examples/run_test_source.py +++ b/airbyte-lib/examples/run_test_source.py @@ -20,6 +20,6 @@ print(source.get_available_streams()) -result = ab.sync(source, cache) +result = source.read_all(cache) print(result.cache.streams) diff --git a/airbyte-lib/tests/integration_tests/test_integration.py b/airbyte-lib/tests/integration_tests/test_integration.py index 0d604865596446..6e69e19931730f 100644 --- a/airbyte-lib/tests/integration_tests/test_integration.py +++ b/airbyte-lib/tests/integration_tests/test_integration.py @@ -62,13 +62,11 @@ def test_sync(): source = ab.get_connector("source-test", config={"apiKey": "test"}) cache = ab.get_in_memory_cache() - result = ab.sync(source, cache) + result = source.read_all(cache) assert result.processed_records == 3 - assert result.cache.streams == { - "stream1": [{"column1": "value1", "column2": 1}, {"column1": "value2", "column2": 2}], - "stream2": [{"column1": "value1", "column2": 1}], - } + assert list(result["stream1"]) == [{"column1": "value1", "column2": 1}, {"column1": "value2", "column2": 2}] + assert list(result["stream2"]) == [{"column1": "value1", "column2": 1}] def test_sync_limited_streams(): @@ -77,12 +75,10 @@ def test_sync_limited_streams(): source.set_streams(["stream2"]) - result = ab.sync(source, cache) + result = source.read_all(cache) assert result.processed_records == 1 - assert result.cache.streams == { - "stream2": [{"column1": "value1", "column2": 1}], - } + assert list(result["stream2"]) == [{"column1": "value1", "column2": 1}] def test_read_stream(): From 19f675e89d15c7e4d7e4454261516e19ddc4d03a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 20 Dec 2023 16:06:11 +0100 Subject: [PATCH 20/22] more refactoring --- airbyte-lib/airbyte_lib/cache.py | 4 ++-- airbyte-lib/airbyte_lib/executor.py | 19 ++++++++++++++++--- airbyte-lib/airbyte_lib/factories.py | 15 ++++++++++++--- airbyte-lib/airbyte_lib/source.py | 2 +- airbyte-lib/airbyte_lib/sync_result.py | 5 +++++ .../integration_tests/test_integration.py | 4 ++-- 6 files changed, 38 insertions(+), 11 deletions(-) diff --git a/airbyte-lib/airbyte_lib/cache.py b/airbyte-lib/airbyte_lib/cache.py index 9115442205f162..10d443cc81dea9 100644 --- a/airbyte-lib/airbyte_lib/cache.py +++ b/airbyte-lib/airbyte_lib/cache.py @@ -25,7 +25,7 @@ def get_sql_table(self, stream: str) -> Any: pass @abstractmethod - def get_sql_engine(self, stream: str) -> Any: + def get_sql_engine(self) -> Any: pass @@ -50,5 +50,5 @@ def get_pandas(self, stream: str) -> Any: def get_sql_table(self, stream: str) -> Any: raise NotImplementedError() - def get_sql_engine(self, stream: str) -> Any: + def get_sql_engine(self) -> Any: raise NotImplementedError() diff --git a/airbyte-lib/airbyte_lib/executor.py b/airbyte-lib/airbyte_lib/executor.py index f0b19028af95ef..98efb850733877 100644 --- a/airbyte-lib/airbyte_lib/executor.py +++ b/airbyte-lib/airbyte_lib/executor.py @@ -24,6 +24,10 @@ def execute(self, args: List[str]) -> Iterable[str]: def ensure_installation(self): pass + @abstractmethod + def install(self): + pass + @contextmanager def _stream_from_subprocess(args: List[str]) -> Generator[Iterable[str], None, None]: @@ -69,6 +73,10 @@ def _stream_from_file(file: IO[str]): class VenvExecutor(Executor): + def __init__(self, metadata: ConnectorMetadata, target_version: str = "latest", install_if_missing: bool = False): + super().__init__(metadata, target_version) + self.install_if_missing = install_if_missing + def _get_venv_name(self): return f".venv-{self.metadata.name}" @@ -80,7 +88,7 @@ def _run_subprocess_and_raise_on_failure(self, args: List[str]): if result.returncode != 0: raise Exception(f"Install process exited with code {result.returncode}") - def _install(self): + def install(self): venv_name = self._get_venv_name() self._run_subprocess_and_raise_on_failure([sys.executable, "-m", "venv", venv_name]) @@ -105,7 +113,9 @@ def ensure_installation(self): venv_name = f".venv-{self.metadata.name}" venv_path = Path(venv_name) if not venv_path.exists(): - self._install() + if not self.install_if_missing: + raise Exception(f"Connector {self.metadata.name} is not available - venv {venv_name} does not exist") + self.install() connector_path = self._get_connector_path() if not connector_path.exists(): @@ -114,7 +124,7 @@ def ensure_installation(self): installed_version = self._get_installed_version() if installed_version != self.target_version: # If the version doesn't match, reinstall - self._install() + self.install() # Check the version again version_after_install = self._get_installed_version() @@ -137,6 +147,9 @@ def ensure_installation(self): except Exception as e: raise Exception(f"Connector {self.metadata.name} is not available - executing it failed: {e}") + def install(self): + raise Exception(f"Connector {self.metadata.name} is not available - cannot install it") + def execute(self, args: List[str]) -> Iterable[str]: with _stream_from_subprocess([self.metadata.name] + args) as stream: yield from stream diff --git a/airbyte-lib/airbyte_lib/factories.py b/airbyte-lib/airbyte_lib/factories.py index 6bd2ecd62426e7..9c982c89b7c2d4 100644 --- a/airbyte-lib/airbyte_lib/factories.py +++ b/airbyte-lib/airbyte_lib/factories.py @@ -13,13 +13,22 @@ def get_in_memory_cache(): return InMemoryCache() -def get_connector(name: str, version: str = "latest", config: Optional[Dict[str, Any]] = None, auto_install: bool = True): +def get_connector( + name: str, + version: str = "latest", + config: Optional[Dict[str, Any]] = None, + use_local_install: bool = False, + install_if_missing: bool = False, +): """ Get a connector by name and version. :param name: connector name :param version: connector version - if not provided, the most recent version will be used :param config: connector config - if not provided, you need to set it later via the set_config method - :param auto_install: whether to use a virtual environment to run the connector. If False, the connector is expected to be available on the path (e.g. installed via pip). If True, the connector will be installed automatically in a virtual environment. + :param use_local_install: whether to use a virtual environment to run the connector. If True, the connector is expected to be available on the path (e.g. installed via pip). If False, the connector will be installed automatically in a virtual environment. + :param install_if_missing: whether to install the connector if it is not available locally. This parameter is ignored if use_local_install is True. """ metadata = get_connector_metadata(name) - return Source(VenvExecutor(metadata, version) if auto_install else PathExecutor(metadata, version), name, config) + return Source( + PathExecutor(metadata, version) if use_local_install else VenvExecutor(metadata, version, use_local_install), name, config + ) diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index 465c96385aea20..8bbb4b17efc83a 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -166,7 +166,7 @@ def check(self): raise Exception(f"Connector did not return check status. Last logs: {self._last_log_messages}") def install(self): - self.executor.ensure_installation() + self.executor.install() def _read(self) -> Iterable[AirbyteRecordMessage]: """ diff --git a/airbyte-lib/airbyte_lib/sync_result.py b/airbyte-lib/airbyte_lib/sync_result.py index 2bfa72d2020217..57c814fdf139fe 100644 --- a/airbyte-lib/airbyte_lib/sync_result.py +++ b/airbyte-lib/airbyte_lib/sync_result.py @@ -1,5 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from typing import Any + from airbyte_lib.cache import Cache @@ -25,3 +27,6 @@ def __init__(self, processed_records: int, cache: Cache) -> None: def __getitem__(self, stream: str) -> Dataset: return Dataset(self._cache, stream) + + def get_sql_engine(self) -> Any: + return self._cache.get_sql_engine() diff --git a/airbyte-lib/tests/integration_tests/test_integration.py b/airbyte-lib/tests/integration_tests/test_integration.py index 6e69e19931730f..2032f747452f5a 100644 --- a/airbyte-lib/tests/integration_tests/test_integration.py +++ b/airbyte-lib/tests/integration_tests/test_integration.py @@ -95,14 +95,14 @@ def test_read_stream_nonexisting(): def test_failing_path_connector(): with pytest.raises(Exception): - ab.get_connector("source-test", config={"apiKey": "test"}, auto_install=False) + ab.get_connector("source-test", config={"apiKey": "test"}, use_local_install=True) def test_succeeding_path_connector(): old_path = os.environ["PATH"] # set path to include the test venv bin folder os.environ["PATH"] = f"{os.path.abspath('.venv-source-test/bin')}:{os.environ['PATH']}" - source = ab.get_connector("source-test", config={"apiKey": "test"}, auto_install=False) + source = ab.get_connector("source-test", config={"apiKey": "test"}, use_local_install=True) source.check() os.environ["PATH"] = old_path From 6d4278ed4418b770ed2b576833d9b8d4ecd97bc6 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 20 Dec 2023 16:52:57 +0100 Subject: [PATCH 21/22] add header to registry request and fix test --- airbyte-lib/airbyte_lib/registry.py | 4 +++- airbyte-lib/airbyte_lib/source.py | 2 +- airbyte-lib/examples/run_test_source.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/airbyte-lib/airbyte_lib/registry.py b/airbyte-lib/airbyte_lib/registry.py index 7ce3efb93682b3..05c107da80b923 100644 --- a/airbyte-lib/airbyte_lib/registry.py +++ b/airbyte-lib/airbyte_lib/registry.py @@ -1,5 +1,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +import importlib.metadata import json import os from dataclasses import dataclass @@ -15,6 +16,7 @@ class ConnectorMetadata: _cache: Optional[Dict[str, ConnectorMetadata]] = None +airbyte_lib_version = importlib.metadata.version("airbyte-lib") REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" @@ -25,7 +27,7 @@ def _update_cache() -> None: with open(str(os.environ.get("AIRBYTE_LOCAL_REGISTRY")), "r") as f: data = json.load(f) else: - response = requests.get(REGISTRY_URL) + response = requests.get(REGISTRY_URL, headers={"User-Agent": f"airbyte-lib-{airbyte_lib_version}"}) response.raise_for_status() data = response.json() _cache = {} diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py index 8bbb4b17efc83a..3a496285fed128 100644 --- a/airbyte-lib/airbyte_lib/source.py +++ b/airbyte-lib/airbyte_lib/source.py @@ -245,7 +245,7 @@ def _process(self, messages: Iterable[AirbyteRecordMessage]): self._processed_records += 1 yield message - def read_all(self, cache: Optional[Cache]) -> SyncResult: + def read_all(self, cache: Optional[Cache] = None) -> SyncResult: if cache is None: cache = InMemoryCache() cache.write(self._process(self._read())) diff --git a/airbyte-lib/examples/run_test_source.py b/airbyte-lib/examples/run_test_source.py index bd6ba23d0c3aeb..76baa8e771dc62 100644 --- a/airbyte-lib/examples/run_test_source.py +++ b/airbyte-lib/examples/run_test_source.py @@ -22,4 +22,5 @@ result = source.read_all(cache) -print(result.cache.streams) +print(result.processed_records) +print(list(result["stream1"])) From 966fe432ca307a60464462f77915403fc53e34c4 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 20 Dec 2023 17:05:50 +0100 Subject: [PATCH 22/22] fix wrong variable used --- airbyte-lib/airbyte_lib/factories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-lib/airbyte_lib/factories.py b/airbyte-lib/airbyte_lib/factories.py index 9c982c89b7c2d4..5b2f543b49ed8c 100644 --- a/airbyte-lib/airbyte_lib/factories.py +++ b/airbyte-lib/airbyte_lib/factories.py @@ -30,5 +30,5 @@ def get_connector( """ metadata = get_connector_metadata(name) return Source( - PathExecutor(metadata, version) if use_local_install else VenvExecutor(metadata, version, use_local_install), name, config + PathExecutor(metadata, version) if use_local_install else VenvExecutor(metadata, version, install_if_missing), name, config )