From ebc8b7640d72ccad6d4fa932b68859b959008c0e Mon Sep 17 00:00:00 2001 From: Deividas J Date: Thu, 18 Nov 2021 03:16:45 +0200 Subject: [PATCH] Source Exchange Rates: add ignore_weekends boolean option (#7936) * Source Exchange Rates: add ignore_weekends boolean option * unformat * backward compatibility for ignore_weekends option Co-authored-by: Marcos Marx * Add changelog Co-authored-by: Marcos Marx --- .../connectors/source-exchange-rates/Dockerfile | 2 +- .../source_exchange_rates/source.py | 11 ++++++----- .../source_exchange_rates/spec.json | 5 +++++ docs/integrations/sources/exchangeratesapi.md | 4 ++-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-exchange-rates/Dockerfile b/airbyte-integrations/connectors/source-exchange-rates/Dockerfile index e041bc679510c0..5233bbfe192c85 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/Dockerfile +++ b/airbyte-integrations/connectors/source-exchange-rates/Dockerfile @@ -16,5 +16,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/source-exchange-rates diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py index 6d243c387bb232..e5361699902737 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py +++ b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py @@ -23,11 +23,12 @@ class ExchangeRates(HttpStream): cursor_field = date_field_name primary_key = "" - def __init__(self, base: Optional[str], start_date: DateTime, access_key: str): + def __init__(self, base: Optional[str], start_date: DateTime, access_key: str, ignore_weekends: Optional[bool]): super().__init__() self._base = base self._start_date = start_date self.access_key = access_key + self.ignore_weekends = ignore_weekends def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -52,7 +53,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: stream_state = stream_state or {} start_date = pendulum.parse(stream_state.get(self.date_field_name, self._start_date)) - return chunk_date_range(start_date) + return chunk_date_range(start_date, self.ignore_weekends) def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): current_stream_state = current_stream_state or {} @@ -62,7 +63,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return current_stream_state -def chunk_date_range(start_date: DateTime) -> Iterable[Mapping[str, Any]]: +def chunk_date_range(start_date: DateTime, ignore_weekends: bool) -> Iterable[Mapping[str, Any]]: """ Returns a list of each day between the start date and now. Ignore weekends since exchanges don't run on weekends. The return value is a list of dicts {'date': date_string}. @@ -71,7 +72,7 @@ def chunk_date_range(start_date: DateTime) -> Iterable[Mapping[str, Any]]: now = pendulum.now() while start_date < now: day_of_week = start_date.day_of_week - if day_of_week != pendulum.SATURDAY and day_of_week != pendulum.SUNDAY: + if day_of_week != pendulum.SATURDAY and day_of_week != pendulum.SUNDAY or not ignore_weekends: days.append({"date": start_date.to_date_string()}) start_date = start_date.add(days=1) @@ -106,4 +107,4 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return False, e def streams(self, config: Mapping[str, Any]) -> List[Stream]: - return [ExchangeRates(config.get("base"), config["start_date"], config["access_key"])] + return [ExchangeRates(config.get("base"), config["start_date"], config["access_key"], config.get("ignore_weekends", True))] diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.json b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.json index 06c9554a37b5e0..328d281ce8b5bd 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.json +++ b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.json @@ -22,6 +22,11 @@ "type": "string", "description": "ISO reference currency. See here. Free plan doesn't support Source Currency Switching, default base currency is EUR", "examples": ["EUR", "USD"] + }, + "ignore_weekends": { + "type": "boolean", + "description": "Ignore weekends? (Exchanges don't run on weekends)", + "default": true } } } diff --git a/docs/integrations/sources/exchangeratesapi.md b/docs/integrations/sources/exchangeratesapi.md index e3cf7c0d90b5a8..4b13d69a3f4f26 100644 --- a/docs/integrations/sources/exchangeratesapi.md +++ b/docs/integrations/sources/exchangeratesapi.md @@ -46,8 +46,8 @@ If you have `free` subscription plan \(you may check it [here](https://manage.ex | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.5 | 2021-11-12 | [7936](https://github.com/airbytehq/airbyte/pull/7936) | Add ignore_weekends boolean option | | 0.2.3 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.2.2 | 2021-05-28 | [3677](https://github.com/airbytehq/airbyte/pull/3677) | Adding clearer error message when a currency isn't supported. access_key field in spec.json was marked as sensitive +| 0.2.2 | 2021-05-28 | [3677](https://github.com/airbytehq/airbyte/pull/3677) | Adding clearer error message when a currency isn't supported. access_key field in spec.json was marked as sensitive | | 0.2.0 | 2021-05-26 | [3566](https://github.com/airbytehq/airbyte/pull/3566) | Move from `api.ratesapi.io/` to `api.exchangeratesapi.io/`. Add required field `access_key` to `config.json`. | | 0.1.0 | 2021-04-19 | [2942](https://github.com/airbytehq/airbyte/pull/2942) | Implement Exchange API using the CDK | -