LITE-27386 support multiply columns in currency converter transformation

connect_transformations/currency_conversion/mixins.py

@@ -4,54 +4,41 @@
 # All rights reserved.
 #
 from decimal import Decimal
+from functools import cached_property
 from typing import List
 
 import httpx
-import requests
 from connect.eaas.core.decorators import router, transformation
 from connect.eaas.core.responses import RowTransformationResponse
 
-from connect_transformations.currency_conversion.exceptions import CurrencyConversionError
 from connect_transformations.currency_conversion.models import Configuration, Currency
-from connect_transformations.currency_conversion.utils import validate_currency_conversion
+from connect_transformations.currency_conversion.utils import (
+    load_currency_rate,
+    validate_currency_conversion,
+)
 from connect_transformations.models import Error, ValidationResult
 from connect_transformations.utils import is_input_column_nullable
 
 
 class CurrencyConverterTransformationMixin:
 
-    def preload_currency_conversion_rates(self, currency_from, currency_to):
-        with self.lock():
-            if hasattr(self, 'currency_conversion_rates'):
-                return
+    def get_currency_rate(self, currency_from, currency_to):
+        if not hasattr(self, '_currency_rates'):
+            self._currency_rates = {}
 
-            self.currency_conversion_rates = {}
+        key = (currency_from, currency_to)
 
-            try:
-                url = 'https://api.exchangerate.host/latest'
-                params = {
-                    'symbols': currency_to,
-                    'base': currency_from,
-                }
-
-                response = requests.get(
-                    url,
-                    params=params,
-                )
-                response.raise_for_status()
-                data = response.json()
-
-                if not data['success']:
-                    raise CurrencyConversionError(
-                        f'Unexpected response calling {url}'
-                        f' with params {params}',
-                    )
-                self.currency_conversion_rates[currency_to] = Decimal(data['rates'][currency_to])
-            except requests.RequestException as exc:
-                raise CurrencyConversionError(
-                    f'An error occurred while requesting {url} with '
-                    f'params {params}: {exc}',
-                )
+        if key not in self._currency_rates:
+            with self.lock():
+                if key not in self._currency_rates:
+                    self._currency_rates[key] = load_currency_rate(currency_from, currency_to)
+
+        return self._currency_rates[key]
+
+    @cached_property
+    def input_columns(self):
+        input_columns = self.transformation_request['transformation']['columns']['input']
+        return {c['id']: c for c in input_columns}
 
     @transformation(
         name='Convert Currency',
@@ -65,32 +52,32 @@ def currency_conversion(
         self,
         row,
     ):
-        trfn_settings = self.transformation_request['transformation']['settings']
-        value = row[trfn_settings['from']['column']]
-        currency = trfn_settings['from']['currency']
-        currency_to = trfn_settings['to']['currency']
+        return_values = {}
+        settings = self.transformation_request['transformation']['settings']
+        if not isinstance(settings, (tuple, list)):
+            settings = [settings]
+
+        for conv_settings in settings:
+            col_name = self.input_columns[conv_settings['from']['column']]['name']
+            value = row[col_name]
+            currency_from = conv_settings['from']['currency']
+            currency_to = conv_settings['to']['currency']
+
+            if (not value) and is_input_column_nullable(
+                self.transformation_request['transformation']['columns']['input'],
+                col_name,
+            ):
+                return RowTransformationResponse.skip()
 
-        try:
-            self.preload_currency_conversion_rates(
-                currency,
-                currency_to,
-            )
-        except Exception as e:
-            return RowTransformationResponse.fail(output=str(e))
-
-        if is_input_column_nullable(
-            self.transformation_request['transformation']['columns']['input'],
-            trfn_settings['from']['column'],
-        ) and not value:
-            return RowTransformationResponse.skip()
-
-        return RowTransformationResponse.done({
-            trfn_settings['to']['column']: (
-                Decimal(value) * self.currency_conversion_rates[currency_to]
-            ).quantize(
-                Decimal('.00001'),
-            ),
-        })
+            try:
+                return_values[conv_settings['to']['column']] = (
+                    Decimal(value) * self.get_currency_rate(currency_from, currency_to)
+                ).quantize(
+                    Decimal('.00001'),
+                )
+            except Exception as e:
+                return RowTransformationResponse.fail(output=str(e))
+        return RowTransformationResponse.done(return_values)
 
 
 class CurrencyConversionWebAppMixin:

connect_transformations/currency_conversion/models.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import List, Optional
 
 from pydantic import BaseModel
 
@@ -26,5 +26,5 @@ class Config:
 
 
 class Configuration(BaseModel):
-    settings: Optional[Settings]
+    settings: Optional[List[Settings]]
     columns: Optional[Columns]

connect_transformations/currency_conversion/utils.py

@@ -3,6 +3,11 @@
 # Copyright (c) 2023, CloudBlue LLC
 # All rights reserved.
 #
+from decimal import Decimal
+
+import requests
+
+from connect_transformations.currency_conversion.exceptions import CurrencyConversionError
 from connect_transformations.utils import (
     build_error_response,
     does_not_contain_required_keys,
@@ -13,43 +18,87 @@
 def validate_currency_conversion(data):
     data = data.dict(by_alias=True)
 
-    if has_invalid_basic_structure(data):
+    if has_invalid_basic_structure(data, data_type=(list, dict)):
         return build_error_response('Invalid input data')
 
+    trf_settings = data['settings']
+    if not isinstance(trf_settings, list):
+        trf_settings = [trf_settings]
+
     input_columns = data['columns']['input']
-    available_input_columns = [c['name'] for c in input_columns]
-    if (
-        does_not_contain_required_keys(
-            data['settings'],
-            ['from', 'to'],
-        ) or does_not_contain_required_keys(
-            data['settings']['from'],
-            ['column', 'currency'],
-        ) or does_not_contain_required_keys(
-            data['settings']['to'],
-            ['column', 'currency'],
-        )
-    ):
-        return build_error_response(
-            'The settings must have `from` with the `column` and the `currency`, and '
-            '`to` with the `column` and `currency` fields',
-        )
+    available_input_columns = {c['id']: c for c in input_columns}
 
-    if data['settings']['from']['currency'] == data['settings']['to']['currency']:
-        return build_error_response(
-            'The settings must have different currencies for `from` and `to`',
-        )
+    overview = ''
 
-    if data['settings']['from']['column'] not in available_input_columns:
-        return build_error_response(
-            'The settings contains an invalid `from` column name'
-            f' "{data["settings"]["from"]["column"]}" that does not exist on '
-            'columns.input',
-        )
+    for row in trf_settings:
+        if (
+            does_not_contain_required_keys(
+                row,
+                ['from', 'to'],
+            ) or does_not_contain_required_keys(
+                row['from'],
+                ['column', 'currency'],
+            ) or does_not_contain_required_keys(
+                row['to'],
+                ['column', 'currency'],
+            )
+        ):
+            return build_error_response(
+                'The settings must have `from` with the `column` and the `currency`, and '
+                '`to` with the `column` and `currency` fields',
+            )
+
+        if row['from']['currency'] == row['to']['currency']:
+            return build_error_response(
+                'The settings must have different currencies for `from` and `to`',
+            )
+
+        if row['from']['column'] not in available_input_columns:
+            return build_error_response(
+                'The settings contains an invalid `from` column name'
+                f' "{row["from"]["column"]}" that does not exist on '
+                'columns.input',
+            )
 
-    overview = 'From Currency = ' + data['settings']['from']['currency'] + '\n'
-    overview += 'To Currency = ' + data['settings']['to']['currency'] + '\n'
+        if overview:
+            overview += '\n'
+
+        overview += 'From: {src} ({src_curr})\nTo: {dst} ({dst_curr})\n'.format(
+            src=available_input_columns[row['from']['column']]['name'],
+            src_curr=row['from']['currency'],
+            dst=row['to']['column'],
+            dst_curr=row['to']['currency'],
+        )
 
     return {
         'overview': overview,
     }
+
+
+def load_currency_rate(currency_from, currency_to):
+    try:
+        url = 'https://api.exchangerate.host/latest'
+        params = {
+            'symbols': currency_to,
+            'base': currency_from,
+        }
+
+        response = requests.get(
+            url,
+            params=params,
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        if not data['success']:
+            raise CurrencyConversionError(
+                f'Unexpected response calling {url}'
+                f' with params {params}',
+            )
+
+        return Decimal(data['rates'][currency_to])
+    except requests.RequestException as exc:
+        raise CurrencyConversionError(
+            f'An error occurred while requesting {url} with '
+            f'params {params}: {exc}',
+        )

connect_transformations/static/transformations/airtable_lookup.html

@@ -1 +1 @@
-<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Airtable Lookup</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/airtable_lookup.378bad986c96d5d867cd.js"></script><link href="../transformations/airtable_lookup.5f5aa442fc131fcaad2c.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app"><div class="main-container"><div class="config-elem api-input"><label for="key-select">API Key</label> <input style="width: 86%;" id="key-input" placeholder="Enter an AirTable key"></div><div class="config-elem"><label for="base-select">AirTable Base</label> <select disabled="disabled" class="list" style="width: 80%;" id="base-select"></select></div><div class="config-elem"><label for="table-select">AirTable table name</label> <select disabled="disabled" class="list" style="width: 80%;" id="table-select"></select></div><div class="config-elem"><label for="table-select">Input column (Key)</label> <select disabled="disabled" class="list" style="width: 80%;" id="input-column-select"></select></div><div class="config-elem"><label for="table-select">AirTable Field Name (Key)</label> <select disabled="disabled" class="list" style="width: 80%;" id="field-select"></select></div></div><b>Columns Mapping</b><div id="content"></div><div class="button-container"><button disabled="disabled" id="add" class="button">ADD COLUMN</button></div></div></body></html>
+<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Airtable Lookup</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/airtable_lookup.378bad986c96d5d867cd.js"></script><link href="../transformations/airtable_lookup.fa2c5fd466f5298cab9e.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app"><div class="main-container"><div class="config-elem api-input"><label for="key-select">API Key</label> <input style="width: 86%;" id="key-input" placeholder="Enter an AirTable key"></div><div class="config-elem"><label for="base-select">AirTable Base</label> <select disabled="disabled" class="list" style="width: 80%;" id="base-select"></select></div><div class="config-elem"><label for="table-select">AirTable table name</label> <select disabled="disabled" class="list" style="width: 80%;" id="table-select"></select></div><div class="config-elem"><label for="table-select">Input column (Key)</label> <select disabled="disabled" class="list" style="width: 80%;" id="input-column-select"></select></div><div class="config-elem"><label for="table-select">AirTable Field Name (Key)</label> <select disabled="disabled" class="list" style="width: 80%;" id="field-select"></select></div></div><b>Columns Mapping</b><div id="content"></div><div class="button-container"><button disabled="disabled" id="add" class="button">ADD COLUMN</button></div></div></body></html>

connect_transformations/static/transformations/attachment_lookup.html

@@ -1 +1 @@
-<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Attachment Lookup</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/attachment_lookup.5e72f506627cfb05e86b.js"></script><link href="../transformations/attachment_lookup.2c9f2595bae22a0caf51.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app" class="hidden"><div class="main-container" id="content"><div class="row"><div class="col"><div class="input-group"><label class="label" for="attachment">Attachment</label> <select materialize id="attachment"></select></div></div><div class="col"><div class="input-group"><label class="label" for="sheet">Sheet</label> <input materialize id="sheet"></div></div></div><div class="row"><div class="col"><div class="input-group"><label class="label" for="input-column">Input Column (Key)</label> <select materialize id="input-column"></select></div></div><div class="col"><div class="input-group"><label class="label" for="attachment-column">Attachment Field name (Key)</label> <input materialize id="attachment-column"></div></div></div><div id="mapping"><div class="subtitle">Columns mapping</div></div></div></div></body></html>
+<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Attachment Lookup</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/attachment_lookup.5e72f506627cfb05e86b.js"></script><link href="../transformations/attachment_lookup.0775117621fee90b28ad.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app" class="hidden"><div class="main-container" id="content"><div class="row"><div class="col"><div class="input-group"><label class="label" for="attachment">Attachment</label> <select materialize id="attachment"></select></div></div><div class="col"><div class="input-group"><label class="label" for="sheet">Sheet</label> <input materialize id="sheet"></div></div></div><div class="row"><div class="col"><div class="input-group"><label class="label" for="input-column">Input Column (Key)</label> <select materialize id="input-column"></select></div></div><div class="col"><div class="input-group"><label class="label" for="attachment-column">Attachment Field name (Key)</label> <input materialize id="attachment-column"></div></div></div><div id="mapping"><div class="subtitle">Columns mapping</div></div></div></div></body></html>

connect_transformations/static/transformations/copy.html

@@ -1 +1 @@
-<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Copy</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/copy.688a33ddba483d0d9c32.js"></script><link href="../transformations/copy.15f999ee385ccc3cdaea.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app" class="application"><div class="main-container" id="content"></div><div class="button-container"><button id="add" class="button">ADD COLUMN</button></div></div></body></html>
+<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Copy</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/copy.688a33ddba483d0d9c32.js"></script><link href="../transformations/copy.a15ed0d89abdd4879a88.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app" class="application"><div class="main-container" id="content"></div><div class="button-container"><button id="add" class="button">ADD COLUMN</button></div></div></body></html>