LITE-28054 Transformations support columns with duplicated names

connect_transformations/formula/mixins.py

@@ -11,7 +11,12 @@
 from connect.eaas.core.responses import RowTransformationResponse
 
 from connect_transformations.formula.models import Configuration
-from connect_transformations.formula.utils import DROP_REGEX, extract_input, validate_formula
+from connect_transformations.formula.utils import (
+    DROP_REGEX,
+    clear_formula,
+    extract_input,
+    validate_formula,
+)
 from connect_transformations.models import Error, StreamsColumn, ValidationResult
 from connect_transformations.utils import cast_value_to_type, deep_convert_type
 
@@ -30,7 +35,8 @@ def precompile(self, row: Dict):
                 formula = expression['formula']
                 if DROP_REGEX.findall(formula):
                     formula = f'def drop_row: "#INSTRUCTION/DELETE_ROW"; {formula}'
-                self.jq_expressions[expression['to']] = jq.compile(formula)
+                clean_formula = clear_formula(formula)
+                self.jq_expressions[expression['to']] = jq.compile(clean_formula)
 
             self.column_converters = []
 

connect_transformations/formula/utils.py

@@ -4,6 +4,7 @@
 # All rights reserved.
 #
 import re
+from collections import defaultdict
 
 import jq
 from fastapi.responses import JSONResponse
@@ -18,6 +19,7 @@
 
 JQ_FIELDS_REGEX = re.compile(r'(\.([a-z_][a-z0-9_]*))|(\."(.+?)")|(\.\["(.+?)"\])', re.I)
 DROP_REGEX = re.compile(r'(?<![."])drop_row(?![."])', re.I)
+COL_ID_REGEX = re.compile(r'\."([^"]+) \(C\d{3,4}\)"')
 
 
 def validate_formula(data):  # noqa: CCR001
@@ -54,38 +56,22 @@ def validate_formula(data):  # noqa: CCR001
             )
 
     output_columns = []
-    available_columns = {col['name']: col['id'] for col in data['columns']['input']}
+    column_converter = get_expression_column_converter(data['columns']['input'])
 
     for expression in data['settings']['expressions']:
         if expression['to'] in output_columns:
             return build_error_response('Each `output column` must be unique.')
 
-        if expression['to'] in available_columns:
-            output_column = list(filter(
-                lambda col: col['name'] == expression['to'],
-                data['columns']['output'],
-            ))
-            if (
-                len(output_column) != 1
-                or not output_column[0].get('id')
-                or output_column[0]['id'] != available_columns[expression['to']]
-            ):
-                return build_error_response(f'Column `{expression["to"]}` already exists.')
-
-        columns = [
-            r[1] or r[3] or r[5]
-            for r in JQ_FIELDS_REGEX.findall(expression['formula'])
-        ]
-
-        for column in columns:
-            if column not in available_columns:
+        for match in JQ_FIELDS_REGEX.findall(expression['formula']):
+            var_name = match[1] or match[3] or match[5]
+            var = column_converter(var_name)
+            if not var:
                 return build_error_response(
                     (
                         f'Settings contains formula `{expression["formula"]}` '
-                        'with column that does not exist on columns.input.'
+                        f'with column `{var_name}` that does not exist on columns.input.'
                     ),
                 )
-
         try:
             formula = expression['formula']
             if DROP_REGEX.findall(formula):
@@ -121,10 +107,53 @@ def extract_input(data):
             },
         )
 
-    input_columns = set()
+    input_columns = []
+    added_columns = {}
+
+    column_converter = get_expression_column_converter(data['columns'])
+
     for expression in data['expressions']:
-        input_columns.update([
-            r[1] or r[3] or r[5]
-            for r in JQ_FIELDS_REGEX.findall(expression['formula'])
-        ])
-    return [column for column in data['columns'] if column['name'] in input_columns]
+        for match in JQ_FIELDS_REGEX.findall(expression['formula']):
+            col_value = match[1] or match[3] or match[5]
+            col = column_converter(col_value)
+            if not col:
+                return JSONResponse(
+                    status_code=400,
+                    content={
+                        'error': f'{col_value} not found in input columns',
+                    },
+                )
+
+            if col['name'] not in added_columns:
+                input_columns.append(col)
+                added_columns[col['name']] = col['id']
+
+    return input_columns
+
+
+def get_expression_column_converter(columns):
+    cols_by_name = defaultdict(list)
+    cols_by_id = {}
+
+    for column in columns:
+        id_suffix = column['id'].split('-')[-1]
+        cols_by_name[column['name']].append(column)
+        cols_by_id[f'(C{id_suffix})'] = column
+
+    def converter(col_name):
+        if col_name in cols_by_name:
+            return cols_by_name[col_name][-1]
+
+        col_name_parts = col_name.split()
+        if (
+            len(col_name_parts) > 1
+            and col_name_parts[-1] in cols_by_id
+            and cols_by_id[col_name_parts[-1]]['name'] == ' '.join(col_name_parts[:-1])
+        ):
+            return cols_by_id[col_name_parts[-1]]
+
+    return converter
+
+
+def clear_formula(expression):
+    return COL_ID_REGEX.sub(r'."\1"', expression)

connect_transformations/static/transformations/airtable_lookup.36093b636dae3ccdc875.js → connect_transformations/static/transformations/airtable_lookup.2852161b91d2511b0373.js

@@ -141,6 +141,12 @@ const getAirtableTables = (key, baseId) => fetch(`/api/airtable_lookup/tables?ap
   },
 }).then((response) => response.json());
 
+const getColumnLabel = (column) => {
+  const colIdParts = column.id.split('-');
+  const colIdSuffix = colIdParts[colIdParts.length - 1];
+
+  return `${column.name} (C${colIdSuffix})`;
+};
 
 ;// CONCATENATED MODULE: ./ui/src/pages/transformations/airtable_lookup.js
 /*
@@ -170,7 +176,7 @@ const createCopyRow = (parent, index, options, input, output) => {
       <select class="list" style="width: 35%;" ${input ? `value="${input.id}"` : ''}>
         ${options.map((column) => `
           <option value="${column.id}" ${input && input.id === column.id ? 'selected' : ''}>
-            ${column.name}
+            ${getColumnLabel(column)}
           </option>`).join(' ')}
       </select>
       <input type="text" placeholder="Copy column name" style="width: 35%;" ${output ? `value="${output.name}"` : ''} />
@@ -428,7 +434,6 @@ const airtable = (app) => {
   .then(airtable);
 
 
-
 /***/ })
 
 /******/ 	});

connect_transformations/static/transformations/airtable_lookup.html

@@ -1 +1 @@
-<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Airtable Lookup</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/airtable_lookup.36093b636dae3ccdc875.js"></script><link href="../transformations/airtable_lookup.5f5aa442fc131fcaad2c.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app"><div class="main-container"><div class="config-elem api-input"><label for="key-select">API Key</label> <input style="width: 86%;" id="key-input" placeholder="Enter an AirTable key"></div><div class="config-elem"><label for="base-select">AirTable Base</label> <select disabled="disabled" class="list" style="width: 80%;" id="base-select"></select></div><div class="config-elem"><label for="table-select">AirTable table name</label> <select disabled="disabled" class="list" style="width: 80%;" id="table-select"></select></div><div class="config-elem"><label for="table-select">Input column (Key)</label> <select disabled="disabled" class="list" style="width: 80%;" id="input-column-select"></select></div><div class="config-elem"><label for="table-select">AirTable Field Name (Key)</label> <select disabled="disabled" class="list" style="width: 80%;" id="field-select"></select></div></div><b>Columns Mapping</b><div id="content"></div><div class="button-container"><button disabled="disabled" id="add" class="button">ADD COLUMN</button></div></div></body></html>
+<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Airtable Lookup</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/airtable_lookup.2852161b91d2511b0373.js"></script><link href="../transformations/airtable_lookup.5f5aa442fc131fcaad2c.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app"><div class="main-container"><div class="config-elem api-input"><label for="key-select">API Key</label> <input style="width: 86%;" id="key-input" placeholder="Enter an AirTable key"></div><div class="config-elem"><label for="base-select">AirTable Base</label> <select disabled="disabled" class="list" style="width: 80%;" id="base-select"></select></div><div class="config-elem"><label for="table-select">AirTable table name</label> <select disabled="disabled" class="list" style="width: 80%;" id="table-select"></select></div><div class="config-elem"><label for="table-select">Input column (Key)</label> <select disabled="disabled" class="list" style="width: 80%;" id="input-column-select"></select></div><div class="config-elem"><label for="table-select">AirTable Field Name (Key)</label> <select disabled="disabled" class="list" style="width: 80%;" id="field-select"></select></div></div><b>Columns Mapping</b><div id="content"></div><div class="button-container"><button disabled="disabled" id="add" class="button">ADD COLUMN</button></div></div></body></html>

connect_transformations/static/transformations/attachment_lookup.ce7b0ca6b8388fab423e.js → connect_transformations/static/transformations/attachment_lookup.073059714271bc4cd2f6.js

@@ -143,6 +143,12 @@ const getAirtableTables = (key, baseId) => fetch(`/api/airtable_lookup/tables?ap
   },
 }).then((response) => response.json());
 
+const getColumnLabel = (column) => {
+  const colIdParts = column.id.split('-');
+  const colIdSuffix = colIdParts[colIdParts.length - 1];
+
+  return `${column.name} (C${colIdSuffix})`;
+};
 
 ;// CONCATENATED MODULE: ./ui/src/pages/transformations/attachment_lookup.js
 /*
@@ -177,7 +183,7 @@ const fillSelect = (options, id, value) => {
   options.forEach((item) => {
     const option = document.createElement('option');
     option.value = item.id;
-    option.text = item.name;
+    option.text = getColumnLabel(item);
     if (item.id === value) {
       option.selected = true;
     }

connect_transformations/static/transformations/attachment_lookup.html

@@ -1 +1 @@
-<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Attachment Lookup</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/attachment_lookup.ce7b0ca6b8388fab423e.js"></script><link href="../transformations/attachment_lookup.2c9f2595bae22a0caf51.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app" class="hidden"><div class="main-container" id="content"><div class="row"><div class="col"><div class="input-group"><label class="label" for="attachment">Attachment</label> <select materialize id="attachment"></select></div></div><div class="col"><div class="input-group"><label class="label" for="sheet">Sheet</label> <input materialize id="sheet"></div></div></div><div class="row"><div class="col"><div class="input-group"><label class="label" for="input-column">Input Column (Key)</label> <select materialize id="input-column"></select></div></div><div class="col"><div class="input-group"><label class="label" for="attachment-column">Attachment Field name (Key)</label> <input materialize id="attachment-column"></div></div></div><div id="mapping"><div class="subtitle">Columns mapping</div></div></div></div></body></html>
+<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Attachment Lookup</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/attachment_lookup.073059714271bc4cd2f6.js"></script><link href="../transformations/attachment_lookup.2c9f2595bae22a0caf51.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app" class="hidden"><div class="main-container" id="content"><div class="row"><div class="col"><div class="input-group"><label class="label" for="attachment">Attachment</label> <select materialize id="attachment"></select></div></div><div class="col"><div class="input-group"><label class="label" for="sheet">Sheet</label> <input materialize id="sheet"></div></div></div><div class="row"><div class="col"><div class="input-group"><label class="label" for="input-column">Input Column (Key)</label> <select materialize id="input-column"></select></div></div><div class="col"><div class="input-group"><label class="label" for="attachment-column">Attachment Field name (Key)</label> <input materialize id="attachment-column"></div></div></div><div id="mapping"><div class="subtitle">Columns mapping</div></div></div></div></body></html>

connect_transformations/static/transformations/copy.1ede3336839fbbe21e63.js → connect_transformations/static/transformations/copy.3f7e33696135d1752653.js

@@ -143,6 +143,12 @@ const getAirtableTables = (key, baseId) => fetch(`/api/airtable_lookup/tables?ap
   },
 }).then((response) => response.json());
 
+const getColumnLabel = (column) => {
+  const colIdParts = column.id.split('-');
+  const colIdSuffix = colIdParts[colIdParts.length - 1];
+
+  return `${column.name} (C${colIdSuffix})`;
+};
 
 ;// CONCATENATED MODULE: ./ui/src/pages/transformations/copy.js
 /*
@@ -166,7 +172,7 @@ const createCopyRow = (parent, index, options, input, output) => {
       <select class="list" style="width: 35%;" ${input ? `value="${input.id}"` : ''}>
         ${options.map((column) => `
           <option value="${column.id}" ${input && input.id === column.id ? 'selected' : ''}>
-            ${column.name}
+            ${getColumnLabel(column)}
           </option>`).join(' ')}
       </select>
       <input type="text" placeholder="Copy column name" style="width: 35%;" ${output ? `value="${output.name}"` : ''} />

connect_transformations/static/transformations/copy.html

@@ -1 +1 @@
-<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Copy</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/copy.1ede3336839fbbe21e63.js"></script><link href="../transformations/copy.15f999ee385ccc3cdaea.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app" class="application"><div class="main-container" id="content"></div><div class="button-container"><button id="add" class="button">ADD COLUMN</button></div></div></body></html>
+<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Copy</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/copy.3f7e33696135d1752653.js"></script><link href="../transformations/copy.15f999ee385ccc3cdaea.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app" class="application"><div class="main-container" id="content"></div><div class="button-container"><button id="add" class="button">ADD COLUMN</button></div></div></body></html>

connect_transformations/static/transformations/currency_conversion.c112386fe452cc74af5b.js → connect_transformations/static/transformations/currency_conversion.345ae7ad31b241768362.js

@@ -79,6 +79,12 @@ const getAirtableTables = (key, baseId) => fetch(`/api/airtable_lookup/tables?ap
   },
 }).then((response) => response.json());
 
+const getColumnLabel = (column) => {
+  const colIdParts = column.id.split('-');
+  const colIdSuffix = colIdParts[colIdParts.length - 1];
+
+  return `${column.name} (C${colIdSuffix})`;
+};
 
 ;// CONCATENATED MODULE: ./ui/src/components.js
 /*
@@ -194,8 +200,8 @@ const convert = (app) => {
 
     columns.forEach(column => {
       const isSelected = settings && column.name === settings.from.column;
-
-      const option = isSelected ? `<option value="${column.id}" selected>${column.name}</option>` : `<option value="${column.id}">${column.name}</option>`;
+      const colLabel = getColumnLabel(column);
+      const option = isSelected ? `<option value="${column.id}" selected>${colLabel}</option>` : `<option value="${column.id}">${colLabel}</option>`;
 
       inputColumnSelect.innerHTML += option;
     });

connect_transformations/static/transformations/currency_conversion.html

@@ -1 +1 @@
-<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Currency Conversion</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/currency_conversion.c112386fe452cc74af5b.js"></script><link href="../transformations/currency_conversion.15f999ee385ccc3cdaea.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app"><div class="main-container" id="content"><form name="convertCurrency" class="convert-currency"><div class="convert-currency__input-group"><div class="convert-currency__column convert-currency__input"><label for="input-column">Input Column</label> <select name="inputColumn" id="input-column"></select></div><div class="convert-currency__input"><label for="from-currency">From Currency</label> <select name="fromCurrency" id="from-currency"></select></div></div><div class="convert-currency__input-group"><div class="convert-currency__column convert-currency__input"><label for="output-column">Output Column</label> <input name="outputColumn" id="output-column"></div><div class="convert-currency__input"><label for="to-currency">To Currency</label> <select name="toCurrency" id="to-currency"></select></div></div></form></div></div></body></html>
+<!doctype html><html><head><meta charset="utf-8"/><title>Transformations/Currency Conversion</title><script defer="defer" src="../vendors.b7829ba6a3fa12b6e5cb.js"></script><script defer="defer" src="../transformations/currency_conversion.345ae7ad31b241768362.js"></script><link href="../transformations/currency_conversion.15f999ee385ccc3cdaea.css" rel="stylesheet"></head><body><div id="loader"></div><div id="app"><div class="main-container" id="content"><form name="convertCurrency" class="convert-currency"><div class="convert-currency__input-group"><div class="convert-currency__column convert-currency__input"><label for="input-column">Input Column</label> <select name="inputColumn" id="input-column"></select></div><div class="convert-currency__input"><label for="from-currency">From Currency</label> <select name="fromCurrency" id="from-currency"></select></div></div><div class="convert-currency__input-group"><div class="convert-currency__column convert-currency__input"><label for="output-column">Output Column</label> <input name="outputColumn" id="output-column"></div><div class="convert-currency__input"><label for="to-currency">To Currency</label> <select name="toCurrency" id="to-currency"></select></div></div></form></div></div></body></html>

connect_transformations/static/transformations/filter_row.97891e81a14862909dfb.js → connect_transformations/static/transformations/filter_row.348ec003c725618baa08.js

@@ -143,6 +143,12 @@ const getAirtableTables = (key, baseId) => fetch(`/api/airtable_lookup/tables?ap
   },
 }).then((response) => response.json());
 
+const getColumnLabel = (column) => {
+  const colIdParts = column.id.split('-');
+  const colIdSuffix = colIdParts[colIdParts.length - 1];
+
+  return `${column.name} (C${colIdSuffix})`;
+};
 
 ;// CONCATENATED MODULE: ./ui/src/pages/transformations/filter_row.js
 /*
@@ -199,7 +205,7 @@ const filterRow = (app) => {
     availableColumns.forEach((column) => {
       const option = document.createElement('option');
       option.value = column.id;
-      option.text = column.name;
+      option.text = getColumnLabel(column);
       document.getElementById('column').appendChild(option);
     });
 
@@ -247,21 +253,20 @@ const filterRow = (app) => {
     hideComponent('app');
 
     const inputSelector = document.getElementById('column');
-    const selectedColumn = inputSelector.options[inputSelector.selectedIndex].text;
     const inputColumn = columns.find((column) => column.id === inputSelector.value);
     const matchCondition = document.getElementById('match').checked;
     data.columns.input.push(inputColumn);
     data.columns.output.push(
       {
-        name: `${selectedColumn}_INSTRUCTIONS`,
+        name: `${inputColumn.name}_INSTRUCTIONS`,
         type: 'string',
         output: false,
       },
     );
 
     const inputValue = document.getElementById('value');
     data.settings = {
-      from: selectedColumn,
+      from: inputColumn.name,
       value: inputValue.value,
       match_condition: matchCondition,
       additional_values: [],