Merge pull request #44 from datayoga-io/43-add_field-remove_field-ren…

…ame_field-blocks-support-list

`add_field`, `remove_field`, `rename_field` blocks - support list #43

README.md

@@ -22,34 +22,29 @@ Use this `example.yaml`:
 steps:
   - uses: add_field
     with:
-      field: full_name
-      language: jmespath
-      expression: '{ "fname": fname, "lname": lname} | join('' '', values(@))'
+      fields:
+        - field: full_name
+          language: jmespath
+          expression: '{ "fname": fname, "lname": lname} | join('' '', values(@))'
+        - field: country
+          language: sql
+          expression: country_code || ' - ' || UPPER(country_name)
   - uses: rename_field
     with:
-      from_field: fname
-      to_field: first_name
-  - uses: rename_field
-    with:
-      from_field: lname
-      to_field: last_name
-  - uses: remove_field
-    with:
-      field: credit_card
-  - uses: add_field
-    with:
-      field: country
-      language: sql
-      expression: country_code || ' - ' || UPPER(country_name)
-  - uses: remove_field
-    with:
-      field: country_name
+      fields:
+        - from_field: fname
+          to_field: first_name
+        - from_field: lname
+          to_field: last_name
   - uses: remove_field
     with:
-      field: country_code
+      fields:
+        - field: credit_card
+        - field: country_name
+        - field: country_code
   - uses: map
     with:
-      object:
+      expression:
         {
           first_name: first_name,
           last_name: last_name,
@@ -95,13 +90,16 @@ As can be seen, the record has been transformed based on the job:
     expression: country_code || ' - ' || UPPER(country_name)
   ```
 
-- Rename field `lname` to `last_name`:
+- Rename `fname` field to `first_name` and `lname` field to `last_name`:
 
   ```yaml
   uses: rename_field
   with:
-    from_field: lname
-    to_field: last_name
+    fields:
+      - from_field: fname
+        to_field: first_name
+      - from_field: lname
+        to_field: last_name
   ```
 
 - Remove `credit_card` field:

datayoga/blocks/add_field/block.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Any, Dict, List
 
+from datayoga import utils
 from datayoga.block import Block as DyBlock
 from datayoga.blocks import expression
 from datayoga.context import Context
@@ -11,12 +12,16 @@
 class Block(DyBlock):
     def init(self):
         logger.debug(f"Initializing {self.get_block_name()}")
-        self.expression = expression.compile(self.properties["language"], self.properties["expression"])
+        self.properties = utils.format_block_properties(self.properties)
+
+        for property in self.properties["fields"]:
+            property["compiled_expression"] = expression.compile(property["language"], property["expression"])
 
     def run(self, data: List[Dict[str, Any]], context: Context = None) -> List[Dict[str, Any]]:
         logger.debug(f"Running {self.get_block_name()}")
 
         for row in data:
-            row[self.properties["field"]] = self.expression.search(row)
+            for property in self.properties["fields"]:
+                row[property["field"]] = property["compiled_expression"].search(row)
 
         return data

datayoga/blocks/add_field/block.schema.json

@@ -1,8 +1,34 @@
 {
-  "description": "Add a field to a record",
-  "title": "Add a field",
+  "description": "Add fields to a record",
+  "title": "Add fields",
   "type": "object",
   "properties": {
+    "fields": {
+      "type": "array",
+      "description": "Fields",
+      "properties": {
+        "items": {
+          "type": "object",
+          "properties": {
+            "field": {
+              "description": "Field",
+              "type": "string"
+            },
+            "expression": {
+              "description": "Expression",
+              "type": "string"
+            },
+            "language": {
+              "description": "Language",
+              "type": "string",
+              "enum": ["jmespath", "sql"]
+            }
+          },
+          "additionalProperties": false,
+          "required": ["field", "expression", "language"]
+        }
+      }
+    },
     "field": {
       "description": "Field",
       "type": "string"
@@ -14,9 +40,12 @@
     "language": {
       "description": "Language",
       "type": "string",
-      "enum": ["jmespath","sql"]
+      "enum": ["jmespath", "sql"]
     }
   },
   "additionalProperties": false,
-  "required": ["field", "expression", "language"]
+  "oneOf": [
+    { "required": ["fields"] },
+    { "required": ["field", "expression", "language"] }
+  ]
 }

datayoga/blocks/filter/block.py

@@ -11,8 +11,7 @@
 class Block(DyBlock):
     def init(self):
         logger.debug(f"Initializing {self.get_block_name()}")
-        self.language = self.properties["language"]
-        self.expression = expression.compile(self.language, self.properties["expression"])
+        self.expression = expression.compile(self.properties["language"], self.properties["expression"])
 
     def run(self, data: List[Dict[str, Any]], context: Context = None) -> List[Dict[str, Any]]:
         logger.debug(f"Running {self.get_block_name()}")

datayoga/blocks/filter/block.schema.json

@@ -10,7 +10,7 @@
     "language": {
       "description": "Language",
       "type": "string",
-      "enum": ["jmespath","sql"]
+      "enum": ["jmespath", "sql"]
     }
   },
   "additionalProperties": false,

datayoga/blocks/map/block.schema.json

@@ -10,7 +10,7 @@
     "language": {
       "description": "Language",
       "type": "string",
-      "enum": ["jmespath","sql"]
+      "enum": ["jmespath", "sql"]
     }
   },
   "additionalProperties": false,

datayoga/blocks/remove_field/block.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Any, Dict, List
 
+from datayoga import utils
 from datayoga.block import Block as DyBlock
 from datayoga.context import Context
 
@@ -10,10 +11,13 @@
 class Block(DyBlock):
     def init(self):
         logger.debug(f"Initializing {self.get_block_name()}")
+        self.properties = utils.format_block_properties(self.properties)
 
     def run(self, data: List[Dict[str, Any]], context: Context = None) -> List[Dict[str, Any]]:
         logger.debug(f"Running {self.get_block_name()}")
 
         for row in data:
-            del row[self.properties["field"]]
+            for property in self.properties["fields"]:
+                del row[property["field"]]
+
         return data

datayoga/blocks/remove_field/block.schema.json

@@ -1,13 +1,30 @@
 {
-  "title": "Remove field",
-  "description": "Remove a field",
+  "title": "Remove fields",
+  "description": "Remove fields",
   "type": "object",
   "properties": {
+    "fields": {
+      "type": "array",
+      "description": "Fields",
+      "properties": {
+        "items": {
+          "type": "object",
+          "properties": {
+            "field": {
+              "description": "Field",
+              "type": "string"
+            }
+          },
+          "additionalProperties": false,
+          "required": ["field"]
+        }
+      }
+    },
     "field": {
       "description": "Field",
       "type": "string"
     }
   },
   "additionalProperties": false,
-  "required": ["field"]
+  "oneOf": [{ "required": ["fields"] }, { "required": ["field"] }]
 }

datayoga/blocks/rename_field/block.py

@@ -1,6 +1,7 @@
 import logging
-from typing import Any, List, Dict
+from typing import Any, Dict, List
 
+from datayoga import utils
 from datayoga.block import Block as DyBlock
 from datayoga.context import Context
 
@@ -10,9 +11,13 @@
 class Block(DyBlock):
     def init(self):
         logger.debug(f"Initializing {self.get_block_name()}")
+        self.properties = utils.format_block_properties(self.properties)
 
     def run(self, data: List[Dict[str, Any]], context: Context = None) -> List[Dict[str, Any]]:
         logger.debug(f"Running {self.get_block_name()}")
+
         for row in data:
-            row[self.properties["to_field"]] = row.pop(self.properties["from_field"])
+            for property in self.properties["fields"]:
+                row[property["to_field"]] = row.pop(property["from_field"])
+
         return data

datayoga/blocks/rename_field/block.schema.json

@@ -1,8 +1,29 @@
 {
-  "title": "Rename field",
-  "description": "Renames a field. All other fields remain unchanged",
+  "title": "Rename fields",
+  "description": "Renames fields. All other fields remain unchanged",
   "type": "object",
   "properties": {
+    "fields": {
+      "type": "array",
+      "description": "Fields",
+      "properties": {
+        "items": {
+          "type": "object",
+          "properties": {
+            "from_field": {
+              "description": "From field",
+              "type": "string"
+            },
+            "to_field": {
+              "description": "To field",
+              "type": "string"
+            }
+          },
+          "additionalProperties": false,
+          "required": ["from_field", "to_field"]
+        }
+      }
+    },
     "from_field": {
       "description": "From field",
       "type": "string"
@@ -13,5 +34,8 @@
     }
   },
   "additionalProperties": false,
-  "required": ["from_field", "to_field"]
+  "oneOf": [
+    { "required": ["fields"] },
+    { "required": ["from_field", "to_field"] }
+  ]
 }

datayoga/schemas/job.schema.json

@@ -14,7 +14,7 @@
           },
           "with": {
             "description": "Properties",
-            "type": "object"
+            "type": ["object", "array"]
           }
         },
         "additionalProperties": false,

datayoga/utils.py

@@ -36,3 +36,15 @@ def read_yaml(filename: str) -> Dict[str, Any]:
             return yaml.safe_load(stream)
     except Exception as e:
         raise ValueError(f"Malformed YAML: {e}")
+
+
+def format_block_properties(properties: Dict[str, Any]) -> Dict[str, Any]:
+    """Adds `fields` array with the passed properties in case it's missing
+
+    Args:
+        properties (Dict[str, Any]): properties
+
+    Returns:
+        Dict[str, Any]: formatted properties with `fields` array
+    """
+    return {"fields": [properties]} if not "fields" in properties else properties

docs/blocks/add_field.md

@@ -3,18 +3,35 @@ parent: Blocks Reference
 layout: page
 ---
 
-# Add a field
+# Add fields
 
-Add a field to a record
+Add fields to a record
 
 
 **Properties**
 
 |Name|Type|Description|Required|
 |----|----|-----------|--------|
-|**field**|`string`|Field<br/>|yes|
-|**expression**|`string`|Expression<br/>|yes|
-|**language**|`string`|Language<br/>Enum: `"jmespath"`, `"sql"`<br/>|yes|
+|[**fields**](#fields)|`array`|Fields<br/>||
+|**field**|`string`|Field<br/>||
+|**expression**|`string`|Expression<br/>||
+|**language**|`string`|Language<br/>Enum: `"jmespath"`, `"sql"`<br/>||
 
 **Additional Properties:** not allowed  
+
+**Option 1 (alternative):** 
+**No properties.**
+
+
+
+**Option 2 (alternative):** 
+**No properties.**
+
+
+<a name="fields"></a>
+## fields\[\]: array
+
+Fields
+
+