Merge branch 'develop' of https://github.com/great-expectations/great…

…_expectations into m/v1-251/email_action
great-expectations · Apr 9, 2024 · 0080e12 · 0080e12
2 parents b61c207 + 2e7469c
commit 0080e12
Show file tree

Hide file tree

Showing 197 changed files with 984 additions and 5 deletions.
diff --git a/great_expectations/core/batch_definition.py b/great_expectations/core/batch_definition.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import re
 from typing import TYPE_CHECKING, Optional
 
 from great_expectations.compatibility import pydantic
@@ -26,6 +27,7 @@ class BatchDefinition(pydantic.BaseModel):
     id: Optional[str] = None
     name: str
     partitioner: Optional[Partitioner] = None
+    batching_regex: Optional[re.Pattern] = None
 
     # private attributes that must be set immediately after instantiation
     _data_asset: DataAsset = pydantic.PrivateAttr()
@@ -43,7 +45,9 @@ def build_batch_request(
     ) -> BatchRequest:
         """Build a BatchRequest from the asset and batch request options."""
         return self.data_asset.build_batch_request(
-            options=batch_request_options, partitioner=self.partitioner
+            options=batch_request_options,
+            partitioner=self.partitioner,
+            batching_regex=self.batching_regex,
         )
 
     def save(self) -> None:

diff --git a/great_expectations/datasource/fluent/batch_request.pyi b/great_expectations/datasource/fluent/batch_request.pyi
@@ -24,6 +24,7 @@ class BatchRequest(pydantic.BaseModel):
         options: Optional[BatchRequestOptions] = None,
         batch_slice: Optional[BatchSlice] = None,
         partitioner: Optional[Partitioner] = None,
+        batching_regex: Optional[re.Pattern] = None,
     ) -> None: ...
     @property
     def batch_slice(self) -> slice: ...

diff --git a/great_expectations/datasource/fluent/file_path_data_asset.py b/great_expectations/datasource/fluent/file_path_data_asset.py
@@ -2,6 +2,7 @@
 
 import copy
 import logging
+import re
 import warnings
 from collections import Counter
 from pprint import pformat as pf
@@ -179,6 +180,7 @@ def build_batch_request(
         options: Optional[BatchRequestOptions] = None,
         batch_slice: Optional[BatchSlice] = None,
         partitioner: Optional[Partitioner] = None,
+        batching_regex: Optional[re.Pattern] = None,
     ) -> BatchRequest:
         """A batch request that can be used to obtain batches for this DataAsset.
 
@@ -189,6 +191,7 @@ def build_batch_request(
             batch_slice: A python slice that can be used to limit the sorted batches by index.
                 e.g. `batch_slice = "[-5:]"` will request only the last 5 batches after the options filter is applied.
             partitioner: A Partitioner used to narrow the data returned from the asset.
+            batching_regex: A Regular Expression used to build batches in path based Assets.
 
         Returns:
             A BatchRequest object that can be used to obtain a batch list from a Datasource by calling the
@@ -229,6 +232,7 @@ def build_batch_request(
             options=options or {},
             batch_slice=batch_slice,
             partitioner=partitioner,
+            batching_regex=batching_regex,
         )
 
     @override

diff --git a/great_expectations/datasource/fluent/interfaces.py b/great_expectations/datasource/fluent/interfaces.py
@@ -4,6 +4,7 @@
 import dataclasses
 import functools
 import logging
+import re
 import uuid
 import warnings
 from pprint import pformat as pf
@@ -218,6 +219,7 @@ def build_batch_request(
         options: Optional[BatchRequestOptions] = None,
         batch_slice: Optional[BatchSlice] = None,
         partitioner: Optional[Partitioner] = None,
+        batching_regex: Optional[re.Pattern] = None,
     ) -> BatchRequest:
         """A batch request that can be used to obtain batches for this DataAsset.
 
@@ -228,6 +230,7 @@ def build_batch_request(
             batch_slice: A python slice that can be used to limit the sorted batches by index.
                 e.g. `batch_slice = "[-5:]"` will request only the last 5 batches after the options filter is applied.
             partitioner: A Partitioner used to narrow the data returned from the asset.
+            batching_regex: A Regular Expression used to build batches in path based Assets.
 
         Returns:
             A BatchRequest object that can be used to obtain a batch list from a Datasource by calling the

diff --git a/great_expectations/datasource/fluent/invalid_datasource.py b/great_expectations/datasource/fluent/invalid_datasource.py
@@ -90,6 +90,7 @@ def build_batch_request(
         options: dict | None = None,
         batch_slice: Any = None,
         partitioner: Any = None,
+        batching_regex: Any = None,
     ) -> NoReturn:
         self._raise_type_error()
 

diff --git a/great_expectations/datasource/fluent/pandas_datasource.py b/great_expectations/datasource/fluent/pandas_datasource.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import re
 import sqlite3
 import uuid
 from pprint import pformat as pf
@@ -177,13 +178,15 @@ def build_batch_request(
         options: Optional[BatchRequestOptions] = None,
         batch_slice: Optional[BatchSlice] = None,
         partitioner: Optional[Partitioner] = None,
+        batching_regex: Optional[re.Pattern] = None,
     ) -> BatchRequest:
         """A batch request that can be used to obtain batches for this DataAsset.
 
         Args:
             options: This is not currently supported and must be {}/None for this data asset.
             batch_slice: This is not currently supported and must be None for this data asset.
             partitioner: This is not currently supported and must be None for this data asset.
+            batching_regex: A Regular Expression used to build batches in path based Assets.
 
         Returns:
             A BatchRequest object that can be used to obtain a batch list from a Datasource by calling the
@@ -208,6 +211,7 @@ def build_batch_request(
             datasource_name=self.datasource.name,
             data_asset_name=self.name,
             options={},
+            batching_regex=batching_regex,
         )
 
     @override
@@ -383,12 +387,13 @@ def _get_reader_options_include(self) -> set[str]:
         version="0.16.15",
     )
     @override
-    def build_batch_request(  # type: ignore[override]
+    def build_batch_request(  # type: ignore[override]  # noqa: PLR0913
         self,
         dataframe: Optional[pd.DataFrame] = None,
         options: Optional[BatchRequestOptions] = None,
         batch_slice: Optional[BatchSlice] = None,
         partitioner: Optional[Partitioner] = None,
+        batching_regex: Optional[re.Pattern] = None,
     ) -> BatchRequest:
         """A batch request that can be used to obtain batches for this DataAsset.
 
@@ -397,6 +402,7 @@ def build_batch_request(  # type: ignore[override]
             options: This is not currently supported and must be {}/None for this data asset.
             batch_slice: This is not currently supported and must be None for this data asset.
             partitioner: This is not currently supported and must be None for this data asset.
+            batching_regex: This is currently not supported and must be None for this data asset.
 
         Returns:
             A BatchRequest object that can be used to obtain a batch list from a Datasource by calling the
@@ -417,6 +423,11 @@ def build_batch_request(  # type: ignore[override]
                 "partitioner is not currently supported and must be None for this DataAsset."
             )
 
+        if batching_regex is not None:
+            raise ValueError(  # noqa: TRY003
+                "batching_regex is not currently supported and must be None for this DataAsset."
+            )
+
         if dataframe is None:
             df = self.dataframe
         else:

diff --git a/great_expectations/datasource/fluent/pandas_datasource.pyi b/great_expectations/datasource/fluent/pandas_datasource.pyi
@@ -1,4 +1,5 @@
 import os
+import re
 import sqlite3
 import typing
 from logging import Logger
@@ -78,6 +79,7 @@ class _PandasDataAsset(DataAsset):
         options: Optional[BatchRequestOptions] = ...,
         batch_slice: Optional[BatchSlice] = ...,
         partitioner: Optional[Partitioner] = ...,
+        batching_regex: Optional[re.Pattern] = ...,
     ) -> BatchRequest: ...
     @override
     def _validate_batch_request(self, batch_request: BatchRequest) -> None: ...

diff --git a/great_expectations/datasource/fluent/schemas/BatchRequest.json b/great_expectations/datasource/fluent/schemas/BatchRequest.json
@@ -50,6 +50,11 @@
                 }
             ]
         },
+        "batching_regex": {
+            "title": "Batching Regex",
+            "type": "string",
+            "format": "regex"
+        },
         "batch_slice": {
             "title": "Batch Slice",
             "anyOf": [

diff --git a/great_expectations/datasource/fluent/schemas/DatabricksSQLDatasource.json b/great_expectations/datasource/fluent/schemas/DatabricksSQLDatasource.json
@@ -373,6 +373,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/..._expectations/datasource/fluent/schemas/DatabricksSQLDatasource/DatabricksTableAsset.json b/..._expectations/datasource/fluent/schemas/DatabricksSQLDatasource/DatabricksTableAsset.json
@@ -341,6 +341,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/great_expectations/datasource/fluent/schemas/DatabricksSQLDatasource/QueryAsset.json b/great_expectations/datasource/fluent/schemas/DatabricksSQLDatasource/QueryAsset.json
@@ -336,6 +336,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/great_expectations/datasource/fluent/schemas/Datasource.json b/great_expectations/datasource/fluent/schemas/Datasource.json
@@ -318,6 +318,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/great_expectations/datasource/fluent/schemas/FabricPowerBIDatasource.json b/great_expectations/datasource/fluent/schemas/FabricPowerBIDatasource.json
@@ -364,6 +364,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/great_expectations/datasource/fluent/schemas/FabricPowerBIDatasource/PowerBIDax.json b/great_expectations/datasource/fluent/schemas/FabricPowerBIDatasource/PowerBIDax.json
@@ -336,6 +336,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/great_expectations/datasource/fluent/schemas/FabricPowerBIDatasource/PowerBIMeasure.json b/great_expectations/datasource/fluent/schemas/FabricPowerBIDatasource/PowerBIMeasure.json
@@ -376,6 +376,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/great_expectations/datasource/fluent/schemas/FabricPowerBIDatasource/PowerBITable.json b/great_expectations/datasource/fluent/schemas/FabricPowerBIDatasource/PowerBITable.json
@@ -360,6 +360,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource.json b/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource.json
@@ -336,6 +336,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/CSVAsset.json b/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/CSVAsset.json
@@ -644,6 +644,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/...t_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/ExcelAsset.json b/...t_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/ExcelAsset.json
@@ -535,6 +535,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/FWFAsset.json b/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/FWFAsset.json
@@ -384,6 +384,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/...expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/FeatherAsset.json b/...expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/FeatherAsset.json
@@ -358,6 +358,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/HDFAsset.json b/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/HDFAsset.json
@@ -396,6 +396,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/HTMLAsset.json b/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/HTMLAsset.json
@@ -456,6 +456,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/JSONAsset.json b/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/JSONAsset.json
@@ -443,6 +443,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [

diff --git a/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/ORCAsset.json b/great_expectations/datasource/fluent/schemas/PandasAzureBlobStorageDatasource/ORCAsset.json
@@ -354,6 +354,11 @@
                             "$ref": "#/definitions/PartitionerConvertedDatetime"
                         }
                     ]
+                },
+                "batching_regex": {
+                    "title": "Batching Regex",
+                    "type": "string",
+                    "format": "regex"
                 }
             },
             "required": [