fix(ingest): workaround great-expectations compatibility issue

Waiting on
great-expectations/great_expectations#3758 for
a full fix.

metadata-ingestion/setup.py

@@ -23,7 +23,7 @@ def get_long_description():
 base_requirements = {
     # Compatability.
     "dataclasses>=0.6; python_version < '3.7'",
-    "typing_extensions>=3.7.4; python_version < '3.8'",
+    "typing_extensions>=3.10.0.2",
     "mypy_extensions>=0.4.3",
     # Actual dependencies.
     "typing-inspect",
@@ -60,7 +60,7 @@ def get_long_description():
     # Required for all SQL sources.
     "sqlalchemy==1.3.24",
     # Required for SQL profiling.
-    "great-expectations==0.13.43",
+    "great-expectations>=0.13.40",
     "greenlet",
 }
 
@@ -144,7 +144,6 @@ def get_long_description():
         # PR is from same author as that of sqlalchemy-trino library below.
         "sqlalchemy-trino"
     },
-
 }
 
 all_exclude_plugins: Set[str] = {
@@ -219,7 +218,11 @@ def get_long_description():
     # The trino plugin only works on Python 3.7 or newer.
     # The trino plugin can be supported on Python 3.6 with minimal changes to opensource sqlalchemy-trino sourcecode.
     base_dev_requirements = base_dev_requirements.union(
-        {dependency for plugin in ["lookml", "trino", "starburst-trino-usage"] for dependency in plugins[plugin]}
+        {
+            dependency
+            for plugin in ["lookml", "trino", "starburst-trino-usage"]
+            for dependency in plugins[plugin]
+        }
     )
 
 dev_requirements = {
@@ -292,7 +295,6 @@ def get_long_description():
         "openapi = datahub.ingestion.source.openapi:OpenApiSource",
         "trino = datahub.ingestion.source.sql.trino:TrinoSource",
         "starburst-trino-usage = datahub.ingestion.source.usage.starburst_trino_usage:TrinoUsageSource",
-
     ],
     "datahub.ingestion.sink.plugins": [
         "file = datahub.ingestion.sink.file:FileSink",

metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py

@@ -10,6 +10,17 @@
 import uuid
 from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union
 
+# Fun compatibility hack! GE version 0.13.44 broke compatibility with SQLAlchemy 1.3.24.
+# This is a temporary workaround until GE fixes the issue on their end.
+# See https://github.com/great-expectations/great_expectations/issues/3758.
+try:
+    import sqlalchemy.engine
+    from sqlalchemy.engine.url import make_url
+
+    sqlalchemy.engine.make_url = make_url  # type: ignore
+except ImportError:
+    pass
+
 import pydantic
 from great_expectations.data_context import BaseDataContext
 from great_expectations.data_context.types.base import (