Merge 64ab69e into 5d444ec

great_expectations/dataset/pandas_dataset.py

@@ -2,20 +2,16 @@
 
 import inspect
 import json
-import re
 from datetime import datetime #, timedelta # Add for case of testing timedelta types
 import logging
-import io
 from datetime import datetime
 from functools import wraps
 import jsonschema
-import sys
 import numpy as np
 import pandas as pd
 from dateutil.parser import parse
 from scipy import stats
 from six import PY3, integer_types, string_types
-from numbers import Number
 
 from .dataset import Dataset
 from great_expectations.data_asset.util import DocInherit, parse_result_format
@@ -25,6 +21,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 class MetaPandasDataset(Dataset):
     """MetaPandasDataset is a thin layer between Dataset and PandasDataset.
 
@@ -292,6 +289,7 @@ class PandasDataset(MetaPandasDataset, pd.DataFrame):
     _internal_names = pd.DataFrame._internal_names + [
         '_batch_kwargs',
         '_expectation_suite',
+        '_config',
         'caching',
         'default_expectation_args',
         'discard_subset_failing_expectations'
@@ -310,7 +308,7 @@ def __finalize__(self, other, method=None, **kwargs):
             self._initialize_expectations(other.get_expectation_suite(
                 discard_failed_expectations=False,
                 discard_result_format_kwargs=False,
-                discard_include_configs_kwargs=False,
+                discard_include_config_kwargs=False,
                 discard_catch_exceptions_kwargs=False))
             # If other was coerced to be a PandasDataset (e.g. via _constructor call during self.copy() operation)
             # then it may not have discard_subset_failing_expectations set. Default to self value

great_expectations/dataset/sparkdf_dataset.py

@@ -193,7 +193,7 @@ def head(self, n=5):
                 discard_failed_expectations=False,
                 discard_result_format_kwargs=False,
                 discard_catch_exceptions_kwargs=False,
-                discard_include_configs_kwargs=False
+                discard_include_config_kwargs=False
             )
         )
 

great_expectations/dataset/sqlalchemy_dataset.py

@@ -241,7 +241,7 @@ def head(self, n=5):
                 discard_failed_expectations=False,
                 discard_result_format_kwargs=False,
                 discard_catch_exceptions_kwargs=False,
-                discard_include_configs_kwargs=False
+                discard_include_config_kwargs=False
             )
         )
 

great_expectations/datasource/sqlalchemy_source.py

@@ -11,11 +11,10 @@
 
 try:
     import sqlalchemy
-    from sqlalchemy import create_engine, MetaData
+    from sqlalchemy import create_engine
 except ImportError:
     sqlalchemy = None
     create_engine = None
-    MetaData = None
     logger.debug("Unable to import sqlalchemy.")
 
 
@@ -63,7 +62,9 @@ def __init__(self, name="default", data_context=None, profile=None, generators=N
 
             # Otherwise, connect using remaining kwargs
             else:
-                self._connect(self._get_sqlalchemy_connection_options(**kwargs))
+                self.engine = create_engine(self._get_sqlalchemy_connection_options(**kwargs))
+                self.engine.connect()
+
         except sqlalchemy.exc.OperationalError as sqlalchemy_error:
             raise DatasourceInitializationError(self._name, str(sqlalchemy_error))
 
@@ -82,11 +83,6 @@ def _get_sqlalchemy_connection_options(self, **kwargs):
         options = sqlalchemy.engine.url.URL(drivername, **credentials)
         return options
 
-    def _connect(self, options):
-        self.engine = create_engine(options)
-        self.engine.connect()
-        self.meta = MetaData()
-
     def _get_generator_class(self, type_):
         if type_ == "queries":
             return QueryGenerator

great_expectations/jupyter_ux/__init__.py

@@ -1,16 +1,16 @@
 """Utility functions for working with great_expectations within jupyter notebooks or jupyter lab.
 """
 
-import json
-import os
 import logging
-import great_expectations as ge
+import sys
+
 import great_expectations.render as render
 from datetime import datetime
 
 import tzlocal
 from IPython.core.display import display, HTML
 
+
 def set_data_source(context, data_source_type=None):
     """
     TODO: Needs a docstring and tests.
@@ -74,9 +74,12 @@ def set_data_source(context, data_source_type=None):
 
     return data_source_name
 
+
 def setup_notebook_logging(logger=None):
-    """
-    TODO: Needs a docstring and tests.
+    """Set up the provided logger for the GE default logging configuration.
+
+    Args:
+        logger - the logger to configure
     """
 
     def posix2local(timestamp, tz=tzlocal.get_localzone()):
@@ -97,18 +100,19 @@ def formatTime(self, record, datefmt=None):
             return s
 
     if not logger:
-        logger = logging.getLogger()
-    chandler = logging.StreamHandler()
+        logger = logging.getLogger("great_expectations")
+    chandler = logging.StreamHandler(stream=sys.stdout)
     chandler.setLevel(logging.DEBUG)
-    chandler.setFormatter(Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s", "%Y-%m-%dT%H:%M:%S%z"))
+    # chandler.setFormatter(Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s", "%Y-%m-%dT%H:%M:%S%z"))
+    chandler.setFormatter(Formatter("%(asctime)s - %(levelname)s - %(message)s", "%Y-%m-%dT%H:%M:%S%z"))
     logger.addHandler(chandler)
-    logger.setLevel(logging.ERROR)
-    # logger.setLevel(logging.INFO)
-    logging.debug("test")
+    logger.setLevel(logging.INFO)
+    logger.info("Great Expectations logging enabled at INFO level by JupyterUX module.")
+    #
+    # # Filter warnings
+    # import warnings
+    # warnings.filterwarnings('ignore')
 
-    # Filter warnings
-    import warnings
-    warnings.filterwarnings('ignore')
 
 def list_available_data_asset_names(context, data_source_name=None):
     """
@@ -196,6 +200,7 @@ def list_available_data_asset_names(context, data_source_name=None):
 </style>
 """
 
+
 def display_column_expectations_as_section(
     expectation_suite,
     column,
@@ -235,6 +240,7 @@ def display_column_expectations_as_section(
     else:
         display(HTML(html_to_display))
 
+
 def display_column_evrs_as_section(
     evrs,
     column,
@@ -272,3 +278,8 @@ def display_column_evrs_as_section(
         return html_to_display
     else:
         display(HTML(html_to_display))
+
+
+# When importing the jupyter_ux module, we set up a preferred logging configuration
+logger = logging.getLogger("great_expectations")
+setup_notebook_logging(logger)

great_expectations/profile/basic_dataset_profiler.py

@@ -5,6 +5,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 class BasicDatasetProfiler(DatasetProfiler):
     """BasicDatasetProfiler is inspired by the beloved pandas_profiling project.
 

tests/datasource/test_datasources.py

@@ -107,7 +107,7 @@ def test_create_sqlalchemy_datasource(data_context):
     type_ = "sqlalchemy"
     connection_kwargs = {
         "drivername": "postgresql",
-        "username": "",
+        "username": "postgres",
         "password": "",
         "host": "localhost",
         "port": 5432,

tests/test_data_asset.py

@@ -324,7 +324,7 @@ def test_get_and_save_expectation_config(self):
         self.assertEqual(
             df.get_expectation_suite(
                 discard_result_format_kwargs=False,
-                discard_include_configs_kwargs=False,
+                discard_include_config_kwargs=False,
                 discard_catch_exceptions_kwargs=False,
             ),
             output_config,
@@ -334,7 +334,7 @@ def test_get_and_save_expectation_config(self):
         df.save_expectation_suite(
             directory_name+'/temp3.json',
             discard_result_format_kwargs=False,
-            discard_include_configs_kwargs=False,
+            discard_include_config_kwargs=False,
             discard_catch_exceptions_kwargs=False,
         )
         temp_file = open(directory_name+'/temp3.json')

tests/test_jupyter_ux.py

@@ -1,14 +1,19 @@
+import logging
+import sys
+
 import great_expectations as ge
 import great_expectations.jupyter_ux as jux
 from great_expectations.profile.basic_dataset_profiler import BasicDatasetProfiler
 
+
 def test_styling_elements_exist():
     assert "<link" in jux.bootstrap_link_element
     assert "bootstrap" in jux.bootstrap_link_element
 
     assert jux.cooltip_style_element[:23] == '<style type="text/css">'
     assert ".cooltip" in jux.cooltip_style_element
 
+
 def test_display_column_expectations_as_section(basic_expectation_suite):
     html_to_display = jux.display_column_expectations_as_section(
         basic_expectation_suite,
@@ -115,6 +120,7 @@ def test_display_column_expectations_as_section(basic_expectation_suite):
     </div>
 </div>"""
 
+
 def test_display_column_evrs_as_section():
     #TODO: We should add a fixture that contains EVRs
     df = ge.read_csv("./tests/test_sets/Titanic.csv")
@@ -133,3 +139,36 @@ def test_display_column_evrs_as_section():
     assert '<div id="section-1" class="ge-section container-fluid">' in html_to_display
     assert '<span class="badge badge-info" >Carlsson, Mr Frans Olof</span>' in html_to_display
     assert '<li class="list-group-item d-flex justify-content-between align-items-center" >expect_column_values_to_be_in_type_list <span class="badge badge-secondary badge-pill" >True</span></li>' in html_to_display
+
+
+def test_configure_logging(caplog):
+    # First, ensure we set the root logger to close-to-jupyter settings (only show warnings)
+    caplog.set_level(logging.WARNING)
+    caplog.set_level(logging.WARNING, logger="great_expectations")
+
+    root = logging.getLogger()  # root logger
+    root.info("do_not_show")
+
+    # This df is used only for logging; we don't want to test against different backends
+    df = ge.dataset.PandasDataset({"a": [1, 2, 3]})
+    df.expect_column_to_exist("a")
+    df.get_expectation_suite()
+
+    res = caplog.text
+    assert "do_not_show" not in res
+
+    assert "expectation_suite" not in res
+    caplog.clear()
+
+    # Now use the logging setup from the notebook
+    logger = logging.getLogger("great_expectations")
+    jux.setup_notebook_logging(logger)
+    df = ge.dataset.PandasDataset({"a": [1, 2, 3]})
+    df.expect_column_to_exist("a")
+    df.get_expectation_suite()
+
+    root.info("do_not_show")
+    res = caplog.text
+    assert "do_not_show" not in res
+
+    assert "expectation_suite" in res