Removes futures from required libs #746

JGSweets · 2023-01-04T22:32:38Z

We are no longer maintaining py2 or <= py3.7. Therefore, this is no longer needed

JGSweets · 2023-01-04T22:33:53Z

dataprofiler/data_readers/text_data.py

@@ -3,8 +3,6 @@
 from io import StringIO
 from typing import Dict, List, Optional, Union, cast

-from past.builtins import basestring


no longer needed

JGSweets · 2023-01-04T22:34:00Z

dataprofiler/data_readers/text_data.py

-            and len(data)
-            and isinstance(data[0], (str, basestring))
-        ):
+        if isinstance(data, list) and len(data) and isinstance(data[0], str):


basestring is for py2

JGSweets · 2023-01-04T22:34:09Z

dataprofiler/labelers/base_model.py

@@ -15,6 +15,8 @@
 class AutoSubRegistrationMeta(abc.ABCMeta):
    """For registering subclasses."""

+    _register_subclass: Callable[[type[AutoSubRegistrationMeta]], None]


proper mypy

JGSweets · 2023-01-04T22:34:29Z

dataprofiler/labelers/base_model.py

@@ -93,7 +95,9 @@ def _register_subclass(cls) -> None:
    @property
    def label_mapping(self) -> dict[str, int]:
        """Return mapping of labels to their encoded values."""
-        return copy.deepcopy(self._label_mapping)


potential bug fix

JGSweets · 2023-01-04T22:34:45Z

dataprofiler/labelers/base_model.py

@@ -154,7 +158,7 @@ def get_class(cls, class_name: str) -> type[BaseModel] | None:

        return cls._BaseModel__subclasses.get(class_name.lower(), None)

-    def get_parameters(self, param_list: list[str] = None) -> dict:


rest in this file are static typing

JGSweets · 2023-01-04T22:35:07Z

dataprofiler/labelers/base_model.py

@@ -189,7 +193,7 @@ def set_params(self, **kwargs: Any) -> None:
        for param in kwargs:
            self._parameters[param] = kwargs[param]

-    def add_label(self, label: str, same_as: str = None) -> None:


this file is static typing

JGSweets · 2023-01-04T22:35:35Z

dataprofiler/labelers/data_processing.py

@@ -97,7 +97,7 @@ def help(cls) -> None:
        """
        raise NotImplementedError()

-    def get_parameters(self, param_list: list[str] = None) -> dict:


this file is static typing

JGSweets · 2023-01-04T22:35:39Z

dataprofiler/labelers/labeler_utils.py

@@ -1,8 +1,10 @@
 """Contains functions for the data labeler."""
+from __future__ import annotations


this file is static typing

JGSweets · 2023-01-04T22:36:12Z

dataprofiler/profilers/base_column_profilers.py

@@ -10,14 +10,13 @@

 import numpy as np
 import pandas as pd
-from future.utils import with_metaclass


this is a main change to fix the metaclass

JGSweets · 2023-01-04T22:36:28Z

dataprofiler/profilers/column_profile_compilers.py

@@ -5,7 +5,6 @@
 from collections import OrderedDict
 from multiprocessing.pool import Pool

-from future.utils import with_metaclass


main change to fix the metaclass

JGSweets · 2023-01-04T22:36:36Z

dataprofiler/profilers/helpers/report_helpers.py

@@ -1,22 +1,23 @@
 """Contains helper functions for generating report."""
+from __future__ import annotations


this file is static typing

JGSweets · 2023-01-04T22:36:41Z

dataprofiler/profilers/int_column_profile.py

@@ -9,7 +9,7 @@
 from .profiler_options import IntOptions


this file is static typing

JGSweets · 2023-01-04T22:36:50Z

dataprofiler/profilers/numerical_column_stats.py

@@ -32,7 +31,7 @@ def __init__(self, function: Callable) -> None:
    __isabstractmethod__ = True


-class NumericStatsMixin(with_metaclass(abc.ABCMeta, object)):  # type: ignore


main change here

JGSweets · 2023-01-04T22:37:20Z

dataprofiler/profilers/profile_builder.py

-        profile: dict = OrderedDict()  # type: ignore[no-redef]
+        report: OrderedDict = OrderedDict()
        if "data_label_profile" not in self.profiles:
            dict_order.remove("data_label")
        for key in dict_order:
            try:
-                profile[key] = unordered_profile[key]
+                report[key] = unordered_profile[key]  # type: ignore
            except KeyError:
-                profile[key] = None
+                report[key] = None  # type: ignore

-        return profile
+        return report


uses report as static typing issues

JGSweets · 2023-01-04T22:37:42Z

dataprofiler/profilers/profile_builder.py

@@ -944,20 +944,24 @@ def _remove_data_labelers(

        # get all profiles, unstructured is a single profile and hence needs to
        # be in a list, whereas structured is already a list
-        profilers = [self._profile]


a lot of this file is static typing fixes

JGSweets · 2023-01-04T22:38:33Z

dataprofiler/profilers/profile_builder.py

+            [
+                col._last_batch_size
+                for col in self._profile
+                if col._last_batch_size is not None


fix to skip if None otherwise min would fail

JGSweets · 2023-01-04T22:38:41Z

dataprofiler/profilers/profile_builder.py

@@ -1791,6 +1814,15 @@ def _get_and_validate_schema_mapping(

        return schema_mapping

+    @property


static typing fix

JGSweets · 2023-01-04T22:38:54Z

dataprofiler/reports/graphs.py

@@ -7,8 +7,9 @@
 from typing import TYPE_CHECKING, List, Union, cast

 if TYPE_CHECKING:
-    from dataprofiler.profilers.float_column_profile import FloatColumn


use same relative format

JGSweets · 2023-01-04T22:39:01Z

dataprofiler/reports/graphs.py

@@ -69,10 +66,11 @@ def plot_histograms(
            "column indexes in the profiler"
        )

+    profile_list: list[StructuredColProfiler] = profiler.profile


static typing fix

JGSweets · 2023-01-04T22:39:07Z

requirements.txt

@@ -1,6 +1,5 @@
 h5py>=2.10.0
 wheel>=0.33.1
-future>=0.18.2


the removal

JGSweets · 2023-01-04T22:39:29Z

setup.cfg

@@ -15,6 +15,7 @@ line_length=88
 warn_return_any = True
 warn_unused_configs = True
 ignore_missing_imports = True
+disable_error_code = override


need to address at a later time, but some mixin / abstraction issues with mypy

We should document this if it is needed in the future

JGSweets added 2 commits January 4, 2023 12:47

fix: remove future

d8ba288

fix: remove future

b825363

JGSweets added Medium Priority Significant improvement or bug / feature reducing overall performance Refactor Code that is being modified to improve the library labels Jan 4, 2023

JGSweets requested review from ksneab7, taylorfturner, micdavis and tyfarnan as code owners January 4, 2023 22:32

JGSweets commented Jan 4, 2023

View reviewed changes

requirements.txt

@@ -1,6 +1,5 @@

h5py>=2.10.0

wheel>=0.33.1

future>=0.18.2

Copy link

Collaborator Author

JGSweets Jan 4, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the removal

JGSweets commented Jan 4, 2023

View reviewed changes

fix: bug

21192d7

fix: 3.8 bug

f02b6c8

ksneab7 approved these changes Jan 5, 2023

View reviewed changes

micdavis approved these changes Jan 5, 2023

View reviewed changes

micdavis merged commit 94a4822 into capitalone:main Jan 5, 2023

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Removes futures from required libs #746

Removes futures from required libs #746

JGSweets commented Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023 •

edited

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023 •

edited

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

JGSweets Jan 4, 2023

ksneab7 Jan 5, 2023

		@@ -154,7 +158,7 @@ def get_class(cls, class_name: str) -> type[BaseModel] \| None:

		return cls._BaseModel__subclasses.get(class_name.lower(), None)

		def get_parameters(self, param_list: list[str] = None) -> dict:

		@@ -1,8 +1,10 @@
		"""Contains functions for the data labeler."""
		from __future__ import annotations

		@@ -1,22 +1,23 @@
		"""Contains helper functions for generating report."""
		from __future__ import annotations

		@@ -32,7 +31,7 @@ def __init__(self, function: Callable) -> None:
		__isabstractmethod__ = True


		class NumericStatsMixin(with_metaclass(abc.ABCMeta, object)): # type: ignore

		@@ -1791,6 +1814,15 @@ def _get_and_validate_schema_mapping(

		return schema_mapping

		@property

Removes futures from required libs #746

Removes futures from required libs #746

Conversation

JGSweets commented Jan 4, 2023

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

JGSweets Jan 4, 2023 • edited

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

JGSweets Jan 4, 2023 • edited

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

JGSweets Jan 4, 2023 •

edited

JGSweets Jan 4, 2023 •

edited