Fixed some deepcopy issues

fireant/dataset/data_blending.py

@@ -1,7 +1,7 @@
 from fireant.dataset.fields import Field
 from fireant.dataset.klass import DataSet
-from fireant.queries.builder import DataSetBlenderQueryBuilder
 from fireant.utils import immutable
+from fireant.queries.builder import DataSetBlenderQueryBuilder
 
 
 class DataSetBlender:

fireant/dataset/fields.py

@@ -2,7 +2,6 @@
 from enum import Enum
 from functools import wraps
 
-from fireant.utils import immutable
 from pypika.enums import Arithmetic
 from pypika.terms import (
     ArithmeticExpression,
@@ -20,6 +19,7 @@
     RangeFilter,
     VoidFilter,
 )
+from ..utils import immutable
 
 
 class DataType(Enum):
@@ -115,6 +115,22 @@ def __init__(
         self.precision = precision
         self.hyperlink_template = hyperlink_template
 
+    def __copy__(self):
+        cls = self.__class__
+        result = cls.__new__(cls)
+        result.__dict__.update(self.__dict__)
+        return result
+
+    def __deepcopy__(self, memo):
+        cls = self.__class__
+        result = cls.__new__(cls)
+        memo[id(self)] = result
+        from copy import deepcopy
+
+        for k, v in self.__dict__.items():
+            setattr(result, k, deepcopy(v, memo))
+        return result
+
     @property
     def is_aggregate(self):
         return self.definition.is_aggregate
@@ -177,7 +193,8 @@ def notin(self, values):
             An iterable of value to constrain the dataset query results by.
 
         :return:
-            A dataset query filter used to filter a dataset query to results where this dimension is *not* one of a set of
+            A dataset query filter used to filter a dataset query to results where this dimension is *not* one of a
+            set of
             values. Opposite of #isin.
         """
         return ExcludesFilter(self, values)
@@ -284,3 +301,4 @@ def get_sql(self, **kwargs):
     @immutable
     def share(self):
         self._share = True
+        return self

fireant/dataset/klass.py

@@ -1,11 +1,11 @@
 import itertools
 
+from fireant.utils import immutable
 from fireant.queries.builder import (
     DataSetQueryBuilder,
     DimensionChoicesQueryBuilder,
     DimensionLatestQueryBuilder,
 )
-from fireant.utils import immutable
 
 
 class _Container(object):
@@ -29,6 +29,14 @@ def __init__(self, items=()):
         for item in items:
             setattr(self, item.alias, item)
 
+    def __copy__(self):
+        return type(self)(self._items)
+
+    def __deepcopy__(self, memodict):
+        for field in self:
+            memodict[id(field)] = field
+        return type(self)(self._items)
+
     def __iter__(self):
         return iter(self._items)
 
@@ -45,10 +53,7 @@ def __contains__(self, item):
             return False
 
         self_item = getattr(self, item.alias, None)
-        if self_item is None:
-            return False
-
-        return str(item.definition) == str(self_item.definition)
+        return item is self_item
 
     def __hash__(self):
         return hash((item for item in self._items))
@@ -71,7 +76,7 @@ def append(self, item):
         setattr(self, item.alias, item)
 
 
-class DataSet(object):
+class DataSet:
     """
     The DataSet class abstracts the query generation, given the fields and what not that were provided, and the
     fetching of the aforementioned query's data.

fireant/dataset/modifiers.py

@@ -1,3 +1,5 @@
+from copy import deepcopy
+
 from fireant.utils import immutable
 from pypika import NullValue
 
@@ -9,6 +11,20 @@ def __init__(self, wrapped):
         wrapped_key = super().__getattribute__("wrapped_key")
         setattr(self, wrapped_key, wrapped)
 
+    def __copy__(self):
+        cls = self.__class__
+        result = cls.__new__(cls)
+        result.__dict__.update(self.__dict__)
+        return result
+
+    def __deepcopy__(self, memo):
+        cls = self.__class__
+        result = cls.__new__(cls)
+        memo[id(self)] = result
+        for k, v in self.__dict__.items():
+            setattr(result, k, deepcopy(v, memo))
+        return result
+
     def __getattr__(self, attr):
         if attr in self.__dict__:
             return super().__getattribute__(attr)
@@ -41,7 +57,7 @@ def __repr__(self):
 
     @immutable
     def for_(self, wrapped):
-        wrapped_key = super().__getattribute__("wrapped_key")
+        wrapped_key = super(type(self), self).__getattribute__("wrapped_key")
         setattr(self, wrapped_key, wrapped)
 
 

fireant/dataset/references.py

@@ -36,7 +36,7 @@ def __init__(self, field, reference_type, delta=False, delta_percent=False):
     def __eq__(self, other):
         return (
             isinstance(self, Reference)
-            and self.field == other.field
+            and self.field is other.field
             and self.alias == other.alias
         )
 

fireant/queries/builder/dataset_blender_query_builder.py

@@ -11,6 +11,7 @@
 from fireant.queries.finders import (
     find_dataset_metrics,
     find_metrics_for_widgets,
+    find_field_in_modified_field,
 )
 from fireant.reference_helpers import reference_alias
 from fireant.utils import alias_selector
@@ -38,9 +39,10 @@ def _flatten_blend_datasets(dataset) -> List:
 
 
 def _replace_field(dimension, field_map, omit_umapped=False):
-    if hasattr(dimension, "dimension"):
+    root_dimension = find_field_in_modified_field(dimension)
+    if root_dimension is not dimension:
         # Handle modified dimensions
-        wrapped_dimension = _replace_field(dimension.dimension, field_map)
+        wrapped_dimension = _replace_field(root_dimension, field_map)
         return dimension.for_(wrapped_dimension)
 
     if field_map is None:

fireant/queries/builder/dataset_query_builder.py

@@ -21,7 +21,6 @@
 from ..field_helper import initialize_orders
 from ..finders import (
     find_and_group_references_for_dimensions,
-    find_and_replace_reference_dimensions,
     find_metrics_for_widgets,
     find_operations_for_widgets,
     find_share_dimensions,
@@ -87,9 +86,6 @@ def sql(self):
         metrics = find_metrics_for_widgets(self._widgets)
         operations = find_operations_for_widgets(self._widgets)
         share_dimensions = find_share_dimensions(self._dimensions, operations)
-        references = find_and_replace_reference_dimensions(
-            self._references, self._dimensions
-        )
         orders = initialize_orders(self._orders, self._dimensions)
 
         return make_slicer_query_with_totals_and_references(
@@ -101,7 +97,7 @@ def sql(self):
             metrics,
             operations,
             self._filters,
-            references,
+            self._references,
             orders,
             share_dimensions=share_dimensions,
         )

fireant/queries/builder/query_builder.py

@@ -1,11 +1,9 @@
 from fireant.dataset.fields import Field
 from fireant.exceptions import DataSetException
-from fireant.utils import (
-    alias_selector,
-    immutable,
-)
+from fireant.utils import immutable
 from pypika import Order
 from ..execution import fetch_data
+from ..finders import find_fields_in_modified_fields
 
 
 class QueryException(DataSetException):
@@ -29,6 +27,28 @@ def get_column_names(database, table):
     return {column_definition[0] for column_definition in column_definitions}
 
 
+def _strip_modifiers(fields):
+    for field in fields:
+        next = field
+        while hasattr(next, "dimension"):
+            next = next.dimension
+        yield next
+
+
+def _validate_fields(fields, dataset):
+    fields = find_fields_in_modified_fields(fields)
+    invalid = [field.alias for field in fields if field not in dataset.fields]
+
+    if not invalid:
+        return
+
+    raise DataSetException(
+        "Only fields from dataset can be used in a dataset query. Found invalid fields: {}.".format(
+            ", ".join(invalid)
+        )
+    )
+
+
 class QueryBuilder(object):
     """
     This is the base class for building dataset queries. This class provides an interface for building dataset queries
@@ -54,6 +74,7 @@ def dimension(self, *dimensions):
         :return:
             A copy of the query with the dimensions added.
         """
+        _validate_fields(dimensions, self.dataset)
         aliases = {dimension.alias for dimension in self._dimensions}
         self._dimensions += [
             dimension for dimension in dimensions if dimension.alias not in aliases
@@ -69,6 +90,7 @@ def filter(self, *filters):
         :return:
             A copy of the query with the filters added.
         """
+        _validate_fields([fltr.field for fltr in filters], self.dataset)
         self._filters += [f for f in filters]
 
     @immutable
@@ -81,6 +103,8 @@ def orderby(self, field: Field, orientation: Order = None):
         :return:
             A copy of the query with the order by added.
         """
+        _validate_fields([field], self.dataset)
+
         if self._orders is None:
             self._orders = []
 
@@ -131,15 +155,15 @@ def fetch(self, hint=None):
         Fetches the data for this query instance and returns it in an instance of `pd.DataFrame`
 
         :param hint:
-            For database vendors that support it, add a query hint to collect analytics on the queries triggerd by
+            For database vendors that support it, add a query hint to collect analytics on the queries triggered by
             fireant.
         """
         queries = add_hints(self.sql, hint)
 
         return fetch_data(self.dataset.database, queries, self._dimensions)
 
 
-class ReferenceQueryBuilderMixin(object):
+class ReferenceQueryBuilderMixin:
     """
     This is a mixin class for building dataset queries that allow references. This class provides an interface for
     building dataset queries via a set of functions which can be chained together.
@@ -159,10 +183,11 @@ def reference(self, *references):
         :return:
             A copy of the query with the references added.
         """
+        _validate_fields([reference.field for reference in references], self.dataset)
         self._references += references
 
 
-class WidgetQueryBuilderMixin(object):
+class WidgetQueryBuilderMixin:
     """
     This is a mixin class for building dataset queries that allow widgets. This class provides an interface for
     building dataset queries via a set of functions which can be chained together.
@@ -187,4 +212,8 @@ def widget(self, *widgets):
         :return:
             A copy of the query with the widgets added.
         """
+        _validate_fields(
+            [field for widget in widgets for field in widget.metrics], self.dataset
+        )
+
         self._widgets += widgets

fireant/queries/finders.py

@@ -1,4 +1,3 @@
-import copy
 from collections import (
     defaultdict,
     namedtuple,
@@ -186,24 +185,18 @@ def find_filters_for_totals(filters):
     return [fltr for fltr in filters if not isinstance(fltr, OmitFromRollup)]
 
 
-def find_and_replace_reference_dimensions(references, dimensions):
+def find_field_in_modified_field(field):
     """
-    Finds the dimension for a reference in the query if there is one and replaces it. This is to force the reference to
-    use the same modifiers with a dimension if it is selected in the query.
-
-    :param references:
-    :param dimensions:
-    :return:
+    Returns the field from a modified field argument (or just the field argument if it is not modified).
     """
-    dimensions_by_key = {dimension.alias: dimension for dimension in dimensions}
+    root = field
+    while hasattr(root, "dimension"):
+        root = root.dimension
+    return root
+
 
-    reference_copies = []
-    for reference in map(copy.deepcopy, references):
-        dimension = dimensions_by_key.get(reference.field.alias)
-        if dimension is not None:
-            reference.field = dimension
-        reference_copies.append(reference)
-    return reference_copies
+def find_fields_in_modified_fields(fields):
+    return [find_field_in_modified_field(field) for field in fields]
 
 
 interval_weekdays = {