From 2430f7939c796e28fa6b171ab87fab19daf5b294 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 19 Jan 2018 10:26:59 +0100 Subject: [PATCH 001/123] Initial commit of 1.0 --- .travis.yml | 1 - fireant/__init__.py | 15 +- fireant/dashboards/__init__.py | 17 - fireant/dashboards/managers.py | 76 - fireant/dashboards/schemas.py | 117 -- fireant/database/__init__.py | 9 +- fireant/database/{database.py => base.py} | 12 +- fireant/database/mysql.py | 17 +- fireant/database/postgresql.py | 13 +- fireant/database/redshift.py | 5 +- fireant/database/vertica.py | 48 +- fireant/settings.py | 1 - fireant/slicer/__init__.py | 57 +- fireant/slicer/base.py | 28 + fireant/slicer/dimensions.py | 125 ++ fireant/slicer/exceptions.py | 13 + fireant/slicer/filters.py | 134 +- fireant/slicer/intervals.py | 31 + fireant/slicer/joins.py | 12 + fireant/slicer/managers.py | 497 ----- fireant/slicer/metrics.py | 32 + fireant/slicer/operations.py | 86 +- fireant/slicer/pagination.py | 19 - fireant/slicer/postprocessors.py | 67 - fireant/slicer/queries/__init__.py | 1 + fireant/slicer/queries/builder.py | 224 +++ fireant/slicer/queries/database.py | 24 + fireant/slicer/queries/logger.py | 3 + fireant/slicer/queries/references.py | 140 ++ fireant/slicer/references.py | 105 +- fireant/slicer/schemas.py | 220 -- fireant/slicer/slicers.py | 62 + fireant/slicer/transformers/__init__.py | 60 - fireant/slicer/transformers/base.py | 15 - fireant/slicer/transformers/datatables.py | 398 ---- fireant/slicer/transformers/highcharts.py | 601 ------ fireant/slicer/transformers/notebooks.py | 128 -- fireant/slicer/widgets/__init__.py | 4 + fireant/slicer/widgets/base.py | 19 + fireant/slicer/widgets/datatables.py | 169 ++ fireant/slicer/widgets/formats.py | 57 + fireant/slicer/widgets/highcharts.py | 34 + fireant/slicer/widgets/matplotlib.py | 5 + fireant/slicer/widgets/pandas.py | 5 + .../tests/dashboards/test_dashboard_api.py | 686 ------- fireant/tests/dashboards/test_widgets.py | 57 - fireant/tests/database/__init__.py | 1 - fireant/tests/database/mock_database.py | 1 - fireant/tests/database/test_databases.py | 1 - fireant/tests/database/test_mysql.py | 48 +- fireant/tests/database/test_postgresql.py | 14 +- fireant/tests/database/test_redshift.py | 2 - fireant/tests/database/test_vertica.py | 30 +- fireant/tests/mock_dataframes.py | 378 ---- fireant/tests/slicer/mocks.py | 226 +++ .../__init__.py => slicer/test_fetch_data.py} | 0 fireant/tests/slicer/test_filters.py | 172 -- fireant/tests/slicer/test_managers.py | 510 ----- fireant/tests/slicer/test_operations.py | 116 -- fireant/tests/slicer/test_pagination.py | 19 - fireant/tests/slicer/test_postprocessors.py | 195 -- fireant/tests/slicer/test_queries.py | 1770 ----------------- fireant/tests/slicer/test_querybuilder.py | 1161 +++++++++++ fireant/tests/slicer/test_slicer_api.py | 1263 ------------ fireant/tests/slicer/transformers/__init__.py | 1 - fireant/tests/slicer/transformers/base.py | 14 - fireant/tests/slicer/transformers/test_csv.py | 262 --- .../slicer/transformers/test_datatables.py | 1120 ----------- .../slicer/transformers/test_highcharts.py | 715 ------- .../slicer/transformers/test_notebooks.py | 480 ----- fireant/tests/slicer/widgets/__init__.py | 0 .../tests/slicer/widgets/test_datatables.py | 480 +++++ fireant/tests/slicer/widgets/test_formats.py | 78 + fireant/utils.py | 48 +- requirements.txt | 12 +- setup.py | 4 +- tox.ini | 2 +- 77 files changed, 3217 insertions(+), 10355 deletions(-) delete mode 100644 fireant/dashboards/__init__.py delete mode 100644 fireant/dashboards/managers.py delete mode 100644 fireant/dashboards/schemas.py rename fireant/database/{database.py => base.py} (82%) create mode 100644 fireant/slicer/base.py create mode 100644 fireant/slicer/dimensions.py create mode 100644 fireant/slicer/exceptions.py create mode 100644 fireant/slicer/intervals.py create mode 100644 fireant/slicer/joins.py delete mode 100644 fireant/slicer/managers.py create mode 100644 fireant/slicer/metrics.py delete mode 100644 fireant/slicer/pagination.py delete mode 100644 fireant/slicer/postprocessors.py create mode 100644 fireant/slicer/queries/__init__.py create mode 100644 fireant/slicer/queries/builder.py create mode 100644 fireant/slicer/queries/database.py create mode 100644 fireant/slicer/queries/logger.py create mode 100644 fireant/slicer/queries/references.py delete mode 100644 fireant/slicer/schemas.py create mode 100644 fireant/slicer/slicers.py delete mode 100644 fireant/slicer/transformers/__init__.py delete mode 100644 fireant/slicer/transformers/base.py delete mode 100644 fireant/slicer/transformers/datatables.py delete mode 100644 fireant/slicer/transformers/highcharts.py delete mode 100644 fireant/slicer/transformers/notebooks.py create mode 100644 fireant/slicer/widgets/__init__.py create mode 100644 fireant/slicer/widgets/base.py create mode 100644 fireant/slicer/widgets/datatables.py create mode 100644 fireant/slicer/widgets/formats.py create mode 100644 fireant/slicer/widgets/highcharts.py create mode 100644 fireant/slicer/widgets/matplotlib.py create mode 100644 fireant/slicer/widgets/pandas.py delete mode 100644 fireant/tests/dashboards/test_dashboard_api.py delete mode 100644 fireant/tests/dashboards/test_widgets.py delete mode 100644 fireant/tests/mock_dataframes.py create mode 100644 fireant/tests/slicer/mocks.py rename fireant/tests/{dashboards/__init__.py => slicer/test_fetch_data.py} (100%) delete mode 100644 fireant/tests/slicer/test_filters.py delete mode 100644 fireant/tests/slicer/test_managers.py delete mode 100644 fireant/tests/slicer/test_operations.py delete mode 100644 fireant/tests/slicer/test_pagination.py delete mode 100644 fireant/tests/slicer/test_postprocessors.py delete mode 100644 fireant/tests/slicer/test_queries.py create mode 100644 fireant/tests/slicer/test_querybuilder.py delete mode 100644 fireant/tests/slicer/test_slicer_api.py delete mode 100644 fireant/tests/slicer/transformers/__init__.py delete mode 100644 fireant/tests/slicer/transformers/base.py delete mode 100644 fireant/tests/slicer/transformers/test_csv.py delete mode 100644 fireant/tests/slicer/transformers/test_datatables.py delete mode 100644 fireant/tests/slicer/transformers/test_highcharts.py delete mode 100644 fireant/tests/slicer/transformers/test_notebooks.py create mode 100644 fireant/tests/slicer/widgets/__init__.py create mode 100644 fireant/tests/slicer/widgets/test_datatables.py create mode 100644 fireant/tests/slicer/widgets/test_formats.py diff --git a/.travis.yml b/.travis.yml index 2265071d..8fadcf99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python python: - - "2.7" - "3.3" - "3.4" - "3.5" diff --git a/fireant/__init__.py b/fireant/__init__.py index 662acbea..c261f00a 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -1,2 +1,13 @@ -# coding: utf-8 -__version__ = '{major}.{minor}.{patch}'.format(major=0, minor=19, patch=3) +# noinspection PyUnresolvedReferences +from .database import * +# noinspection PyUnresolvedReferences +from .slicer import * +# noinspection PyUnresolvedReferences +from .slicer.widgets import ( + DataTablesJS, + HighCharts, + Matplotlib, + Pandas, +) + +__version__ = '{major}.{minor}.{patch}'.format(major=1, minor=0, patch=0) diff --git a/fireant/dashboards/__init__.py b/fireant/dashboards/__init__.py deleted file mode 100644 index 1f9de589..00000000 --- a/fireant/dashboards/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# coding: utf-8 - -from .schemas import ( - AreaChartWidget, - AreaPercentageChartWidget, - BarChartWidget, - ColumnChartWidget, - ColumnIndexCSVWidget, - ColumnIndexTableWidget, - LineChartWidget, - PieChartWidget, - RowIndexCSVWidget, - RowIndexTableWidget, - StackedBarChartWidget, - StackedColumnChartWidget, - WidgetGroup, -) diff --git a/fireant/dashboards/managers.py b/fireant/dashboards/managers.py deleted file mode 100644 index dfd22103..00000000 --- a/fireant/dashboards/managers.py +++ /dev/null @@ -1,76 +0,0 @@ -# coding: utf-8 -import pandas as pd - -from fireant import utils -from fireant.slicer.managers import Totals - - -class WidgetGroupManager(object): - def __init__(self, widget_group): - self.widget_group = widget_group - - def _schema(self, dimensions=None, metric_filters=None, dimension_filters=None, - references=None, operations=None, pagination=None): - return dict( - metrics=[metric - for widget in self.widget_group.widgets - for metric in utils.flatten(widget.metrics)], - dimensions=dimensions, - metric_filters=self.widget_group.metric_filters + (metric_filters or []), - dimension_filters=self.widget_group.dimension_filters + (dimension_filters or []), - references=references, - operations=operations, - pagination=pagination - ) - - def render(self, dimensions=None, metric_filters=None, dimension_filters=None, - references=None, operations=None, pagination=None): - dimensions = utils.filter_duplicates(self.widget_group.dimensions + (dimensions or [])) - references = utils.filter_duplicates(self.widget_group.references + (references or [])) - operations = utils.filter_duplicates(self.widget_group.operations + (operations or [])) - - for widget in self.widget_group.widgets: - widget.transformer.prevalidate_request(self.widget_group.slicer, widget.metrics, dimensions, - metric_filters, dimension_filters, references, operations) - - schema = self._schema(dimensions, metric_filters, dimension_filters, references, operations, pagination) - dataframe = self.widget_group.slicer.manager.data(**schema) - - _args = [dataframe, schema['dimensions'], schema['references'], schema['operations']] - return [self._transform_widget(widget, *_args) - for widget in self.widget_group.widgets] - - def query_string(self, dimensions=None, metric_filters=None, dimension_filters=None, - references=None, operations=None, pagination=None): - dimensions = utils.filter_duplicates(self.widget_group.dimensions + (dimensions or [])) - references = utils.filter_duplicates(self.widget_group.references + (references or [])) - operations = utils.filter_duplicates(self.widget_group.operations + (operations or [])) - - schema = self._schema(dimensions, metric_filters, dimension_filters, references, operations, pagination) - return self.widget_group.slicer.manager.query_string(**schema) - - def _transform_widget(self, widget, dataframe, dimensions, references, operations): - display_schema = self.widget_group.slicer.manager.display_schema( - metrics=widget.metrics, - dimensions=dimensions, - references=references, - operations=operations, - ) - - # Temporary fix to enable operations to get output properly. Can removed when the Fireant API is refactored. - operation_columns = ['{}_{}'.format(operation.metric_key, operation.key) - for operation in operations if operation.key != Totals.key] - - columns = utils.flatten(widget.metrics) + operation_columns - - if references: - # This escapes a pandas bug where a data frame subset of columns still returns the columns of the - # original data frame - reference_keys = [''] + [ref.key for ref in references] - subset_columns = pd.MultiIndex.from_product([reference_keys, columns]) - subset = pd.DataFrame(dataframe[subset_columns], columns=subset_columns) - - else: - subset = dataframe[columns] - - return widget.transformer.transform(subset, display_schema) diff --git a/fireant/dashboards/schemas.py b/fireant/dashboards/schemas.py deleted file mode 100644 index 37b40968..00000000 --- a/fireant/dashboards/schemas.py +++ /dev/null @@ -1,117 +0,0 @@ -# coding: utf-8 -from fireant.dashboards.managers import WidgetGroupManager - -from fireant.slicer.transformers import * - - -class Widget(object): - """ - The `Widget` class represents a chart or table widget in one of the dashboard's sections. - - Attributes: - * label: human readable text. - * metrics: list of metric names/identifiers - """ - - def __init__(self, metrics=tuple()): - self.metrics = metrics - - -class LineChartWidget(Widget): - """ - The `LineChartWidget` class represents a Highcharts line chart. - """ - transformer = HighchartsLineTransformer() - - -class AreaChartWidget(Widget): - """ - The `AreaChartWidget` class represents a Highcharts area chart. - """ - transformer = HighchartsAreaTransformer() - - -class AreaPercentageChartWidget(Widget): - """ - The `AreaPercentageChartWidget` class represents a Highcharts area percentage chart. - """ - transformer = HighchartsAreaPercentageTransformer() - - -class BarChartWidget(Widget): - """ - The `BarChartWidget` class represents a Highcharts bar chart. - """ - transformer = HighchartsBarTransformer() - - -class StackedBarChartWidget(Widget): - """ - The `StackedBarChartWidget` class represents a Highcharts stacked bar chart. - """ - transformer = HighchartsStackedBarTransformer() - - -class ColumnChartWidget(Widget): - """ - The `ColumnChartWidget` class represents a Highcharts column chart. - """ - transformer = HighchartsColumnTransformer() - - -class StackedColumnChartWidget(Widget): - """ - The `StackedColumnChartWidget` class represents a Highcharts stacked column chart. - """ - transformer = HighchartsStackedColumnTransformer() - - -class PieChartWidget(Widget): - """ - The `PieChartWidget` class represents a Highcharts Pie chart. - """ - transformer = HighchartsPieTransformer() - - -class RowIndexTableWidget(Widget): - """ - The `RowIndexTableWidget` class represents datatables.js data table with row-indexed dimensions. - """ - transformer = DataTablesRowIndexTransformer() - - -class ColumnIndexTableWidget(Widget): - """ - The `ColumnIndexTableWidget` class represents datatables.js data table with column-indexed dimensions. - """ - transformer = DataTablesColumnIndexTransformer() - - -class RowIndexCSVWidget(Widget): - """ - The `RowIndexCSVWidget` class outputs data as a CSV string with row-indexed dimensions. - """ - transformer = CSVRowIndexTransformer() - - -class ColumnIndexCSVWidget(Widget): - """ - The `ColumnIndexCSVWidget` class outputs data as a CSV string with column-indexed dimensions. - """ - transformer = CSVColumnIndexTransformer() - - -class WidgetGroup(object): - def __init__(self, slicer, widgets=None, dimensions=None, - metric_filters=None, dimension_filters=None, - references=None, operations=None): - self.slicer = slicer - self.widgets = widgets - - self.dimensions = dimensions or [] - self.metric_filters = metric_filters or [] - self.dimension_filters = dimension_filters or [] - self.references = references or [] - self.operations = operations or [] - - self.manager = WidgetGroupManager(self) diff --git a/fireant/database/__init__.py b/fireant/database/__init__.py index 8fe12a15..cbcd0712 100644 --- a/fireant/database/__init__.py +++ b/fireant/database/__init__.py @@ -1,8 +1,5 @@ -# coding: utf-8 - -from .database import Database +from .base import Database from .mysql import MySQLDatabase -from .redshift import RedshiftDatabase from .postgresql import PostgreSQLDatabase -from .vertica import (Vertica, - VerticaDatabase) +from .redshift import RedshiftDatabase +from .vertica import VerticaDatabase diff --git a/fireant/database/database.py b/fireant/database/base.py similarity index 82% rename from fireant/database/database.py rename to fireant/database/base.py index 59c6ad4c..d0334818 100644 --- a/fireant/database/database.py +++ b/fireant/database/base.py @@ -1,11 +1,15 @@ -# coding: utf-8 - import pandas as pd -from pypika import Query +from pypika import ( + Query, + terms, +) class Database(object): + """ + WRITEME + """ # The pypika query class to use for constructing queries query_cls = Query @@ -15,7 +19,7 @@ def connect(self): def trunc_date(self, field, interval): raise NotImplementedError - def date_add(self, date_part, interval, field): + def date_add(self, field: terms.Term, date_part: str, interval: int): """ Database specific function for adding or subtracting dates """ raise NotImplementedError diff --git a/fireant/database/mysql.py b/fireant/database/mysql.py index e3baeced..d0b06bc3 100644 --- a/fireant/database/mysql.py +++ b/fireant/database/mysql.py @@ -1,12 +1,11 @@ -# coding: utf-8 import pandas as pd + from pypika import ( Dialects, MySQLQuery, terms, ) - -from fireant.database import Database +from .base import Database class Trunc(terms.Function): @@ -51,12 +50,10 @@ def __init__(self, database=None, host='localhost', port=3306, def connect(self): import pymysql - return pymysql.connect( - host=self.host, port=self.port, db=self.database, - user=self.user, password=self.password, - charset=self.charset, - cursorclass=pymysql.cursors.Cursor, - ) + return pymysql.connect(host=self.host, port=self.port, db=self.database, + user=self.user, password=self.password, + charset=self.charset, + cursorclass=pymysql.cursors.Cursor) def fetch(self, query): with self.connect().cursor() as cursor: @@ -69,7 +66,7 @@ def fetch_dataframe(self, query): def trunc_date(self, field, interval): return Trunc(field, interval) - def date_add(self, date_part, interval, field): + def date_add(self, field, date_part, interval): # adding an extra 's' as MySQL's interval doesn't work with 'year', 'week' etc, it expects a plural interval_term = terms.Interval(**{'{}s'.format(str(date_part)): interval, 'dialect': Dialects.MYSQL}) return DateAdd(field, interval_term) diff --git a/fireant/database/postgresql.py b/fireant/database/postgresql.py index 6137f6bb..9027974f 100644 --- a/fireant/database/postgresql.py +++ b/fireant/database/postgresql.py @@ -1,12 +1,11 @@ -# coding: utf-8 import pandas as pd + from pypika import ( PostgreSQLQuery, functions as fn, terms, ) - -from fireant.database import Database +from .base import Database class Trunc(terms.Function): @@ -40,10 +39,8 @@ def __init__(self, database=None, host='localhost', port=5432, def connect(self): import psycopg2 - return psycopg2.connect( - host=self.host, port=self.port, dbname=self.database, - user=self.user, password=self.password, - ) + return psycopg2.connect(host=self.host, port=self.port, dbname=self.database, + user=self.user, password=self.password) def fetch(self, query): with self.connect().cursor() as cursor: @@ -56,5 +53,5 @@ def fetch_dataframe(self, query): def trunc_date(self, field, interval): return Trunc(field, interval) - def date_add(self, date_part, interval, field): + def date_add(self, field, date_part, interval): return fn.DateAdd(date_part, interval, field) diff --git a/fireant/database/redshift.py b/fireant/database/redshift.py index 187bd3e6..65e4d9cf 100644 --- a/fireant/database/redshift.py +++ b/fireant/database/redshift.py @@ -1,7 +1,4 @@ -# coding: utf-8 -from pypika import ( - RedshiftQuery, -) +from pypika import RedshiftQuery from .postgresql import PostgreSQLDatabase diff --git a/fireant/database/vertica.py b/fireant/database/vertica.py index 015cd928..419c7bf6 100644 --- a/fireant/database/vertica.py +++ b/fireant/database/vertica.py @@ -1,12 +1,17 @@ -# coding: utf-8 +from fireant.slicer import ( + annually, + daily, + hourly, + monthly, + quarterly, + weekly, +) from pypika import ( VerticaQuery, functions as fn, terms, ) - -from fireant.database import Database -from fireant.slicer import DatetimeInterval +from .base import Database class Trunc(terms.Function): @@ -30,16 +35,15 @@ class VerticaDatabase(Database): query_cls = VerticaQuery DATETIME_INTERVALS = { - 'hour': DatetimeInterval('HH'), - 'day': DatetimeInterval('DD'), - 'week': DatetimeInterval('IW'), - 'month': DatetimeInterval('MM'), - 'quarter': DatetimeInterval('Q'), - 'year': DatetimeInterval('Y') + hourly: 'HH', + daily: 'DD', + weekly: 'IW', + monthly: 'MM', + quarterly: 'Q', + annually: 'Y' } - def __init__(self, host='localhost', port=5433, database='vertica', - user='vertica', password=None, + def __init__(self, host='localhost', port=5433, database='vertica', user='vertica', password=None, read_timeout=None): self.host = host self.port = port @@ -51,21 +55,13 @@ def __init__(self, host='localhost', port=5433, database='vertica', def connect(self): import vertica_python - return vertica_python.connect( - host=self.host, port=self.port, database=self.database, - user=self.user, password=self.password, - read_timeout=self.read_timeout, - ) + return vertica_python.connect(host=self.host, port=self.port, database=self.database, + user=self.user, password=self.password, + read_timeout=self.read_timeout) def trunc_date(self, field, interval): - interval = self.DATETIME_INTERVALS[interval] - return Trunc(field, interval.size) + trunc_date_interval = self.DATETIME_INTERVALS.get(interval, 'DD') + return Trunc(field, trunc_date_interval) - def date_add(self, date_part, interval, field): + def date_add(self, field, date_part, interval): return fn.TimestampAdd(date_part, interval, field) - - -def Vertica(*args, **kwargs): - from warnings import warn - warn('The Vertica class is now deprecated. Please use VerticaDatabase instead!') - return VerticaDatabase(*args, **kwargs) diff --git a/fireant/settings.py b/fireant/settings.py index a05ba7fc..289437b7 100644 --- a/fireant/settings.py +++ b/fireant/settings.py @@ -1,4 +1,3 @@ -# coding: utf-8 highcharts_colors = 'kayak' matplotlib_figsize = (14, 5) diff --git a/fireant/slicer/__init__.py b/fireant/slicer/__init__.py index 1f1c56df..e97e3b7f 100644 --- a/fireant/slicer/__init__.py +++ b/fireant/slicer/__init__.py @@ -1,29 +1,40 @@ -# coding: utf-8 -from .managers import SlicerException -from .filters import ( - BooleanFilter, - ContainsFilter, - EqualityFilter, - ExcludesFilter, - RangeFilter, - WildcardFilter, -) -from .pagination import Paginator -from .schemas import ( - DimensionValue, - EqualityOperator, - Join, - Metric, - Slicer, -) -from .schemas import ( - DatetimeInterval, - NumericInterval, -) -from .schemas import ( +from .dimensions import ( BooleanDimension, CategoricalDimension, ContinuousDimension, DatetimeDimension, + Dimension, + DimensionValue, UniqueDimension, ) +from .exceptions import ( + QueryException, + SlicerException, +) +from .intervals import ( + DatetimeInterval, + NumericInterval, + annually, + daily, + hourly, + monthly, + quarterly, + weekly, +) +from .joins import Join +from .metrics import Metric +from .operations import ( + CumAvg, + CumSum, + L1Loss, + L2Loss, +) +from .references import ( + DayOverDay, + MonthOverMonth, + QuarterOverQuarter, + Reference, + WeekOverWeek, + YearOverYear, +) +from .slicers import Slicer diff --git a/fireant/slicer/base.py b/fireant/slicer/base.py new file mode 100644 index 00000000..6f99f693 --- /dev/null +++ b/fireant/slicer/base.py @@ -0,0 +1,28 @@ +class SlicerElement(object): + """ + The `SlicerElement` class represents an element of the slicer, either a metric or dimension, which contains + information about such as how to query it from the database. + """ + def __init__(self, key, label=None, definition=None): + """ + :param key: + The unique identifier of the slicer element, used in the Slicer manager API to reference a defined element. + + :param label: + A displayable representation of the column. Defaults to the key capitalized. + + :param definition: + The definition of the element as a PyPika expression which defines how to query it from the database. + """ + self.key = key + self.label = label or key + self.definition = definition + + def __unicode__(self): + return self.key + + def __repr__(self): + return self.key + + def __str__(self): + return self.key diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py new file mode 100644 index 00000000..fd1a6eaa --- /dev/null +++ b/fireant/slicer/dimensions.py @@ -0,0 +1,125 @@ +from fireant.utils import immutable +from .base import SlicerElement +from .exceptions import QueryException +from .filters import ( + BooleanFilter, + ContainsFilter, + ExcludesFilter, + RangeFilter, + WildcardFilter, +) +from .intervals import ( + NumericInterval, + daily, +) + + +class Dimension(SlicerElement): + """ + The `Dimension` class represents a dimension in the `Slicer` object. + """ + + def __init__(self, key, label=None, definition=None): + super(Dimension, self).__init__(key, label, definition) + + +class BooleanDimension(Dimension): + """ + This is a dimension that represents a boolean true/false value. The expression should always result in a boolean + value. + """ + + def __init__(self, key, label=None, definition=None): + super(BooleanDimension, self).__init__(key=key, label=label, definition=definition) + + def is_(self, value): + return BooleanFilter(self.definition, value) + + +class CategoricalDimension(Dimension): + """ + This is a dimension that represents an enum-like database field, with a finite list of options to chose from. It + provides support for configuring a display value for each of the possible values. + """ + + def __init__(self, key, label=None, definition=None, display_values=()): + super(CategoricalDimension, self).__init__(key=key, label=label, definition=definition) + self.display_values = dict(display_values) + + def isin(self, values): + return ContainsFilter(self.definition, values) + + def notin(self, values): + return ExcludesFilter(self.definition, values) + + +class UniqueDimension(Dimension): + """ + This is a dimension that represents a field in a database which is a unique identifier, such as a primary/foreign + key. It provides support for a display value field which is selected and used in the results. + """ + + def __init__(self, key, label=None, definition=None, display_definition=None): + super(UniqueDimension, self).__init__(key=key, label=label, definition=definition) + self.display_key = '{}_display'.format(key) + self.display_definition = display_definition + + def isin(self, values, use_display=False): + if use_display and self.display_definition is None: + raise QueryException('No value set for display_definition.') + filter_field = self.display_definition if use_display else self.definition + return ContainsFilter(filter_field, values) + + def notin(self, values, use_display=False): + if use_display and self.display_definition is None: + raise QueryException('No value set for display_definition.') + filter_field = self.display_definition if use_display else self.definition + return ExcludesFilter(filter_field, values) + + def wildcard(self, pattern): + if self.display_definition is None: + raise QueryException('No value set for display_definition.') + return WildcardFilter(self.display_definition, pattern) + + +class ContinuousDimension(Dimension): + """ + This is a dimension that represents a field in the database which is a continuous value, such as a decimal, integer, + or date/time. It requires the use of an interval which is the window over the values. + """ + + def __init__(self, key, label=None, definition=None, default_interval=NumericInterval(1, 0)): + super(ContinuousDimension, self).__init__(key=key, label=label, definition=definition) + self.interval = default_interval + + +class DatetimeDimension(ContinuousDimension): + """ + WRITEME + """ + + def __init__(self, key, label=None, definition=None, default_interval=daily): + super(DatetimeDimension, self).__init__(key=key, label=label, definition=definition, + default_interval=default_interval) + self.references = [] + + @immutable + def __call__(self, interval): + self.interval = interval + + @immutable + def reference(self, reference): + self.references.append(reference) + + def between(self, start, stop): + return RangeFilter(self.definition, start, stop) + + +class DimensionValue(object): + """ + An option belongs to a categorical dimension which specifies a fixed set of values + """ + + def __init__(self, key, label=None): + self.key = key + self.label = label or key diff --git a/fireant/slicer/exceptions.py b/fireant/slicer/exceptions.py new file mode 100644 index 00000000..32a03ffc --- /dev/null +++ b/fireant/slicer/exceptions.py @@ -0,0 +1,13 @@ +class SlicerException(Exception): + pass + + +class QueryException(SlicerException): + pass + + +class MissingTableJoinException(SlicerException): + pass + +class CircularJoinsException(SlicerException): + pass diff --git a/fireant/slicer/filters.py b/fireant/slicer/filters.py index d3cc3c12..a3d52172 100644 --- a/fireant/slicer/filters.py +++ b/fireant/slicer/filters.py @@ -1,115 +1,71 @@ -# coding: utf8 -from fireant import utils +from pypika import Not class Filter(object): - def __init__(self, element_key): - self.element_key = element_key - - -class EqualityFilter(Filter): - def __init__(self, element_key, operator, value): - super(EqualityFilter, self).__init__(element_key) - self.operator = operator - self.value = value - - def schemas(self, elements, **kwargs): - elements = utils.wrap_list(elements) - value = utils.wrap_list(self.value) - - criterion = None - for element, value in zip(elements, value): - crit = getattr(element, self.operator)(value) - if criterion: - criterion = criterion & crit - else: - criterion = crit - - return criterion + def __init__(self, definition): + self.definition = definition def __eq__(self, other): return isinstance(other, self.__class__) \ - and self.element_key == other.element_key \ - and self.operator == other.operator \ - and self.value == other.value + and str(self.definition) == str(other.definition) + def __str__(self): + return str(self.definition) -class BooleanFilter(Filter): - def __init__(self, element_key, value): - super(BooleanFilter, self).__init__(element_key) - self.value = value + def __repr__(self): + return str(self.definition) - def schemas(self, element): - if not self.value: - return element.negate() - return element + def __unicode__(self): + return str(self.definition) -class ContainsFilter(Filter): - def __init__(self, element_key, values): - super(ContainsFilter, self).__init__(element_key) - self.values = values +class DimensionFilter(Filter): + pass - def schemas(self, element): - return element.isin(self.values) - def __eq__(self, other): - return isinstance(other, self.__class__) \ - and self.element_key == other.element_key \ - and self.values == other.values +class MetricFilter(Filter): + pass -class ExcludesFilter(Filter): - def __init__(self, element_key, values): - super(ExcludesFilter, self).__init__(element_key) - self.values = values +class ComparatorFilter(MetricFilter): + class Operator(object): + eq = 'eq' + ne = 'ne' + gt = 'gt' + lt = 'lt' + gte = 'gte' + lte = 'lte' - def schemas(self, element): - return element.notin(self.values) - - def __eq__(self, other): - return isinstance(other, self.__class__) \ - and self.element_key == other.element_key \ - and self.values == other.values + def __init__(self, metric_definition, operator, value): + definition = getattr(metric_definition, operator)(value) + super(ComparatorFilter, self).__init__(definition) -class RangeFilter(Filter): - def __init__(self, element_key, start, stop): - super(RangeFilter, self).__init__(element_key) - self.start = start - self.stop = stop +class BooleanFilter(DimensionFilter): + def __init__(self, element_key, value): + definition = element_key if value else Not(element_key) + super(BooleanFilter, self).__init__(definition) - def schemas(self, element): - element = utils.wrap_list(element) - starts = utils.wrap_list(self.start) - stops = utils.wrap_list(self.stop) - criterion = None - for el, start, stop in zip(element, starts, stops): - crit = el[start:stop] - if criterion: - criterion = criterion & crit - else: - criterion = crit +class ContainsFilter(DimensionFilter): + def __init__(self, dimension_definition, values): + definition = dimension_definition.isin(values) + super(DimensionFilter, self).__init__(definition) - return criterion - def __eq__(self, other): - return isinstance(other, self.__class__) \ - and self.element_key == other.element_key \ - and self.start == other.start \ - and self.stop == other.stop +class ExcludesFilter(DimensionFilter): + def __init__(self, dimension_definition, values): + definition = dimension_definition.notin(values) + super(DimensionFilter, self).__init__(definition) -class WildcardFilter(Filter): - def __init__(self, element_key, value): - super(WildcardFilter, self).__init__(element_key) - self.value = value +class RangeFilter(DimensionFilter): + def __init__(self, dimension_definition, start, stop): + definition = dimension_definition[start:stop] + super(RangeFilter, self).__init__(definition) - def schemas(self, element): - return element.like(self.value) - def __eq__(self, other): - return isinstance(other, self.__class__) \ - and self.element_key == other.element_key \ - and self.value == other.value +class WildcardFilter(DimensionFilter): + def __init__(self, dimension_definition, pattern): + definition = dimension_definition.like(pattern) + super(WildcardFilter, self).__init__(definition) diff --git a/fireant/slicer/intervals.py b/fireant/slicer/intervals.py new file mode 100644 index 00000000..a8c86630 --- /dev/null +++ b/fireant/slicer/intervals.py @@ -0,0 +1,31 @@ +class NumericInterval(object): + def __init__(self, size=1, offset=0): + self.size = size + self.offset = offset + + def __eq__(self, other): + return all([isinstance(other, NumericInterval), + self.size == other.size, + self.offset == other.offset]) + + def __hash__(self): + return hash('NumericInterval %d %d' % (self.size, self.offset)) + + def __str__(self): + return 'NumericInterval(size=%d,offset=%d)' % (self.size, self.offset) + + +class DatetimeInterval(object): + def __init__(self, key): + self.key = key + + def __hash__(self): + return hash(self.key) + + +hourly = DatetimeInterval('hourly') +daily = DatetimeInterval('daily') +weekly = DatetimeInterval('weekly') +monthly = DatetimeInterval('monthly') +quarterly = DatetimeInterval('quarterly') +annually = DatetimeInterval('annually') diff --git a/fireant/slicer/joins.py b/fireant/slicer/joins.py new file mode 100644 index 00000000..6b463635 --- /dev/null +++ b/fireant/slicer/joins.py @@ -0,0 +1,12 @@ +from pypika import JoinType + + +class Join(object): + """ + WRITEME + """ + + def __init__(self, table, criterion, join_type=JoinType.inner): + self.table = table + self.criterion = criterion + self.join_type = join_type diff --git a/fireant/slicer/managers.py b/fireant/slicer/managers.py deleted file mode 100644 index b86df708..00000000 --- a/fireant/slicer/managers.py +++ /dev/null @@ -1,497 +0,0 @@ -# coding: utf-8 - -import functools -import itertools -from collections import OrderedDict - -import numpy as np -import pandas as pd -from fireant import utils -from fireant.slicer.operations import Totals -from pypika import functions as fn - -from .postprocessors import OperationManager -from .queries import QueryManager - - -class SlicerException(Exception): - pass - - -class SlicerManager(QueryManager, OperationManager): - def __init__(self, slicer): - """ - :param slicer: - """ - super(SlicerManager, self).__init__(database=slicer.database) - self.slicer = slicer - - def data(self, metrics=(), dimensions=(), - metric_filters=(), dimension_filters=(), - references=(), operations=(), pagination=None): - """ - :param metrics: - Type: list or tuple - A set of metrics to include in the query. - - :param dimensions: - Type: list or tuple - A set of dimensions to split the metrics into groups. - - :param metric_filters: - Type: list or tuple - A set of filters to constrain the data with by metric thresholds. - - :param dimension_filters: - Type: list or tuple - A set of filters to constrain the data with by dimension. - - :param references: - Type: list or tuple - A set of comparisons to include in the query. - - :param operations: - Type: list or tuple - A set of operations to perform on the response. - - :param pagination: - Type: ``fireant.slicer.pagination.Paginator`` object - An object detailing the pagination to apply to the query - - :return: - A transformed response that is queried based on the slicer and the format. - """ - if operations and pagination: - raise SlicerException('Pagination cannot be used when operations are defined!') - - metrics, dimensions = map(utils.filter_duplicates, (utils.flatten(metrics), dimensions)) - - query_schema = self.data_query_schema(metrics=metrics, dimensions=dimensions, - metric_filters=metric_filters, dimension_filters=dimension_filters, - references=references, operations=operations, pagination=pagination) - operation_schema = self.operation_schema(operations) - - dataframe = self.query_data(**query_schema) - dataframe = self.post_process(dataframe, operation_schema) - - # Filter additional metrics from the dataframe that were needed for operations - final_columns = metrics + ['%s_%s' % (os['metric'], os['key']) for os in operation_schema] - if not references: - return dataframe[final_columns] - - reference_columns = [''] + [r.key for r in references] - return dataframe[list(itertools.product(reference_columns, final_columns))] - - def get_query(self, metrics=(), dimensions=(), - metric_filters=(), dimension_filters=(), - references=(), operations=(), pagination=None): - """ Returns the PyPika query object after building the query from the given params """ - metrics = utils.filter_duplicates(utils.flatten(metrics)) - dimensions = utils.filter_duplicates(dimensions) - - query_schema = self.data_query_schema(metrics=metrics, dimensions=dimensions, - metric_filters=metric_filters, dimension_filters=dimension_filters, - references=references, operations=operations, pagination=pagination) - - return self._build_data_query(**query_schema) - - def query_string(self, metrics=(), dimensions=(), - metric_filters=(), dimension_filters=(), - references=(), operations=(), pagination=None): - """ - :param metrics: - Type: list or tuple - A set of metrics to include in the query. - - :param dimensions: - Type: list or tuple - A set of dimensions to split the metrics into groups. - - :param metric_filters: - Type: list or tuple - A set of filters to constrain the data with by metric thresholds. - - :param dimension_filters: - Type: list or tuple - A set of filters to constrain the data with by dimension. - - :param references: - Type: list or tuple - A set of comparisons to include in the query. - - :param operations: - Type: list or tuple - A set of operations to perform on the response. - - :param pagination: - Type: ``fireant.slicer.pagination.Paginator`` object - An object detailing the pagination to apply to the query - - :return: - The query that would generate the data. - """ - return str(self.get_query(metrics, dimensions, - metric_filters, dimension_filters, - references, operations, pagination)) - - def dimension_options(self, dimension, filters, limit=None): - dimopt_schema = self.dimension_option_schema(dimension, filters, limit) - return self.query_dimension_options(**dimopt_schema) - - def data_query_schema(self, metrics=(), dimensions=(), - metric_filters=(), dimension_filters=(), - references=(), operations=(), pagination=None): - """ - Builds a `dict` model of the schema parts required for executing a data query given a request. - - :param metrics: - :param dimensions: - :param metric_filters: - :param dimension_filters: - :param references: - :param operations: - :param pagination: - - :return: - """ - metric_joins_schema = self._joins_schema(set(metrics) | {mf.element_key for mf in metric_filters}, - self.slicer.metrics) - dimension_joins_schema = self._joins_schema(set(dimensions) | {df.element_key for df in dimension_filters}, - self.slicer.dimensions) - return { - 'database': self.slicer.database, - 'table': self.slicer.table, - - 'metrics': self._metrics_schema(metrics, operations), - 'dimensions': self._dimensions_schema(dimensions), - - 'mfilters': self._filters_schema(self.slicer.metrics, metric_filters, - self._default_metric_definition, - element_label='metric'), - 'dfilters': self._filters_schema(self.slicer.dimensions, - dimension_filters, - self._default_dimension_definition), - - 'joins': list(metric_joins_schema | dimension_joins_schema), - 'references': self._references_schema(references), - 'rollup': self._totals_schema(dimensions, operations), - 'pagination': pagination - } - - def dimension_option_schema(self, dimension, filters, limit=None): - dimensions = [dimension] - - schema_dimensions = self._dimensions_schema(dimensions) - schema_filters = self._filters_schema(self.slicer.dimensions, filters, self._default_dimension_definition) - schema_joins = self._joins_schema(set(dimensions) | {df.element_key for df in filters}, - self.slicer.dimensions) - - return { - 'database': self.slicer.database, - 'table': self.slicer.hint_table or self.slicer.table, - 'joins': schema_joins, - 'dimensions': schema_dimensions, - 'filters': schema_filters, - 'limit': limit, - } - - def display_schema(self, metrics=(), dimensions=(), references=(), operations=()): - """ - Builds a display schema for a request. This is used in combination with the results of the query that is - executed by the Slicer manager to transform the results with display labels. The display schema carries - meta-data pertaining to displaying the results of the query. - - :param metrics: - Type: list[str] - The requested list of metrics. This must match a Metric contained in the slicer. - - :param dimensions: - Type: list[str or tuple/list] - The requested list of dimensions. For simple dimensions, the string key is passed as a parameter, otherwise - a tuple of list with the first element equal to the key and the subsequent elements equal to the parameters. - - :param references: - Type: list[tuple] - A list of tuples describing reference options, where the first element in the tuple is the key of the - reference operation and the second value is the dimension to perform the comparasion along. - - :return: - A dictionary describing how to transform the resulting data frame for the same request. - """ - return { - 'metrics': self._display_metrics(metrics, operations), - 'dimensions': self._display_dimensions(dimensions, operations), - 'references': OrderedDict([(reference.key, reference.label) - for reference in references]), - } - - def operation_schema(self, operations): - results = [] - for operation in operations: - schema = operation.schemas() - - if schema is not None: - results.append(schema) - - return results - - def _metrics_schema(self, metrics=(), operations=()): - keys = list(metrics) + [metric - for op in operations - for metric in op.metrics()] - - if not keys: - raise SlicerException('At least one metric is required requests. Please add a metric.') - - invalid_metrics = {utils.slice_first(key) - for key in keys} - set(self.slicer.metrics) - if invalid_metrics: - raise SlicerException('Invalid metrics included in request: ' - '[%s]' % ', '.join(invalid_metrics)) - - schema_metrics = OrderedDict() - for key in keys: - schema_metric = self.slicer.metrics.get(key) - - for metric_key, definition in schema_metric.schemas(): - schema_metrics[metric_key] = definition or self._default_metric_definition(metric_key) - - return schema_metrics - - def _dimensions_schema(self, keys): - invalid_dimensions = {utils.slice_first(key) for key in keys} - set(self.slicer.dimensions) - if invalid_dimensions: - raise SlicerException('Invalid dimensions included in request: ' - '[%s]' % ', '.join(invalid_dimensions)) - - dimensions = OrderedDict() - for dimension in keys: - # unpack tuples for args - if isinstance(dimension, (list, tuple)): - dimension, args = dimension[0], dimension[1:] - else: - args = [] - - schema_dimension = self.slicer.dimensions.get(dimension) - - for key, definition in schema_dimension.schemas(*args, database=self.slicer.database): - dimensions[key] = definition or self._default_dimension_definition(key) - - return dimensions - - def _joins_schema(self, keys, elements): - """ - - :param keys: - The keys of the schema elements to retrieve joins for. - :param elements: - The elements to retrieve the joins from, either slicer.metrics or slicer.dimensions. - :return: - A `set` of join schemas containing the join table and the join criterion. - """ - joins = set() - - for key in keys: - element = elements.get(utils.slice_first(key)) - if element and element.joins: - joins |= set(element.joins) - - return {(self.slicer.joins[key].table, - self.slicer.joins[key].criterion, - self.slicer.joins[key].join_type) for key in joins} - - def _filters_schema(self, elements, filters, default_value_func, element_label='dimension'): - filters_schema = [] - for filter_item in filters: - if isinstance(filter_item.element_key, (tuple, list)): - element_key, modifier = filter_item.element_key - else: - element_key, modifier = filter_item.element_key, None - - element = elements.get(element_key) - if not element: - raise SlicerException( - 'Unable to apply filter [{filter}]. ' - 'No such {element} with key [{key}].'.format( - filter=filter_item, - element=element_label, - key=filter_item.element_key - )) - - if hasattr(element, 'display_field') and 'display' == modifier: - definition = element.display_field - - else: - definition = element.definition or default_value_func(element.key) - - filters_schema.append(filter_item.schemas(definition)) - - return filters_schema - - def _display_dimensions(self, dimensions, operations): - req_dimension_keys = [utils.slice_first(dimension) - for dimension in dimensions] - - display_dims = OrderedDict() - for key in req_dimension_keys: - dimension = self.slicer.dimensions[key] - display_dim = {'label': dimension.label} - - if hasattr(dimension, 'display_options'): - display_dim['display_options'] = {opt.key: opt.label - for opt in dimension.display_options} - display_dim['display_options'].update({pd.NaT: '', np.nan: ''}) - - if hasattr(dimension, 'display_field') and dimension.display_field: - display_dim['display_field'] = '%s_display' % dimension.key - - display_dims[key] = display_dim - - return display_dims - - def _references_schema(self, references): - schema_references = OrderedDict() - for reference in references: - if reference.element_key not in self.slicer.dimensions.keys(): - raise SlicerException( - 'Unable to query with [{reference}]. ' - 'No such dimension [{dimension}].'.format(reference=str(reference), - dimension=reference.element_key)) - - from .schemas import DatetimeDimension - if not isinstance(self.slicer.dimensions[reference.element_key], DatetimeDimension): - raise SlicerException( - 'Unable to query with [{reference}]. ' - 'Dimension [{dimension}] must be a DatetimeDimension.'.format(reference=str(reference), - dimension=reference.element_key)) - - schema_references[reference.key] = { - 'dimension': reference.element_key, - 'definition': self.slicer.dimensions[reference.element_key].definition, - 'modifier': reference.modifier, - 'time_unit': reference.time_unit, - 'interval': reference.interval, - } - - return schema_references - - def _default_dimension_definition(self, key): - return fn.Coalesce(self.slicer.table.field(key), 'None') - - def _default_metric_definition(self, key): - return fn.Sum(self.slicer.table.field(key)) - - def _display_metrics(self, metrics, operations): - display = OrderedDict() - - axis = 0 - for metrics_level in metrics: - for metric_key in utils.wrap_list(metrics_level): - schema = self.slicer.metrics[metric_key] - display[metric_key] = {attr: getattr(schema, attr) - for attr in ['label', 'precision', 'prefix', 'suffix'] - if getattr(schema, attr) is not None} - display[metric_key]["axis"] = axis - axis += 1 - - for operation in operations: - if not hasattr(operation, 'metric_key'): - continue - - metric_key = operation.metric_key - metric_schema = self.slicer.metrics[metric_key] - - key = '{}_{}'.format(metric_key, operation.key) - display[key] = {attr: getattr(metric_schema, attr) - for attr in ['precision', 'prefix', 'suffix'] - if getattr(metric_schema, attr) is not None} - display[key]['label'] = '{} {}'.format(metric_schema.label, operation.label) - display[key]["axis"] = axis - axis += 1 - - return display - - def _totals_schema(self, dimensions, operations): - dimension_set = set(utils.slice_first(dimension) for dimension in dimensions) - totals, missing_dimensions = [], set() - - for operation in operations: - if operation.key != Totals.key: - continue - - missing_dimensions |= set(operation.dimension_keys) - dimension_set - totals += [[level for level in self.slicer.dimensions[dimension].levels()] - for dimension in operation.dimension_keys - ] - - if missing_dimensions: - raise SlicerException("Missing dimensions with keys: {}".format(", ".join(missing_dimensions))) - - return totals - - -class TransformerManager(object): - def __init__(self, manager, transformers): - self.manager = manager - - # Creates a function on the slicer for each transformer - for tx_key, tx in transformers.items(): - setattr(self, tx_key, functools.partial(self._get_and_transform_data, tx)) - - def _get_and_transform_data(self, tx, metrics=(), dimensions=(), - metric_filters=(), dimension_filters=(), - references=(), operations=(), pagination=None): - """ - Handles a request and applies a transformation to the result. This is the implementation of all of the - transformer manager methods, which are constructed in the __init__ function of this class for each transformer. - - The request is first validated with the transformer then the request is executed via the SlicerManager and then - lastly the result is transformed and returned. - - :param tx: - The transformer to use - - :param metrics: - See ``fireant.slicer.SlicerManager`` - A list of metrics to include in the query. - - :param dimensions: - See ``fireant.slicer.SlicerManager`` - A list of dimensions to include in the query. - - :param metric_filters: - See ``fireant.slicer.SlicerManager`` - A list of metrics filters to apply to the query. - - :param dimension_filters: - See ``fireant.slicer.SlicerManager`` - A list of dimension filters to apply to the query. - - :param references: - See ``fireant.slicer.SlicerManager`` - A list of references to include in the query - - :param operations: - See ``fireant.slicer.SlicerManager`` - A list of post-operations to apply to the result before transformation. - - :param pagination: - See: ``fireant.slicer.pagination.Paginator`` object - An object detailing the pagination to apply to the query - - :return: - The transformed result of the request. - """ - tx.prevalidate_request(self.manager.slicer, metrics=metrics, - dimensions=[utils.slice_first(dimension) - for dimension in dimensions], - metric_filters=metric_filters, dimension_filters=dimension_filters, - references=references, operations=operations) - - # Loads data and transforms it with a given transformer. - dataframe = self.manager.data(metrics=utils.flatten(metrics), dimensions=dimensions, - metric_filters=metric_filters, dimension_filters=dimension_filters, - references=references, operations=operations, pagination=pagination) - display_schema = self.manager.display_schema(metrics, dimensions, references, operations) - - return tx.transform(dataframe, display_schema) diff --git a/fireant/slicer/metrics.py b/fireant/slicer/metrics.py new file mode 100644 index 00000000..85bb84f6 --- /dev/null +++ b/fireant/slicer/metrics.py @@ -0,0 +1,32 @@ +from .base import SlicerElement +from .filters import ComparatorFilter + + +class Metric(SlicerElement): + """ + The `Metric` class represents a metric in the `Slicer` object. + """ + + def __init__(self, key, definition, label=None, precision=None, prefix=None, suffix=None): + super(Metric, self).__init__(key, label, definition) + self.precision = precision + self.prefix = prefix + self.suffix = suffix + + def __eq__(self, other): + return ComparatorFilter(self.definition, ComparatorFilter.Operator.eq, other) + + def __ne__(self, other): + return ComparatorFilter(self.definition, ComparatorFilter.Operator.ne, other) + + def __gt__(self, other): + return ComparatorFilter(self.definition, ComparatorFilter.Operator.gt, other) + + def __ge__(self, other): + return ComparatorFilter(self.definition, ComparatorFilter.Operator.gte, other) + + def __lt__(self, other): + return ComparatorFilter(self.definition, ComparatorFilter.Operator.lt, other) + + def __le__(self, other): + return ComparatorFilter(self.definition, ComparatorFilter.Operator.lte, other) diff --git a/fireant/slicer/operations.py b/fireant/slicer/operations.py index a515311b..5c5c3f4d 100644 --- a/fireant/slicer/operations.py +++ b/fireant/slicer/operations.py @@ -1,85 +1,43 @@ -# coding: utf8 +from .metrics import Metric class Operation(object): """ The `Operation` class represents an operation in the `Slicer` API. """ - key = None - label = None + pass - def schemas(self): - pass +class _Loss(Operation): + def __init__(self, expected, actual): + self.expected = expected + self.actual = actual + + @property def metrics(self): - return [] + return [metric + for metric in [self.expected, self.actual] + if isinstance(metric, Metric)] -class Totals(Operation): - """ - `Operation` for rolling up totals across dimensions in queries. This will append the totals across a dimension to - a dimension. - """ - key = '_total' - label = 'Total' +class L1Loss(_Loss): pass - def __init__(self, *dimension_keys): - self.dimension_keys = dimension_keys +class L2Loss(_Loss): pass -class L1Loss(Operation): - """ - Performs L1 Loss (mean abs. error) operation on a metric using another metric as the target. - """ - key = 'l1loss' - label = 'L1 loss' - def __init__(self, metric_key, target_metric_key): - self.metric_key = metric_key - self.target_metric_key = target_metric_key - - def schemas(self): - return { - 'key': self.key, - 'metric': self.metric_key, - 'target': self.target_metric_key, - } +class _Cumulative(Operation): + def __init__(self, arg): + self.arg = arg + @property def metrics(self): - return [self.metric_key, self.target_metric_key] - - -class L2Loss(L1Loss): - """ - Performs L2 Loss (mean sqr. error) operation on a metric using another metric as the target. - """ - key = 'l2loss' - label = 'L2 loss' + return [metric + for metric in [self.arg] + if isinstance(metric, Metric)] -class CumSum(Operation): - """ - Accumulates the sum of one or more metrics. - """ - key = 'cumsum' - label = 'cum. sum' +class CumSum(_Cumulative): pass - def __init__(self, metric_key): - self.metric_key = metric_key - def schemas(self): - return { - 'key': self.key, - 'metric': self.metric_key, - } - - def metrics(self): - return (self.metric_key,) - - -class CumMean(CumSum): - """ - Accumulates the mean of one or more metrics - """ - key = 'cummean' - label = 'cum. mean' +class CumAvg(_Cumulative): pass diff --git a/fireant/slicer/pagination.py b/fireant/slicer/pagination.py deleted file mode 100644 index 271ed070..00000000 --- a/fireant/slicer/pagination.py +++ /dev/null @@ -1,19 +0,0 @@ -class Paginator(object): - def __init__(self, limit=0, offset=0, order=()): - """ - Class for keeping track of pagination parameters - - :param limit: The number of rows that should be returned - :param offset: The number of rows to offset the query by - :param order: Collection of tuples in the format - (, ``pypika.Order.desc`` or ``pypika.Order.asc``) - """ - self.offset = offset - self.limit = limit - self.order = order - - def __str__(self): - return 'offset: {offset} limit: {limit} order: {order}'.format(offset=self.offset, - limit=self.limit, - order=[(key, orderby.name) - for key, orderby in self.order]) diff --git a/fireant/slicer/postprocessors.py b/fireant/slicer/postprocessors.py deleted file mode 100644 index 750739d7..00000000 --- a/fireant/slicer/postprocessors.py +++ /dev/null @@ -1,67 +0,0 @@ -# coding: utf-8 -import pandas as pd - - -def get_cum_metric(dataframe, schema, reference=None): - metric = schema['metric'] - if reference is None: - return dataframe[metric] - return dataframe[reference, metric] - - -def get_loss_metric(dataframe, schema, reference=None): - target, metric = ( - (schema['target'], schema['metric']) - if reference is None - else ((reference, schema['target']), (reference, schema['metric'])) - ) - - error_series = dataframe[target] - dataframe[metric] - error_series.name = (schema['metric'] - if reference is None - else (reference, schema['metric'])) - return error_series - - -FUNCTIONS = { - 'cumsum': (get_cum_metric, lambda x: x.expanding(min_periods=1).sum()), - 'cummean': (get_cum_metric, lambda x: x.expanding(min_periods=1).mean()), - 'l1loss': (get_loss_metric, lambda x: x.abs().expanding(min_periods=1).mean()), - 'l2loss': (get_loss_metric, lambda x: x.pow(2).expanding(min_periods=1).mean()), -} - - -class OperationManager(object): - def post_process(self, dataframe, operation_schema): - dataframe = dataframe.copy() - - for schema in operation_schema: - key = schema['key'] - - value_func, operation_func = FUNCTIONS.get(schema['key']) - if not value_func or not operation_func: - continue - - self._perform_operation(dataframe, key, schema, value_func, operation_func) - - return dataframe - - def _perform_operation(self, dataframe, key, schema, value_func, operation): - # Check for references - references = (dataframe.columns.get_level_values(0).tolist() - if isinstance(dataframe.columns, pd.MultiIndex) - else [None]) - - for reference in references: - metric_df = value_func(dataframe, schema, reference=reference) - - operation_key = ('{}_{}'.format(metric_df.name, key) - if reference is None - else (reference, '{}_{}'.format(metric_df.name[1], key))) - - if isinstance(dataframe.index, pd.MultiIndex): - unstack_levels = list(range(1, len(dataframe.index.levels))) - dataframe[operation_key] = metric_df.groupby(level=unstack_levels).apply(operation) - - else: - dataframe[operation_key] = operation(metric_df) diff --git a/fireant/slicer/queries/__init__.py b/fireant/slicer/queries/__init__.py new file mode 100644 index 00000000..f53ef456 --- /dev/null +++ b/fireant/slicer/queries/__init__.py @@ -0,0 +1 @@ +from .builder import QueryBuilder \ No newline at end of file diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py new file mode 100644 index 00000000..fe2e5111 --- /dev/null +++ b/fireant/slicer/queries/builder.py @@ -0,0 +1,224 @@ +from collections import defaultdict + +from toposort import ( + CircularDependencyError, + toposort_flatten, +) + +from fireant.utils import ( + immutable, + ordered_distinct_list, + ordered_distinct_list_by_attr, +) +from .database import fetch_data +from .references import join_reference +from ..exceptions import ( + CircularJoinsException, + MissingTableJoinException, +) +from ..filters import DimensionFilter +from ..intervals import DatetimeInterval + + +def _build_dimension_definition(dimension, interval_func): + if hasattr(dimension, 'interval') and isinstance(dimension.interval, DatetimeInterval): + return interval_func(dimension.definition, + dimension.interval).as_(dimension.key) + + return dimension.definition.as_(dimension.key) + + +class QueryBuilder(object): + """ + + """ + + def __init__(self, slicer): + self.slicer = slicer + self._widgets = [] + self._dimensions = [] + self._filters = [] + self._orders = [] + + @immutable + def widget(self, widget): + """ + + :param widget: + :return: + """ + self._widgets.append(widget) + + @immutable + def dimension(self, dimension): + """ + + :param dimension: + :param rollup: + :return: + """ + self._dimensions.append(dimension) + + @immutable + def filter(self, filter): + """ + + :param filter: + :return: + """ + self._filters.append(filter) + + @property + def tables(self): + """ + :return: + A collection of tables required to execute a query, + """ + return ordered_distinct_list([table + for group in [self.metrics, self._dimensions, self._filters] + for element in group + for attr in [getattr(element, 'definition', None), + getattr(element, 'display_definition', None)] + if attr is not None + for table in attr.tables_]) + + @property + def metrics(self): + """ + :return: + an ordered, distinct list of metrics used in all widgets as part of this query. + """ + return ordered_distinct_list_by_attr([metric + for widget in self._widgets + for metric in widget.metrics]) + + @property + def joins(self): + """ + Given a set of tables required for a slicer query, this function finds the joins required for the query and + sorts them topologically. + + :return: + A list of joins in the order that they must be joined to the query. + :raises: + MissingTableJoinException - If a table is required but there is no join for that table + CircularJoinsException - If there is a circular dependency between two or more joins + """ + dependencies = defaultdict(set) + slicer_joins = {join.table: join + for join in self.slicer.joins} + tables_to_eval = list(self.tables) + + while tables_to_eval: + table = tables_to_eval.pop() + + if self.slicer.table == table: + continue + + if table not in slicer_joins: + raise MissingTableJoinException('Could not find a join for table {table}' + .format(table=str(table))) + + join = slicer_joins[table] + tables_required_for_join = set(join.criterion.tables_) - {self.slicer.table, join.table} + + dependencies[join] |= {slicer_joins[table] + for table in tables_required_for_join} + tables_to_eval += tables_required_for_join - {d.table for d in dependencies} + + try: + return toposort_flatten(dependencies) + except CircularDependencyError as e: + raise CircularJoinsException(str(e)) + + @property + def query(self): + """ + WRITEME + """ + query = self.slicer.database.query_cls.from_(self.slicer.table) + + # Add joins + for join in self.joins: + query = query.join(join.table, how=join.join_type).on(join.criterion) + + # Add dimensions + for dimension in self._dimensions: + dimension_definition = _build_dimension_definition(dimension, self.slicer.database.trunc_date) + query = query.select(dimension_definition).groupby(dimension_definition) + + # Add display definition field + if hasattr(dimension, 'display_definition'): + dimension_display_definition = dimension.display_definition.as_(dimension.display_key) + query = query.select(dimension_display_definition).groupby(dimension_display_definition) + + # Add metrics + query = query.select(*[metric.definition.as_(metric.key) + for metric in self.metrics]) + + # Add filters + for filter_ in self._filters: + query = query.where(filter_.definition) \ + if isinstance(filter_, DimensionFilter) \ + else query.having(filter_.definition) + + # Add references + references = [(reference, dimension) + for dimension in self._dimensions + if hasattr(dimension, 'references') + for reference in dimension.references] + if references: + query = self._join_references(query, references) + + # Add ordering + order = self._orders if self._orders else self._dimensions + query = query.orderby(*[element.definition.as_(element.key) + for element in order]) + + return str(query) + + def _join_references(self, query, references): + original_query = query.as_('base') + + def original_query_field(key): + return original_query.field(key).as_(key) + + outer_query = self.slicer.database.query_cls.from_(original_query) + + # Add dimensions + for dimension in self._dimensions: + outer_query = outer_query.select(original_query_field(dimension.key)) + + if hasattr(dimension, 'display_definition'): + outer_query = outer_query.select(original_query_field(dimension.display_key)) + + # Add metrics + outer_query = outer_query.select(*[original_query_field(metric.key) + for metric in self.metrics]) + + # Build nested reference queries + for reference, dimension in references: + outer_query = join_reference(reference, + self.metrics, + self._dimensions, + dimension, + self.slicer.database.date_add, + original_query, + outer_query) + + return outer_query + + def render(self): + """ + + :return: + """ + dataframe = fetch_data(self.slicer.database, self.query, index_levels=[dimension.key + for dimension in self._dimensions]) + + # Apply operations + ... + + # Apply transformations + return [widget.transform(dataframe, self.slicer) + for widget in self._widgets] diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py new file mode 100644 index 00000000..3403a433 --- /dev/null +++ b/fireant/slicer/queries/database.py @@ -0,0 +1,24 @@ +import time + +from .logger import logger + + +def fetch_data(database, query, index_levels): + """ + Returns a Pandas Dataframe built from the result of the query. + The query is also logged along with its duration. + + :param database: Database object + :param query: PyPika query object + :return: Pandas Dataframe built from the result of the query + """ + start_time = time.time() + logger.debug(query) + + dataframe = database.fetch_dataframe(query) + + logger.info('[{duration} seconds]: {query}' + .format(duration=round(time.time() - start_time, 4), + query=query)) + + return dataframe.set_index(index_levels) diff --git a/fireant/slicer/queries/logger.py b/fireant/slicer/queries/logger.py new file mode 100644 index 00000000..c1944a63 --- /dev/null +++ b/fireant/slicer/queries/logger.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger('fireant.query_log$') \ No newline at end of file diff --git a/fireant/slicer/queries/references.py b/fireant/slicer/queries/references.py new file mode 100644 index 00000000..89013b5b --- /dev/null +++ b/fireant/slicer/queries/references.py @@ -0,0 +1,140 @@ +from functools import partial + +from typing import ( + Callable, + Iterator, +) + +from pypika import ( + JoinType, + + functions as fn, +) +from pypika.queries import QueryBuilder +from pypika.terms import ( + ComplexCriterion, + Criterion, + Term, +) +from ..dimensions import Dimension +from ..metrics import Metric +from ..references import Reference + + +def join_reference(reference: Reference, + metrics: Iterator[Metric], + dimensions: Iterator[Dimension], + ref_dimension: Dimension, + date_add: Callable, + original_query, + outer_query: QueryBuilder): + ref_query = original_query.as_(reference.key) + date_add = partial(date_add, date_part=reference.time_unit, interval=reference.interval) + + # FIXME this is a bit hacky, need to replace the ref dimension term in all of the filters with the offset + if ref_query._wheres: + ref_query._wheres = _apply_to_term_in_criterion(ref_dimension.definition, + date_add(ref_dimension.definition), + ref_query._wheres) + + # Join inner query + join_criterion = _build_reference_join_criterion(ref_dimension, + dimensions, + original_query, + ref_query, + date_add) + outer_query = outer_query.join(ref_query, JoinType.left).on(join_criterion) + # Add metrics + ref_metric = _reference_metric(reference, + original_query, + ref_query) + outer_query = outer_query.select(*[ref_metric(metric).as_("{}_{}".format(metric.key, reference.key)) + for metric in metrics]) + return outer_query + + +def _apply_to_term_in_criterion(target: Term, + replacement: Term, + criterion: Criterion): + """ + Finds and replaces a term within a criterion. This is necessary for adapting filters used in reference queries + where the reference dimension must be offset by some value. The target term is found inside the criterion and + replaced with the replacement. + + :param target: + The target term to replace in the criterion. It will be replaced in all locations within the criterion with + the func applied to itself. + :param replacement: + The replacement for the term. + :param criterion: + The criterion to replace the term in. + :return: + A criterion identical to the original criterion arg except with the target term replaced by the replacement arg. + """ + if isinstance(criterion, ComplexCriterion): + criterion.left = _apply_to_term_in_criterion(target, replacement, criterion.left) + criterion.right = _apply_to_term_in_criterion(target, replacement, criterion.right) + return criterion + + for attr in ['term', 'left', 'right']: + if hasattr(criterion, attr) and str(getattr(criterion, attr)) == str(target): + setattr(criterion, attr, replacement) + + return criterion + + +def _build_reference_join_criterion(dimension: Dimension, + all_dimensions: Iterator[Dimension], + original_query: QueryBuilder, + ref_query: QueryBuilder, + offset_func: Callable): + """ + This builds the criterion for joining a reference query to the base query. It matches the referenced dimension + in the base query to the offset referenced dimension in the reference query and all other dimensions. + + :param dimension: + The referenced dimension + :param original_query: + The base query, the original query despite the references. + :param ref_query: + The reference query, a copy of the base query with the referenced dimension replaced. + :param offset_func: + The offset function for shifting the referenced dimension. + :return: + pypika.Criterion + """ + join_criterion = original_query.field(dimension.key) == offset_func(field=ref_query.field(dimension.key)) + + for dimension_ in all_dimensions: + if dimension == dimension_: + continue + + join_criterion &= original_query.field(dimension_.key) == ref_query.field(dimension_.key) + + return join_criterion + + +def _reference_metric(reference: Reference, + original_query: QueryBuilder, + ref_query: QueryBuilder): + """ + WRITEME + + :param reference: + :param original_query: + :param ref_query: + :return: + """ + + def ref_field(metric): + return ref_query.field(metric.key) + + if reference.is_delta: + if reference.is_percent: + return lambda metric: (original_query.field(metric.key) - ref_field(metric)) \ + * \ + (100 / fn.NullIf(ref_field(metric), 0)) + + return lambda metric: original_query.field(metric.key) - ref_field(metric) + + return ref_field diff --git a/fireant/slicer/references.py b/fireant/slicer/references.py index afab10c9..3ac32a12 100644 --- a/fireant/slicer/references.py +++ b/fireant/slicer/references.py @@ -1,94 +1,23 @@ -# coding: utf-8 -from pypika import functions as fn - - class Reference(object): - key = None - label = None - modifier = None # Modifier calculation function e.g. delta percentage arithmetic - time_unit = None # The time unit to perform the reference on - WoW is week, YoY is year etc. - interval = None # The number unit used in the interval e.g. for 1 week it is 1 - - def __init__(self, element_key): - self.element_key = element_key - - def __eq__(self, other): - if not isinstance(other, Reference): - return False - - return all([ - self.key == other.key, - self.element_key == other.element_key, - ]) - - def __hash__(self): - return hash('%s:%s' % (self.key, self.element_key)) - - -class Delta(Reference): - def __init__(self, reference): - super(Delta, self).__init__(reference.element_key) - self.key = self.generate_key(reference.key) - self.label = '%s Δ' % reference.label - self.time_unit = reference.time_unit - self.interval = reference.interval - - @staticmethod - def modifier(field, join_field): - return field - join_field - - @staticmethod - def generate_key(reference_key): - return '%s_delta' % reference_key - - -class DeltaPercentage(Reference): - def __init__(self, reference): - super(DeltaPercentage, self).__init__(reference.element_key) - self.key = self.generate_key(reference.key) - self.label = '%s Δ%%' % reference.label - self.time_unit = reference.time_unit - self.interval = reference.interval - - @staticmethod - def modifier(field, join_field): - return ((field - join_field) * 100) / fn.NullIf(join_field, 0) - - @staticmethod - def generate_key(reference_key): - return '%s_delta_percent' % reference_key - - -class DoD(Reference): - key = 'dod' - label = 'DoD' - time_unit = 'day' - interval = 1 - - -class WoW(Reference): - key = 'wow' - label = 'WoW' - time_unit = 'week' - interval = 1 - + def __init__(self, key, time_unit, interval, delta=False, percent=False): + self.key = key + if delta: + self.key += '_delta' + if percent: + self.key += '_percent' -class MoM(Reference): - key = 'mom' - label = 'MoM' - time_unit = 'month' - interval = 1 + self.time_unit = time_unit + self.interval = interval + self.is_delta = delta + self.is_percent = percent -class QoQ(Reference): - key = 'qoq' - label = 'QoQ' - time_unit = 'quarter' - interval = 1 + def delta(self, percent=False): + return Reference(self.key, self.time_unit, self.interval, delta=True, percent=percent) -class YoY(Reference): - key = 'yoy' - label = 'YoY' - time_unit = 'year' - interval = 1 +DayOverDay = Reference('dod', 'day', 1) +WeekOverWeek = Reference('wow', 'week', 1) +MonthOverMonth = Reference('mom', 'month', 1) +QuarterOverQuarter = Reference('qoq', 'quarter', 1) +YearOverYear = Reference('yoy', 'year', 1) diff --git a/fireant/slicer/schemas.py b/fireant/slicer/schemas.py deleted file mode 100644 index 08e6b555..00000000 --- a/fireant/slicer/schemas.py +++ /dev/null @@ -1,220 +0,0 @@ -# coding: utf-8 -from fireant.slicer import transformers -from fireant.slicer.managers import SlicerManager, TransformerManager -from pypika import JoinType -from pypika.terms import Mod - - -class SlicerElement(object): - """The `SlicerElement` class represents an element of the slicer, either a metric or dimension, which contains - information about such as how to query it from the database.""" - - def __init__(self, key, label=None, definition=None, joins=None): - """ - :param key: - The unique identifier of the slicer element, used in the Slicer manager API to reference a defined element. - - :param label: - A displayable representation of the column. Defaults to the key capitalized. - - :param definition: - The definition of the element as a PyPika expression which defines how to query it from the database. - - :param joins: - A list of Join keys required by this element. - """ - self.key = key - self.label = label or key - self.definition = definition - self.joins = joins - - def __unicode__(self): - return self.key - - def __repr__(self): - return self.key - - def schemas(self, *args, **kwargs): - return [ - (self.key, self.definition) - ] - - -class Metric(SlicerElement): - """ - The `Metric` class represents a metric in the `Slicer` object. - """ - - def __init__(self, key, label=None, definition=None, joins=None, precision=None, prefix=None, suffix=None): - super(Metric, self).__init__(key, label, definition, joins) - self.precision = precision - self.prefix = prefix - self.suffix = suffix - - -class Dimension(SlicerElement): - """ - The `Dimension` class represents a dimension in the `Slicer` object. - """ - - def __init__(self, key, label=None, definition=None, joins=None): - super(Dimension, self).__init__(key, label, definition, joins) - - def levels(self): - return [self.key] - - -class NumericInterval(object): - def __init__(self, size=1, offset=0): - self.size = size - self.offset = offset - - def __eq__(self, other): - return isinstance(other, NumericInterval) and self.size == other.size and self.offset == other.offset - - def __hash__(self): - return hash('NumericInterval %d %d' % (self.size, self.offset)) - - def __str__(self): - return 'NumericInterval(size=%d,offset=%d)' % (self.size, self.offset) - - -class ContinuousDimension(Dimension): - def __init__(self, key, label=None, definition=None, default_interval=NumericInterval(1, 0), joins=None): - super(ContinuousDimension, self).__init__(key=key, label=label, definition=definition, joins=joins) - self.default_interval = default_interval - - def schemas(self, *args, **kwargs): - size, offset = args if args else (self.default_interval.size, self.default_interval.offset) - return [(self.key, Mod(self.definition + offset, size))] - - -class DatetimeInterval(object): - def __init__(self, size): - self.size = size - - def __eq__(self, other): - return isinstance(other, DatetimeInterval) and self.size == other.size - - def __hash__(self): - return hash('DatetimeInterval %s' % self.size) - - def __str__(self): - return 'DatetimeInterval(interval=%s)' % self.size - - -class DatetimeDimension(ContinuousDimension): - hour = 'hour' - day = 'day' - week = 'week' - month = 'month' - quarter = 'quarter' - year = 'year' - - def __init__(self, key, label=None, definition=None, default_interval=day, joins=None): - super(DatetimeDimension, self).__init__(key=key, label=label, definition=definition, joins=joins, - default_interval=default_interval) - - def schemas(self, *args, **kwargs): - interval = args[0] if args else self.default_interval - return [(self.key, kwargs['database'].trunc_date(self.definition, interval))] - - -class CategoricalDimension(Dimension): - def __init__(self, key, label=None, definition=None, display_options=tuple(), joins=None): - super(CategoricalDimension, self).__init__(key=key, label=label, definition=definition, joins=joins) - self.display_options = display_options - - -class UniqueDimension(Dimension): - def __init__(self, key, label=None, definition=None, display_field=None, joins=None): - super(UniqueDimension, self).__init__(key=key, label=label, definition=definition, joins=joins) - self.display_field = display_field - - def display_key(self): - return '{key}_display'.format(key=self.key) - - def schemas(self, *args, **kwargs): - schemas = [('{key}'.format(key=self.key), self.definition)] - - if self.display_field: - schemas.append((self.display_key(), self.display_field)) - - return schemas - - def levels(self): - if self.display_field is not None: - return [self.key, self.display_key()] - return super(UniqueDimension, self).levels() - - -class BooleanDimension(Dimension): - def __init__(self, key, label=None, definition=None, joins=None): - super(BooleanDimension, self).__init__(key=key, label=label, definition=definition, joins=joins) - - -class DimensionValue(object): - """ - An option belongs to a categorical dimension which specifies a fixed set of values - """ - - def __init__(self, key, label=None): - self.key = key - self.label = label or key - - -class Join(object): - def __init__(self, key, table, criterion, join_type=JoinType.inner): - self.key = key - self.table = table - self.criterion = criterion - self.join_type = join_type - - -class Slicer(object): - def __init__(self, table, database, metrics=tuple(), dimensions=tuple(), joins=tuple(), hint_table=None): - """ - Constructor for a slicer. Contains all the fields to initialize the slicer. - - :param table: (Required) - A Pypika Table reference. The primary table that this slicer will retrieve data from. - - :param database: (Required) - A Database reference. Holds the connection details used by this slicer to execute queries. - - :param metrics: (Required: At least one) - A list of metrics which can be queried. Metrics are the types of data that are displayed. - - :param dimensions: (Optional) - A list of dimensions used for grouping metrics. Dimensions are used as the axes in charts, the indices in - tables, and also for splitting charts into multiple lines. - - :param joins: (Optional) - A list of join descriptions for joining additional tables. Joined tables are only used when querying a - metric or dimension which requires it. - - :param hint_table: (Optional) - A hint table used for querying dimension options. If not present, the table will be used. The hint_table - must have the same definition as the table omitting dimensions which do not have a set of options (such as - datetime dimensions) and the metrics. This is provided to more efficiently query dimension options. - """ - self.table = table - self.database = database - - self.metrics = {metric.key: metric for metric in metrics} - self.dimensions = {dimension.key: dimension for dimension in dimensions} - self.joins = {join.key: join for join in joins} - self.hint_table = hint_table - - self.manager = SlicerManager(self) - for name, bundle in transformers.BUNDLES.items(): - setattr(self, name, TransformerManager(self.manager, bundle)) - - -class EqualityOperator(object): - eq = 'eq' - ne = 'ne' - gt = 'gt' - lt = 'lt' - gte = 'gte' - lte = 'lte' diff --git a/fireant/slicer/slicers.py b/fireant/slicer/slicers.py new file mode 100644 index 00000000..2796e16a --- /dev/null +++ b/fireant/slicer/slicers.py @@ -0,0 +1,62 @@ +from .queries import QueryBuilder + + +class _Container(object): + def __init__(self, items): + self._items = items + for item in items: + setattr(self, item.key, item) + + def __iter__(self): + return iter(self._items) + + +class Slicer(object): + """ + WRITEME + """ + + class Dimensions(_Container): + pass + + class Metrics(_Container): + pass + + def __init__(self, table, database, joins=(), dimensions=(), metrics=(), hint_table=None): + """ + Constructor for a slicer. Contains all the fields to initialize the slicer. + + :param table: (Required) + A Pypika Table reference. The primary table that this slicer will retrieve data from. + + :param database: (Required) + A Database reference. Holds the connection details used by this slicer to execute queries. + + :param metrics: (Required: At least one) + A list of metrics which can be queried. Metrics are the types of data that are displayed. + + :param dimensions: (Optional) + A list of dimensions used for grouping metrics. Dimensions are used as the axes in charts, the indices in + tables, and also for splitting charts into multiple lines. + + :param joins: (Optional) + A list of join descriptions for joining additional tables. Joined tables are only used when querying a + metric or dimension which requires it. + + :param hint_table: (Optional) + A hint table used for querying dimension options. If not present, the table will be used. The hint_table + must have the same definition as the table omitting dimensions which do not have a set of options (such as + datetime dimensions) and the metrics. This is provided to more efficiently query dimension options. + """ + self.table = table + self.database = database + self.joins = joins + self.dimensions = Slicer.Dimensions(dimensions) + self.metrics = Slicer.Metrics(metrics) + self.hint_table = hint_table + + def query(self): + """ + WRITEME + """ + return QueryBuilder(self) diff --git a/fireant/slicer/transformers/__init__.py b/fireant/slicer/transformers/__init__.py deleted file mode 100644 index 11de6969..00000000 --- a/fireant/slicer/transformers/__init__.py +++ /dev/null @@ -1,60 +0,0 @@ -# coding: utf-8 - -from .base import (Transformer, - TransformationException) -from .datatables import (DataTablesRowIndexTransformer, - DataTablesColumnIndexTransformer) -from .datatables import (DataTablesRowIndexTransformer, - DataTablesColumnIndexTransformer, - CSVRowIndexTransformer, - CSVColumnIndexTransformer) -from .highcharts import (HighchartsLineTransformer, - HighchartsAreaTransformer, - HighchartsAreaPercentageTransformer, - HighchartsColumnTransformer, - HighchartsStackedColumnTransformer, - HighchartsBarTransformer, - HighchartsStackedBarTransformer, - HighchartsPieTransformer) -from .notebooks import (PandasRowIndexTransformer, - PandasColumnIndexTransformer, - MatplotlibLineChartTransformer, - MatplotlibBarChartTransformer) - -ROW_INDEX_TABLE = 'row_index_table' -ROW_INDEX_CSV = 'row_index_csv' -COLUMN_INDEX_TABLE = 'column_index_table' -COLUMN_INDEX_CSV = 'column_index_csv' -LINE_CHART = 'line_chart' -BAR_CHART = 'bar_chart' -AREA_CHART = 'area_chart' -AREA_PERCENTAGE_CHART = 'area_percentage_chart' -COLUMN_CHART = 'column_chart' -STACKED_COLUMN_CHART = 'stacked_column_chart' -STACKED_BAR_CHART = 'stacked_bar_chart' -PIE_CHART = 'pie_chart' - -BUNDLES = { - 'notebooks': { - ROW_INDEX_TABLE: PandasRowIndexTransformer(), - COLUMN_INDEX_TABLE: PandasColumnIndexTransformer(), - LINE_CHART: MatplotlibLineChartTransformer(), - BAR_CHART: MatplotlibBarChartTransformer(), - }, - 'highcharts': { - LINE_CHART: HighchartsLineTransformer(), - AREA_CHART: HighchartsAreaTransformer(), - AREA_PERCENTAGE_CHART: HighchartsAreaPercentageTransformer(), - COLUMN_CHART: HighchartsColumnTransformer(), - STACKED_COLUMN_CHART: HighchartsStackedColumnTransformer(), - BAR_CHART: HighchartsBarTransformer(), - STACKED_BAR_CHART: HighchartsStackedBarTransformer(), - PIE_CHART: HighchartsPieTransformer(), - }, - 'datatables': { - ROW_INDEX_TABLE: DataTablesRowIndexTransformer(), - COLUMN_INDEX_TABLE: DataTablesColumnIndexTransformer(), - ROW_INDEX_CSV: CSVRowIndexTransformer(), - COLUMN_INDEX_CSV: CSVColumnIndexTransformer(), - }, -} diff --git a/fireant/slicer/transformers/base.py b/fireant/slicer/transformers/base.py deleted file mode 100644 index bed2d5e7..00000000 --- a/fireant/slicer/transformers/base.py +++ /dev/null @@ -1,15 +0,0 @@ -# coding: utf-8 - - -class Transformer(object): - def prevalidate_request(self, slicer, metrics, dimensions, - metric_filters, dimension_filters, - references, operations): - pass - - def transform(self, dataframe, display_schema): - raise NotImplementedError - - -class TransformationException(Exception): - pass diff --git a/fireant/slicer/transformers/datatables.py b/fireant/slicer/transformers/datatables.py deleted file mode 100644 index 45493f4c..00000000 --- a/fireant/slicer/transformers/datatables.py +++ /dev/null @@ -1,398 +0,0 @@ -# coding: utf-8 -import locale as lc -from datetime import time - -import numpy as np -import pandas as pd -from fireant import settings -from fireant.slicer.operations import Totals -from fireant.slicer.transformers import Transformer - -NO_TIME = time(0) - - -def _safe(value): - if isinstance(value, pd.Timestamp): - if value.time() == NO_TIME: - return value.strftime('%Y-%m-%d') - else: - return value.strftime('%Y-%m-%dT%H:%M:%S') - - if value is None or (isinstance(value, float) and np.isnan(value)) or pd.isnull(value): - return None - - if isinstance(value, np.int64): - # Cannot transform np.int64 to json - return int(value) - - if isinstance(value, np.float64): - return float(value) - - return value - - -def _pretty(value, schema): - precision = schema.get('precision') - - if isinstance(value, float): - if precision is not None: - float_format = '%d' if precision == 0 else '%.{}f'.format(precision) - value = lc.format(float_format, value, grouping=True) - elif value.is_integer(): - float_format = '%d' - value = lc.format(float_format, value, grouping=True) - else: - float_format = '%f' - # Stripping trailing zeros is necessary because %f can add them if no precision is set - value = lc.format(float_format, value, grouping=True).rstrip('.0') - - if isinstance(value, int): - float_format = '%d' - value = lc.format(float_format, value, grouping=True) - - return '{prefix}{value}{suffix}'.format( - prefix=schema.get('prefix', ''), - value=str(value), - suffix=schema.get('suffix', ''), - ) - - -def _format_value(value, metric): - raw_value = _safe(value) - return {'value': raw_value, 'display': _pretty(raw_value, metric) if raw_value is not None else None} - - -def _format_column_display(csv_df, metrics, dimensions): - column_display = [] - for idx in list(csv_df.columns): - metric_value, dimension_values = idx[0], idx[1:] - - dimension_displays = [] - for dimension_level, dimension_value in zip(csv_df.columns.names[1:], dimension_values): - if 'display_options' in dimensions[dimension_level]: - dimension_display = dimensions[dimension_level]['display_options'].get(dimension_value, - dimension_value) - else: - dimension_display = dimension_value - - dimension_display = _safe(dimension_display) - - if dimension_display != Totals.label: - dimension_displays.append(dimension_display) - - if dimension_displays: - column_display.append('{metric} ({dimensions})'.format( - metric=metrics[metric_value]['label'], - dimensions=', '.join(dimension_displays), - )) - else: - column_display.append(metrics[metric_value]['label']) - - return column_display - - -class DataTablesRowIndexTransformer(Transformer): - def transform(self, dataframe, display_schema): - dataframe = self._prepare_dataframe(dataframe, display_schema['dimensions']) - - return { - 'columns': self._render_columns(dataframe, display_schema), - 'data': self._render_data(dataframe, display_schema), - } - - def _prepare_dataframe(self, dataframe, dimensions): - # Replaces invalid values and unstacks the data frame for column_index tables. - return dataframe.replace([np.inf, -np.inf], np.nan) - - def _render_columns(self, dataframe, display_schema): - maxcols = settings.datatables_maxcols - dimension_columns = [self._render_dimension_column(dimension_key, display_schema) - for dimension_key in dataframe.index.names[:maxcols] - if dimension_key in display_schema['dimensions']] - - maxcols -= len(dimension_columns) - metric_columns = [self._render_column_level(metric_column, display_schema) - for metric_column in list(dataframe.columns)[:maxcols]] - - return dimension_columns + metric_columns - - def _render_dimension_column(self, key, display_schema): - dimension = display_schema['dimensions'][key] - - if 'display_field' in dimension or 'display_options' in dimension: - render = { - '_': 'display', - 'type': 'display', - 'sort': 'display' - } - else: - render = { - '_': 'value', - 'type': 'value', - 'sort': 'value' - } - - return {'title': dimension['label'], - 'data': '{}'.format(key), - 'render': render} - - def _render_column_level(self, metric_column, display_schema): - metrics = display_schema['metrics'] - if not isinstance(metric_column, tuple): - return {'title': metrics[metric_column]['label'], - 'data': metric_column, - 'render': {'type': 'value', '_': 'display', 'sort': 'value'}} - - references = display_schema.get('references') - metric_key_idx = 1 if references else 0 - if metric_key_idx and metric_column[0]: - reference_key = metric_column[0] - metric_key = metric_column[1] - data_keys = [reference_key, metric_key] - metric_label = '{metric} {reference}'.format( - metric=metrics[metric_key]['label'], - reference=display_schema['references'][reference_key] - ) - - else: - metric_key = metric_column[metric_key_idx] - data_keys = [metric_key] - metric_label = metrics[metric_key]['label'] - - path = '.'.join(map(str, data_keys)) - return { - 'title': metric_label, - 'data': path, - 'render': {'type': 'value', '_': 'display', 'sort': 'value'} - } - - def _render_data(self, dataframe, display_schema): - n = len(dataframe.index.levels) if isinstance(dataframe.index, pd.MultiIndex) else 1 - dimensions = list(display_schema['dimensions'].items()) - row_dimensions, column_dimensions = dimensions[:n], dimensions[n:] - - data = [] - for idx, df_row in dataframe.iterrows(): - row = {} - - if not isinstance(idx, tuple): - idx = (idx,) - - for key, value in self._render_dimension_data(idx, row_dimensions): - row[key] = value - - for key, value in self._render_metric_data(df_row, column_dimensions, - display_schema['metrics'], display_schema.get('references')): - row[key] = value - - data.append(row) - - return data - - def _render_dimension_data(self, idx, dimensions): - i = 0 - for key, dimension in dimensions: - dimension_value = _safe(idx[i]) - - if 'display_field' in dimension: - i += 1 - yield key, {'display': _safe(idx[i]), 'value': dimension_value} - - elif 'display_options' in dimension: - display = dimension['display_options'].get(dimension_value, dimension_value) - yield key, {'display': display, 'value': dimension_value} - - else: - yield key, {'value': dimension_value} - - i += 1 - - def _render_metric_data(self, dataframe, dimensions, metrics, references): - if not isinstance(dataframe.index, pd.MultiIndex): - for metric_key, label in metrics.items(): - yield metric_key, _format_value(dataframe[metric_key], metrics[metric_key]) - - if references: - for reference in [''] + list(references): - for key, value in self._recurse_dimensions(dataframe[reference], dimensions, metrics, - reference or None): - yield key, value - return - - for key, value in self._recurse_dimensions(dataframe, dimensions, metrics): - yield key, value - - def _recurse_dimensions(self, dataframe, dimensions, metrics, reference=None): - if reference: - return [(reference, dict(self._recurse_dimensions(dataframe, dimensions, metrics)))] - - return [(metric_key, _format_value(dataframe[metric_key], metrics[metric_key])) - for metric_key, label in metrics.items()] - - -class DataTablesColumnIndexTransformer(DataTablesRowIndexTransformer): - def _prepare_dataframe(self, dataframe, dimensions): - # Replaces invalid values and unstacks the data frame for column_index tables. - dataframe = super(DataTablesColumnIndexTransformer, self)._prepare_dataframe(dataframe, dimensions) - - unstack_levels = [] - for key, dimension in list(dimensions.items())[1:]: - unstack_levels.append(key) - if 'display_field' in dimension: - unstack_levels.append(dimension['display_field']) - - dataframe.replace('', 'N/A', inplace=True) - - # TODO: Remove the extra NaN row that is created for each column when totals are being applied to a - # unique dimension. - return dataframe.unstack(level=unstack_levels, fill_value='N/A') - - def _render_column_level(self, metric_column, display_schema): - column = super(DataTablesColumnIndexTransformer, self)._render_column_level(metric_column, display_schema) - - # Iterate through the pivoted levels - i = 2 if display_schema.get('references') else 1 - data_keys, levels = [], [] - for dimension in list(display_schema['dimensions'].values())[1:]: - level_key = metric_column[i] - - if 'display_options' in dimension: - level_display = dimension['display_options'].get(level_key, level_key) - else: - if 'display_field' in dimension: - i += 1 - - level_display = metric_column[i] - - # the metric key must remain last - if not (isinstance(level_key, float) and np.isnan(level_key)): - if level_key != Totals.key: - levels.append(level_display) - data_keys.append(level_key) - - i += 1 - - if levels: - metric_label = '{metric} ({levels})'.format( - metric=column['title'], - levels=', '.join(map(str, levels)) - ) - else: - metric_label = column['title'] - - if data_keys and '.' in column['data']: - # Squeeze the dimension levels between the reference and the metric - data = column['data'].replace('.', '.' + ('.'.join(map(str, data_keys))) + '.') - else: - # Prepend the dimension levels to the metric - data = '.'.join(map(str, data_keys + [column['data']])) - - return { - 'title': metric_label, - 'data': data, - 'render': {'type': 'value', '_': 'display', 'sort': 'value'} - } - - def _recurse_dimensions(self, dataframe, dimensions, metrics, reference=None): - if dataframe.empty: - return [] - - if reference or not dimensions: - return super(DataTablesColumnIndexTransformer, self)._recurse_dimensions(dataframe, dimensions, metrics, - reference) - - data = [] - - if 'display_field' in dimensions[0][1]: - # Returning indexes per level does not guarantee they are ordered, therefore iteritems() must be used - levels = [row[0][1:] for row in dataframe.iteritems()] - format_key = lambda level: level[0] - slice_metric_data = lambda level: dataframe[:, level[0], level[1]] - else: - levels = dataframe.index.levels[1] - format_key = str - slice_metric_data = lambda level: dataframe[:, level] - - for level in levels: - metric_data = dict(self._recurse_dimensions(slice_metric_data(level), dimensions[1:], metrics)) - - if not metric_data: - continue - - data.append((format_key(level), metric_data)) - - return data - - -class CSVRowIndexTransformer(DataTablesRowIndexTransformer): - def transform(self, dataframe, display_schema): - csv_df = self._format_columns(dataframe, display_schema['metrics'], display_schema['dimensions']) - - if isinstance(dataframe.index, pd.RangeIndex): - # If there are no dimensions, just serialize to csv without the index - return csv_df.to_csv(index=False) - - csv_df = self._format_index(csv_df, display_schema['dimensions']) - - row_dimension_labels = self._format_row_dimension_labels(display_schema['dimensions']) - return csv_df.to_csv(index_label=row_dimension_labels) - - def _format_index(self, csv_df, dimensions): - levels = list(dimensions.items())[:None if isinstance(csv_df.index, pd.MultiIndex) else 1] - - csv_df.index = [self.get_level_values(csv_df, key, dimension) - for key, dimension in levels] - csv_df.index.names = [key - for key, dimension in levels] - return csv_df - - def get_level_values(self, csv_df, key, dimension): - if 'display_options' in dimension: - return [_safe(dimension['display_options'].get(value, value)) - for value in csv_df.index.get_level_values(key)] - - if 'display_field' in dimension: - return [_safe(data_point) - for data_point in csv_df.index.get_level_values(dimension['display_field'])] - - return [_safe(data_point) - for data_point in csv_df.index.get_level_values(key)] - - @staticmethod - def _format_dimension_label(idx, dim_ordinal, dimension): - if 'display_field' in dimension: - display_field = dimension['display_field'] - return idx[dim_ordinal[display_field]] - - if isinstance(idx, tuple): - id_field = dimension['definition'][0] - dimension_display = idx[dim_ordinal[id_field]] - - else: - dimension_display = idx - - if 'display_options' in dimension: - return dimension['display_options'].get(dimension_display, dimension_display) - - return dimension_display - - def _format_columns(self, dataframe, metrics, dimensions): - return dataframe.rename(columns=lambda metric: metrics[metric].get('label', metric)) - - def _format_row_dimension_labels(self, dimensions): - return [dimension['label'] - for dimension in dimensions.values()] - - -class CSVColumnIndexTransformer(DataTablesColumnIndexTransformer, CSVRowIndexTransformer): - def _format_columns(self, dataframe, metrics, dimensions): - if 1 < len(dimensions): - csv_df = self._prepare_dataframe(dataframe, dimensions) - csv_df.columns = _format_column_display(csv_df, metrics, dimensions) - return csv_df - - return super(CSVColumnIndexTransformer, self)._format_columns(dataframe, metrics, dimensions) - - def _format_row_dimension_labels(self, dimensions): - return [dimension['label'] - for dimension in list(dimensions.values())[:1]] diff --git a/fireant/slicer/transformers/highcharts.py b/fireant/slicer/transformers/highcharts.py deleted file mode 100644 index 5a40e39e..00000000 --- a/fireant/slicer/transformers/highcharts.py +++ /dev/null @@ -1,601 +0,0 @@ -# coding: utf-8 -from collections import Counter - -import numpy as np -import pandas as pd - -from fireant import ( - settings, - utils, -) -from fireant.slicer.operations import Totals -from .base import ( - TransformationException, - Transformer, -) - -COLORS = { - 'kayak': ['#FF690F', '#3083F0', '#00B86B', '#D10244', '#FFDBE5', '#7B4F4B', '#B903AA', '#B05B6F'], - 'dark-blue': ["#DDDF0D", "#55BF3B", "#DF5353", "#7798BF", "#AAEEEE", "#FF0066", "#EEAAEE", - "#55BF3B", "#DF5353", "#7798BF", "#AAEEEE"], - 'dark-green': ["#DDDF0D", "#55BF3B", "#DF5353", "#7798BF", "#AAEEEE", "#FF0066", "#EEAAEE", - "#55BF3B", "#DF5353", "#7798BF", "#AAEEEE"], - 'dark-unica': ["#2B908F", "#90EE7E", "#F45B5B", "#7798BF", "#AAEEEE", "#FF0066", "#EEAAEE", - "#55BF3B", "#DF5353", "#7798BF", "#AAEEEE"], - 'gray': ["#DDDF0D", "#7798BF", "#55BF3B", "#DF5353", "#AAEEEE", "#FF0066", "#EEAAEE", - "#55BF3B", "#DF5353", "#7798BF", "#AAEEEE"], - 'grid-light': ["#7CB5EC", "#F7A35C", "#90EE7E", "#7798BF", "#AAEEEE", "#FF0066", "#EEAAEE", - "#55BF3B", "#DF5353", "#7798BF", "#AAEEEE"], - 'grid': ['#058DC7', '#50B432', '#ED561B', '#DDDF00', '#24CBE5', '#64E572', '#FF9655', '#FFF263', '#6AF9C4'], - 'sand-signika': ["#F45B5B", "#8085E9", "#8D4654", "#7798BF", "#AAEEEE", "#FF0066", "#EEAAEE", - "#55BF3B", "#DF5353", "#7798BF", "#AAEEEE"], - 'skies': ["#514F78", "#42A07B", "#9B5E4A", "#72727F", "#1F949A", "#82914E", "#86777F", "#42A07B"], -} - - -def _format_data_point(value): - if isinstance(value, pd.Timestamp): - return int(value.asm8) // int(1e6) - if value is None or (isinstance(value, (float, int)) and np.isnan(value)): - return None - if isinstance(value, np.int64): - # Cannot serialize np.int64 to json - return int(value) - return value - - -def _color(i): - colors = COLORS.get(settings.highcharts_colors, 'grid') - n_colors = len(colors) - - return colors[i % n_colors] - - -MISSING_CONT_DIM_MESSAGE = ('Highcharts line charts require a continuous dimension as the first ' - 'dimension. Please add a continuous dimension from your Slicer to ' - 'your request.') - - -class HighchartsLineTransformer(Transformer): - """ - Transforms data frames into Highcharts format for several chart types, particularly line or bar charts. - """ - - chart_type = 'line' - - def prevalidate_request(self, slicer, metrics, dimensions, - metric_filters, dimension_filters, - references, operations): - if not dimensions or not slicer.dimensions: - raise TransformationException(MISSING_CONT_DIM_MESSAGE) - - from fireant.slicer import ContinuousDimension - first_dimension = utils.slice_first(dimensions[0]) - dimension0 = slicer.dimensions[first_dimension] - if not isinstance(dimension0, ContinuousDimension): - raise TransformationException(MISSING_CONT_DIM_MESSAGE) - - def transform(self, dataframe, display_schema): - has_references = isinstance(dataframe.columns, pd.MultiIndex) - - dim_ordinal = {name: ordinal - for ordinal, name in enumerate(dataframe.index.names)} - dataframe = self._prepare_dataframe(dataframe, dim_ordinal, display_schema['dimensions']) - - if has_references: - series = sum( - [self._make_series(dataframe[level], dim_ordinal, display_schema, reference=level or None) - for level in dataframe.columns.levels[0]], - [] - ) - - else: - series = self._make_series(dataframe, dim_ordinal, display_schema) - - return { - 'chart': {'type': self.chart_type, 'zoomType': 'x'}, - 'title': {'text': None}, - 'plotOptions': {}, - 'xAxis': self.xaxis_options(dataframe, dim_ordinal, display_schema), - 'yAxis': self.yaxis_options(dataframe, dim_ordinal, display_schema), - 'tooltip': {'shared': True, 'useHTML': True}, - 'legend': {'useHTML': True}, - 'series': series - } - - def xaxis_options(self, dataframe, dim_ordinal, display_schema): - return { - 'type': 'datetime' if isinstance(dataframe.index, pd.DatetimeIndex) else 'linear' - } - - def yaxis_options(self, dataframe, dim_ordinal, display_schema): - metrics_per_axis = Counter((metric['axis'] for metric in display_schema['metrics'].values())) - - metric_idx = 0 - axis_options = {} - - for axes_id, metrics_count in metrics_per_axis.items(): - metric_idx += metrics_count - axis_options[axes_id] = { - 'opposite': metrics_count > 1, 'color': 'black' if metrics_count > 1 else _color(metric_idx - 1) - } - - # Axes with just one metric have the same color of their line. References and multiple metrics' axis - # will default to black. - axis = [ - { - 'id': axes_id, 'opposite': options['opposite'], 'title': {'text': None}, - 'labels': { - 'style': { - 'color': options['color'] - } - } - } - for axes_id, options in axis_options.items() - ] - - reference_keys = display_schema.get('references', {}).keys() - - # Create only one axes per available modifier (i.e. delta percent, percent and none). - reference_axes_ids = set(self._reference_axes_id(reference_key) for reference_key in reference_keys) - - for i, axes_id in enumerate(reference_axes_ids): - axis.append({'id': axes_id, 'title': {'text': 'References' if i == 0 else None}, 'opposite': True}) - - return axis - - def _make_series(self, dataframe, dim_ordinal, display_schema, reference=None): - metrics = list(dataframe.columns.levels[0] - if isinstance(dataframe.columns, pd.MultiIndex) - else dataframe.columns) - - return [self._make_series_item(idx, item, dim_ordinal, display_schema, metrics, reference, _color(i)) - for i, (idx, item) in enumerate(dataframe.iteritems())] - - def _make_series_item(self, idx, item, dim_ordinal, display_schema, metrics, reference, color='#000'): - metric_key = utils.slice_first(idx) - - return { - 'name': self._format_label(idx, dim_ordinal, display_schema, reference), - 'data': self._format_data(item), - 'tooltip': self._format_tooltip(display_schema['metrics'][metric_key]), - 'yAxis': display_schema['metrics'][metric_key].get('axis', 0) - if not reference else self._reference_axes_id(reference), - 'color': color, - 'dashStyle': 'Dot' if reference else 'Solid' - } - - @staticmethod - def _reference_axes_id(reference_key): - if '_' in reference_key: - # Replace dod, wow, mom and yoy with reference, since each modifier type should have only one axes. - modifier = '_'.join(reference_key.split('_')[1:]) - return 'reference_{}'.format(modifier) - return 'reference' - - @staticmethod - def _make_categories(dataframe, dim_ordinal, display_schema): - return None - - def _format_tooltip(self, metric_schema): - tooltip = {} - - if 'precision' in metric_schema: - tooltip['valueDecimals'] = metric_schema['precision'] - if 'prefix' in metric_schema: - tooltip['valuePrefix'] = metric_schema['prefix'] - if 'suffix' in metric_schema: - tooltip['valueSuffix'] = metric_schema['suffix'] - - return tooltip - - def _prepare_dataframe(self, dataframe, dim_ordinal, dimensions): - # Replaces invalid values and unstacks the data frame for line charts. - - # Force all fields to be float (Safer for highcharts) - dataframe = dataframe.astype(np.float).replace([np.inf, -np.inf], np.nan) - - # Unstack multi-indices - if 1 < len(dimensions): - # We need to unstack all of the dimensions here after the first dimension, which is the first dimension in - # the dimensions list, not necessarily the one in the dataframe - unstack_levels = list(self._unstack_levels(list(dimensions.items())[1:], dim_ordinal)) - dataframe = dataframe.unstack(level=unstack_levels) - - return dataframe - - def _format_label(self, idx, dim_ordinal, display_schema, reference): - is_multidimensional = isinstance(idx, tuple) - - metric_idx = idx[0] if is_multidimensional else idx - metric = display_schema['metrics'].get(metric_idx, {}) - metric_label = metric.get('label', metric_idx) - - if reference: - metric_label += ' {}'.format(display_schema['references'][reference]) - - if not is_multidimensional: - return metric_label - - dimension_labels = [self._format_dimension_display(dim_ordinal, key, dimension, idx) - for key, dimension in list(display_schema['dimensions'].items())[1:]] - - # filter out the Totals - dimension_labels = [dimension_label for dimension_label in dimension_labels - if (dimension_label or dimension_label is False) - and dimension_label is not Totals.label] - - return ( - '{metric} ({dimensions})'.format( - metric=metric_label, - dimensions=', '.join(map(str, dimension_labels)) - ) - if dimension_labels - else metric_label - ) - - @staticmethod - def _format_dimension_display(dim_ordinal, key, dimension, idx): - if not isinstance(idx, (list, tuple)): - idx = [idx] - - if 'display_field' in dimension: - display_field = dimension['display_field'] - return idx[dim_ordinal[display_field]] - - dimension_value = idx[dim_ordinal[key]] - - if pd.isnull(dimension_value) or dimension_value == '': - dimension_value = 'Null' - - if 'display_options' in dimension: - dimension_value = dimension['display_options'].get(dimension_value, dimension_value) - - return dimension_value - - def _format_data(self, column): - if isinstance(column, float): - return [_format_data_point(column)] - - return [self._format_point(key, value) - for key, value in column.iteritems() - if not (isinstance(value, (float, int)) and np.isnan(value)) and not pd.isnull(key)] - - @staticmethod - def _format_point(x, y): - return (_format_data_point(x), _format_data_point(y)) - - def _unstack_levels(self, dimensions, dim_ordinal): - for key, dimension in dimensions: - yield dim_ordinal[key] - - if 'display_field' in dimension: - yield dim_ordinal[dimension['display_field']] - - -class HighchartsAreaTransformer(HighchartsLineTransformer): - """ - Transformer for a Highcharts Area chart - http://www.highcharts.com/demo/area-basic - """ - chart_type = 'area' - - def _make_series_item(self, idx, item, dim_ordinal, display_schema, metrics, reference, color='#000'): - """ - Overriding the parent class' _make_series_item to remove the yAxis key as area charts - only really make sense on a single y axis - """ - metric_key = utils.slice_first(idx) - return { - 'name': self._format_label(idx, dim_ordinal, display_schema, reference), - 'data': self._format_data(item), - 'tooltip': self._format_tooltip(display_schema['metrics'][metric_key]), - 'color': color, - 'dashStyle': 'Dot' if reference else 'Solid' - } - - def yaxis_options(self, dataframe, dim_ordinal, display_schema): - return [{'title': None}] - - -class HighchartsAreaPercentageTransformer(HighchartsAreaTransformer): - """ - Transformer for a Highcharts Area Percentage chart - http://www.highcharts.com/demo/area-stacked-percent - """ - chart_type = 'area' - - def transform(self, dataframe, display_schema): - config = super(HighchartsAreaPercentageTransformer, self).transform(dataframe, display_schema) - config['plotOptions'] = { - 'area': { - 'stacking': 'percent', - } - } - return config - - def _format_tooltip(self, metric_schema): - tooltip = super(HighchartsAreaPercentageTransformer, self)._format_tooltip(metric_schema) - # Add the percentage to the default tooltip point format - tooltip['pointFormat'] = '\u25CF {series.name}: ' \ - '{point.y} - {point.percentage:.1f}%
' - return tooltip - - -class HighchartsColumnTransformer(HighchartsLineTransformer): - """ - Transformer for a Highcharts Column chart - http://www.highcharts.com/demo/column-basic - """ - chart_type = 'column' - - def prevalidate_request(self, slicer, metrics, dimensions, - metric_filters, dimension_filters, - references, operations): - if dimensions and 2 < len(dimensions): - # Too many dimensions - raise TransformationException('Highcharts bar and column charts support at a maximum two dimensions. ' - 'Request included %d dimensions.' % len(dimensions)) - - if dimensions and 2 == len(dimensions) and metrics and 1 < len(metrics): - # Too many metrics - raise TransformationException('Highcharts bar and column charts support at a maximum one metric with ' - 'two dimensions. Please remove some dimensions or metrics. ' - 'Request included %d metrics and %d dimensions.' % (len(metrics), - len(dimensions))) - - def _make_series_item(self, idx, item, dim_ordinal, display_schema, metrics, reference, color='#000'): - metric_key = utils.slice_first(idx) - return { - 'name': self._format_label(idx, dim_ordinal, display_schema, reference), - 'data': self._format_data(item), - 'tooltip': self._format_tooltip(display_schema['metrics'][metric_key]), - 'yAxis': display_schema['metrics'][metric_key].get('axis', 0) - if not reference else self._reference_axes_id(reference), - 'color': color - } - - def xaxis_options(self, dataframe, dim_ordinal, display_schema): - if isinstance(dataframe.index, pd.DatetimeIndex): - return {'type': 'datetime'} - - result = {'type': 'categorical'} - - categories = self._make_categories(dataframe, dim_ordinal, display_schema) - if categories is not None: - result['categories'] = categories - - return result - - def _validate_dimensions(self, dataframe, dimensions): - if 1 < len(dimensions) and 1 < len(dataframe.columns): - raise TransformationException('Cannot transform %s chart. ' - 'No more than 1 dimension or 2 dimensions ' - 'with 1 metric are allowed.' % self.chart_type) - - def _prepare_dataframe(self, dataframe, dim_ordinal, dimensions): - # Replaces invalid values and unstacks the data frame for line charts. - - # Force all fields to be float (Safer for highcharts) - dataframe = dataframe.replace([np.inf, -np.inf], np.nan) - - # Unstack multi-indices - if 1 < len(dimensions): - unstack_levels = list(self._unstack_levels(list(dimensions.items())[1:], dim_ordinal)) - dataframe = dataframe.unstack(level=unstack_levels) - - return dataframe - - @staticmethod - def _make_categories(dataframe, dim_ordinal, display_schema): - if not display_schema['dimensions']: - return None - - category_dimension = list(display_schema['dimensions'].values())[0] - - if 'display_field' in category_dimension: - display_field = category_dimension['display_field'] - return dataframe.index.get_level_values(display_field).tolist() - - display_options = category_dimension.get('category_dimension', {}) - return [display_options.get(value, value) for value in dataframe.index] - - -class HighchartsStackedColumnTransformer(HighchartsColumnTransformer): - """ - Transformer for a Highcharts Stacked Column chart - http://www.highcharts.com/demo/column-stacked - """ - - def _make_series_item(self, idx, item, dim_ordinal, display_schema, metrics, reference, color='#000'): - metric_key = utils.slice_first(idx) - return { - 'name': self._format_label(idx, dim_ordinal, display_schema, reference), - 'data': self._format_data(item), - 'tooltip': self._format_tooltip(display_schema['metrics'][metric_key]), - 'color': color - } - - def transform(self, dataframe, display_schema): - result = super(HighchartsStackedColumnTransformer, self).transform(dataframe, display_schema) - result['plotOptions'] = result.get('plotOptions', {}) - result['plotOptions'].update({ - 'column': { - 'stacking': 'normal' - } - }) - - return result - - def yaxis_options(self, dataframe, dim_ordinal, display_schema): - return [{'title': None}] - - -class HighchartsBarTransformer(HighchartsColumnTransformer): - """ - Transformer for a Highcharts Bar chart - http://www.highcharts.com/demo/bar-basic - """ - chart_type = 'bar' - - -class HighchartsStackedBarTransformer(HighchartsBarTransformer): - """ - Transformer for a Highcharts Stacked Bar chart - http://www.highcharts.com/demo/bar-stacked - """ - - def _make_series_item(self, idx, item, dim_ordinal, display_schema, metrics, reference, color='#000'): - metric_key = utils.slice_first(idx) - return { - 'name': self._format_label(idx, dim_ordinal, display_schema, reference), - 'data': self._format_data(item), - 'tooltip': self._format_tooltip(display_schema['metrics'][metric_key]), - 'color': color - } - - def transform(self, dataframe, display_schema): - result = super(HighchartsStackedBarTransformer, self).transform(dataframe, display_schema) - result['plotOptions'] = result.get('plotOptions', {}) - result['plotOptions'].update({ - 'series': { - 'stacking': 'normal' - } - }) - - return result - - def yaxis_options(self, dataframe, dim_ordinal, display_schema): - return [{'title': None}] - - -class HighchartsPieTransformer(HighchartsLineTransformer): - """ - Transformer to create the correct JSON payload for Highcharts Pie Charts - http://www.highcharts.com/demo/pie-basic - """ - chart_type = 'pie' - - def prevalidate_request(self, slicer, metrics, dimensions, - metric_filters, dimension_filters, - references, operations): - """ - Ensure no references or operations are passed and that there is no more than one enabled metric - """ - - if len(references) > 0 or len(operations) > 0: - raise TransformationException('References and Operations cannot be used with ' - '{} charts'.format(self.chart_type)) - - if len(utils.flatten(metrics)) > 1: - raise TransformationException('Only one metric can be specified when using ' - '{} charts'.format(self.chart_type)) - - def transform(self, dataframe, display_schema): - """ - Transform the dataframe into the format Highcharts expects for Pie charts - - :param dataframe: Dataframe containing queried data - :param display_schema: Dictionary defining how the metrics and dimensions should be displayed - :return: Dictionary in the required Highcharts Pie chart format ready to be dumped into JSON - """ - dim_ordinal = {name: ordinal - for ordinal, name in enumerate(dataframe.index.names)} - dataframe = self._prepare_dataframe(dataframe, dim_ordinal, display_schema['dimensions']) - series = self._make_series(dataframe, dim_ordinal, display_schema) - - # Issues have been seen when showing over 900 data points on a Highcharts pie chart. - max_data_points = 900 - num_data_points = len(series['data']) - if num_data_points > max_data_points: - raise TransformationException('You have reached the maximum number of data points that can be shown ' - 'on a pie chart. Maximum number of data points: {}. ' - 'Current number of data points: {}.'.format(max_data_points, num_data_points)) - - metric_key = self._get_metric_key(dataframe) - tooltip = self._format_tooltip(display_schema['metrics'][metric_key]) - tooltip['useHTML'] = True - - return { - 'chart': {'type': self.chart_type}, - 'legend': {'useHTML': True}, - 'title': {'text': None}, - 'tooltip': tooltip, - 'series': [series], - 'plotOptions': { - 'pie': { - 'allowPointSelect': True, # Allows users to select a piece of pie and it separates out - 'cursor': 'pointer', - 'dataLabels': { - 'enabled': True, - 'format': '{point.name}: {point.percentage:.1f} %', - 'style': { - 'color': COLORS.get(settings.highcharts_colors, 'grid') - } - } - } - }, - } - - def _prepare_dataframe(self, dataframe, dim_ordinal, dimensions): - """ - Force all fields to be float (Safer for highcharts) - - :param dataframe: Dataframe containing queried data - :param dim_ordinal: Dictionary containing dimensions with their associated order - :param dimensions: OrderedDict of dimensions along with their display options/display field - :return: Dataframe with float values and without np.inf values - """ - return dataframe.astype(np.float).replace([np.inf, -np.inf], np.nan) - - @staticmethod - def _get_metric_key(dataframe): - """ - Get the metric label (There will only ever be one metric) - - :param dataframe: Dataframe containing queried data - """ - metrics = list(dataframe.columns.levels[0] - if isinstance(dataframe.columns, pd.MultiIndex) - else dataframe.columns) - return metrics[0] - - def _make_series(self, dataframe, dim_ordinal, display_schema, reference=None): - """ - Create the series. Pie charts only ever have a single series. - - :param dataframe: Dataframe containing queried data - :param dim_ordinal: Dictionary containing dimensions with their associated order - :param display_schema: Dictionary defining how the metrics and dimensions should be displayed - :param reference: Not used - set here to match parent classes' function signature - :return: Dictonary containing the series name and data list - """ - metric_key = self._get_metric_key(dataframe) - return { - 'name': display_schema['metrics'][metric_key].get('label', metric_key), - 'data': [(self._format_label(idx, dim_ordinal, display_schema, reference), _format_data_point(item)) - for idx, item in dataframe[metric_key].iteritems()] - } - - def _format_label(self, idx, dim_ordinal, display_schema, reference): - """ - Create the labels for the pie pieces. If there is more than one dimension, - these will be shown in the format (dim1, dim2). Otherwise, it will show just the - individual dimension without parentheses. - - :param idx: Dataframe index - :param dim_ordinal: Dictionary containing dimensions with their associated order - :param display_schema: Dictionary defining how the metrics and dimensions should be displayed - :param reference: Not used - set here to match parent classes' function signature - :return: Dimension label string - """ - dimension_labels = [self._format_dimension_display(dim_ordinal, key, dimension, idx) - for key, dimension in list(display_schema['dimensions'].items())] - - label = ( - '{dimensions}'.format( - dimensions=', '.join(map(str, dimension_labels)) - ) - if dimension_labels else '' - ) - return '({})'.format(label) if len(dimension_labels) > 1 else label diff --git a/fireant/slicer/transformers/notebooks.py b/fireant/slicer/transformers/notebooks.py deleted file mode 100644 index 00aed887..00000000 --- a/fireant/slicer/transformers/notebooks.py +++ /dev/null @@ -1,128 +0,0 @@ -# coding: utf-8 -import pandas as pd - -from fireant import settings -from . import Transformer, TransformationException - - -def _format_dimension_labels(dimension): - if 'display_field' in dimension: - return ['%s ID' % dimension['label'], dimension['label']] - - return [dimension['label']] - - -class PandasRowIndexTransformer(Transformer): - def transform(self, dataframe, display_schema): - dataframe = self._set_display_options(dataframe, display_schema) - if display_schema['dimensions']: - dataframe = self._set_dimension_labels(dataframe, display_schema['dimensions']) - dataframe = self._set_metric_labels(dataframe, display_schema['metrics'], display_schema.get('references')) - - if isinstance(dataframe.index, pd.MultiIndex): - drop_levels = ['%s ID' % dimension['label'] - for key, dimension in display_schema['dimensions'].items() - if dimension.get('display_field')] - dataframe.reset_index(level=drop_levels, drop=True, inplace=True) - - return dataframe - - def _set_display_options(self, dataframe, display_schema): - """ - Replaces the dimension options with those that the user has specified manually e.g. change 'm' to 'mobile' - """ - dataframe = dataframe.copy() - - for key, dimension in display_schema['dimensions'].items(): - if 'display_options' in dimension: - display_values = [dimension['display_options'].get(value, value) - for value in dataframe.index.get_level_values(key).unique()] - - if not display_values: - continue - - if isinstance(dataframe.index, pd.MultiIndex): - dataframe.index.set_levels(display_values, key, inplace=True) - - else: - dataframe.index = pd.Index(display_values) - - return dataframe - - def _set_dimension_labels(self, dataframe, dimensions): - dataframe = dataframe.copy() - dataframe.index.names = [label - for key, dimension in dimensions.items() - for label in _format_dimension_labels(dimension)] - - return dataframe - - def _set_metric_labels(self, dataframe, metrics, references): - dataframe = dataframe.copy() - - labels = [metric['label'] for metric in metrics.values()] - if isinstance(dataframe.columns, pd.MultiIndex): - dataframe.columns = dataframe.columns.reorder_levels((1, 0)) \ - .set_levels([labels, - [None] + list(references.values())]) \ - .set_names('Reference', 1) - - else: - dataframe.columns = labels - - return dataframe - - -class PandasColumnIndexTransformer(PandasRowIndexTransformer): - def transform(self, dataframe, display_schema): - dataframe = super(PandasColumnIndexTransformer, self).transform(dataframe, display_schema) - - if isinstance(dataframe.index, pd.MultiIndex): - drop_levels = [dimension['label'] - for dimension in list(display_schema['dimensions'].values())[1:]] - - dataframe = dataframe.unstack(level=drop_levels) - - return dataframe - - -class MatplotlibLineChartTransformer(PandasColumnIndexTransformer): - def transform(self, dataframe, display_schema): - self._validate_dimensions(dataframe, display_schema['dimensions']) - dataframe = super(MatplotlibLineChartTransformer, self).transform(dataframe, display_schema) - - metrics = list(display_schema['metrics'].values()) - height, width = settings.matplotlib_figsize or (14, 5) - figsize = (height, width * len(metrics)) - - if 1 == len(metrics): - return dataframe.plot.line(figsize=figsize) \ - .legend(loc='center left', bbox_to_anchor=(1, 0.5)) \ - .set_title(metrics[0]['label']) - - try: - import matplotlib.pyplot as plt - except ImportError: - raise TransformationException('Missing library: matplotlib') - - fig, axes = plt.subplots(len(metrics), sharex=True, figsize=figsize) - for metric, axis in zip(metrics, axes): - label = metric['label'] - dataframe[label].plot.line(ax=axis) \ - .legend(loc='center left', bbox_to_anchor=(1, 0.5)) \ - .set_title(label) - - return axes - - def _validate_dimensions(self, dataframe, dimensions): - if not 0 < len(dimensions): - raise TransformationException('Cannot transform line chart. ' - 'At least one dimension is required.') - - -class MatplotlibBarChartTransformer(PandasColumnIndexTransformer): - def transform(self, dataframe, display_schema): - dataframe = super(MatplotlibBarChartTransformer, self).transform(dataframe, display_schema) - - return dataframe.plot.bar(figsize=settings.matplotlib_figsize or (14, 5)) \ - .legend(loc='center left', bbox_to_anchor=(1, 0.5)) diff --git a/fireant/slicer/widgets/__init__.py b/fireant/slicer/widgets/__init__.py new file mode 100644 index 00000000..0104753a --- /dev/null +++ b/fireant/slicer/widgets/__init__.py @@ -0,0 +1,4 @@ +from .datatables import DataTablesJS +from .highcharts import HighCharts +from .matplotlib import Matplotlib +from .pandas import Pandas diff --git a/fireant/slicer/widgets/base.py b/fireant/slicer/widgets/base.py new file mode 100644 index 00000000..6e11aa5c --- /dev/null +++ b/fireant/slicer/widgets/base.py @@ -0,0 +1,19 @@ +from fireant.utils import immutable + + +class Widget(object): + def __init__(self, metrics=()): + self._metrics = list(metrics) + + @immutable + def metric(self, metric): + self._metrics.append(metric) + + @property + def metrics(self): + return [metric + for group in self._metrics + for metric in (group.metrics if hasattr(group, 'metrics') else [group])] + + def transform(self, data_frame, slicer): + raise NotImplementedError() diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py new file mode 100644 index 00000000..ac5a3996 --- /dev/null +++ b/fireant/slicer/widgets/datatables.py @@ -0,0 +1,169 @@ +import itertools + +import pandas as pd + +from fireant import ( + ContinuousDimension, + utils, +) +from . import formats +from .base import Widget + + +def _format_dimension_cell(dimension_value, display_values): + dimension_cell = {'value': formats.safe(dimension_value)} + + if display_values is not None: + dimension_cell['display'] = display_values.get(dimension_value, dimension_value) + + return dimension_cell + + +def _format_dimensional_metric_cell(row_data, metric): + level = {} + for key, next_row in row_data.groupby(level=-1): + next_row.reset_index(level=-1, drop=True, inplace=True) + + safe_key = formats.safe(key) + + level[safe_key] = _format_dimensional_metric_cell(next_row, metric) \ + if isinstance(next_row.index, pd.MultiIndex) \ + else _format_metric_cell(next_row[metric.key], metric) + + return level + + +def _format_metric_cell(value, metric): + raw_value = formats.safe(value) + return { + 'value': raw_value, + 'display': formats.display(raw_value, + prefix=metric.prefix, + suffix=metric.suffix, + precision=metric.precision) + if raw_value is not None + else None + } + + +HARD_MAX_COLUMNS = 24 + + +class DataTablesJS(Widget): + def __init__(self, metrics=(), pivot=False, max_columns=None): + super(DataTablesJS, self).__init__(metrics) + self.pivot = pivot + self.max_columns = min(max_columns, HARD_MAX_COLUMNS) \ + if max_columns is not None \ + else HARD_MAX_COLUMNS + + def transform(self, data_frame, slicer): + """ + + :param data_frame: + :param slicer: + :return: + """ + dimension_keys = list(filter(None, data_frame.index.names)) + dimensions = [getattr(slicer.dimensions, key) + for key in dimension_keys] + dimension_display_values = self._dimension_display_values(dimensions, data_frame) + + metric_keys = [metric.key for metric in self.metrics] + data_frame = data_frame[metric_keys] + + pivot_index_to_columns = self.pivot and isinstance(data_frame.index, pd.MultiIndex) + if pivot_index_to_columns: + levels = list(range(1, len(dimensions))) + data_frame = data_frame \ + .unstack(level=levels) \ + .fillna(value=0) + + columns = self._dimension_columns(dimensions[:1]) \ + + self._metric_columns_pivoted(data_frame.columns, dimension_display_values) + + else: + columns = self._dimension_columns(dimensions) + self._metric_columns() + + data = [self._data_row(dimensions, dimension_values, dimension_display_values, row_data) + for dimension_values, row_data in data_frame.iterrows()] + + return dict(columns=columns, data=data) + + def _dimension_display_values(self, dimensions, data_frame): + dv_by_dimension = {} + + for dimension in dimensions: + dkey = dimension.key + if hasattr(dimension, 'display_values'): + dv_by_dimension[dkey] = dimension.display_values + elif hasattr(dimension, 'display_key'): + dv_by_dimension[dkey] = data_frame[dimension.display_key].groupby(level=dkey).first() + + return dv_by_dimension + + def _dimension_columns(self, dimensions): + """ + + :param data_frame: + :param slicer: + :return: + """ + columns = [] + for dimension in dimensions: + column = dict(title=dimension.label or dimension.key, + data=dimension.key, + render=dict(_='value')) + + if not isinstance(dimension, ContinuousDimension): + column['render']['display'] = 'display' + + columns.append(column) + + return columns + + def _metric_columns(self): + """ + + :return: + """ + columns = [] + for metric in self.metrics: + columns.append(dict(title=metric.label or metric.key, + data=metric.key, + render=dict(_='value', display='display'))) + return columns + + def _metric_columns_pivoted(self, df_columns, display_values): + """ + + :param index_levels: + :param display_values: + :return: + """ + columns = [] + for metric in self.metrics: + dimension_keys = df_columns.names[1:] + for level_values in itertools.product(*map(list, df_columns.levels[1:])): + level_display_values = [utils.deep_get(display_values, [key, raw_value], raw_value) + for key, raw_value in zip(dimension_keys, level_values)] + + columns.append(dict(title="{metric} ({dimensions})".format(metric=metric.label or metric.key, + dimensions=", ".join(level_display_values)), + data='.'.join([metric.key] + [str(x) for x in level_values]), + render=dict(_='value', display='display'))) + + return columns + + def _data_row(self, dimensions, dimension_values, dimension_display_values, row_data): + row = {} + + for dimension, dimension_value in zip(dimensions, utils.wrap_list(dimension_values)): + row[dimension.key] = _format_dimension_cell(dimension_value, dimension_display_values.get(dimension.key)) + + for metric in self.metrics: + row[metric.key] = _format_dimensional_metric_cell(row_data, metric) \ + if isinstance(row_data.index, pd.MultiIndex) \ + else _format_metric_cell(row_data[metric.key], metric) + + return row diff --git a/fireant/slicer/widgets/formats.py b/fireant/slicer/widgets/formats.py new file mode 100644 index 00000000..42ad941c --- /dev/null +++ b/fireant/slicer/widgets/formats.py @@ -0,0 +1,57 @@ +import locale + +import numpy as np +import pandas as pd +from datetime import ( + date, + datetime, + time, +) + +NO_TIME = time(0) + + +def safe(value): + if isinstance(value, date): + if not hasattr(value, 'time') or value.time() == NO_TIME: + return value.strftime('%Y-%m-%d') + else: + return value.strftime('%Y-%m-%dT%H:%M:%S') + + if value is None or (isinstance(value, float) and np.isnan(value)) or pd.isnull(value): + return None + + if isinstance(value, np.int64): + # Cannot transform np.int64 to json + return int(value) + + if isinstance(value, np.float64): + return float(value) + + return value + + +def display(value, prefix=None, suffix=None, precision=None): + if isinstance(value, float): + if precision is not None: + float_format = '%d' if precision == 0 else '%.{}f'.format(precision) + value = locale.format(float_format, value, grouping=True) + + elif value.is_integer(): + float_format = '%d' + value = locale.format(float_format, value, grouping=True) + + else: + float_format = '%f' + # Stripping trailing zeros is necessary because %f can add them if no precision is set + value = locale.format(float_format, value, grouping=True).rstrip('.0') + + if isinstance(value, int): + float_format = '%d' + value = locale.format(float_format, value, grouping=True) + + return '{prefix}{value}{suffix}'.format( + prefix=prefix or '', + value=str(value), + suffix=suffix or '', + ) diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py new file mode 100644 index 00000000..704905ca --- /dev/null +++ b/fireant/slicer/widgets/highcharts.py @@ -0,0 +1,34 @@ +from fireant.utils import immutable +from .base import Widget + + +class HighCharts(Widget): + class PieChart(Widget): + pass + + class LineChart(Widget): + pass + + class ColumnChart(Widget): + pass + + class BarChart(Widget): + pass + + def __init__(self, axes=()): + self.axes = list(axes) + + def metric(self, metric): + raise NotImplementedError() + + @immutable + def axis(self, axis): + self.axes.append(axis) + + @property + def metrics(self): + seen = set() + return [metric + for axis in self.axes + for metric in axis.metrics + if not (metric.key in seen or seen.add(metric.key))] diff --git a/fireant/slicer/widgets/matplotlib.py b/fireant/slicer/widgets/matplotlib.py new file mode 100644 index 00000000..4e746133 --- /dev/null +++ b/fireant/slicer/widgets/matplotlib.py @@ -0,0 +1,5 @@ +from .base import Widget + + +class Matplotlib(Widget): + pass diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py new file mode 100644 index 00000000..f01f4b17 --- /dev/null +++ b/fireant/slicer/widgets/pandas.py @@ -0,0 +1,5 @@ +from .base import Widget + + +class Pandas(Widget): + pass diff --git a/fireant/tests/dashboards/test_dashboard_api.py b/fireant/tests/dashboards/test_dashboard_api.py deleted file mode 100644 index 1b979f23..00000000 --- a/fireant/tests/dashboards/test_dashboard_api.py +++ /dev/null @@ -1,686 +0,0 @@ -# coding: utf-8 -from datetime import date -from unittest import TestCase - -import pandas as pd -from mock import Mock, patch, call - -from fireant.dashboards import * -from fireant.slicer import * -from fireant.slicer.managers import SlicerManager -from fireant.slicer.references import WoW -from fireant.slicer.transformers import TransformationException -from fireant.tests.database.mock_database import TestDatabase -from pypika import ( - Table, - functions as fn, - Order, -) - - -class DashboardTests(TestCase): - maxDiff = None - - @classmethod - def setUpClass(cls): - test_table = Table('test_table') - cls.test_slicer = Slicer( - table=test_table, - database=TestDatabase(), - - metrics=[ - # Metric with defaults - Metric('clicks', 'Clicks'), - Metric('conversions', 'Conversions'), - Metric('roi', 'ROI', definition=fn.Sum(test_table.revenue) / fn.Sum(test_table.cost)), - Metric('rpc', 'RPC', definition=fn.Sum(test_table.revenue) / fn.Sum(test_table.clicks)), - Metric('cpc', 'CPC', definition=fn.Sum(test_table.cost) / fn.Sum(test_table.clicks)), - ], - - dimensions=[ - # Continuous date dimension - DatetimeDimension('date', definition=test_table.dt), - - # Continuous integer dimension - ContinuousDimension('clicks', label='Clicks CUSTOM LABEL', definition=test_table.clicks), - - # Categorical dimension with fixed options - CategoricalDimension('locale', 'Locale', definition=test_table.locale, - display_options=[DimensionValue('us', 'United States'), - DimensionValue('de', 'Germany')]), - - # Unique Dimension with single ID field - UniqueDimension('account', 'Account', definition=test_table.account_id, - display_field=test_table.account_name), - ] - ) - - cls.test_slicer.manager.data = Mock() - cls.test_slicer.manager.display_schema = Mock() - - def assert_slicer_queried(self, metrics, dimensions=None, mfilters=None, dfilters=None, - references=None, operations=None, pagination=None): - self.test_slicer.manager.data.assert_called_with( - metrics=metrics, - dimensions=dimensions or [], - metric_filters=mfilters or [], - dimension_filters=dfilters or [], - references=references or [], - operations=operations or [], - pagination=pagination, - ) - - def assert_result_transformed(self, widgets, dimensions, mock_transformer, tx_generator, references=(), - operations=()): - # Assert that there is a result for each widget - self.assertEqual(len(widgets), len(list(tx_generator))) - - self.test_slicer.manager.display_schema.assert_has_calls( - [call( - dimensions=dimensions or [], - metrics=widget.metrics, - references=list(references), - operations=list(operations), - ) for widget in widgets] - ) - - # Assert that a transformation was performed for each widget - self.assertEqual(len(mock_transformer.transform.call_args), len(widgets)) - - for idx, widget in enumerate(widgets): - self.assertIsInstance(mock_transformer.transform.call_args_list[idx][0][0], pd.DataFrame) - - metrics = widget.metrics - if references: - metrics = [(reference.key if reference else '', metric) - for reference in [None] + references - for metric in metrics] - - self.assertListEqual(list(mock_transformer.transform.call_args_list[idx][0][0].columns), metrics) - - -class DashboardSchemaTests(DashboardTests): - def test_if_the_dashboard_query_string_is_equal_to_its_slicer_query_string_given_same_metrics(self): - metrics = ['clicks'] - - test_render = WidgetGroup( - slicer=self.test_slicer, - widgets=[ - LineChartWidget(metrics=metrics), - ] - ) - - result = test_render.manager.query_string() - - self.assertEqual(self.test_slicer.manager.query_string(metrics=metrics), result) - - @patch.object(SlicerManager, 'query_string') - def test_paginator_sent_to_query_string_is_added_to_query(self, mock_slicer_query_string): - metrics = ['clicks'] - paginator = Paginator(limit=10, offset=10, order=[('clicks', Order.desc)]) - - test_render = WidgetGroup( - slicer=self.test_slicer, - widgets=[ - LineChartWidget(metrics=metrics), - ], - ) - - test_render.manager.query_string(pagination=paginator) - - mock_slicer_query_string.assert_called_once_with(metrics=metrics, dimensions=[], - metric_filters=[], dimension_filters=[], - operations=[], references=[], pagination=paginator) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_metric_widgets(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = [] - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=metrics) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ] - ) - - result = test_render.manager.render() - - self.assert_slicer_queried( - metrics, - ) - self.assert_result_transformed(test_render.widgets, [], mock_transformer, result) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_categorical_dim(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = ['locale'] - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=metrics) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimensions=dimensions, - ) - - result = test_render.manager.render() - - self.assert_slicer_queried( - ['clicks', 'conversions'], - dimensions=dimensions, - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_datetime_dim(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = [('date', DatetimeDimension.week)] - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=metrics) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimensions=dimensions, - ) - - result = test_render.manager.render() - - self.assert_slicer_queried( - ['clicks', 'conversions'], - dimensions=dimensions, - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, - result) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_eq_filter_dim(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = [] - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=metrics) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - eq_filter = EqualityFilter('device_type', EqualityOperator.eq, 'desktop') - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimension_filters=[eq_filter], - ) - - result = test_render.manager.render() - - self.assert_slicer_queried( - metrics, - dfilters=[eq_filter], - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_contains_filter_dim(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = [] - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=metrics) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - contains_filter = ContainsFilter('device_type', ['desktop', 'mobile']) - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimension_filters=[contains_filter], - ) - - result = test_render.manager.render() - - self.assert_slicer_queried( - metrics, - dfilters=[contains_filter], - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_pagination_in_widgetgroup_render(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = ['locale'] - paginator = Paginator(offset=10, limit=10, order=('locale', Order.desc)) - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=metrics) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimensions=dimensions, - ) - - result = test_render.manager.render(pagination=paginator) - - self.assert_slicer_queried( - ['clicks', 'conversions'], - dimensions=dimensions, - pagination=paginator - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_range_filter_date(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = [] - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=metrics) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - range_filter = RangeFilter('date', date(2000, 1, 1), date(2000, 3, 1)) - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimension_filters=[range_filter], - ) - - result = test_render.manager.render() - - self.assert_slicer_queried( - metrics, - dfilters=[range_filter], - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_wildcard_filter_date(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = [] - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=metrics) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - wildcard_filter = WildcardFilter('locale', 'U%') - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimension_filters=[wildcard_filter], - ) - - result = test_render.manager.render() - - self.assert_slicer_queried( - metrics, - dfilters=[wildcard_filter], - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_reference_with_dim(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = ['date'] - references = [WoW('date')] - self.test_slicer.manager.data.return_value = pd.DataFrame( - columns=pd.MultiIndex.from_product([['', 'wow'], metrics]) - ) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': {'wow': 'date'}}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': {'wow': 'date'}}, - ] - - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimensions=dimensions, - ) - - result = test_render.manager.render(references=references) - - self.assert_slicer_queried( - metrics, - dimensions=dimensions, - references=references, - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result, references) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_reference_in_widgetgroup(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = ['date'] - references = [WoW('date')] - self.test_slicer.manager.data.return_value = pd.DataFrame( - columns=pd.MultiIndex.from_product([['', 'wow'], metrics]) - ) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': {'wow': 'date'}}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': {'wow': 'date'}}, - ] - - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimensions=dimensions, - references=references, - ) - - result = test_render.manager.render() - - self.assert_slicer_queried( - metrics, - dimensions=dimensions, - references=references, - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result, references) - - -class DashboardAPITests(DashboardTests): - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_api_with_dimension(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = ['date'] - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=metrics) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ] - ) - - result = test_render.manager.render( - dimensions=dimensions, - ) - - self.assert_slicer_queried( - metrics, - dimensions=dimensions, - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_api_with_filter(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = [] - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - eq_filter = EqualityFilter('device_type', EqualityOperator.eq, 'desktop') - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ] - ) - - result = test_render.manager.render( - dimension_filters=[eq_filter], - ) - - self.assert_slicer_queried( - ['clicks', 'conversions'], - dfilters=[eq_filter], - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_api_with_reference(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = ['date'] - self.test_slicer.manager.data.return_value = pd.DataFrame( - columns=pd.MultiIndex.from_product([['', 'wow'], metrics]) - ) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': {}}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': {}}, - ] - - references = [WoW('date')] - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimensions=dimensions - ) - - result = test_render.manager.render( - references=references, - ) - - self.assert_slicer_queried( - metrics, - dimensions=dimensions, - references=references, - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result, references) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_remove_duplicated_dimension_keys(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = ['date'] - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=metrics) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimensions=dimensions, - ) - - result = test_render.manager.render( - dimensions=dimensions, - ) - - self.assert_slicer_queried( - metrics, - dimensions=dimensions, - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_remove_duplicated_dimension_keys_with_intervals_in_schema(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = [('date', DatetimeDimension.day)] - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=metrics) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimensions=dimensions, - ) - - result = test_render.manager.render( - dimensions=['date'], - ) - - self.assert_slicer_queried( - metrics, - dimensions=dimensions, - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_remove_duplicated_dimension_keys_with_intervals_in_api(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = [('date', DatetimeDimension.day)] - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=metrics) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimensions=dimensions, - ) - - result = test_render.manager.render( - dimensions=[('date', DatetimeDimension.week)], - ) - - self.assert_slicer_queried( - metrics, - dimensions=dimensions, - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result) - - @patch('fireant.dashboards.LineChartWidget.transformer') - def test_remove_duplicated_dimension_keys_with_intervals_in_api2(self, mock_transformer): - metrics = ['clicks', 'conversions'] - dimensions = ['date'] - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=metrics) - self.test_slicer.manager.display_schema.side_effect = [ - {'metrics': metrics[:1], 'dimensions': dimensions, 'references': []}, - {'metrics': metrics[1:], 'dimensions': dimensions, 'references': []}, - ] - - test_render = WidgetGroup( - slicer=self.test_slicer, - - widgets=[ - LineChartWidget(metrics=metrics[:1]), - LineChartWidget(metrics=metrics[1:]), - ], - - dimensions=dimensions, - ) - - result = test_render.manager.render( - dimensions=[('date', DatetimeDimension.week)], - ) - - self.assert_slicer_queried( - metrics, - dimensions=dimensions, - ) - self.assert_result_transformed(test_render.widgets, dimensions, mock_transformer, result) - - -class PrevalidationTests(DashboardTests): - @classmethod - def setUpClass(cls): - super(PrevalidationTests, cls).setUpClass() - - cls.test_wg = WidgetGroup( - slicer=cls.test_slicer, - - widgets=[ - LineChartWidget(metrics=['clicks']), - LineChartWidget(metrics=['conversions']), - ] - ) - - def test_raises_exception_for_linechart_with_no_dimensions(self): - with self.assertRaises(TransformationException): - self.test_wg.manager.render(dimensions=[]) - - @patch('fireant.dashboards.LineChartWidget.transformer.transform') - def test_raises_exception_for_linechart_without_continuous_first_dimension(self, mock_transform): - self.test_slicer.manager.data.return_value = pd.DataFrame(columns=['clicks', 'conversions']) - - self.test_wg.manager.render(dimensions=['date']) - self.test_wg.manager.render(dimensions=['clicks']) - - with self.assertRaises(TransformationException): - self.test_wg.manager.render(dimensions=['locale']) - with self.assertRaises(TransformationException): - self.test_wg.manager.render(dimensions=['account']) diff --git a/fireant/tests/dashboards/test_widgets.py b/fireant/tests/dashboards/test_widgets.py deleted file mode 100644 index 44cf748e..00000000 --- a/fireant/tests/dashboards/test_widgets.py +++ /dev/null @@ -1,57 +0,0 @@ -# coding: utf-8 -from unittest import TestCase - -from fireant.dashboards import * -from fireant.slicer.transformers import ( - CSVColumnIndexTransformer, - CSVRowIndexTransformer, - DataTablesColumnIndexTransformer, - DataTablesRowIndexTransformer, - HighchartsBarTransformer, - HighchartsColumnTransformer, - HighchartsLineTransformer, -) - - -class DashboardTests(TestCase): - def _widget_test(self, widget_type, metrics, transformer_type): - widget = widget_type(metrics=metrics) - - self.assertListEqual(metrics, widget.metrics) - self.assertEqual(type(transformer_type), type(widget.transformer)) - return widget - - def test_line_chart(self): - self._widget_test(LineChartWidget, ['clicks', 'conversions'], HighchartsLineTransformer()) - - def test_bar_chart(self): - tx = HighchartsBarTransformer() - widget = self._widget_test(BarChartWidget, ['clicks', 'conversions'], tx) - - self.assertEqual(HighchartsBarTransformer.chart_type, widget.transformer.chart_type) - - def test_column_chart(self): - tx = HighchartsColumnTransformer() - widget = self._widget_test(ColumnChartWidget, ['clicks', 'conversions'], tx) - - self.assertEqual(HighchartsColumnTransformer.chart_type, widget.transformer.chart_type) - - def test_row_index_table(self): - tx = DataTablesRowIndexTransformer() - widget = self._widget_test(RowIndexTableWidget, ['clicks', 'conversions'], tx) - self.assertEqual(DataTablesRowIndexTransformer, type(widget.transformer)) - - def test_column_index_table(self): - tx = DataTablesColumnIndexTransformer() - widget = self._widget_test(ColumnIndexTableWidget, ['clicks', 'conversions'], tx) - self.assertEqual(DataTablesColumnIndexTransformer, type(widget.transformer)) - - def test_row_index_csv(self): - tx = CSVRowIndexTransformer() - widget = self._widget_test(RowIndexCSVWidget, ['clicks', 'conversions'], tx) - self.assertEqual(CSVRowIndexTransformer, type(widget.transformer)) - - def test_column_index_csv(self): - tx = CSVColumnIndexTransformer() - widget = self._widget_test(ColumnIndexCSVWidget, ['clicks', 'conversions'], tx) - self.assertEqual(CSVColumnIndexTransformer, type(widget.transformer)) diff --git a/fireant/tests/database/__init__.py b/fireant/tests/database/__init__.py index 07761d90..11fb1c15 100644 --- a/fireant/tests/database/__init__.py +++ b/fireant/tests/database/__init__.py @@ -1,4 +1,3 @@ -# coding: utf-8 __author__ = "Timothy Heys" __email__ = "theys@kayak.com" diff --git a/fireant/tests/database/mock_database.py b/fireant/tests/database/mock_database.py index a9fa2605..1c02692b 100644 --- a/fireant/tests/database/mock_database.py +++ b/fireant/tests/database/mock_database.py @@ -1,4 +1,3 @@ -# coding: utf-8 from fireant.database.vertica import VerticaDatabase diff --git a/fireant/tests/database/test_databases.py b/fireant/tests/database/test_databases.py index 42262a90..d7d17028 100644 --- a/fireant/tests/database/test_databases.py +++ b/fireant/tests/database/test_databases.py @@ -1,4 +1,3 @@ -# coding: utf-8 from unittest import TestCase from mock import patch, MagicMock diff --git a/fireant/tests/database/test_mysql.py b/fireant/tests/database/test_mysql.py index 27201fe7..b85f1054 100644 --- a/fireant/tests/database/test_mysql.py +++ b/fireant/tests/database/test_mysql.py @@ -1,22 +1,26 @@ -# coding: utf-8 from unittest import TestCase -from mock import patch, Mock, ANY -from pypika import Field +from mock import ( + ANY, + Mock, + patch, +) from fireant.database import MySQLDatabase +from pypika import Field class TestMySQLDatabase(TestCase): + @classmethod + def setUpClass(cls): + cls.mysql = MySQLDatabase(database='testdb') def test_defaults(self): - mysql = MySQLDatabase(database='testdb') - - self.assertEqual('localhost', mysql.host) - self.assertEqual(3306, mysql.port) - self.assertEqual('utf8mb4', mysql.charset) - self.assertIsNone(mysql.user) - self.assertIsNone(mysql.password) + self.assertEqual('localhost', self.mysql.host) + self.assertEqual(3306, self.mysql.port) + self.assertEqual('utf8mb4', self.mysql.charset) + self.assertIsNone(self.mysql.user) + self.assertIsNone(self.mysql.password) def test_connect(self): mock_pymysql = Mock() @@ -34,61 +38,61 @@ def test_connect(self): ) def test_trunc_hour(self): - result = MySQLDatabase(database='testdb').trunc_date(Field('date'), 'hour') + result = self.mysql.trunc_date(Field('date'), 'hour') self.assertEqual('dashmore.TRUNC("date",\'hour\')', str(result)) def test_trunc_day(self): - result = MySQLDatabase(database='testdb').trunc_date(Field('date'), 'day') + result = self.mysql.trunc_date(Field('date'), 'day') self.assertEqual('dashmore.TRUNC("date",\'day\')', str(result)) def test_trunc_week(self): - result = MySQLDatabase(database='testdb').trunc_date(Field('date'), 'week') + result = self.mysql.trunc_date(Field('date'), 'week') self.assertEqual('dashmore.TRUNC("date",\'week\')', str(result)) def test_trunc_month(self): - result = MySQLDatabase(database='testdb').trunc_date(Field('date'), 'month') + result = self.mysql.trunc_date(Field('date'), 'month') self.assertEqual('dashmore.TRUNC("date",\'month\')', str(result)) def test_trunc_quarter(self): - result = MySQLDatabase(database='testdb').trunc_date(Field('date'), 'quarter') + result = self.mysql.trunc_date(Field('date'), 'quarter') self.assertEqual('dashmore.TRUNC("date",\'quarter\')', str(result)) def test_trunc_year(self): - result = MySQLDatabase(database='testdb').trunc_date(Field('date'), 'year') + result = self.mysql.trunc_date(Field('date'), 'year') self.assertEqual('dashmore.TRUNC("date",\'year\')', str(result)) def test_date_add_hour(self): - result = MySQLDatabase(database='testdb').date_add('hour', 1, Field('date')) + result = self.mysql.date_add(Field('date'), 'hour', 1) self.assertEqual('DATE_ADD("date",INTERVAL 1 HOUR)', str(result)) def test_date_add_day(self): - result = MySQLDatabase(database='testdb').date_add('day', 1, Field('date')) + result = self.mysql.date_add(Field('date'), 'day', 1) self.assertEqual('DATE_ADD("date",INTERVAL 1 DAY)', str(result)) def test_date_add_week(self): - result = MySQLDatabase(database='testdb').date_add('week', 1, Field('date')) + result = self.mysql.date_add(Field('date'), 'week', 1) self.assertEqual('DATE_ADD("date",INTERVAL 1 WEEK)', str(result)) def test_date_add_month(self): - result = MySQLDatabase(database='testdb').date_add('month', 1, Field('date')) + result = self.mysql.date_add(Field('date'), 'month', 1) self.assertEqual('DATE_ADD("date",INTERVAL 1 MONTH)', str(result)) def test_date_add_quarter(self): - result = MySQLDatabase(database='testdb').date_add('quarter', 1, Field('date')) + result = self.mysql.date_add(Field('date'), 'quarter', 1) self.assertEqual('DATE_ADD("date",INTERVAL 1 QUARTER)', str(result)) def test_date_add_year(self): - result = MySQLDatabase(database='testdb').date_add('year', 1, Field('date')) + result = self.mysql.date_add(Field('date'), 'year', 1) self.assertEqual('DATE_ADD("date",INTERVAL 1 YEAR)', str(result)) diff --git a/fireant/tests/database/test_postgresql.py b/fireant/tests/database/test_postgresql.py index 31d2d429..7ff5889e 100644 --- a/fireant/tests/database/test_postgresql.py +++ b/fireant/tests/database/test_postgresql.py @@ -1,4 +1,3 @@ -# coding: utf-8 from unittest import TestCase from mock import ( @@ -13,7 +12,6 @@ class TestPostgreSQL(TestCase): @classmethod def setUpClass(cls): - super(TestPostgreSQL, cls).setUpClass() cls.database = PostgreSQLDatabase() def test_defaults(self): @@ -63,31 +61,31 @@ def test_trunc_year(self): self.assertEqual('date_trunc(\'year\',"date")', str(result)) def test_date_add_hour(self): - result = self.database.date_add('hour', 1, Field('date')) + result = self.database.date_add(Field('date'), 'hour', 1) self.assertEqual('DATE_ADD(\'hour\',1,"date")', str(result)) def test_date_add_day(self): - result = self.database.date_add('day', 1, Field('date')) + result = self.database.date_add(Field('date'), 'day', 1) self.assertEqual('DATE_ADD(\'day\',1,"date")', str(result)) def test_date_add_week(self): - result = self.database.date_add('week', 1, Field('date')) + result = self.database.date_add(Field('date'), 'week', 1) self.assertEqual('DATE_ADD(\'week\',1,"date")', str(result)) def test_date_add_month(self): - result = self.database.date_add('month', 1, Field('date')) + result = self.database.date_add(Field('date'), 'month', 1) self.assertEqual('DATE_ADD(\'month\',1,"date")', str(result)) def test_date_add_quarter(self): - result = self.database.date_add('quarter', 1, Field('date')) + result = self.database.date_add(Field('date'), 'quarter', 1) self.assertEqual('DATE_ADD(\'quarter\',1,"date")', str(result)) def test_date_add_year(self): - result = self.database.date_add('year', 1, Field('date')) + result = self.database.date_add(Field('date'), 'year', 1) self.assertEqual('DATE_ADD(\'year\',1,"date")', str(result)) diff --git a/fireant/tests/database/test_redshift.py b/fireant/tests/database/test_redshift.py index d0e658ee..eb3f8e59 100644 --- a/fireant/tests/database/test_redshift.py +++ b/fireant/tests/database/test_redshift.py @@ -1,4 +1,3 @@ -# coding: utf-8 from mock import ( Mock, @@ -13,7 +12,6 @@ class TestRedshift(TestPostgreSQL): """ Inherits from TestPostgreSQL as Redshift is almost identical to PostgreSQL so the tests are similar """ @classmethod def setUpClass(cls): - super(TestRedshift, cls).setUpClass() cls.database = RedshiftDatabase() def test_defaults(self): diff --git a/fireant/tests/database/test_vertica.py b/fireant/tests/database/test_vertica.py index ce1559fe..06d27690 100644 --- a/fireant/tests/database/test_vertica.py +++ b/fireant/tests/database/test_vertica.py @@ -1,8 +1,14 @@ -# coding: utf-8 from unittest import TestCase from mock import patch, Mock +from fireant import ( + hourly, + daily, + weekly, + quarterly, + annually, +) from fireant.database import VerticaDatabase from pypika import Field @@ -35,57 +41,57 @@ def test_connect(self): ) def test_trunc_hour(self): - result = VerticaDatabase().trunc_date(Field('date'), 'hour') + result = VerticaDatabase().trunc_date(Field('date'), hourly) self.assertEqual('TRUNC("date",\'HH\')', str(result)) def test_trunc_day(self): - result = VerticaDatabase().trunc_date(Field('date'), 'day') + result = VerticaDatabase().trunc_date(Field('date'), daily) self.assertEqual('TRUNC("date",\'DD\')', str(result)) def test_trunc_week(self): - result = VerticaDatabase().trunc_date(Field('date'), 'week') + result = VerticaDatabase().trunc_date(Field('date'), weekly) self.assertEqual('TRUNC("date",\'IW\')', str(result)) def test_trunc_quarter(self): - result = VerticaDatabase().trunc_date(Field('date'), 'quarter') + result = VerticaDatabase().trunc_date(Field('date'), quarterly) self.assertEqual('TRUNC("date",\'Q\')', str(result)) def test_trunc_year(self): - result = VerticaDatabase().trunc_date(Field('date'), 'year') + result = VerticaDatabase().trunc_date(Field('date'), annually) self.assertEqual('TRUNC("date",\'Y\')', str(result)) def test_date_add_hour(self): - result = VerticaDatabase().date_add('hour', 1, Field('date')) + result = VerticaDatabase().date_add(Field('date'), 'hour', 1) self.assertEqual('TIMESTAMPADD(\'hour\',1,"date")', str(result)) def test_date_add_day(self): - result = VerticaDatabase().date_add('day', 1, Field('date')) + result = VerticaDatabase().date_add(Field('date'), 'day', 1) self.assertEqual('TIMESTAMPADD(\'day\',1,"date")', str(result)) def test_date_add_week(self): - result = VerticaDatabase().date_add('week', 1, Field('date')) + result = VerticaDatabase().date_add(Field('date'), 'week', 1) self.assertEqual('TIMESTAMPADD(\'week\',1,"date")', str(result)) def test_date_add_month(self): - result = VerticaDatabase().date_add('month', 1, Field('date')) + result = VerticaDatabase().date_add(Field('date'), 'month', 1) self.assertEqual('TIMESTAMPADD(\'month\',1,"date")', str(result)) def test_date_add_quarter(self): - result = VerticaDatabase().date_add('quarter', 1, Field('date')) + result = VerticaDatabase().date_add(Field('date'), 'quarter', 1) self.assertEqual('TIMESTAMPADD(\'quarter\',1,"date")', str(result)) def test_date_add_year(self): - result = VerticaDatabase().date_add('year', 1, Field('date')) + result = VerticaDatabase().date_add(Field('date'), 'year', 1) self.assertEqual('TIMESTAMPADD(\'year\',1,"date")', str(result)) diff --git a/fireant/tests/mock_dataframes.py b/fireant/tests/mock_dataframes.py deleted file mode 100644 index e90c4daf..00000000 --- a/fireant/tests/mock_dataframes.py +++ /dev/null @@ -1,378 +0,0 @@ -# coding: utf-8 -from collections import OrderedDict -from datetime import date - -import numpy as np -import pandas as pd - -from fireant.slicer.operations import Totals - - -def rollup(dataframe, levels): - roll = dataframe.groupby(level=levels).sum() - rolled_levels = [name - for i, name in enumerate(dataframe.index.names) - if i not in levels] - - for rolled_level in rolled_levels: - roll[rolled_level] = Totals.key - - return dataframe.append(roll.set_index(rolled_levels, append=True)).sort_index() - - -bool_dim = {'axis': 0, 'label': 'Bool'} -cont_dim = {'axis': 0, 'label': 'Cont'} -datetime_dim = {'axis': 0, 'label': 'Date'} -uni_dim = {'axis': 0, 'label': 'Uni', 'display_field': 'uni_label'} -cat1_dim = {'axis': 0, 'label': 'Cat1', 'display_options': {'a': 'A', 'b': 'B'}} -cat2_dim = {'axis': 0, 'label': 'Cat2', 'display_options': {'y': 'Y', 'z': 'Z'}} -cat1_rollup_dim = {'axis': 0, 'label': 'Cat1', 'display_options': {'a': 'A', 'b': 'B', Totals.key: Totals.label}} -cat2_rollup_dim = {'axis': 0, 'label': 'Cat2', 'display_options': {'y': 'Y', 'z': 'Z', Totals.key: Totals.label}} - -shortcuts = { - 'a': 'A', - 'b': 'B', - 'c': 'C', - 'd': 'D', - 'y': 'Y', - 'z': 'Z', - Totals.key: Totals.label -} - -bool_idx = pd.Index([None, False, True], name='bool') -cont_idx = pd.Index([0, 1, 2, 3, 4, 5, 6, 7], name='cont') -cat1_idx = pd.Index([u'a', u'b'], name='cat1') -cat2_idx = pd.Index([u'y', u'z'], name='cat2') - -uni_idx = pd.MultiIndex.from_tuples([(1, u'Aa'), (2, u'Bb'), (3, u'Cc')], - names=['uni', 'uni_label']) - -datetime_idx = pd.DatetimeIndex(pd.date_range(start=date(2000, 1, 1), periods=8), name='date') -cont_cat_idx = pd.MultiIndex.from_product([cont_idx, cat1_idx], names=['cont', 'cat1']) -cont_bool_idx = pd.MultiIndex.from_product([cont_idx, bool_idx], names=['cont', 'bool']) -cont_uni_idx = pd.MultiIndex.from_product([cont_idx, uni_idx.levels[0]], - names=['cont', 'uni']) -cat_cat_idx = pd.MultiIndex.from_product([cat1_idx, cat2_idx], names=['cat1', 'cat2']) -cont_cat_cat_idx = pd.MultiIndex.from_product([cont_idx, cat1_idx, cat2_idx], names=['cont', 'cat1', 'cat2']) - -cont_cat_uni_idx = pd.MultiIndex.from_product([cont_idx, cat1_idx, uni_idx.levels[0]], - names=['cont', 'cat1', 'uni']) - -# Mock DF with single metric column and no dimension -no_dims_single_metric_df = pd.DataFrame( - np.array([ - np.arange(1), - ]), - columns=['one'], -) -no_dims_single_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'})]), - 'dimensions': OrderedDict() -} - -# Mock DF with single continuous dimension and one metric column -no_dims_multi_metric_df = pd.DataFrame( - np.array([ - np.arange(8), - ]), - columns=['one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight'], -) -no_dims_multi_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'}), ('two', {'axis': 0, 'label': 'Two'}), - ('three', {'axis': 0, 'label': 'Three'}), ('four', {'axis': 0, 'label': 'Four'}), - ('five', {'axis': 0, 'label': 'Five'}), ('six', {'axis': 0, 'label': 'Six'}), - ('seven', {'axis': 0, 'label': 'Seven'}), ('eight', {'axis': 0, 'label': 'Eight'})]), - 'dimensions': OrderedDict() -} - -# Mock DF with single continuous dimension and one metric column -cont_dim_single_metric_df = pd.DataFrame( - np.array([ - np.arange(8), - ]).T, - columns=['one'], - index=cont_idx -) -cont_dim_single_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'})]), - 'dimensions': OrderedDict([('cont', cont_dim)]) -} - -# Mock DF with single continuous dimension and two metric columns -cont_dim_multi_metric_df = pd.DataFrame( - np.array([ - np.arange(8), - 2 * np.arange(8), - ]).T, - columns=['one', 'two'], - index=cont_idx -) -cont_dim_multi_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'}), ('two', {'axis': 0, 'label': 'Two'})]), - 'dimensions': OrderedDict([('cont', cont_dim)]) -} - -# Mock DF with single unique dimension and one metric column -uni_dim_single_metric_df = pd.DataFrame( - np.array([ - np.arange(3), - ]).T, - columns=['one'], - index=uni_idx -) -uni_dim_single_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'})]), - 'dimensions': OrderedDict([('uni', uni_dim)]) -} - -# Mock DF with single unique dimension and two metric columns -uni_dim_multi_metric_df = pd.DataFrame( - np.array([ - np.arange(3), - 2 * np.arange(3), - ]).T, - columns=['one', 'two'], - index=uni_idx -) -uni_dim_multi_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'}), ('two', {'axis': 0, 'label': 'Two'})]), - 'dimensions': OrderedDict([('uni', uni_dim)]) -} - -# Mock DF with single unique dimension and a single metric columns with pretty prefix/suffix/precision settings -uni_dim_pretty_df = pd.DataFrame( - np.array([np.arange(3)]).T, - columns=['pretty'], - index=uni_idx -) -uni_dim_pretty_schema = { - 'metrics': OrderedDict([('pretty', {'axis': 0, 'label': 'One', 'prefix': '!', 'suffix': '~', 'precision': 1})]), - 'dimensions': OrderedDict([('uni', uni_dim)]) -} - -# Mock DF with single categorical dimension and one metric column -cat_dim_single_metric_df = pd.DataFrame( - np.array([ - np.arange(2), - ]).T, - columns=['one'], - index=cat1_idx -) -cat_dim_single_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'})]), - 'dimensions': OrderedDict([('cat1', cat1_dim)]) -} - -# Mock DF with single categorical dimension and two metric columns -cat_dim_multi_metric_df = pd.DataFrame( - np.array([ - np.arange(2), - 2 * np.arange(2), - ]).T, - columns=['one', 'two'], - index=cat1_idx -) -cat_dim_multi_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'}), ('two', {'axis': 0, 'label': 'Two'})]), - 'dimensions': OrderedDict([('cat1', cat1_dim)]) -} - -# Mock DF with single continuous time dimension and one metric column -time_dim_single_metric_df = pd.DataFrame( - np.array([ - np.arange(8), - ]).T, - columns=['one'], - index=datetime_idx -) -time_dim_single_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'})]), - 'dimensions': OrderedDict([('date', datetime_dim)]) -} -time_dim_single_metric_ref_df = pd.DataFrame( - np.array([ - np.arange(8), - 2 * np.arange(8), - ]).T, - columns=[['', 'wow'], ['one', 'one']], - index=datetime_idx -) -time_dim_single_metric_ref_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'})]), - 'dimensions': OrderedDict([('date', datetime_dim)]), - 'references': {'wow': 'WoW'} -} - -# Mock DF with continuous and categorical dimensions and one metric column -cont_cat_dims_single_metric_df = pd.DataFrame( - np.array([ - np.arange(16), - ]).T, - columns=['one'], - index=cont_cat_idx -) -cont_cat_dims_single_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'})]), - 'dimensions': OrderedDict([('cont', cont_dim), ('cat1', cat1_dim)]) -} - -# Mock DF with continuous and categorical dimensions and two metric columns -cont_cat_dims_multi_metric_df = pd.DataFrame( - np.array([ - np.arange(16), - 2 * np.arange(16), - ]).T, - columns=['one', 'two'], - index=cont_cat_idx -) -cont_cat_dims_multi_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'}), ('two', {'axis': 0, 'label': 'Two'})]), - 'dimensions': OrderedDict([('cont', cont_dim), ('cat1', cat1_dim)]) -} - -# Mock DF with continuous and boolean dimensions and one metric column -cont_bool_dims_single_metric_df = pd.DataFrame( - np.array([ - np.arange(24), - ]).T, - columns=['one'], - index=cont_bool_idx -) -cont_bool_dims_single_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'})]), - 'dimensions': OrderedDict([('cont', cont_dim), ('bool', bool_dim)]) -} - -# Mock DF with continuous and boolean dimensions and two metric columns -cont_bool_dims_multi_metric_df = pd.DataFrame( - np.array([ - np.arange(24), - 2 * np.arange(24), - ]).T, - columns=['one', 'two'], - index=cont_bool_idx -) -cont_bool_dims_multi_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'}), ('two', {'axis': 0, 'label': 'Two'})]), - 'dimensions': OrderedDict([('cont', cont_dim), ('bool', bool_dim)]) -} - -# Mock DF with continuous and unique dimensions and two metric columns -_cont_uni = pd.DataFrame( - np.array([np.arange(24), np.arange(100, 124)]).T, - columns=['one', 'two'], - index=cont_uni_idx -) -_cont_uni['uni_label'] = None -for uni_id, label in uni_idx: - _cont_uni.loc[pd.IndexSlice[:, uni_id], ['uni_label']] = label - -cont_uni_dims_multi_metric_df = _cont_uni.set_index(['uni_label'], append=True) -cont_uni_dims_multi_metric_df = pd.DataFrame(cont_uni_dims_multi_metric_df) -cont_uni_dims_multi_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'}), ('two', {'axis': 0, 'label': 'Two'})]), - 'dimensions': OrderedDict([('cont', cont_dim), ('uni', uni_dim)]) -} - -# Mock DF with continuous and unique dimensions and one metric column -cont_uni_dims_single_metric_df = pd.DataFrame(cont_uni_dims_multi_metric_df['one']) -cont_uni_dims_single_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'})]), - 'dimensions': OrderedDict([('cont', cont_dim), ('uni', uni_dim)]) -} - -# Mock DF with two categorical dimensions and one metric column -cat_cat_dims_single_metric_df = pd.DataFrame( - np.array([ - np.arange(4), - ]).T, - columns=['one'], - index=cat_cat_idx, -) - -cat_cat_dims_single_metric_empty_df = pd.DataFrame([], columns=['one'], - index=pd.MultiIndex([[], []], [[], []], names=['cat1', 'cat2'])) - -cat_cat_dims_single_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'})]), - 'dimensions': OrderedDict([('cat1', cat1_dim), ('cat2', cat2_dim)]) -} - -# Mock DF with two categorical dimensions and two metric columns -cat_cat_dims_multi_metric_df = pd.DataFrame( - np.array([ - np.arange(4), - 2 * np.arange(4), - ]).T, - columns=['one', 'two'], - index=cat_cat_idx, -) -cat_cat_dims_multi_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'}), ('two', {'axis': 0, 'label': 'Two'})]), - 'dimensions': OrderedDict([('cat1', cat1_dim), ('cat2', cat2_dim)]) -} - -# Mock DF with continuous and two categorical dimensions and two metric columns -cont_cat_cat_dims_multi_metric_df = pd.DataFrame( - np.array([ - np.arange(32), - 2 * np.arange(32), - ]).T, - columns=['one', 'two'], - index=cont_cat_cat_idx -) -cont_cat_cat_dims_multi_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'}), ('two', {'axis': 0, 'label': 'Two'})]), - 'dimensions': OrderedDict([('cont', cont_dim), ('cat1', cat1_dim), ('cat2', cat2_dim)]) -} - -# Mock DF with continuous and two categorical dimensions and two metric columns -_cont_cat_uni = pd.DataFrame( - np.array([np.arange(48), np.arange(100, 148)]).T, - columns=['one', 'two'], - index=cont_cat_uni_idx -) -_cont_cat_uni['uni_label'] = None -for uni_id, label in uni_idx: - _cont_cat_uni.loc[pd.IndexSlice[:, :, uni_id], ['uni_label']] = label - -cont_cat_uni_dims_multi_metric_df = _cont_cat_uni.set_index('uni_label', append=True) -cont_cat_uni_dims_multi_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'}), ('two', {'axis': 0, 'label': 'Two'})]), - 'dimensions': OrderedDict([('cont', cont_dim), ('cat1', cat1_dim), ('uni', uni_dim)]) -} - -# Mock DF with continuous and two categorical dimensions and two metric columns using rollup for totals -rollup_cont_cat_cat_dims_multi_metric_df = pd.DataFrame( - np.array([ - np.arange(32), - 2 * np.arange(32), - ]).T, - columns=['one', 'two'], - index=cont_cat_cat_idx -) -rollup_cont_cat_cat_dims_multi_metric_df = rollup(rollup_cont_cat_cat_dims_multi_metric_df, [0, 1]) -rollup_cont_cat_cat_dims_multi_metric_df = rollup(rollup_cont_cat_cat_dims_multi_metric_df, [0]) -rollup_cont_cat_cat_dims_multi_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'}), ('two', {'axis': 0, 'label': 'Two'})]), - 'dimensions': OrderedDict([('cont', cont_dim), ('cat1', cat1_rollup_dim), ('cat2', cat2_rollup_dim)]) -} - -# Mock DF with single unique dimension and one metric column using rollup for totals -rollup_cont_cat_uni_dims_multi_metric_df = rollup(_cont_cat_uni.set_index('uni_label', append=True), [0, 1]) -rollup_cont_cat_uni_dims_multi_metric_df = rollup(rollup_cont_cat_uni_dims_multi_metric_df, [0]) -rollup_cont_cat_uni_dims_multi_metric_schema = { - 'metrics': OrderedDict([('one', {'axis': 0, 'label': 'One'}), ('two', {'axis': 0, 'label': 'Two'})]), - 'dimensions': OrderedDict([('cont', cont_dim), ('cat1', cat1_dim), ('uni', uni_dim)]) -} - -# Mock DF with single continuous dimension and two metric columns -cont_dim_pretty_df = pd.DataFrame( - np.array([0.12345, 0.23456, 0.34567, 0.45678, 0.56789, 0.67891, 0.78912, 0.89123]).T, - columns=['pretty'], - index=cont_idx -) -cont_dim_pretty_schema = { - 'metrics': OrderedDict([('pretty', {'axis': 0, 'label': 'One', 'prefix': '!', 'suffix': '~', 'precision': 1})]), - 'dimensions': OrderedDict([('cont', cont_dim)]) -} diff --git a/fireant/tests/slicer/mocks.py b/fireant/tests/slicer/mocks.py new file mode 100644 index 00000000..0a2e4919 --- /dev/null +++ b/fireant/tests/slicer/mocks.py @@ -0,0 +1,226 @@ +from collections import ( + OrderedDict, + namedtuple, +) +from unittest.mock import Mock + +import pandas as pd +from datetime import date + +from fireant import * +from fireant import VerticaDatabase +from pypika import ( + JoinType, + Table, + functions as fn, +) + + +class TestDatabase(VerticaDatabase): + # Vertica client that uses the vertica_python driver. + + connect = Mock() + + +test_database = TestDatabase() +politicians_table = Table('politician', schema='politics') +voters_table = Table('voter', schema='politics') +state_table = Table('state', schema='locations') +district_table = Table('district', schema='locations') +deep_join_table = Table('deep', schema='test') + +slicer = Slicer( + table=politicians_table, + database=test_database, + + joins=( + Join(table=district_table, + criterion=politicians_table.district_id == district_table.id, + join_type=JoinType.outer), + Join(table=state_table, + criterion=district_table.state_id == state_table.id), + Join(table=voters_table, + criterion=district_table.id == voters_table.district_id), + Join(table=deep_join_table, + criterion=deep_join_table.id == state_table.ref_id), + ), + + dimensions=( + DatetimeDimension('timestamp', + label='Timestamp', + definition=politicians_table.timestamp), + CategoricalDimension('political_party', + label='Party', + definition=politicians_table.political_party, + display_values=( + ('d', 'Democrat'), + ('r', 'Republican'), + ('i', 'Independent'), + ('l', 'Libertarian'), + ('g', 'Green'), + ('c', 'Constitution'))), + UniqueDimension('candidate', + label='Candidate', + definition=politicians_table.candidate_id, + display_definition=politicians_table.candidate_name), + UniqueDimension('election', + label='Election', + definition=politicians_table.election_id, + display_definition=politicians_table.election_year), + UniqueDimension('district', + label='District', + definition=politicians_table.district_id, + display_definition=district_table.district_name), + UniqueDimension('state', + label='State', + definition=district_table.state_id, + display_definition=state_table.state_name), + BooleanDimension('winner', + label='Winner', + definition=politicians_table.is_winner), + UniqueDimension('deepjoin', + definition=deep_join_table.id), + ), + + metrics=( + Metric('votes', + label='Votes', + definition=fn.Sum(politicians_table.votes)), + Metric('wins', + label='Wins', + definition=fn.Sum(politicians_table.is_winner)), + Metric('voters', + label='Voters', + definition=fn.Count(voters_table.id)), + Metric('turnout', + label='Turnout', + definition=fn.Sum(politicians_table.votes) / fn.Count(voters_table.id)), + ), +) + +political_parties = OrderedDict((('d', 'Democrat'), + ('r', 'Republican'), + ('i', 'Independent'), + ('l', 'Libertarian'), + ('g', 'Green'), + ('c', 'Constitution'))) + +candidates = OrderedDict(((1, 'Bill Clinton'), + (2, 'Bob Dole'), + (3, 'Ross Perot'), + (4, 'George Bush'), + (5, 'Al Gore'), + (6, 'John Kerry'), + (7, 'Barrack Obama'), + (8, 'John McCain'), + (9, 'Mitt Romney'), + (10, 'Donald Trump'), + (11, 'Hillary Clinton'))) + +states = OrderedDict(((1, 'Texas'), + (2, 'California'))) + +elections = OrderedDict(((1, '1996'), + (2, '2000'), + (3, '2004'), + (4, '2008'), + (5, '2012'), + (6, '2016'))) + +election_candidates = { + 1: {'candidates': [1, 2, 3], 'winner': 1}, + 2: {'candidates': [4, 5], 'winner': 4}, + 3: {'candidates': [4, 6], 'winner': 4}, + 4: {'candidates': [7, 8], 'winner': 7}, + 5: {'candidates': [7, 9], 'winner': 7}, + 6: {'candidates': [10, 11], 'winner': 10}, +} + +candidate_parties = { + 1: 'd', + 2: 'r', + 3: 'i', + 4: 'r', + 5: 'd', + 6: 'd', + 7: 'd', + 8: 'r', + 9: 'r', + 10: 'r', + 11: 'd', +} + +election_candidate_state_votes = { + # Texas + (1, 1, 1): 2459683, + (1, 2, 1): 2736167, + (1, 3, 1): 378537, + (2, 4, 1): 3799639, + (2, 5, 1): 2433746, + (3, 4, 1): 4526917, + (3, 6, 1): 2832704, + (4, 7, 1): 3528633, + (4, 8, 1): 4479328, + (5, 7, 1): 4569843, + (5, 9, 1): 3308124, + (6, 10, 1): 4685047, + (6, 11, 1): 387868, + + # California + (1, 1, 2): 5119835, + (1, 2, 2): 3828380, + (1, 3, 2): 697847, + (2, 4, 2): 4567429, + (2, 5, 2): 5861203, + (3, 4, 2): 5509826, + (3, 6, 2): 6745485, + (4, 7, 2): 8274473, + (4, 8, 2): 5011781, + (5, 7, 2): 7854285, + (5, 9, 2): 4839958, + (6, 10, 2): 8753788, + (6, 11, 2): 4483810, +} + +election_candidate_wins = { + (1, 1): True, + (1, 2): False, + (1, 3): False, + (2, 4): True, + (2, 5): False, + (3, 4): True, + (3, 6): False, + (4, 7): True, + (4, 8): False, + (5, 7): True, + (5, 9): False, + (6, 10): True, + (6, 11): False, +} + +columns = ['timestamp', + 'candidate', 'candidate_display', + 'political_party', + 'election', 'election_display', + 'state', 'state_display', + 'winner', + 'votes', + 'wins'] +PoliticsRow = namedtuple('PoliticsRow', columns) + +records = [] +for (election_id, candidate_id, state_id), votes in election_candidate_state_votes.items(): + election_year = elections[election_id] + winner = election_candidate_wins[(election_id, candidate_id)] + records.append(PoliticsRow( + timestamp=date(int(election_year), 1, 1), + candidate=candidate_id, candidate_display=candidates[candidate_id], + political_party=candidate_parties[candidate_id], + election=election_id, election_display=elections[election_id], + state=state_id, state_display=states[state_id], + winner=winner, + votes=votes, + wins=(1 if winner else 0), + )) + +mock_politics_database = pd.DataFrame.from_records(records, columns=columns) diff --git a/fireant/tests/dashboards/__init__.py b/fireant/tests/slicer/test_fetch_data.py similarity index 100% rename from fireant/tests/dashboards/__init__.py rename to fireant/tests/slicer/test_fetch_data.py diff --git a/fireant/tests/slicer/test_filters.py b/fireant/tests/slicer/test_filters.py deleted file mode 100644 index f4ce970a..00000000 --- a/fireant/tests/slicer/test_filters.py +++ /dev/null @@ -1,172 +0,0 @@ -# coding: utf-8 -from datetime import date -from unittest import TestCase - -from fireant.slicer import ( - BooleanFilter, - ContainsFilter, - EqualityFilter, - EqualityOperator, - ExcludesFilter, - RangeFilter, - WildcardFilter, -) -from pypika import Table - - -class FilterTests(TestCase): - test_table = Table('abc') - - def test_equality_filter_equals_number(self): - eq_filter = EqualityFilter('test', EqualityOperator.eq, 1) - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo"=1', str(schemas)) - - def test_equality_filter_equals_char(self): - eq_filter = EqualityFilter('test', EqualityOperator.eq, 'a') - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo"=\'a\'', str(schemas)) - - def test_equality_filter_equals_date(self): - eq_filter = EqualityFilter('test', EqualityOperator.eq, date(2000, 1, 1)) - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo"=\'2000-01-01\'', str(schemas)) - - def test_equality_filter_equals_string(self): - eq_filter = EqualityFilter('test', EqualityOperator.eq, 'test') - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo"=\'test\'', str(schemas)) - - def test_boolean_filter_with_true(self): - bool_filter = BooleanFilter('test', True) - - schemas = bool_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo"', str(schemas)) - - def test_boolean_filter_with_false(self): - bool_filter = BooleanFilter('test', False) - - schemas = bool_filter.schemas(self.test_table.foo) - - self.assertEqual('NOT "foo"', str(schemas)) - - def test_contains_filter_numbers(self): - eq_filter = ContainsFilter('test', [1, 2, 3]) - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" IN (1,2,3)', str(schemas)) - - def test_contains_filter_characters(self): - eq_filter = ContainsFilter('test', ['a', 'b', 'c']) - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" IN (\'a\',\'b\',\'c\')', str(schemas)) - - def test_contains_filter_dates(self): - eq_filter = ContainsFilter('test', [date(2000, 1, 1), date(2000, 1, 2), date(2000, 1, 3)]) - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" IN (\'2000-01-01\',\'2000-01-02\',\'2000-01-03\')', str(schemas)) - - def test_contains_filter_strings(self): - eq_filter = ContainsFilter('test', ['abc', 'efg', 'hij']) - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" IN (\'abc\',\'efg\',\'hij\')', str(schemas)) - - def test_excludes_filter_numbers(self): - eq_filter = ExcludesFilter('test', [1, 2, 3]) - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" NOT IN (1,2,3)', str(schemas)) - - def test_excludes_filter_characters(self): - eq_filter = ExcludesFilter('test', ['a', 'b', 'c']) - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" NOT IN (\'a\',\'b\',\'c\')', str(schemas)) - - def test_excludes_filter_dates(self): - eq_filter = ExcludesFilter('test', [date(2000, 1, 1), date(2000, 1, 2), date(2000, 1, 3)]) - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" NOT IN (\'2000-01-01\',\'2000-01-02\',\'2000-01-03\')', str(schemas)) - - def test_excludes_filter_strings(self): - eq_filter = ExcludesFilter('test', ['abc', 'efg', 'hij']) - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" NOT IN (\'abc\',\'efg\',\'hij\')', str(schemas)) - - def test_range_filter_numbers(self): - eq_filter = RangeFilter('test', 0, 1) - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" BETWEEN 0 AND 1', str(schemas)) - - def test_range_filter_characters(self): - eq_filter = RangeFilter('test', 'A', 'Z') - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" BETWEEN \'A\' AND \'Z\'', str(schemas)) - - def test_range_filter_dates(self): - eq_filter = RangeFilter('test', date(2000, 1, 1), date(2000, 12, 31)) - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" BETWEEN \'2000-01-01\' AND \'2000-12-31\'', str(schemas)) - - def test_range_filter_strings(self): - eq_filter = RangeFilter('test', 'ABC', 'XYZ') - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" BETWEEN \'ABC\' AND \'XYZ\'', str(schemas)) - - def test_wildcard_filter_suffix(self): - eq_filter = WildcardFilter('test', '%xyz') - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" LIKE \'%xyz\'', str(schemas)) - - def test_wildcard_filter_prefix(self): - eq_filter = WildcardFilter('test', 'abc%') - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" LIKE \'abc%\'', str(schemas)) - - def test_wildcard_filter_circumfix(self): - eq_filter = WildcardFilter('test', 'lm%no') - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" LIKE \'lm%no\'', str(schemas)) - - def test_wildcard_filter_infix(self): - eq_filter = WildcardFilter('test', '%lmn%') - - schemas = eq_filter.schemas(self.test_table.foo) - - self.assertEqual('"foo" LIKE \'%lmn%\'', str(schemas)) diff --git a/fireant/tests/slicer/test_managers.py b/fireant/tests/slicer/test_managers.py deleted file mode 100644 index 2a7cb3d8..00000000 --- a/fireant/tests/slicer/test_managers.py +++ /dev/null @@ -1,510 +0,0 @@ -# coding: utf-8 -import copy -import itertools -from unittest import TestCase - -import pandas as pd -from fireant.slicer import * -from fireant.slicer.managers import SlicerManager -from fireant.slicer.operations import CumSum -from fireant.slicer.references import WoW -from fireant.slicer.transformers import * -from fireant.tests.database.mock_database import TestDatabase -from mock import ( - MagicMock, - patch, -) -from pypika import ( - Table, - Query, -) - - -class ManagerInitializationTests(TestCase): - def setUp(self): - self.test_table = Table('test') - self.test_database = TestDatabase() - self.slicer = Slicer( - self.test_table, - self.test_database, - - metrics=[ - Metric('foo'), - Metric('bar'), - ], - - dimensions=[ - ContinuousDimension('cont'), - DatetimeDimension('date'), - CategoricalDimension('cat'), - UniqueDimension('uni', display_field=self.test_table.uni_label), - ] - ) - self.paginator = Paginator(offset=10, limit=10) - - def test_transformers(self): - self.assertTrue(hasattr(self.slicer, 'manager')) - - self.assertTrue(hasattr(self.slicer, 'notebooks')) - self.assertTrue(hasattr(self.slicer.notebooks, 'line_chart')) - self.assertTrue(hasattr(self.slicer.notebooks, 'bar_chart')) - - self.assertTrue(hasattr(self.slicer, 'highcharts')) - self.assertTrue(hasattr(self.slicer.highcharts, 'line_chart')) - self.assertTrue(hasattr(self.slicer.highcharts, 'area_chart')) - self.assertTrue(hasattr(self.slicer.highcharts, 'column_chart')) - self.assertTrue(hasattr(self.slicer.highcharts, 'bar_chart')) - self.assertTrue(hasattr(self.slicer.highcharts, 'pie_chart')) - - self.assertTrue(hasattr(self.slicer, 'datatables')) - self.assertTrue(hasattr(self.slicer.datatables, 'row_index_table')) - self.assertTrue(hasattr(self.slicer.datatables, 'column_index_table')) - self.assertTrue(hasattr(self.slicer.datatables, 'row_index_csv')) - self.assertTrue(hasattr(self.slicer.datatables, 'column_index_csv')) - - @patch('fireant.slicer.managers.SlicerManager.post_process') - @patch('fireant.slicer.managers.SlicerManager.query_data') - @patch('fireant.slicer.managers.SlicerManager.operation_schema') - @patch('fireant.slicer.managers.SlicerManager.data_query_schema') - def test_data(self, mock_query_schema, mock_operation_schema, mock_query_data, mock_post_process): - mock_args = {'metrics': ['a'], 'dimensions': ['e'], - 'metric_filters': [2], 'dimension_filters': [3], - 'references': [WoW('d')], 'operations': [5], 'pagination': None} - mock_query_schema.return_value = {'a': 1, 'b': 2} - mock_query_data.return_value = mock_post_process.return_value = pd.DataFrame( - columns=itertools.product(['', 'wow'], ['a', 'c', 'm_test'])) - mock_operation_schema.return_value = [{'metric': 'm', 'key': 'test'}] - - result = self.slicer.manager.data(**mock_args) - - self.assertIsInstance(result, pd.DataFrame) - self.assertListEqual([('', 'a'), ('', 'm_test'), ('wow', 'a'), ('wow', 'm_test')], list(result.columns)) - mock_query_schema.assert_called_once_with(**mock_args) - mock_query_data.assert_called_once_with(a=1, b=2) - mock_operation_schema.assert_called_once_with(mock_args['operations']) - - @patch('fireant.slicer.managers.SlicerManager._build_data_query') - @patch('fireant.slicer.managers.SlicerManager.data_query_schema') - def test_query_string(self, mock_query_schema, mock_build_query_string): - mock_args = {'metrics': [0], 'dimensions': [1], - 'metric_filters': [2], 'dimension_filters': [3], - 'references': [4], 'operations': [5], 'pagination': self.paginator} - mock_query_schema.return_value = {'database': 'db1', 'a': 1, 'b': 2} - mock_build_query_string.return_value = 1 - - result = self.slicer.manager.query_string(**mock_args) - - self.assertEqual(str(mock_build_query_string.return_value), result) - mock_query_schema.assert_called_once_with(**mock_args) - mock_build_query_string.assert_called_once_with(database='db1', a=1, b=2) - - def missing_database_config(self): - with self.assertRaises(SlicerException): - self.slicer.manager.data() - - def test_require_metrics(self): - with self.assertRaises(SlicerException): - self.slicer.manager._metrics_schema([]) - - def test_require_valid_metrics(self): - with self.assertRaises(SlicerException): - self.slicer.manager._metrics_schema(['fizz', 'buzz']) - - def test_require_valid_dimensions(self): - with self.assertRaises(SlicerException): - self.slicer.manager._dimensions_schema(['fizz', 'buzz']) - - @patch.object(SlicerManager, 'display_schema') - @patch.object(SlicerManager, 'data') - def _test_transform(self, test_func, mock_transform, request, mock_sm_data, mock_sm_ds): - mock_sm_data.return_value = mock_df = MagicMock() - mock_sm_ds.return_value = mock_schema = { - 'metrics': [] - } - mock_transform.return_value = mock_return = 'OK' - - result = test_func(**request) - - self.assertEqual(mock_return, result) - mock_sm_data.assert_called_once_with(**request) - mock_sm_ds.assert_called_once_with(request['metrics'], request['dimensions'], request.get('references', ()), ()) - mock_transform.assert_called_once_with(mock_df, mock_schema) - - @patch.object(HighchartsLineTransformer, 'transform') - def test_transform_highcharts_line_chart(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['cont'], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - self._test_transform(self.slicer.highcharts.line_chart, mock_transform, request) - - @patch.object(HighchartsLineTransformer, 'transform') - def test_transform_highcharts_line_chart_date(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['date'], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - self._test_transform(self.slicer.highcharts.line_chart, mock_transform, request) - - @patch.object(HighchartsLineTransformer, 'transform') - def test_transform_highcharts_require_cont_dim(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['cat'], - } - - with self.assertRaises(TransformationException): - self._test_transform(self.slicer.highcharts.line_chart, mock_transform, request) - - @patch.object(HighchartsLineTransformer, 'transform') - def test_transform_highcharts_require_cont_dim_on_slicer_with_no_dims(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': [], - } - - slicer = copy.deepcopy(self.slicer) - slicer.dimensions = [] - with self.assertRaises(TransformationException): - self._test_transform(self.slicer.highcharts.line_chart, mock_transform, request) - - @patch.object(HighchartsAreaTransformer, 'transform') - def test_transform_highcharts_area_chart(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['cont'], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - self._test_transform(self.slicer.highcharts.area_chart, mock_transform, request) - - @patch.object(HighchartsAreaTransformer, 'transform') - def test_transform_highcharts_area_chart_date(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['date'], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - self._test_transform(self.slicer.highcharts.area_chart, mock_transform, request) - - @patch.object(HighchartsAreaTransformer, 'transform') - def test_transform_highcharts_area_chart_require_cont_dim(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['cat'], - } - - with self.assertRaises(TransformationException): - self._test_transform(self.slicer.highcharts.area_chart, mock_transform, request) - - @patch.object(HighchartsAreaTransformer, 'transform') - def test_transform_highcharts_area_chart_require_cont_dim_on_slicer_with_no_dims(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': [], - } - - slicer = copy.deepcopy(self.slicer) - slicer.dimensions = [] - with self.assertRaises(TransformationException): - self._test_transform(self.slicer.highcharts.area_chart, mock_transform, request) - - @patch.object(HighchartsAreaPercentageTransformer, 'transform') - def test_transform_highcharts_area_percentage_chart(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['cont'], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - self._test_transform(self.slicer.highcharts.area_percentage_chart, mock_transform, request) - - @patch.object(HighchartsAreaPercentageTransformer, 'transform') - def test_transform_highcharts_area_percentage_chart_date(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['date'], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - self._test_transform(self.slicer.highcharts.area_percentage_chart, mock_transform, request) - - @patch.object(HighchartsAreaPercentageTransformer, 'transform') - def test_transform_highcharts_area_chart_require_cont_dim(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['cat'], - } - - with self.assertRaises(TransformationException): - self._test_transform(self.slicer.highcharts.area_percentage_chart, mock_transform, request) - - @patch.object(HighchartsAreaPercentageTransformer, 'transform') - def test_transform_highcharts_area_percentage_chart_require_cont_dim_on_slicer_with_no_dims(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': [], - } - - slicer = copy.deepcopy(self.slicer) - slicer.dimensions = [] - with self.assertRaises(TransformationException): - self._test_transform(self.slicer.highcharts.area_percentage_chart, mock_transform, request) - - @patch.object(HighchartsPieTransformer, 'transform') - def test_transform_highcharts_pie_chart(self, mock_transform): - request = { - 'metrics': ['foo'], - 'dimensions': ['cont'], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - self._test_transform(self.slicer.highcharts.pie_chart, mock_transform, request) - - @patch.object(HighchartsPieTransformer, 'transform') - def test_transform_highcharts_pie_chart_date(self, mock_transform): - request = { - 'metrics': ['foo'], - 'dimensions': ['date'], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - self._test_transform(self.slicer.highcharts.pie_chart, mock_transform, request) - - @patch.object(HighchartsPieTransformer, 'transform') - def test_transform_highcharts_pie_chart_does_not_allow_two_metrics(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['cat'], - } - - with self.assertRaises(TransformationException): - self._test_transform(self.slicer.highcharts.pie_chart, mock_transform, request) - - @patch.object(HighchartsPieTransformer, 'transform') - def test_transform_highcharts_pie_chart_multiple_dimensions_one_metric(self, mock_transform): - request = { - 'metrics': ['foo'], - 'dimensions': ['date', 'cat'], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - - self._test_transform(self.slicer.highcharts.pie_chart, mock_transform, request) - - @patch.object(HighchartsColumnTransformer, 'transform') - def test_transform_highcharts_column_chart(self, mock_transform): - request = { - 'metrics': ['foo'], - 'dimensions': [], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - self._test_transform(self.slicer.highcharts.column_chart, mock_transform, request) - - @patch.object(HighchartsColumnTransformer, 'transform') - def test_transform_highcharts_max_dims(self, mock_transform): - request = { - 'metrics': ['foo'], - 'dimensions': ['cat', 'uni', 'cont'], - } - - with self.assertRaises(TransformationException): - self._test_transform(self.slicer.highcharts.column_chart, mock_transform, request) - - @patch.object(HighchartsColumnTransformer, 'transform') - def test_transform_highcharts_max_metrics(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['cat', 'uni'], - } - - with self.assertRaises(TransformationException): - self._test_transform(self.slicer.highcharts.column_chart, mock_transform, request) - - @patch.object(HighchartsBarTransformer, 'transform') - def test_transform_highcharts_bar_chart(self, mock_transform): - request = { - 'metrics': ['foo'], 'dimensions': [], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - self._test_transform(self.slicer.highcharts.bar_chart, mock_transform, request) - - @patch.object(HighchartsColumnTransformer, 'transform') - def test_transform_highcharts_max_dims(self, mock_transform): - request = { - 'metrics': ['foo'], - 'dimensions': ['cat', 'uni', 'cont'], - } - - with self.assertRaises(TransformationException): - self._test_transform(self.slicer.highcharts.bar_chart, mock_transform, request) - - @patch.object(HighchartsColumnTransformer, 'transform') - def test_transform_highcharts_max_metrics(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['cat', 'uni'], - } - - with self.assertRaises(TransformationException): - self._test_transform(self.slicer.highcharts.bar_chart, mock_transform, request) - - @patch.object(DataTablesRowIndexTransformer, 'transform') - def test_transform_datatables_row_index_table(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['cat', 'uni'], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - - self._test_transform(self.slicer.datatables.row_index_table, mock_transform, request) - - @patch.object(DataTablesColumnIndexTransformer, 'transform') - def test_transform_datatables_col_index_table(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['cat', 'uni'], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - - self._test_transform(self.slicer.datatables.column_index_table, mock_transform, request) - - @patch.object(CSVRowIndexTransformer, 'transform') - def test_transform_datatables_row_index_table(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['cat', 'uni'], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - - self._test_transform(self.slicer.datatables.row_index_csv, mock_transform, request) - - @patch.object(CSVColumnIndexTransformer, 'transform') - def test_transform_datatables_col_index_table(self, mock_transform): - request = { - 'metrics': ['foo', 'bar'], - 'dimensions': ['cat', 'uni'], - 'metric_filters': (), 'dimension_filters': (), - 'references': (), 'operations': (), 'pagination': self.paginator, - } - - self._test_transform(self.slicer.datatables.column_index_csv, mock_transform, request) - - @patch.object(SlicerManager, 'query_data') - @patch.object(SlicerManager, 'data_query_schema') - def test_remove_duplicate_metric_keys(self, mock_query_schema, mock_query_data): - self.slicer.manager.data( - metrics=['foo', 'foo'] - ) - - mock_query_schema.assert_called_once_with( - metrics=['foo'], - dimensions=[], - metric_filters=(), dimension_filters=(), - references=(), operations=(), pagination=None - ) - - @patch.object(SlicerManager, 'query_data') - @patch.object(SlicerManager, 'data_query_schema') - def test_slicer_exception_raised_with_operations_and_pagination(self, mock_query_schema, mock_query_data): - with self.assertRaises(SlicerException): - self.slicer.manager.data( - metrics=['foo', 'foo'], - operations=[CumSum('foo')], - pagination=Paginator(offset=10, limit=10) - ) - - @patch.object(SlicerManager, 'query_data') - @patch.object(SlicerManager, 'data_query_schema') - def test_remove_duplicate_dimension_keys(self, mock_query_schema, mock_query_data): - self.slicer.manager.data( - metrics=['foo'], - dimensions=['fizz', 'fizz'], - ) - - mock_query_schema.assert_called_once_with( - metrics=['foo'], - dimensions=['fizz'], - metric_filters=(), dimension_filters=(), - references=(), operations=(), pagination=None - ) - - @patch.object(SlicerManager, 'query_data') - @patch.object(SlicerManager, 'data_query_schema') - def test_remove_duplicate_dimension_keys_with_interval(self, mock_query_schema, mock_query_data): - self.slicer.manager.data( - metrics=['foo'], - dimensions=['fizz', ('fizz', DatetimeDimension.week)], - ) - - mock_query_schema.assert_called_once_with( - metrics=['foo'], - dimensions=['fizz'], - metric_filters=(), dimension_filters=(), - references=(), operations=(), pagination=None - ) - - @patch.object(SlicerManager, 'query_data') - @patch.object(SlicerManager, 'data_query_schema') - def test_remove_duplicate_dimension_keys_with_interval_backwards(self, mock_query_schema, mock_query_data): - mock_query_schema.reset() - self.slicer.manager.data( - metrics=['foo'], - dimensions=[('fizz', DatetimeDimension.week), 'fizz'], - ) - - mock_query_schema.assert_called_once_with( - metrics=['foo'], - dimensions=[('fizz', DatetimeDimension.week)], - metric_filters=(), dimension_filters=(), - references=(), operations=(), pagination=None - ) - - @patch.object(TestDatabase, 'fetch_dataframe') - @patch('fireant.slicer.queries.query_logger') - def test_get_dataframe_from_query_logs_query_before_and_query_after_with_duration(self, mock_logging, *args): - query = Query.from_('customers').select('id') - self.slicer.manager._get_dataframe_from_query(TestDatabase(), query) - self.assertEqual(mock_logging.debug.call_count, 1) - self.assertEqual(mock_logging.info.call_count, 1) - - @patch.object(SlicerManager, '_get_dataframe_from_query') - @patch.object(SlicerManager, '_build_data_query') - def test_query_data_calls_build_data_query_with_correct_args(self, mock_build_query, *args): - db = TestDatabase() - metrics = {'bar': self.slicer.metrics.get('bar')} - dimensions = {'cat': self.slicer.dimensions.get('cat')} - dfilters = ContainsFilter('cat', ['a', 'b']) - references = {'date': WoW('date')} - args = { - 'database': db, - 'table': self.slicer.table, - 'joins': self.slicer.joins, - 'metrics': metrics, - 'dimensions': dimensions, - 'dfilters': dfilters, - 'references': references - } - self.slicer.manager.query_data(**args) - - mock_build_query.assert_called_once_with(db, self.slicer.table, self.slicer.joins, - metrics, dimensions, dfilters, None, references, None, None) - - @patch.object(SlicerManager, '_get_dataframe_from_query') - @patch.object(SlicerManager, '_build_data_query') - def test_query_data_calls_get_dataframe_from_query_with_correct_args(self, mock_query, mock_get_dataframe): - query = Query.from_('customers').select('id') - mock_query.return_value = query - db = TestDatabase() - self.slicer.manager.query_data(db, self.slicer.table) - mock_get_dataframe.assert_called_once_with(db, query) diff --git a/fireant/tests/slicer/test_operations.py b/fireant/tests/slicer/test_operations.py deleted file mode 100644 index 2ed79967..00000000 --- a/fireant/tests/slicer/test_operations.py +++ /dev/null @@ -1,116 +0,0 @@ -# coding: utf-8 -from unittest import TestCase - -import numpy as np -import pandas as pd - -from fireant.slicer import Slicer, Metric, CategoricalDimension, SlicerException -from fireant.slicer.operations import Totals, CumSum, L1Loss -from fireant.tests.database.mock_database import TestDatabase -from pypika import Table - - -class TotalsTests(TestCase): - @classmethod - def setUpClass(cls): - cls.test_table = Table('test_table') - cls.test_slicer = Slicer( - table=Table('test_table'), - database=TestDatabase(), - joins=[], - metrics=[ - Metric('foo'), - Metric('bar'), - ], - - dimensions=[ - CategoricalDimension('dim'), - ] - ) - - def test_totals_init(self): - totals = Totals('date') - self.assertEqual('_total', totals.key) - - def test_data_query_schema__totals_dim_set_in_rollup(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo', 'bar'], - dimensions=['dim'], - operations=[Totals('dim')], - ) - self.assertListEqual(query_schema['rollup'], [['dim']]) - - def test_data_query_schema__exception_when_missing_totals_dim(self): - with self.assertRaises(SlicerException): - print(self.test_slicer.manager.data_query_schema( - metrics=['foo', 'bar'], - dimensions=[], - operations=[Totals('dim')], - )) - - def test_totals_display_schema__no_extra_metrics_selected(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['foo', 'bar'], - dimensions=['dim'], - operations=[Totals('dim')] - ) - self.assertDictEqual(display_schema['metrics'], {'foo': {'label': 'foo', 'axis': 0}, - 'bar': {'label': 'bar', 'axis': 1}}) - self.assertDictEqual(display_schema['dimensions'], { - 'dim': {'label': 'dim', 'display_options': {np.nan: '', pd.NaT: ''}} - }) - self.assertDictEqual(display_schema['references'], {}) - - -class MetricOperationTests(TestCase): - @classmethod - def setUpClass(cls): - cls.test_table = Table('test_table') - cls.test_slicer = Slicer( - table=Table('test_table'), - database=TestDatabase(), - joins=[], - metrics=[ - Metric('foo'), - Metric('bar'), - ], - - dimensions=[ - CategoricalDimension('dim'), - ] - ) - - def test_metric_included_in_query_schema_for_operation(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['dim'], - operations=[CumSum('foo')], - ) - - # Metric included - self.assertTrue('foo' in query_schema['metrics']) - - # No rollup added - self.assertEqual(len(query_schema['rollup']), 0) - - def test_metric_included_in_query_schema_for_operation_without_metrics(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=[], - dimensions=['dim'], - operations=[CumSum('foo')], - ) - - # Metric included - self.assertTrue('foo' in query_schema['metrics']) - - def test_all_metrics_included_in_query_schema_for_operation_without_metrics(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=[], - dimensions=['dim'], - operations=[L1Loss('foo','bar')], - ) - - # Metric included - self.assertTrue('foo' in query_schema['metrics']) - self.assertTrue('bar' in query_schema['metrics']) - diff --git a/fireant/tests/slicer/test_pagination.py b/fireant/tests/slicer/test_pagination.py deleted file mode 100644 index 360e1023..00000000 --- a/fireant/tests/slicer/test_pagination.py +++ /dev/null @@ -1,19 +0,0 @@ -from pypika import Order - -from fireant.slicer import Paginator -from unittest import TestCase - - -class PaginatorObjectTests(TestCase): - def test_offset_and_limit_have_default_0(self): - paginator = Paginator() - self.assertEqual(paginator.limit, 0) - self.assertEqual(paginator.offset, 0) - - def test_orderby_empty_tuple_by_default(self): - paginator = Paginator() - self.assertEqual(paginator.order, ()) - - def test_object__str__formatting(self): - paginator = Paginator(offset=10, limit=20, order=[('clicks', Order.desc)]) - self.assertEqual(str(paginator), 'offset: 10 limit: 20 order: [(\'clicks\', \'desc\')]') diff --git a/fireant/tests/slicer/test_postprocessors.py b/fireant/tests/slicer/test_postprocessors.py deleted file mode 100644 index 97300e14..00000000 --- a/fireant/tests/slicer/test_postprocessors.py +++ /dev/null @@ -1,195 +0,0 @@ -# coding: utf-8 -from unittest import TestCase - -import numpy as np - -from fireant.slicer.postprocessors import OperationManager -from fireant.tests import mock_dataframes as mock_df - - -class PostProcessingTests(object): - manager = OperationManager() - maxDiff = None - - -class CumulativeSumOperationTests(PostProcessingTests, TestCase): - @property - def op_key(self): - return 'cumsum' - - def operation(self, df): - return list(df.expanding(min_periods=1).sum()) - - def test_single_dim(self): - df = mock_df.time_dim_single_metric_df - result_df = self.manager.post_process(df, [{'key': self.op_key, 'metric': 'one'}]) - - # original DF unchanged - self.assertListEqual(['one'], list(df.columns)) - - operation_key = 'one_%s' % self.op_key - self.assertListEqual(['one', operation_key], list(result_df.columns)) - np.testing.assert_array_almost_equal(self.operation(df['one']), list(result_df[operation_key])) - - def test_single_dim_extra_metrics(self): - df = mock_df.cont_dim_multi_metric_df - result_df = self.manager.post_process(df, [{'key': self.op_key, 'metric': 'one'}]) - - # original DF unchanged - self.assertListEqual(['one', 'two'], list(df.columns)) - - operation_key = 'one_%s' % self.op_key - self.assertListEqual(['one', 'two', operation_key], list(result_df.columns)) - np.testing.assert_array_almost_equal(self.operation(df['one']), list(result_df[operation_key])) - np.testing.assert_array_almost_equal(list(df['two']), list(result_df['two'])) - - def test_single_dim_both_metrics(self): - df = mock_df.cont_dim_multi_metric_df - result_df = self.manager.post_process(df, [{'key': self.op_key, 'metric': 'one'}, - {'key': self.op_key, 'metric': 'two'}]) - # original DF unchanged - self.assertListEqual(['one', 'two'], list(df.columns)) - - operation1_key = 'one_%s' % self.op_key - operation2_key = 'two_%s' % self.op_key - self.assertListEqual(['one', 'two', operation1_key, operation2_key], list(result_df.columns)) - np.testing.assert_array_almost_equal(self.operation(df['one']), list(result_df[operation1_key])) - np.testing.assert_array_almost_equal(self.operation(df['two']), list(result_df[operation2_key])) - - def test_single_dim_with_ref(self): - df = mock_df.time_dim_single_metric_ref_df - result_df = self.manager.post_process(df, [{'key': self.op_key, 'metric': 'one'}]) - - # original DF unchanged - self.assertListEqual([('', 'one'), ('wow', 'one')], list(df.columns)) - - operation_key = 'one_%s' % self.op_key - self.assertListEqual([('', 'one'), ('wow', 'one'), ('', operation_key), ('wow', operation_key)], - list(result_df.columns)) - np.testing.assert_array_almost_equal(self.operation(df['', 'one']), - list(result_df[('', operation_key)])) - np.testing.assert_array_almost_equal(self.operation(df['wow', 'one']), - list(result_df[('wow', operation_key)])) - - def test_multi_dim(self): - df = mock_df.cont_cat_dims_single_metric_df - result_df = self.manager.post_process(df, [{'key': self.op_key, 'metric': 'one'}]) - - # original DF unchanged - self.assertListEqual(['one'], list(df.columns)) - - operation_key = 'one_%s' % self.op_key - self.assertListEqual(['one', operation_key], list(result_df.columns)) - np.testing.assert_array_almost_equal(self.operation(df.loc[(slice(None), 'a'), 'one']), - list(result_df.loc[(slice(None), 'a'), operation_key])) - np.testing.assert_array_almost_equal(self.operation(df.loc[(slice(None), 'b'), 'one']), - list(result_df.loc[(slice(None), 'b'), operation_key])) - - -class CumulativeMeanOperationTests(CumulativeSumOperationTests, TestCase): - @property - def op_key(self): - return 'cummean' - - def operation(self, df): - return list(df.expanding(min_periods=1).mean()) - - -class L1LossOperationTests(PostProcessingTests, TestCase): - @property - def op_key(self): - return 'l1loss' - - def operation(self, metric_df, target_df): - return list((metric_df - target_df).abs().expanding(min_periods=1).mean()) - - def test_single_dim(self): - df = mock_df.time_dim_single_metric_df.copy() - df['target'] = [2, 0, 4, 0, 6, 0, 8, 0] - - result_df = self.manager.post_process(df, [{'key': self.op_key, 'metric': 'one', 'target': 'target'}]) - - # original DF unchanged - self.assertListEqual(['one', 'target'], list(df.columns)) - - operation_key = 'one_%s' % self.op_key - self.assertListEqual(['one', 'target', operation_key], list(result_df.columns)) - np.testing.assert_array_almost_equal(self.operation(df['one'], df['target']), list(result_df[operation_key])) - - def test_single_dim_extra_metrics(self): - df = mock_df.cont_dim_multi_metric_df.copy() - df['target'] = [2, 0, 4, 0, 6, 0, 8, 0] - - result_df = self.manager.post_process(df, [{'key': self.op_key, 'metric': 'one', 'target': 'target'}]) - - # original DF unchanged - self.assertListEqual(['one', 'two', 'target'], list(df.columns)) - - operation_key = 'one_%s' % self.op_key - self.assertListEqual(['one', 'two', 'target', operation_key], list(result_df.columns)) - np.testing.assert_array_almost_equal(self.operation(df['one'], df['target']), list(result_df[operation_key])) - np.testing.assert_array_almost_equal(list(df['two']), list(result_df['two'])) - - def test_single_dim_both_metrics(self): - df = mock_df.cont_dim_multi_metric_df.copy() - df['target'] = [2, 0, 4, 0, 6, 0, 8, 0] - - result_df = self.manager.post_process(df, [{'key': self.op_key, 'metric': 'one', 'target': 'target'}, - {'key': self.op_key, 'metric': 'two', 'target': 'target'}]) - # original DF unchanged - self.assertListEqual(['one', 'two', 'target'], list(df.columns)) - - operation1_key = 'one_%s' % self.op_key - operation2_key = 'two_%s' % self.op_key - self.assertListEqual(['one', 'two', 'target', operation1_key, operation2_key], list(result_df.columns)) - np.testing.assert_array_almost_equal(self.operation(df['one'], df['target']), list(result_df[operation1_key])) - np.testing.assert_array_almost_equal(self.operation(df['two'], df['target']), list(result_df[operation2_key])) - - def test_single_dim_with_ref(self): - df = mock_df.time_dim_single_metric_ref_df.copy() - df['', 'target'] = [2, 0, 4, 0, 6, 0, 8, 0] - df['wow', 'target'] = [2, 0, 4, 0, 6, 0, 8, 0] - - result_df = self.manager.post_process(df, [{'key': self.op_key, 'metric': 'one', 'target': 'target'}]) - - # original DF unchanged - self.assertListEqual([('', 'one'), ('wow', 'one'), ('', 'target'), ('wow', 'target')], list(df.columns)) - - operation_key = 'one_%s' % self.op_key - self.assertListEqual([('', 'one'), ('wow', 'one'), - ('', 'target'), ('wow', 'target'), - ('', operation_key), ('wow', operation_key)], - list(result_df.columns)) - np.testing.assert_array_almost_equal(self.operation(df['', 'one'], df['', 'target']), - list(result_df[('', operation_key)])) - np.testing.assert_array_almost_equal(self.operation(df['wow', 'one'], df['wow', 'target']), - list(result_df[('wow', operation_key)])) - - def test_multi_dim(self): - df = mock_df.cont_cat_dims_single_metric_df.copy() - df['target'] = df['one'] + 1 - - result_df = self.manager.post_process(df, [{'key': self.op_key, 'metric': 'one', 'target': 'target'}]) - - # original DF unchanged - self.assertListEqual(['one', 'target'], list(df.columns)) - - operation_key = 'one_%s' % self.op_key - self.assertListEqual(['one', 'target', operation_key], list(result_df.columns)) - - slice_a = df.loc[(slice(None), 'a'), :] - np.testing.assert_array_almost_equal(self.operation(slice_a['one'], slice_a['target']), - list(result_df.loc[(slice(None), 'a'), operation_key])) - - slice_b = df.loc[(slice(None), 'b'), :] - np.testing.assert_array_almost_equal(self.operation(slice_b['one'], slice_b['target']), - list(result_df.loc[(slice(None), 'b'), operation_key])) - - -class L2LossOperationTests(L1LossOperationTests, TestCase): - @property - def op_key(self): - return 'l2loss' - - def operation(self, metric_df, target_df): - return list((metric_df - target_df).pow(2).expanding(min_periods=1).mean()) diff --git a/fireant/tests/slicer/test_queries.py b/fireant/tests/slicer/test_queries.py deleted file mode 100644 index 736c083e..00000000 --- a/fireant/tests/slicer/test_queries.py +++ /dev/null @@ -1,1770 +0,0 @@ -# coding: utf-8 -import unittest -from collections import OrderedDict -from datetime import date - -from mock import patch -from pypika import ( - JoinType, - Order, - Tables, - functions as fn, -) - -from fireant import settings -from fireant.database import ( - MySQLDatabase, - PostgreSQLDatabase, - RedshiftDatabase, -) -from fireant.slicer import references -from fireant.slicer.pagination import Paginator -from fireant.slicer.queries import ( - QueryManager, - QueryNotSupportedError, -) -from fireant.tests.database.mock_database import TestDatabase - - -class QueryTests(unittest.TestCase): - manager = QueryManager(database=TestDatabase()) - maxDiff = None - - mock_table, mock_join1, mock_join2 = Tables('test_table', 'test_join1', 'test_join2') - - @classmethod - def setUpClass(cls): - settings.database = TestDatabase() - - -class ExampleTests(QueryTests): - def test_full_example(self): - """ - This is an example using several features of the slicer. It demonstrates usage of metrics, dimensions, filters, - references and also joining metrics from additional tables. - - The _build_query function takes a dictionary parameter that expresses all of the options of the slicer aside - from Operations which are done in a post-processing step. - - Many of the fields are defined as a dict. In the examples and other tests an OrderedDict is used in many places - so that the field order is maintained for the assertions. In a real example, a regular dict can be used. - - The fields in the dictionary are as follows: - - :param 'table': - The primary table to query data from. This is the table used for the FROM clause in the query. - :param 'joins': - A list or tuple of tuples. This lists the tables that need to be joined in the query and the criterion to - use to join them. The inner tuples must contain two elements, the first element is the table to join and - the second element is a criterion to join the tables with. - :param 'metrics': - A dict containing the SELECT clause. The values can be strings (as a short cut for a column of the primary - table), a field instance, or an expression containing functions, arithmetic, or anything else supported by - pypika. - :param 'dimensions': - A dict containing the INDEX of the query. These fields will be included in the SELECT clause, the GROUP BY - clause, and the ORDER BY clause. For comparisons, they are also used to join the nested query. - :param 'filters': - A list containing criterion expressions for the WHERE clause. Multiple filters will be combined with an - 'AND' operator. - :param 'references': - A dict containing comparison operators. The keys of this dict must match a supported comparison operation. - The value of the key must match the key from the dimensions table. The dimension must also be of the - supported type, for example 'yoy' requires a DATE type dimension. - :param 'rollup': - A list of dimensions to rollup, or provide the totals across groups. When multiple dimensions are included, - rollup works from the last to the first dimension, providing the totals across the dimensions in a tree - structure. - - See pypika documentation for more examples of query expressions: http://pypika.readthedocs.io/en/latest/ - """ - dt = self.mock_table.dt - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[ - (self.mock_join1, self.mock_table.join1_id == self.mock_join1.id, JoinType.inner), - (self.mock_join2, self.mock_table.join2_id == self.mock_join2.id, JoinType.left), - ], - metrics=OrderedDict([ - # Examples using a field of a table - ('foo', fn.Sum(self.mock_table.foo)), - - # Examples using a field of a table - ('bar', fn.Avg(self.mock_join1.bar)), - - # Example using functions and Arithmetic - ('ratio', fn.Sum(self.mock_table.numerator) / fn.Sum(self.mock_table.denominator)), - ]), - dimensions=OrderedDict([ - # Example of using a continuous datetime dimension, where the values are truncated to the nearest day - ('date', settings.database.trunc_date(dt, 'day')), - - # Example of using a categorical dimension from a joined table - ('fiz', self.mock_join2.fiz), - ]), - mfilters=[ - fn.Sum(self.mock_join2.buz) > 100 - ], - dfilters=[ - # Example of filtering the query to a date range - dt[date(2016, 1, 1):date(2016, 12, 31)], - - # Example of filtering the query to certain categories - self.mock_join2.fiz.isin(['a', 'b', 'c']), - ], - references=OrderedDict([ - # Example of adding a Week-over-Week comparison to the query - (references.WoW.key, { - 'dimension': 'date', 'definition': dt, - 'time_unit': references.WoW.time_unit, 'interval': references.WoW.interval - }) - ]), - rollup=[], - pagination=None, - ) - - self.assertEqual('SELECT ' - # Dimensions - '"sq0"."date" "date","sq0"."fiz" "fiz",' - # Metrics - '"sq0"."foo" "foo","sq0"."bar" "bar","sq0"."ratio" "ratio",' - # Reference Dimension - # Currently not selected - # '"sq1"."dt" "dt_wow",' - # Reference Metrics - '"sq1"."foo" "foo_wow","sq1"."bar" "bar_wow","sq1"."ratio" "ratio_wow" ' - 'FROM (' - # Main Query - 'SELECT ' - 'TRUNC("test_table"."dt",\'DD\') "date","test_join2"."fiz" "fiz",' - 'SUM("test_table"."foo") "foo",' - 'AVG("test_join1"."bar") "bar",' - 'SUM("test_table"."numerator")/SUM("test_table"."denominator") "ratio" ' - 'FROM "test_table" ' - 'JOIN "test_join1" ON "test_table"."join1_id"="test_join1"."id" ' - 'LEFT JOIN "test_join2" ON "test_table"."join2_id"="test_join2"."id" ' - 'WHERE "test_table"."dt" BETWEEN \'2016-01-01\' AND \'2016-12-31\' ' - 'AND "test_join2"."fiz" IN (\'a\',\'b\',\'c\') ' - 'GROUP BY "date","fiz" ' - 'HAVING SUM("test_join2"."buz")>100' - ') "sq0" ' - 'LEFT JOIN (' - # Reference Query - 'SELECT ' - 'TRUNC("test_table"."dt",\'DD\') "date","test_join2"."fiz" "fiz",' - 'SUM("test_table"."foo") "foo",' - 'AVG("test_join1"."bar") "bar",' - 'SUM("test_table"."numerator")/SUM("test_table"."denominator") "ratio" ' - 'FROM "test_table" ' - 'JOIN "test_join1" ON "test_table"."join1_id"="test_join1"."id" ' - 'LEFT JOIN "test_join2" ON "test_table"."join2_id"="test_join2"."id" ' - 'WHERE TIMESTAMPADD(\'week\',1,"test_table"."dt") BETWEEN \'2016-01-01\' AND \'2016-12-31\' ' - 'AND "test_join2"."fiz" IN (\'a\',\'b\',\'c\') ' - 'GROUP BY "date","fiz" ' - 'HAVING SUM("test_join2"."buz")>100' - ') "sq1" ON "sq0"."date"=TIMESTAMPADD(\'week\',1,"sq1"."date") ' - 'AND "sq0"."fiz"="sq1"."fiz" ' - 'ORDER BY "date","fiz"', str(query)) - - -class MetricsTests(QueryTests): - def test_metrics(self): - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ]), - dimensions={}, - mfilters=[], - dfilters=[], - references={}, - rollup=[], - pagination=None, - ) - - self.assertEqual('SELECT SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table"', str(query)) - - def test_metrics_dimensions(self): - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ]), - dimensions=OrderedDict([ - ('device_type', self.mock_table.device_type) - ]), - mfilters=[], - dfilters=[], - references={}, - rollup=[], - pagination=None, - ) - - self.assertEqual( - 'SELECT "device_type" "device_type",SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "device_type" ' - 'ORDER BY "device_type"', str(query)) - - def test_metrics_filters(self): - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ]), - dimensions=OrderedDict([ - ('device_type', self.mock_table.device_type) - ]), - mfilters=[], - dfilters=[ - self.mock_table.dt[date(2000, 1, 1):date(2001, 1, 1)] - ], - references={}, - rollup=[], - pagination=None, - ) - - self.assertEqual( - 'SELECT "device_type" "device_type",SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE "dt" BETWEEN \'2000-01-01\' AND \'2001-01-01\' ' - 'GROUP BY "device_type" ' - 'ORDER BY "device_type"', str(query)) - - def test_metrics_dimensions_filters(self): - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', (fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost))), - ]), - dimensions=OrderedDict([ - ('device_type', self.mock_table.device_type), - ('locale', self.mock_table.locale), - ]), - mfilters=[], - dfilters=[ - self.mock_table.locale.isin(['US', 'CA', 'UK']) - ], - references={}, - rollup=[], - pagination=None, - ) - - self.assertEqual( - 'SELECT ' - '"device_type" "device_type",' - '"locale" "locale",' - 'SUM("clicks") "clicks",' - 'SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE "locale" IN (\'US\',\'CA\',\'UK\') ' - 'GROUP BY "device_type","locale" ' - 'ORDER BY "device_type","locale"', str(query)) - - -class DimensionTests(QueryTests): - def _test_truncated_timeseries(self, increment): - truncated_dt = settings.database.trunc_date(self.mock_table.dt, increment) - - return self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ]), - dimensions=OrderedDict([ - ('date', truncated_dt) - ]), - mfilters=[], - dfilters=[], - references={}, - rollup=[], - pagination=None, - ) - - def test_timeseries_hour(self): - query = self._test_truncated_timeseries('hour') - - self.assertEqual( - 'SELECT TRUNC("dt",\'HH\') "date",SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "date" ' - 'ORDER BY "date"', str(query)) - - def test_timeseries_DD(self): - query = self._test_truncated_timeseries('day') - - self.assertEqual( - 'SELECT TRUNC("dt",\'DD\') "date",SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "date" ' - 'ORDER BY "date"', str(query)) - - def test_timeseries_week(self): - query = self._test_truncated_timeseries('week') - - self.assertEqual( - 'SELECT TRUNC("dt",\'IW\') "date",SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "date" ' - 'ORDER BY "date"', str(query)) - - def test_timeseries_month(self): - query = self._test_truncated_timeseries('month') - - self.assertEqual( - 'SELECT TRUNC("dt",\'MM\') "date",SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "date" ' - 'ORDER BY "date"', str(query)) - - def test_timeseries_quarter(self): - query = self._test_truncated_timeseries('quarter') - - self.assertEqual( - 'SELECT TRUNC("dt",\'Q\') "date",SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "date" ' - 'ORDER BY "date"', str(query)) - - def test_timeseries_year(self): - query = self._test_truncated_timeseries('year') - - self.assertEqual( - 'SELECT TRUNC("dt",\'Y\') "date",SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "date" ' - 'ORDER BY "date"', str(query)) - - def test_multidimension_categorical(self): - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ]), - dimensions=OrderedDict([ - ('device_type', self.mock_table.device_type), - ('locale', self.mock_table.locale), - ]), - mfilters=[], - dfilters=[], - references={}, - rollup=[], - pagination=None, - ) - - self.assertEqual( - 'SELECT "device_type" "device_type","locale" "locale",' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "device_type","locale" ' - 'ORDER BY "device_type","locale"', str(query)) - - def test_multidimension_timeseries_categorical(self): - truncated_dt = settings.database.trunc_date(self.mock_table.dt, 'day') - device_type = self.mock_table.device_type - - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ]), - dimensions=OrderedDict([ - ('date', truncated_dt), - ('device_type', device_type), - ]), - mfilters=[], - dfilters=[], - references={}, - rollup=[], - pagination=None, - ) - - self.assertEqual( - 'SELECT TRUNC("dt",\'DD\') "date","device_type" "device_type",' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "date","device_type" ' - 'ORDER BY "date","device_type"', str(query)) - - def test_metrics_with_joins(self): - truncated_dt = settings.database.trunc_date(self.mock_table.dt, 'day') - locale = self.mock_table.locale - - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[ - (self.mock_join1, self.mock_table.hotel_id == self.mock_join1.hotel_id, JoinType.left), - ], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ('hotel_name', self.mock_join1.hotel_name), - ('hotel_address', self.mock_join1.address), - ('city_id', self.mock_join1.ctid), - ('city_name', self.mock_join1.city_name), - ]), - dimensions=OrderedDict([ - ('date', truncated_dt), - ('locale', locale), - ]), - mfilters=[], - dfilters=[], - references={}, - rollup=[], - pagination=None, - ) - - self.assertEqual('SELECT ' - 'TRUNC("test_table"."dt",\'DD\') "date","test_table"."locale" "locale",' - 'SUM("test_table"."clicks") "clicks",' - 'SUM("test_table"."revenue")/SUM("test_table"."cost") "roi",' - '"test_join1"."hotel_name" "hotel_name","test_join1"."address" "hotel_address",' - '"test_join1"."ctid" "city_id","test_join1"."city_name" "city_name" ' - 'FROM "test_table" ' - 'LEFT JOIN "test_join1" ON "test_table"."hotel_id"="test_join1"."hotel_id" ' - 'GROUP BY "date","locale" ' - 'ORDER BY "date","locale"', str(query)) - - -class FilterTests(QueryTests): - def test_single_dimension_filter(self): - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', (fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost))), - ]), - dimensions=OrderedDict([ - ('locale', self.mock_table.locale), - ]), - mfilters=[], - dfilters=[ - self.mock_table.locale.isin(['US', 'CA', 'UK']) - ], - references={}, - rollup=[], - pagination=None, - ) - - self.assertEqual('SELECT ' - '"locale" "locale",' - 'SUM("clicks") "clicks",' - 'SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE "locale" IN (\'US\',\'CA\',\'UK\') ' - 'GROUP BY "locale" ' - 'ORDER BY "locale"', str(query)) - - def test_multi_dimension_filter(self): - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', (fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost))), - ]), - dimensions=OrderedDict([ - ('locale', self.mock_table.locale), - ]), - mfilters=[], - dfilters=[ - self.mock_table.locale.isin(['US', 'CA', 'UK']), - self.mock_table.device_type == 'desktop', - self.mock_table.dt > date(2016, 1, 1), - ], - references={}, - rollup=[], - pagination=None, - ) - - self.assertEqual('SELECT ' - '"locale" "locale",' - 'SUM("clicks") "clicks",' - 'SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE "locale" IN (\'US\',\'CA\',\'UK\') ' - 'AND "device_type"=\'desktop\' ' - 'AND "dt">\'2016-01-01\' ' - 'GROUP BY "locale" ' - 'ORDER BY "locale"', str(query)) - - def test_single_metric_filter(self): - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', (fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost))), - ]), - dimensions=OrderedDict([ - ('locale', self.mock_table.locale), - ]), - mfilters=[ - fn.Sum(self.mock_table.clicks) > 100 - ], - dfilters=[], - references={}, - rollup=[], - pagination=None, - ) - - self.assertEqual('SELECT ' - '"locale" "locale",' - 'SUM("clicks") "clicks",' - 'SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "locale" ' - 'HAVING SUM("clicks")>100 ' - 'ORDER BY "locale"', str(query)) - - def test_multi_metric_filter(self): - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', (fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost))), - ]), - dimensions=OrderedDict([ - ('locale', self.mock_table.locale), - ]), - mfilters=[ - fn.Sum(self.mock_table.clicks) > 100, - (fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)) < 0.7, - fn.Sum(self.mock_table.conversions) >= 10, - ], - dfilters=[], - references={}, - rollup=[], - pagination=None, - ) - - self.assertEqual('SELECT ' - '"locale" "locale",' - 'SUM("clicks") "clicks",' - 'SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "locale" ' - 'HAVING SUM("clicks")>100 ' - 'AND SUM("revenue")/SUM("cost")<0.7 ' - 'AND SUM("conversions")>=10 ' - 'ORDER BY "locale"', str(query)) - - -class ReferenceTests(QueryTests): - intervals = { - 'yoy': '1', - 'qoq': '1 QUARTER', - 'mom': '1 MONTH', - 'wow': '1 WEEK', - 'dod': '1 DAY', - } - - def _get_compare_query(self, ref): - dt = self.mock_table.dt - device_type = self.mock_table.device_type - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ]), - dimensions=OrderedDict([ - ('date', settings.database.trunc_date(dt, 'day')), - ('device_type', device_type), - ]), - mfilters=[], - dfilters=[ - dt[date(2000, 1, 1):date(2000, 3, 1)] - ], - references=OrderedDict([ - (ref.key, { - 'dimension': ref.element_key, 'definition': dt, 'interval': ref.interval, - 'modifier': ref.modifier, 'time_unit': ref.time_unit - }) - ]), - rollup=[], - pagination=None, - ) - return query - - def assert_reference(self, query, key, time_unit): - self.assertEqual( - 'SELECT ' - '"sq0"."date" "date","sq0"."device_type" "device_type",' - '"sq0"."clicks" "clicks","sq0"."roi" "roi",' - '"sq1"."clicks" "clicks_{key}",' - '"sq1"."roi" "roi_{key}" ' - 'FROM (' - 'SELECT ' - 'TRUNC("dt",\'DD\') "date","device_type" "device_type",' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE "dt" BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "date","device_type"' - ') "sq0" ' - 'LEFT JOIN (' - 'SELECT ' - 'TRUNC("dt",\'DD\') "date","device_type" "device_type",' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE TIMESTAMPADD(\'{time_unit}\',1,"dt") BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "date","device_type"' - ') "sq1" ON "sq0"."date"=TIMESTAMPADD(\'{time_unit}\',1,"sq1"."date") ' - 'AND "sq0"."device_type"="sq1"."device_type" ' - 'ORDER BY "date","device_type"'.format( - key=key, - time_unit=time_unit - ), str(query) - ) - - def assert_reference_delta(self, query, key, time_unit): - self.assertEqual( - 'SELECT ' - '"sq0"."date" "date","sq0"."device_type" "device_type",' - '"sq0"."clicks" "clicks","sq0"."roi" "roi",' - '"sq0"."clicks"-"sq1"."clicks" "clicks_{key}_delta",' - '"sq0"."roi"-"sq1"."roi" "roi_{key}_delta" ' - 'FROM (' - 'SELECT ' - 'TRUNC("dt",\'DD\') "date","device_type" "device_type",' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE "dt" BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "date","device_type"' - ') "sq0" ' - 'LEFT JOIN (' - 'SELECT ' - 'TRUNC("dt",\'DD\') "date","device_type" "device_type",' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE TIMESTAMPADD(\'{time_unit}\',1,"dt") BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "date","device_type"' - ') "sq1" ON "sq0"."date"=TIMESTAMPADD(\'{time_unit}\',1,"sq1"."date") ' - 'AND "sq0"."device_type"="sq1"."device_type" ' - 'ORDER BY "date","device_type"'.format( - key=key, - time_unit=time_unit - ), str(query) - ) - - def assert_reference_delta_percent(self, query, key, time_unit): - self.assertEqual( - 'SELECT ' - '"sq0"."date" "date","sq0"."device_type" "device_type",' - '"sq0"."clicks" "clicks","sq0"."roi" "roi",' - '("sq0"."clicks"-"sq1"."clicks")*100/NULLIF("sq1"."clicks",0) "clicks_{key}_delta_percent",' - '("sq0"."roi"-"sq1"."roi")*100/NULLIF("sq1"."roi",0) "roi_{key}_delta_percent" ' - 'FROM (' - 'SELECT ' - 'TRUNC("dt",\'DD\') "date","device_type" "device_type",' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE "dt" BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "date","device_type"' - ') "sq0" ' - 'LEFT JOIN (' - 'SELECT ' - 'TRUNC("dt",\'DD\') "date","device_type" "device_type",' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE TIMESTAMPADD(\'{time_unit}\',1,"dt") BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "date","device_type"' - ') "sq1" ON "sq0"."date"=TIMESTAMPADD(\'{time_unit}\',1,"sq1"."date") ' - 'AND "sq0"."device_type"="sq1"."device_type" ' - 'ORDER BY "date","device_type"'.format( - key=key, - time_unit=time_unit - ), str(query) - ) - - def test_metrics_dimensions_filters_references__yoy(self): - reference = references.YoY('date') - query = self._get_compare_query(reference) - self.assert_reference(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__qoq(self): - reference = references.QoQ('date') - query = self._get_compare_query(reference) - self.assert_reference(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__mom(self): - reference = references.MoM('date') - query = self._get_compare_query(reference) - self.assert_reference(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__wow(self): - reference = references.WoW('date') - query = self._get_compare_query(reference) - self.assert_reference(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__dod(self): - reference = references.DoD('date') - query = self._get_compare_query(reference) - self.assert_reference(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__yoy_delta(self): - reference = references.YoY('date') - query = self._get_compare_query(references.Delta(reference)) - self.assert_reference_delta(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__qoq_delta(self): - reference = references.QoQ('date') - query = self._get_compare_query(references.Delta(reference)) - self.assert_reference_delta(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__mom_delta(self): - reference = references.MoM('date') - query = self._get_compare_query(references.Delta(reference)) - self.assert_reference_delta(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__wow_delta(self): - reference = references.WoW('date') - query = self._get_compare_query(references.Delta(reference)) - self.assert_reference_delta(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__dod_delta(self): - reference = references.DoD('date') - query = self._get_compare_query(references.Delta(reference)) - self.assert_reference_delta(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__yoy_delta_percent(self): - reference = references.YoY('date') - query = self._get_compare_query(references.DeltaPercentage(reference)) - self.assert_reference_delta_percent(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__qoq_delta_percent(self): - reference = references.QoQ('date') - query = self._get_compare_query(references.DeltaPercentage(reference)) - self.assert_reference_delta_percent(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__mom_delta_percent(self): - reference = references.MoM('date') - query = self._get_compare_query(references.DeltaPercentage(reference)) - self.assert_reference_delta_percent(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__wow_delta_percent(self): - reference = references.WoW('date') - query = self._get_compare_query(references.DeltaPercentage(reference)) - self.assert_reference_delta_percent(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__dod_delta_percent(self): - reference = references.DoD('date') - query = self._get_compare_query(references.DeltaPercentage(reference)) - self.assert_reference_delta_percent(query, reference.key, reference.time_unit) - - def test_metrics_dimensions_filters_references__no_date_dimension(self): - ref = references.DoD('date') - dt = self.mock_table.dt - device_type = self.mock_table.device_type - - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ]), - dimensions=OrderedDict([ - # NO Date Dimension - ('device_type', device_type), - ]), - mfilters=[], - dfilters=[ - dt[date(2000, 1, 1):date(2000, 3, 1)] - ], - references=OrderedDict([ - (ref.key, { - 'dimension': ref.element_key, 'definition': self.mock_table.dt, - 'modifier': ref.modifier, - 'time_unit': ref.time_unit, 'interval': ref.interval - }) - ]), - rollup=[], - pagination=None, - ) - - self.assertEqual( - 'SELECT ' - '"sq0"."device_type" "device_type",' - '"sq0"."clicks" "clicks","sq0"."roi" "roi",' - '"sq1"."clicks" "clicks_{key}",' - '"sq1"."roi" "roi_{key}" ' - 'FROM (' - 'SELECT ' - '"device_type" "device_type",' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE "dt" BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "device_type"' - ') "sq0" ' - 'LEFT JOIN (' - 'SELECT ' - '"device_type" "device_type",' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE TIMESTAMPADD(\'day\',1,"dt") BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "device_type"' - ') "sq1" ON "sq0"."device_type"="sq1"."device_type" ' - 'ORDER BY "device_type"'.format( - key=ref.key, - ), str(query) - ) - - def test_metrics_dimensions_filters_references__no_dimensions(self): - ref = references.DoD('date') - dt = self.mock_table.dt - - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ]), - dimensions={}, - mfilters=[], - dfilters=[ - dt[date(2000, 1, 1):date(2000, 3, 1)] - ], - references=OrderedDict([ - (ref.key, { - 'dimension': ref.element_key, 'definition': self.mock_table.dt, - 'modifier': ref.modifier, - 'time_unit': ref.time_unit, 'interval': ref.interval - - }) - ]), - rollup=[], - pagination=None, - ) - - self.assertEqual( - 'SELECT ' - '"sq0"."clicks" "clicks","sq0"."roi" "roi",' - '"sq1"."clicks" "clicks_{key}",' - '"sq1"."roi" "roi_{key}" ' - 'FROM (' - 'SELECT ' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE "dt" BETWEEN \'2000-01-01\' AND \'2000-03-01\'' - ') "sq0",(' - 'SELECT ' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'WHERE TIMESTAMPADD(\'day\',1,"dt") BETWEEN \'2000-01-01\' AND \'2000-03-01\'' - ') "sq1"'.format( - key=ref.key, - ), str(query) - ) - - -class TotalsQueryTests(QueryTests): - def test_add_rollup_one_dimension(self): - truncated_dt = settings.database.trunc_date(self.mock_table.dt, 'day') - locale = self.mock_table.locale - - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ]), - dimensions=OrderedDict([ - ('date', truncated_dt), - ('locale', locale), - ]), - mfilters=[], - dfilters=[], - references={}, - rollup=[['locale']], - pagination=None, - ) - - self.assertEqual('SELECT ' - 'TRUNC("dt",\'DD\') "date","locale" "locale",' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "date",ROLLUP(("locale")) ' - 'ORDER BY "date","locale"', str(query)) - - def test_add_rollup_two_dimensions(self): - truncated_dt = settings.database.trunc_date(self.mock_table.dt, 'day') - locale = self.mock_table.locale - device_type = self.mock_table.device_type - - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ]), - dimensions=OrderedDict([ - ('date', truncated_dt), - ('locale', locale), - ('device_type', device_type), - ]), - mfilters=[], - dfilters=[], - references={}, - rollup=[['locale'], ['device_type']], - pagination=None, - ) - - self.assertEqual('SELECT ' - 'TRUNC("dt",\'DD\') "date",' - '"locale" "locale",' - '"device_type" "device_type",' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "date",ROLLUP(("locale"),("device_type")) ' - 'ORDER BY "date","locale","device_type"', str(query)) - - def test_add_rollup_two_dimensions_partial(self): - truncated_dt = settings.database.trunc_date(self.mock_table.dt, 'day') - locale = self.mock_table.locale - device_type = self.mock_table.device_type - - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ]), - dimensions=OrderedDict([ - ('date', truncated_dt), - ('locale', locale), - ('device_type', device_type), - ]), - mfilters=[], - dfilters=[], - references={}, - rollup=[['locale']], - pagination=None, - ) - - self.assertEqual('SELECT ' - 'TRUNC("dt",\'DD\') "date",' - '"device_type" "device_type",' - '"locale" "locale",' # Order is changed, rollup dims move to end - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "date","device_type",ROLLUP(("locale")) ' - 'ORDER BY "date","locale","device_type"', str(query)) - - def test_add_rollup_uni_dimension(self): - truncated_dt = settings.database.trunc_date(self.mock_table.dt, 'day') - locale = self.mock_table.locale - locale_display = self.mock_table.locale_display - - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('roi', fn.Sum(self.mock_table.revenue) / fn.Sum(self.mock_table.cost)), - ]), - dimensions=OrderedDict([ - ('date', truncated_dt), - ('locale', locale), - ('locale_display', locale_display), - ]), - mfilters=[], - dfilters=[], - references={}, - rollup=[['locale', 'locale_display']], - pagination=None, - ) - - self.assertEqual('SELECT ' - 'TRUNC("dt",\'DD\') "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",SUM("revenue")/SUM("cost") "roi" ' - 'FROM "test_table" ' - 'GROUP BY "date",ROLLUP(("locale","locale_display")) ' - 'ORDER BY "date","locale","locale_display"', str(query)) - - -class DimensionOptionTests(QueryTests): - def test_dimension_options(self): - locale = self.mock_table.locale - - query = self.manager._build_dimension_query( - table=self.mock_table, - joins=[], - dimensions=OrderedDict([ - ('locale', locale), - ]), - filters=[], - ) - - self.assertEqual('SELECT DISTINCT ' - '"locale" "locale" ' - 'FROM "test_table"', str(query)) - - def test_dimension_options_with_query_with_limit(self): - locale = self.mock_table.locale - - query = self.manager._build_dimension_query( - table=self.mock_table, - joins=[], - dimensions=OrderedDict([ - ('locale', locale), - ]), - filters=[], - limit=10, - ) - - self.assertEqual('SELECT DISTINCT ' - '"locale" "locale" ' - 'FROM "test_table" ' - 'LIMIT 10', str(query)) - - def test_dimension_options_with_query_with_filter(self): - locale = self.mock_table.locale - - query = self.manager._build_dimension_query( - table=self.mock_table, - joins=[], - dimensions=OrderedDict([ - ('locale', locale), - ]), - filters=[ - self.mock_table.device_type == 'desktop', - ], - ) - - self.assertEqual('SELECT DISTINCT ' - '"locale" "locale" ' - 'FROM "test_table" ' - 'WHERE "device_type"=\'desktop\'', str(query)) - - def test_dimension_options_with_multiple_dimensions(self): - account_id = self.mock_table.account_id - account_name = self.mock_table.account_name - - query = self.manager._build_dimension_query( - table=self.mock_table, - joins=[], - dimensions=OrderedDict([ - ('account_id', account_id), - ('account_name', account_name), - ]), - filters=[], - limit=10 - ) - - self.assertEqual('SELECT DISTINCT ' - '"account_id" "account_id",' - '"account_name" "account_name" ' - 'FROM "test_table" ' - 'LIMIT 10', str(query)) - - def test_dimension_options_with_joins(self): - account_id = self.mock_table.account_id - - query = self.manager._build_dimension_query( - table=self.mock_table, - joins=[ - (self.mock_join1, account_id == self.mock_join1.account_id, JoinType.left), - ], - dimensions=OrderedDict([ - ('account_id', account_id), - ('account_name', self.mock_join1.account_name), - ]), - filters=[], - ) - - self.assertEqual('SELECT DISTINCT ' - '"test_table"."account_id" "account_id",' - '"test_join1"."account_name" "account_name" ' - 'FROM "test_table" ' - 'LEFT JOIN "test_join1" ' - 'ON "test_table"."account_id"="test_join1"."account_id"', str(query)) - - @patch.object(MySQLDatabase, 'fetch_dataframe') - def test_exception_raised_if_rollup_requested_for_a_mysql_database(self, mock_db): - db = MySQLDatabase(database='testdb') - manager = QueryManager(database=db) - - with self.assertRaises(QueryNotSupportedError): - manager.query_data(db, self.mock_table, rollup=[['locale']]) - - @patch.object(PostgreSQLDatabase, 'fetch_dataframe') - def test_exception_raised_if_rollup_requested_for_a_postgres_database(self, mock_db): - db = PostgreSQLDatabase(database='testdb') - manager = QueryManager(database=db) - - with self.assertRaises(QueryNotSupportedError): - manager.query_data(db, self.mock_table, rollup=[['locale']]) - - @patch.object(RedshiftDatabase, 'fetch_dataframe') - def test_exception_raised_if_rollup_requested_for_a_redshift_database(self, mock_db): - db = RedshiftDatabase(database='testdb') - manager = QueryManager(database=db) - - with self.assertRaises(QueryNotSupportedError): - manager.query_data(db, self.mock_table, rollup=[['locale']]) - - def test_yoy_week_interval(self): - ref = references.YoY('date') - dt = self.mock_table.dt - - query = self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ]), - dimensions=OrderedDict([ - ('date', settings.database.trunc_date(dt, 'week')) - ]), - mfilters=[], - dfilters=[], - references=OrderedDict([ - (ref.key, { - 'dimension': ref.element_key, 'definition': dt, - 'modifier': ref.modifier, - 'time_unit': ref.time_unit, 'interval': ref.interval - }) - ]), - rollup=[], - pagination=None, - ) - - self.assertEqual('SELECT "sq0"."date" "date","sq0"."clicks" "clicks","sq1"."clicks" "clicks_yoy" ' - 'FROM (' - 'SELECT TRUNC("dt",\'IW\') "date",SUM("clicks") "clicks" ' - 'FROM "test_table" ' - 'GROUP BY "date"' - ') "sq0" ' - 'LEFT JOIN (' - 'SELECT TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"dt"),\'IW\')) "date",' - 'SUM("clicks") "clicks" ' - 'FROM "test_table" ' - 'GROUP BY "date") ' - '"sq1" ' - 'ON "sq0"."date"=TIMESTAMPADD(\'year\',1,"sq1"."date") ' - 'ORDER BY "date"', str(query)) - - -class PaginationNonReferenceQueryTests(QueryTests): - def get_non_reference_query(self, paginator): - return self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('impressions', fn.Sum(self.mock_table.impressions)), - ]), - dimensions=OrderedDict([ - ('locale', self.mock_table.locale), - ('locale_display', self.mock_table.locale_display), - ]), - mfilters=[], - dfilters=[], - references={}, - rollup=[], - pagination=paginator, - ) - - def test_offset_0_limit_0_no_orderby_applied_to_query(self): - query = self.get_non_reference_query(Paginator(offset=0, limit=0)) - - self.assertEqual('SELECT ' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "locale","locale_display"', str(query)) - - def test_offset_0_limit_50_no_orderby_applied_to_query(self): - query = self.get_non_reference_query(Paginator(offset=0, limit=50)) - - self.assertEqual('SELECT ' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "locale","locale_display" ' - 'LIMIT 50', str(query)) - - def test_offset_10_limit_50_no_orderby_applied_to_query(self): - query = self.get_non_reference_query(Paginator(offset=10, limit=50)) - - self.assertEqual('SELECT ' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "locale","locale_display" ' - 'LIMIT 50 OFFSET 10', str(query)) - - def test_offset_0_limit_10_with_single_dim_orderby_applied_to_query(self): - query = self.get_non_reference_query(Paginator(offset=0, limit=50, order=[('locale', Order.desc)])) - - self.assertEqual('SELECT ' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "locale","locale_display" ' - 'ORDER BY "locale" DESC ' - 'LIMIT 50', str(query)) - - def test_offset_0_limit_10_with_single_metric_orderby_applied_to_query(self): - query = self.get_non_reference_query(Paginator(offset=0, limit=50, order=[('clicks', Order.desc)])) - - self.assertEqual('SELECT ' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "locale","locale_display" ' - 'ORDER BY "clicks" DESC ' - 'LIMIT 50', str(query)) - - def test_offset_0_limit_0_with_multiple_dim_orderby_applied_to_query(self): - query = self.get_non_reference_query(Paginator(offset=0, limit=0, order=[('locale', Order.desc), - ('locale_display', Order.asc)])) - - self.assertEqual('SELECT ' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "locale","locale_display" ' - 'ORDER BY "locale" DESC,"locale_display" ASC', str(query)) - - def test_offset_0_limit_0_with_multiple_metric_orderby_applied_to_query(self): - query = self.get_non_reference_query(Paginator(offset=0, limit=0, order=[('clicks', Order.desc), - ('impressions', Order.asc)])) - - self.assertEqual('SELECT ' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "locale","locale_display" ' - 'ORDER BY "clicks" DESC,"impressions" ASC', str(query)) - - def test_offset_0_limit_10_with_multiple_dim_orderby_applied_to_query(self): - query = self.get_non_reference_query(Paginator(offset=0, limit=50, order=[('locale', Order.desc), - ('locale_display', Order.asc)])) - - self.assertEqual('SELECT ' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "locale","locale_display" ' - 'ORDER BY "locale" DESC,"locale_display" ASC ' - 'LIMIT 50', str(query)) - - def test_offset_10_limit_50_with_multiple_dim_orderby_applied_to_query(self): - query = self.get_non_reference_query(Paginator(offset=10, limit=50, order=[('locale', Order.desc), - ('locale_display', Order.asc)])) - - self.assertEqual('SELECT ' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "locale","locale_display" ' - 'ORDER BY "locale" DESC,"locale_display" ASC ' - 'LIMIT 50 OFFSET 10', str(query)) - - def test_offset_10_limit_50_with_multiple_dim_and_metric_orderby_applied_to_query(self): - query = self.get_non_reference_query(Paginator(offset=10, limit=50, order=[('clicks', Order.desc), - ('locale_display', Order.asc)])) - - self.assertEqual('SELECT ' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "locale","locale_display" ' - 'ORDER BY "clicks" DESC,"locale_display" ASC ' - 'LIMIT 50 OFFSET 10', str(query)) - - -class PaginationReferenceQueryTests(QueryTests): - def get_reference_query(self, paginator): - ref = references.YoY('date') - dt = self.mock_table.dt - - return self.manager._build_data_query( - database=settings.database, - table=self.mock_table, - joins=[], - metrics=OrderedDict([ - ('clicks', fn.Sum(self.mock_table.clicks)), - ('impressions', fn.Sum(self.mock_table.impressions)), - ]), - dimensions=OrderedDict([ - ('date', dt), - ('locale', self.mock_table.locale), - ('locale_display', self.mock_table.locale_display), - ]), - mfilters=[], - dfilters=[], - references=OrderedDict([ - (ref.key, { - 'dimension': ref.element_key, 'definition': dt, - 'modifier': ref.modifier, - 'time_unit': ref.time_unit, 'interval': ref.interval - }) - ]), - rollup=[], - pagination=paginator, - ) - - def test_offset_0_limit_0_no_orderby_applied_to_query(self): - query = self.get_reference_query(Paginator(offset=0, limit=0)) - - self.assertEqual('SELECT ' - '"sq0"."date" "date",' - '"sq0"."locale" "locale",' - '"sq0"."locale_display" "locale_display",' - '"sq0"."clicks" "clicks",' - '"sq0"."impressions" "impressions",' - '"sq1"."clicks" "clicks_yoy",' - '"sq1"."impressions" "impressions_yoy" ' - 'FROM ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq0" ' - 'LEFT JOIN ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq1" ' - 'ON "sq0"."date"=TIMESTAMPADD(\'year\',1,"sq1"."date") ' - 'AND "sq0"."locale"="sq1"."locale" ' - 'AND "sq0"."locale_display"="sq1"."locale_display"', str(query)) - - def test_offset_0_limit_50_no_orderby_applied_to_query(self): - query = self.get_reference_query(Paginator(offset=0, limit=50)) - - self.assertEqual('SELECT ' - '"sq0"."date" "date",' - '"sq0"."locale" "locale",' - '"sq0"."locale_display" "locale_display",' - '"sq0"."clicks" "clicks",' - '"sq0"."impressions" "impressions",' - '"sq1"."clicks" "clicks_yoy",' - '"sq1"."impressions" "impressions_yoy" ' - 'FROM ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq0" ' - 'LEFT JOIN ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq1" ' - 'ON "sq0"."date"=TIMESTAMPADD(\'year\',1,"sq1"."date") ' - 'AND "sq0"."locale"="sq1"."locale" ' - 'AND "sq0"."locale_display"="sq1"."locale_display" ' - 'LIMIT 50', str(query)) - - def test_offset_10_limit_50_no_orderby_applied_to_query(self): - query = self.get_reference_query(Paginator(offset=10, limit=50)) - - self.assertEqual('SELECT ' - '"sq0"."date" "date",' - '"sq0"."locale" "locale",' - '"sq0"."locale_display" "locale_display",' - '"sq0"."clicks" "clicks",' - '"sq0"."impressions" "impressions",' - '"sq1"."clicks" "clicks_yoy",' - '"sq1"."impressions" "impressions_yoy" ' - 'FROM ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq0" ' - 'LEFT JOIN ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq1" ' - 'ON "sq0"."date"=TIMESTAMPADD(\'year\',1,"sq1"."date") ' - 'AND "sq0"."locale"="sq1"."locale" ' - 'AND "sq0"."locale_display"="sq1"."locale_display" ' - 'LIMIT 50 OFFSET 10', str(query)) - - def test_offset_0_limit_10_with_single_dim_orderby_applied_to_query(self): - query = self.get_reference_query(Paginator(offset=0, limit=50, order=[('locale', Order.desc)])) - - self.assertEqual('SELECT ' - '"sq0"."date" "date",' - '"sq0"."locale" "locale",' - '"sq0"."locale_display" "locale_display",' - '"sq0"."clicks" "clicks",' - '"sq0"."impressions" "impressions",' - '"sq1"."clicks" "clicks_yoy",' - '"sq1"."impressions" "impressions_yoy" ' - 'FROM ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq0" ' - 'LEFT JOIN ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq1" ' - 'ON "sq0"."date"=TIMESTAMPADD(\'year\',1,"sq1"."date") ' - 'AND "sq0"."locale"="sq1"."locale" ' - 'AND "sq0"."locale_display"="sq1"."locale_display" ' - 'ORDER BY "locale" DESC ' - 'LIMIT 50', str(query)) - - def test_offset_0_limit_10_with_single_metric_orderby_applied_to_query(self): - query = self.get_reference_query(Paginator(offset=0, limit=50, order=[('clicks', Order.desc)])) - - self.assertEqual('SELECT ' - '"sq0"."date" "date",' - '"sq0"."locale" "locale",' - '"sq0"."locale_display" "locale_display",' - '"sq0"."clicks" "clicks",' - '"sq0"."impressions" "impressions",' - '"sq1"."clicks" "clicks_yoy",' - '"sq1"."impressions" "impressions_yoy" ' - 'FROM ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq0" ' - 'LEFT JOIN ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq1" ' - 'ON "sq0"."date"=TIMESTAMPADD(\'year\',1,"sq1"."date") ' - 'AND "sq0"."locale"="sq1"."locale" ' - 'AND "sq0"."locale_display"="sq1"."locale_display" ' - 'ORDER BY "clicks" DESC ' - 'LIMIT 50', str(query)) - - def test_offset_0_limit_0_with_multiple_dim_orderby_applied_to_query(self): - query = self.get_reference_query(Paginator(offset=0, limit=0, order=[('locale', Order.desc), - ('locale_display', Order.asc)])) - - self.assertEqual('SELECT ' - '"sq0"."date" "date",' - '"sq0"."locale" "locale",' - '"sq0"."locale_display" "locale_display",' - '"sq0"."clicks" "clicks",' - '"sq0"."impressions" "impressions",' - '"sq1"."clicks" "clicks_yoy",' - '"sq1"."impressions" "impressions_yoy" ' - 'FROM ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq0" ' - 'LEFT JOIN ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq1" ' - 'ON "sq0"."date"=TIMESTAMPADD(\'year\',1,"sq1"."date") ' - 'AND "sq0"."locale"="sq1"."locale" ' - 'AND "sq0"."locale_display"="sq1"."locale_display" ' - 'ORDER BY "locale" DESC,"locale_display" ASC', str(query)) - - def test_offset_0_limit_0_with_multiple_metric_orderby_applied_to_query(self): - query = self.get_reference_query(Paginator(offset=0, limit=0, order=[('clicks', Order.desc), - ('impressions', Order.asc)])) - - self.assertEqual('SELECT ' - '"sq0"."date" "date",' - '"sq0"."locale" "locale",' - '"sq0"."locale_display" "locale_display",' - '"sq0"."clicks" "clicks",' - '"sq0"."impressions" "impressions",' - '"sq1"."clicks" "clicks_yoy",' - '"sq1"."impressions" "impressions_yoy" ' - 'FROM ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq0" ' - 'LEFT JOIN ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq1" ' - 'ON "sq0"."date"=TIMESTAMPADD(\'year\',1,"sq1"."date") ' - 'AND "sq0"."locale"="sq1"."locale" ' - 'AND "sq0"."locale_display"="sq1"."locale_display" ' - 'ORDER BY "clicks" DESC,"impressions" ASC', str(query)) - - def test_offset_0_limit_10_with_multiple_dim_orderby_applied_to_query(self): - query = self.get_reference_query(Paginator(offset=0, limit=50, order=[('locale', Order.desc), - ('locale_display', Order.asc)])) - - self.assertEqual('SELECT ' - '"sq0"."date" "date",' - '"sq0"."locale" "locale",' - '"sq0"."locale_display" "locale_display",' - '"sq0"."clicks" "clicks",' - '"sq0"."impressions" "impressions",' - '"sq1"."clicks" "clicks_yoy",' - '"sq1"."impressions" "impressions_yoy" ' - 'FROM ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq0" ' - 'LEFT JOIN ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq1" ' - 'ON "sq0"."date"=TIMESTAMPADD(\'year\',1,"sq1"."date") ' - 'AND "sq0"."locale"="sq1"."locale" ' - 'AND "sq0"."locale_display"="sq1"."locale_display" ' - 'ORDER BY "locale" DESC,"locale_display" ASC ' - 'LIMIT 50', str(query)) - - def test_offset_10_limit_50_with_multiple_dim_orderby_applied_to_query(self): - query = self.get_reference_query(Paginator(offset=10, limit=50, order=[('locale', Order.desc), - ('locale_display', Order.asc)])) - - self.assertEqual('SELECT ' - '"sq0"."date" "date",' - '"sq0"."locale" "locale",' - '"sq0"."locale_display" "locale_display",' - '"sq0"."clicks" "clicks",' - '"sq0"."impressions" "impressions",' - '"sq1"."clicks" "clicks_yoy",' - '"sq1"."impressions" "impressions_yoy" ' - 'FROM ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq0" ' - 'LEFT JOIN ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq1" ' - 'ON "sq0"."date"=TIMESTAMPADD(\'year\',1,"sq1"."date") ' - 'AND "sq0"."locale"="sq1"."locale" ' - 'AND "sq0"."locale_display"="sq1"."locale_display" ' - 'ORDER BY "locale" DESC,"locale_display" ASC ' - 'LIMIT 50 OFFSET 10', str(query)) - - def test_offset_10_limit_50_with_multiple_dim_and_metric_orderby_applied_to_query(self): - query = self.get_reference_query(Paginator(offset=10, limit=50, order=[('clicks', Order.desc), - ('locale_display', Order.asc)])) - - self.assertEqual('SELECT ' - '"sq0"."date" "date",' - '"sq0"."locale" "locale",' - '"sq0"."locale_display" "locale_display",' - '"sq0"."clicks" "clicks",' - '"sq0"."impressions" "impressions",' - '"sq1"."clicks" "clicks_yoy",' - '"sq1"."impressions" "impressions_yoy" ' - 'FROM ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq0" ' - 'LEFT JOIN ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq1" ' - 'ON "sq0"."date"=TIMESTAMPADD(\'year\',1,"sq1"."date") ' - 'AND "sq0"."locale"="sq1"."locale" ' - 'AND "sq0"."locale_display"="sq1"."locale_display" ' - 'ORDER BY "clicks" DESC,"locale_display" ASC ' - 'LIMIT 50 OFFSET 10', str(query)) - - def test_offset_10_limit_50_with_reference_orderby_applied_to_query(self): - query = self.get_reference_query(Paginator(offset=10, limit=50, order=[('clicks_yoy', Order.desc)])) - - self.assertEqual('SELECT ' - '"sq0"."date" "date",' - '"sq0"."locale" "locale",' - '"sq0"."locale_display" "locale_display",' - '"sq0"."clicks" "clicks",' - '"sq0"."impressions" "impressions",' - '"sq1"."clicks" "clicks_yoy",' - '"sq1"."impressions" "impressions_yoy" ' - 'FROM ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq0" ' - 'LEFT JOIN ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq1" ' - 'ON "sq0"."date"=TIMESTAMPADD(\'year\',1,"sq1"."date") ' - 'AND "sq0"."locale"="sq1"."locale" ' - 'AND "sq0"."locale_display"="sq1"."locale_display" ' - 'ORDER BY "clicks_yoy" DESC ' - 'LIMIT 50 OFFSET 10', str(query)) - - def test_offset_10_limit_50_with_multiple_reference_orderby_applied_to_query(self): - query = self.get_reference_query(Paginator(offset=10, limit=50, order=[('clicks_yoy', Order.desc), - ('impressions_yoy', Order.desc)])) - - self.assertEqual('SELECT ' - '"sq0"."date" "date",' - '"sq0"."locale" "locale",' - '"sq0"."locale_display" "locale_display",' - '"sq0"."clicks" "clicks",' - '"sq0"."impressions" "impressions",' - '"sq1"."clicks" "clicks_yoy",' - '"sq1"."impressions" "impressions_yoy" ' - 'FROM ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq0" ' - 'LEFT JOIN ' - '(SELECT "dt" "date",' - '"locale" "locale",' - '"locale_display" "locale_display",' - 'SUM("clicks") "clicks",' - 'SUM("impressions") "impressions" ' - 'FROM "test_table" ' - 'GROUP BY "date","locale","locale_display") "sq1" ' - 'ON "sq0"."date"=TIMESTAMPADD(\'year\',1,"sq1"."date") ' - 'AND "sq0"."locale"="sq1"."locale" ' - 'AND "sq0"."locale_display"="sq1"."locale_display" ' - 'ORDER BY "clicks_yoy" DESC,"impressions_yoy" DESC ' - 'LIMIT 50 OFFSET 10', str(query)) diff --git a/fireant/tests/slicer/test_querybuilder.py b/fireant/tests/slicer/test_querybuilder.py new file mode 100644 index 00000000..58a8f2a0 --- /dev/null +++ b/fireant/tests/slicer/test_querybuilder.py @@ -0,0 +1,1161 @@ +from unittest import TestCase + +from datetime import date + +import fireant as f +from .mocks import slicer + + +class SlicerShortcutTests(TestCase): + maxDiff = None + + def test_get_attr_from_slicer_dimensions_returns_dimension(self): + timestamp_dimension = slicer.dimensions.timestamp + self.assertTrue(hasattr(timestamp_dimension, 'definition')) + + def test_get_attr_from_slicer_metrics_returns_metric(self): + votes_metric = slicer.metrics.votes + self.assertTrue(hasattr(votes_metric, 'definition')) + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderMetricTests(TestCase): + maxDiff = None + + def test_build_query_with_single_metric(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician"', query) + + def test_build_query_with_multiple_metrics(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes, slicer.metrics.wins])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes",' + 'SUM("is_winner") "wins" ' + 'FROM "politics"."politician"', query) + + def test_build_query_with_multiple_visualizations(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS([slicer.metrics.wins])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes",' + 'SUM("is_winner") "wins" ' + 'FROM "politics"."politician"', query) + + def test_build_query_for_chart_visualization_with_single_axis(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[ + f.HighCharts.PieChart(metrics=[slicer.metrics.votes]) + ])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician"', query) + + def test_build_query_for_chart_visualization_with_multiple_axes(self): + query = slicer.query() \ + .widget(f.HighCharts() + .axis(f.HighCharts.PieChart(metrics=[slicer.metrics.votes])) + .axis(f.HighCharts.PieChart(metrics=[slicer.metrics.wins]))) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes",' + 'SUM("is_winner") "wins" ' + 'FROM "politics"."politician"', query) + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderDimensionTests(TestCase): + maxDiff = None + + def test_build_query_with_datetime_dimension(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp"', query) + + def test_build_query_with_datetime_dimension_hourly(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp(f.hourly)) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'HH\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp"', query) + + def test_build_query_with_datetime_dimension_daily(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp(f.daily)) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp"', query) + + def test_build_query_with_datetime_dimension_weekly(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp(f.weekly)) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'IW\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp"', query) + + def test_build_query_with_datetime_dimension_monthly(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp(f.monthly)) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'MM\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp"', query) + + def test_build_query_with_datetime_dimension_quarterly(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp(f.quarterly)) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'Q\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp"', query) + + def test_build_query_with_datetime_dimension_annually(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp(f.annually)) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'Y\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp"', query) + + def test_build_query_with_boolean_dimension(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.winner) \ + .query + + self.assertEqual('SELECT ' + '"is_winner" "winner",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "winner" ' + 'ORDER BY "winner"', query) + + def test_build_query_with_categorical_dimension(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.political_party) \ + .query + + self.assertEqual('SELECT ' + '"political_party" "political_party",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "political_party" ' + 'ORDER BY "political_party"', query) + + def test_build_query_with_unique_dimension(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.election) \ + .query + + self.assertEqual('SELECT ' + '"election_id" "election",' + '"election_year" "election_display",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "election","election_display" ' + 'ORDER BY "election"', query) + + def test_build_query_with_multiple_dimensions(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .dimension(slicer.dimensions.candidate) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + '"candidate_id" "candidate",' + '"candidate_name" "candidate_display",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp","candidate","candidate_display" ' + 'ORDER BY "timestamp","candidate"', query) + + def test_build_query_with_multiple_dimensions_and_visualizations(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes, slicer.metrics.wins])) \ + .widget(f.HighCharts( + axes=[ + f.HighCharts.PieChart(metrics=[slicer.metrics.votes]), + f.HighCharts.ColumnChart(metrics=[slicer.metrics.wins]), + ])) \ + .dimension(slicer.dimensions.timestamp) \ + .dimension(slicer.dimensions.political_party) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + '"political_party" "political_party",' + 'SUM("votes") "votes",' + 'SUM("is_winner") "wins" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp","political_party" ' + 'ORDER BY "timestamp","political_party"', query) + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderDimensionFilterTests(TestCase): + maxDiff = None + + def test_build_query_with_filter_isin_categorical_dim(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.political_party.isin(['d'])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "political_party" IN (\'d\')', query) + + def test_build_query_with_filter_notin_categorical_dim(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.political_party.notin(['d'])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "political_party" NOT IN (\'d\')', query) + + def test_build_query_with_filter_isin_unique_dim(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.candidate.isin([1])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_id" IN (1)', query) + + def test_build_query_with_filter_notin_unique_dim(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.candidate.notin([1])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_id" NOT IN (1)', query) + + def test_build_query_with_filter_isin_unique_dim_display(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.candidate.isin(['Donald Trump'], use_display=True)) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" IN (\'Donald Trump\')', query) + + def test_build_query_with_filter_notin_unique_dim_display(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.candidate.notin(['Donald Trump'], use_display=True)) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" NOT IN (\'Donald Trump\')', query) + + def test_build_query_with_filter_wildcard_unique_dim(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.candidate.wildcard('%Trump')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" LIKE \'%Trump\'', query) + + def test_build_query_with_filter_isin_raise_exception_when_display_definition_undefined(self): + with self.assertRaises(f.QueryException): + slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.deepjoin.isin([1], use_display=True)) + + def test_build_query_with_filter_notin_raise_exception_when_display_definition_undefined(self): + with self.assertRaises(f.QueryException): + slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.deepjoin.notin([1], use_display=True)) + + def test_build_query_with_filter_wildcard_raise_exception_when_display_definition_undefined(self): + with self.assertRaises(f.QueryException): + slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.deepjoin.wildcard('test')) + + def test_build_query_with_filter_range_datetime_dimension(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.timestamp.between(date(2009, 1, 20), date(2017, 1, 20))) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2009-01-20\' AND \'2017-01-20\'', query) + + def test_build_query_with_filter_boolean_true(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.winner.is_(True)) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "is_winner"', query) + + def test_build_query_with_filter_boolean_false(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.winner.is_(False)) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE NOT "is_winner"', query) + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderMetricFilterTests(TestCase): + maxDiff = None + + def test_build_query_with_metric_filter_eq(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.metrics.votes == 5) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")=5', query) + + def test_build_query_with_metric_filter_eq_left(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(5 == slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")=5', query) + + def test_build_query_with_metric_filter_ne(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.metrics.votes != 5) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")<>5', query) + + def test_build_query_with_metric_filter_ne_left(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(5 != slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")<>5', query) + + def test_build_query_with_metric_filter_gt(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.metrics.votes > 5) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")>5', query) + + def test_build_query_with_metric_filter_gt_left(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(5 < slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")>5', query) + + def test_build_query_with_metric_filter_gte(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.metrics.votes >= 5) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")>=5', query) + + def test_build_query_with_metric_filter_gte_left(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(5 <= slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")>=5', query) + + def test_build_query_with_metric_filter_lt(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.metrics.votes < 5) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")<5', query) + + def test_build_query_with_metric_filter_lt_left(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(5 > slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")<5', query) + + def test_build_query_with_metric_filter_lte(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.metrics.votes <= 5) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")<=5', query) + + def test_build_query_with_metric_filter_lte_left(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(5 >= slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")<=5', query) + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderOperationTests(TestCase): + maxDiff = None + + def test_build_query_with_cumsum_operation(self): + query = slicer.query() \ + .widget(f.DataTablesJS([f.CumSum(slicer.metrics.votes)])) \ + .dimension(slicer.dimensions.timestamp) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp"', query) + + def test_build_query_with_cumavg_operation(self): + query = slicer.query() \ + .widget(f.DataTablesJS([f.CumAvg(slicer.metrics.votes)])) \ + .dimension(slicer.dimensions.timestamp) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp"', query) + + def test_build_query_with_l1loss_operation_constant(self): + query = slicer.query() \ + .widget(f.DataTablesJS([f.L1Loss(slicer.metrics.turnout, 1)])) \ + .dimension(slicer.dimensions.timestamp) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("politician"."timestamp",\'DD\') "timestamp",' + 'SUM("politician"."votes")/COUNT("voter"."id") "turnout" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'JOIN "politics"."voter" ' + 'ON "district"."id"="voter"."district_id" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp"', query) + + def test_build_query_with_l2loss_operation_constant(self): + query = slicer.query() \ + .widget(f.DataTablesJS([f.L2Loss(slicer.metrics.turnout, 1)])) \ + .dimension(slicer.dimensions.timestamp) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("politician"."timestamp",\'DD\') "timestamp",' + 'SUM("politician"."votes")/COUNT("voter"."id") "turnout" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'JOIN "politics"."voter" ' + 'ON "district"."id"="voter"."district_id" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp"', query) + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderDatetimeReferenceTests(TestCase): + maxDiff = None + + def test_dimension_with_single_reference_dod(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.DayOverDay)) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', query) + + def test_dimension_with_single_reference_wow(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.WeekOverWeek)) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_wow" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'week\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', query) + + def test_dimension_with_single_reference_mom(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.MonthOverMonth)) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_mom" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'month\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', query) + + def test_dimension_with_single_reference_qoq(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.QuarterOverQuarter)) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_qoq" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'quarter\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', query) + + def test_dimension_with_single_reference_yoy(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.YearOverYear)) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_yoy" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', query) + + def test_dimension_with_single_reference_as_a_delta(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.DayOverDay.delta())) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"base"."votes"-"sq1"."votes" "votes_dod_delta" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', query) + + def test_dimension_with_single_reference_as_a_delta_percentage(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.DayOverDay.delta(percent=True))) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '("base"."votes"-"sq1"."votes")*100/NULLIF("sq1"."votes",0) "votes_dod_delta_percent" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', query) + + def test_dimension_with_multiple_references(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.DayOverDay) + .reference(f.YearOverYear.delta(percent=True))) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_dod",' + '("base"."votes"-"sq2"."votes")*100/NULLIF("sq2"."votes",0) "votes_yoy_delta_percent" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq2" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq2"."timestamp") ' + 'ORDER BY "timestamp"', query) + + def test_reference_joins_nested_query_on_dimensions(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.YearOverYear)) \ + .dimension(slicer.dimensions.political_party) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."political_party" "political_party",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_yoy" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + '"political_party" "political_party",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp","political_party"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + '"political_party" "political_party",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp","political_party"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' + 'AND "base"."political_party"="sq1"."political_party" ' + 'ORDER BY "timestamp","political_party"', query) + + def test_reference_with_unique_dimension_includes_display_definition(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.YearOverYear)) \ + .dimension(slicer.dimensions.candidate) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."candidate" "candidate",' + '"base"."candidate_display" "candidate_display",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_yoy" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + '"candidate_id" "candidate",' + '"candidate_name" "candidate_display",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp","candidate","candidate_display"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + '"candidate_id" "candidate",' + '"candidate_name" "candidate_display",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp","candidate","candidate_display"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' + 'AND "base"."candidate"="sq1"."candidate" ' + 'ORDER BY "timestamp","candidate"', query) + + def test_adjust_reference_dimension_filters_in_reference_query(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.DayOverDay)) \ + .filter(slicer.dimensions.timestamp + .between(date(2018, 1, 1), date(2018, 1, 31))) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', query) + + def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_filters(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.DayOverDay)) \ + .filter(slicer.dimensions.timestamp + .between(date(2018, 1, 1), date(2018, 1, 31))) \ + .filter(slicer.dimensions.political_party + .isin(['d'])) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' + 'AND "political_party" IN (\'d\') ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' + 'AND "political_party" IN (\'d\') ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', query) + + def test_references_adapt_for_leap_year(self): + pass + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderJoinTests(TestCase): + maxDiff = None + + def test_dimension_with_join_includes_join_in_query(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .dimension(slicer.dimensions.district) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("politician"."timestamp",\'DD\') "timestamp",' + '"politician"."district_id" "district",' + '"district"."district_name" "district_display",' + 'SUM("politician"."votes") "votes" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'GROUP BY "timestamp","district","district_display" ' + 'ORDER BY "timestamp","district"', query) + + def test_dimension_with_recursive_join_joins_all_join_tables(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .dimension(slicer.dimensions.state) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("politician"."timestamp",\'DD\') "timestamp",' + '"district"."state_id" "state",' + '"state"."state_name" "state_display",' + 'SUM("politician"."votes") "votes" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'JOIN "locations"."state" ' + 'ON "district"."state_id"="state"."id" ' + 'GROUP BY "timestamp","state","state_display" ' + 'ORDER BY "timestamp","state"', query) + + def test_metric_with_join_includes_join_in_query(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.voters])) \ + .dimension(slicer.dimensions.district) \ + .query + + self.assertEqual('SELECT ' + '"politician"."district_id" "district",' + '"district"."district_name" "district_display",' + 'COUNT("voter"."id") "voters" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'JOIN "politics"."voter" ' + 'ON "district"."id"="voter"."district_id" ' + 'GROUP BY "district","district_display" ' + 'ORDER BY "district"', query) + + def test_dimension_filter_with_join_on_display_definition_does_not_include_join_in_query(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.district.isin([1])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "district_id" IN (1)', query) + + def test_dimension_filter_display_field_with_join_includes_join_in_query(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.district.isin(['District 4'], use_display=True)) \ + .query + + self.assertEqual('SELECT ' + 'SUM("politician"."votes") "votes" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'WHERE "district"."district_name" IN (\'District 4\')', query) + + def test_dimension_filter_with_recursive_join_includes_join_in_query(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.state.isin([1])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("politician"."votes") "votes" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'WHERE "district"."state_id" IN (1)', query) + + def test_dimension_filter_with_deep_recursive_join_includes_joins_in_query(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.deepjoin.isin([1])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("politician"."votes") "votes" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'JOIN "locations"."state" ' + 'ON "district"."state_id"="state"."id" ' + 'JOIN "test"."deep" ' + 'ON "deep"."id"="state"."ref_id" ' + 'WHERE "deep"."id" IN (1)', query) + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderValidationTests(TestCase): + maxDiff = None + + def test_query_requires_at_least_one_metric(self): + pass diff --git a/fireant/tests/slicer/test_slicer_api.py b/fireant/tests/slicer/test_slicer_api.py deleted file mode 100644 index 2679ddfd..00000000 --- a/fireant/tests/slicer/test_slicer_api.py +++ /dev/null @@ -1,1263 +0,0 @@ -# coding: utf-8 -from unittest import TestCase - -import numpy as np -import pandas as pd -from datetime import date - -from fireant.slicer import * -from fireant.slicer.operations import * -from fireant.slicer.references import * -from fireant.tests.database.mock_database import TestDatabase -from pypika import JoinType -from pypika import functions as fn, Tables, Case - -QUERY_BUILDER_PARAMS = {'table', 'database', 'joins', 'metrics', 'dimensions', 'mfilters', 'dfilters', 'references', - 'rollup', 'pagination'} - - -class SlicerSchemaTests(TestCase): - maxDiff = None - - @classmethod - def setUpClass(cls): - cls.test_table, cls.test_join_table = Tables('test_table', 'test_join_table') - cls.test_table.alias = 'test' - cls.test_join_table.alias = 'join' - cls.test_db = TestDatabase() - - cls.test_slicer = Slicer( - table=cls.test_table, - database=cls.test_db, - - joins=[ - Join('join1', cls.test_join_table, cls.test_table.join_id == cls.test_join_table.id) - ], - - metrics=[ - # Metric with defaults - Metric('foo'), - - # Metric with custom label and definition - Metric('bar', label='FizBuz', definition=fn.Sum(cls.test_table.fiz + cls.test_table.buz)), - - # Metric with complex definition - Metric('weirdcase', label='Weird Case', - definition=( - Case().when(cls.test_table.case == 1, 'a') - .when(cls.test_table.case == 2, 'b') - .else_('weird'))), - - # Metric with joins - Metric('piddle', definition=fn.Sum(cls.test_join_table.piddle), joins=['join1']), - - # Metric with custom label and definition - Metric('paddle', definition=fn.Sum(cls.test_join_table.paddle + cls.test_table.foo), joins=['join1']), - - # Metric with rounding - Metric('decimal', precision=2), - - # Metric with prefix - Metric('dollar', prefix='$'), - - # Metric with suffix - Metric('euro', suffix='€'), - - # Metric with suffix - Metric('join_metric', definition=fn.Sum(cls.test_join_table.join_metric), joins=['join1']), - ], - - dimensions=[ - # Continuous date dimension - DatetimeDimension('date', definition=cls.test_table.dt), - - # Continuous integer dimension - ContinuousDimension('clicks', label='My Clicks', definition=cls.test_table.clicks), - - # Categorical dimension with display options - CategoricalDimension('locale', 'Locale', definition=cls.test_table.locale, - display_options=[DimensionValue('us', 'United States'), - DimensionValue('de', 'Germany')]), - - # Unique Dimension with single ID field - UniqueDimension('account', 'Account', definition=cls.test_table.account_id, - display_field=cls.test_table.account_name), - - # Unique Dimension with composite ID field - UniqueDimension('keyword', 'Keyword', definition=cls.test_table.keyword_id, - display_field=cls.test_table.keyword_name), - - # Dimension with joined columns - UniqueDimension('join_dimension', 'Join Dimension', definition=cls.test_join_table.join_dimension, - display_field=cls.test_join_table.join_dimension_display, joins=['join1']), - ] - ) - - -class SlicerSchemaMetricTests(SlicerSchemaTests): - def test_metric_with_default_definition(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - ) - - self.assertTrue({'table', 'metrics'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - def test_metric_with_custom_definition(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['bar'], - ) - - self.assertTrue({'table', 'metrics'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'bar'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."fiz"+"test"."buz")', str(query_schema['metrics']['bar'])) - - def test_metrics_added_for_cumsum(self): - query_schema = self.test_slicer.manager.data_query_schema( - operations=[CumSum('foo', )] - ) - - self.assertTrue({'table', 'metrics'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - - def test_metrics_added_for_cummean(self): - query_schema = self.test_slicer.manager.data_query_schema( - operations=[CumMean('foo')] - ) - - self.assertTrue({'table', 'metrics'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - - def test_metrics_added_for_l1loss(self): - query_schema = self.test_slicer.manager.data_query_schema( - operations=[L1Loss('foo', 'bar')] - ) - - self.assertTrue({'table', 'metrics'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo', 'bar'}, set(query_schema['metrics'].keys())) - - def test_metrics_added_for_l2loss(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=[], - operations=[L2Loss('foo', 'bar')] - ) - - self.assertTrue({'table', 'metrics'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo', 'bar'}, set(query_schema['metrics'].keys())) - - -class SlicerSchemaDimensionTests(SlicerSchemaTests): - def test_date_dimension_default_interval(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['date'], - ) - - self.assertTrue({'table', 'metrics', 'dimensions'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'date'}, set(query_schema['dimensions'].keys())) - self.assertEqual('TRUNC("test"."dt",\'DD\')', str(query_schema['dimensions']['date'])) - - def test_date_dimension_custom_week_interval(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - # TODO This could be improved by using an object - dimensions=[('date', DatetimeDimension.week)], - ) - - self.assertTrue({'table', 'metrics', 'dimensions'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'date'}, set(query_schema['dimensions'].keys())) - self.assertEqual('TRUNC("test"."dt",\'IW\')', str(query_schema['dimensions']['date'])) - - def test_date_dimension_year_interval_uses_correct_trunc_statement(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=[('date', DatetimeDimension.year)], - ) - self.assertSetEqual({'date'}, set(query_schema['dimensions'].keys())) - self.assertEqual('TRUNC("test"."dt",\'Y\')', str(query_schema['dimensions']['date'])) - - def test_date_dimension_quarter_interval_uses_correct_trunc_statement(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=[('date', DatetimeDimension.quarter)], - ) - self.assertSetEqual({'date'}, set(query_schema['dimensions'].keys())) - self.assertEqual('TRUNC("test"."dt",\'Q\')', str(query_schema['dimensions']['date'])) - - def test_date_dimension_month_interval_uses_correct_trunc_statement(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=[('date', DatetimeDimension.month)], - ) - self.assertSetEqual({'date'}, set(query_schema['dimensions'].keys())) - self.assertEqual('TRUNC("test"."dt",\'MM\')', str(query_schema['dimensions']['date'])) - - def test_date_dimension_day_interval_uses_correct_trunc_statement(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=[('date', DatetimeDimension.day)], - ) - self.assertSetEqual({'date'}, set(query_schema['dimensions'].keys())) - self.assertEqual('TRUNC("test"."dt",\'DD\')', str(query_schema['dimensions']['date'])) - - def test_date_dimension_hour_interval_uses_correct_trunc_statement(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=[('date', DatetimeDimension.hour)], - ) - self.assertSetEqual({'date'}, set(query_schema['dimensions'].keys())) - self.assertEqual('TRUNC("test"."dt",\'HH\')', str(query_schema['dimensions']['date'])) - - def test_numeric_dimension_default_interval(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['clicks'], - ) - - self.assertTrue({'table', 'metrics', 'dimensions'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'clicks'}, set(query_schema['dimensions'].keys())) - self.assertEqual('MOD("test"."clicks"+0,1)', str(query_schema['dimensions']['clicks'])) - - def test_numeric_dimension_custom_interval(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - # TODO This could be improved by using an object - dimensions=[('clicks', 100, 25)], - ) - - self.assertTrue({'table', 'metrics', 'dimensions'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'clicks'}, set(query_schema['dimensions'].keys())) - self.assertEqual('MOD("test"."clicks"+25,100)', str(query_schema['dimensions']['clicks'])) - - def test_categorical_dimension(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - ) - - self.assertTrue({'table', 'metrics', 'dimensions'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - def test_unique_dimension_id(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['account'], - ) - - self.assertTrue({'table', 'metrics', 'dimensions'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'account', 'account_display'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."account_id"', str(query_schema['dimensions']['account'])) - self.assertEqual('"test"."account_name"', str(query_schema['dimensions']['account_display'])) - - def test_multiple_metrics_and_dimensions(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo', 'bar'], - dimensions=[('date', DatetimeDimension.month), ('clicks', 50, 100), 'locale', 'account'], - ) - - self.assertTrue({'table', 'metrics', 'dimensions'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo', 'bar'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - self.assertEqual('SUM("test"."fiz"+"test"."buz")', str(query_schema['metrics']['bar'])) - - self.assertSetEqual({'date', 'clicks', 'locale', 'account', 'account_display'}, - set(query_schema['dimensions'].keys())) - self.assertEqual('MOD("test"."clicks"+100,50)', str(query_schema['dimensions']['clicks'])) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - self.assertEqual('"test"."account_id"', str(query_schema['dimensions']['account'])) - self.assertEqual('"test"."account_name"', str(query_schema['dimensions']['account_display'])) - - -class SlicerSchemaFilterTests(SlicerSchemaTests): - def test_cat_dimension_filter_eq(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - dimension_filters=[EqualityFilter('locale', EqualityOperator.eq, 'en')], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['"test"."locale"=\'en\''], [str(f) for f in query_schema['dfilters']]) - - def test_cat_dimension_filter_ne(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - dimension_filters=[ - EqualityFilter('locale', EqualityOperator.ne, 'en'), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['"test"."locale"<>\'en\''], [str(f) for f in query_schema['dfilters']]) - - def test_cat_dimension_filter_in(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - dimension_filters=[ - ContainsFilter('locale', ['en', 'es', 'de']), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['"test"."locale" IN (\'en\',\'es\',\'de\')'], [str(f) for f in query_schema['dfilters']]) - - def test_cat_dimension_filter_like(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - dimension_filters=[ - WildcardFilter('locale', 'e%'), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['"test"."locale" LIKE \'e%\''], [str(f) for f in query_schema['dfilters']]) - - def test_cat_dimension_filter_gt(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - dimension_filters=[ - EqualityFilter('date', EqualityOperator.gt, date(2000, 1, 1)), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['"test"."dt">\'2000-01-01\''], [str(f) for f in query_schema['dfilters']]) - - def test_cat_dimension_filter_lt(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - dimension_filters=[ - EqualityFilter('date', EqualityOperator.lt, date(2000, 1, 1)), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['"test"."dt"<\'2000-01-01\''], [str(f) for f in query_schema['dfilters']]) - - def test_cat_dimension_filter_gte(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - dimension_filters=[ - EqualityFilter('date', EqualityOperator.gte, date(2000, 1, 1)), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['"test"."dt">=\'2000-01-01\''], [str(f) for f in query_schema['dfilters']]) - - def test_cat_dimension_filter_lte(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - dimension_filters=[ - EqualityFilter('date', EqualityOperator.lte, date(2000, 1, 1)), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['"test"."dt"<=\'2000-01-01\''], [str(f) for f in query_schema['dfilters']]) - - def test_cat_dimension_filter_daterange(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - dimension_filters=[ - RangeFilter('date', date(2000, 1, 1), date(2000, 3, 1)), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertListEqual(['"test"."dt" BETWEEN \'2000-01-01\' AND \'2000-03-01\''], - [str(f) for f in query_schema['dfilters']]) - - def test_unique_dimension_eq_filter(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['account'], - dimension_filters=[EqualityFilter('account', EqualityOperator.eq, 1)], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'account', 'account_display'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."account_id"', str(query_schema['dimensions']['account'])) - self.assertEqual('"test"."account_name"', str(query_schema['dimensions']['account_display'])) - - self.assertListEqual(['"test"."account_id"=1'], [str(f) for f in query_schema['dfilters']]) - - def test_unique_dimension_display_eq_filter(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['account'], - dimension_filters=[EqualityFilter(('account', 'display'), EqualityOperator.eq, 'abc')], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'account', 'account_display'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."account_id"', str(query_schema['dimensions']['account'])) - self.assertEqual('"test"."account_name"', str(query_schema['dimensions']['account_display'])) - - self.assertListEqual(['"test"."account_name"=\'abc\''], [str(f) for f in query_schema['dfilters']]) - - def test_unique_dimension_contains_filter(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['account'], - dimension_filters=[ContainsFilter('account', [1, 2, 3])], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'account', 'account_display'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."account_id"', str(query_schema['dimensions']['account'])) - self.assertEqual('"test"."account_name"', str(query_schema['dimensions']['account_display'])) - - self.assertListEqual(['"test"."account_id" IN (1,2,3)'], [str(f) for f in query_schema['dfilters']]) - - def test_unique_dimension_wildcard_filter_label(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['account'], - dimension_filters=[WildcardFilter(('account', 'display'), 'nam%')], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'account', 'account_display'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."account_id"', str(query_schema['dimensions']['account'])) - self.assertEqual('"test"."account_name"', str(query_schema['dimensions']['account_display'])) - - self.assertListEqual(['"test"."account_name" LIKE \'nam%\''], [str(f) for f in query_schema['dfilters']]) - - def test_metric_filter_eq(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - metric_filters=[EqualityFilter('foo', EqualityOperator.eq, 0)], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['SUM("test"."foo")=0'], [str(f) for f in query_schema['mfilters']]) - - def test_metric_filter_ne(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - metric_filters=[ - EqualityFilter('foo', EqualityOperator.ne, 0), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['SUM("test"."foo")<>0'], [str(f) for f in query_schema['mfilters']]) - - def test_metric_filter_gt(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - metric_filters=[ - EqualityFilter('foo', EqualityOperator.gt, 100), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['SUM("test"."foo")>100'], [str(f) for f in query_schema['mfilters']]) - - def test_metric_filter_lt(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - metric_filters=[ - EqualityFilter('foo', EqualityOperator.lt, 100), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['SUM("test"."foo")<100'], [str(f) for f in query_schema['mfilters']]) - - def test_metric_filter_gte(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - metric_filters=[ - EqualityFilter('foo', EqualityOperator.gte, 100), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['SUM("test"."foo")>=100'], [str(f) for f in query_schema['mfilters']]) - - def test_metric_filter_lte(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - metric_filters=[ - EqualityFilter('foo', EqualityOperator.lte, 100), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'locale'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) - - self.assertListEqual(['SUM("test"."foo")<=100'], [str(f) for f in query_schema['mfilters']]) - - def test_invalid_dimensions_raise_exception(self): - with self.assertRaises(SlicerException): - self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - dimension_filters=[ - EqualityFilter('blahblahblah', EqualityOperator.eq, 0), - ], - ) - - def test_invalid_metrics_raise_exception(self): - with self.assertRaises(SlicerException): - self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - metric_filters=[ - EqualityFilter('blahblahblah', EqualityOperator.eq, 0), - ], - ) - - def test_metrics_dont_work_for_dimensions(self): - with self.assertRaises(SlicerException): - self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - dimension_filters=[ - EqualityFilter('foo', EqualityOperator.gt, 100), - ], - ) - - def test_dimensions_dont_work_for_metrics(self): - with self.assertRaises(SlicerException): - self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - metric_filters=[ - EqualityFilter('locale', EqualityOperator.eq, 'US'), - ], - ) - - def test_joined_metric_filter(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - metric_filters=[ - EqualityFilter('join_metric', EqualityOperator.eq, 0), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertSetEqual(set(), set(query_schema['dimensions'].keys())) - - self.assertEqual(1, len(query_schema['joins'])) - self.assertEqual(3, len(query_schema['joins'][0])) - self.assertEqual(self.test_join_table, query_schema['joins'][0][0]) - self.assertEqual('"test"."join_id"="join"."id"', str(query_schema['joins'][0][1])) - self.assertEqual(JoinType.inner, query_schema['joins'][0][2]) - - def test_joined_dimension_filter(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimension_filters=[ - EqualityFilter('join_dimension', EqualityOperator.eq, 'test'), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertSetEqual(set(), set(query_schema['dimensions'].keys())) - - self.assertEqual(1, len(query_schema['joins'])) - self.assertEqual(3, len(query_schema['joins'][0])) - self.assertEqual(self.test_join_table, query_schema['joins'][0][0]) - self.assertEqual('"test"."join_id"="join"."id"', str(query_schema['joins'][0][1])) - self.assertEqual(JoinType.inner, query_schema['joins'][0][2]) - - def test_joined_unique_dimension_display_filter(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimension_filters=[ - EqualityFilter(('join_dimension', 'display'), EqualityOperator.eq, 'test'), - ], - ) - - self.assertSetEqual(QUERY_BUILDER_PARAMS, set(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertSetEqual(set(), set(query_schema['dimensions'].keys())) - - self.assertEqual(1, len(query_schema['joins'])) - self.assertEqual(3, len(query_schema['joins'][0])) - self.assertEqual(self.test_join_table, query_schema['joins'][0][0]) - self.assertEqual('"test"."join_id"="join"."id"', str(query_schema['joins'][0][1])) - self.assertEqual(JoinType.inner, query_schema['joins'][0][2]) - - -class SlicerSchemaReferenceTests(SlicerSchemaTests): - def _reference_test_with_date(self, reference): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['date'], - references=[reference], - ) - self.assertTrue({'table', 'metrics', 'dimensions', 'references'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - self.assertSetEqual({'date'}, set(query_schema['dimensions'].keys())) - self.assertEqual('TRUNC("test"."dt",\'DD\')', str(query_schema['dimensions']['date'])) - - # Cast definition to string for comparison - query_schema['references'][reference.key]['definition'] = str( - query_schema['references'][reference.key]['definition'] - ) - - self.assertDictEqual({ - reference.key: { - 'dimension': reference.element_key, - 'definition': '"test"."dt"', - 'modifier': reference.modifier, - 'time_unit': reference.time_unit, - 'interval': reference.interval, - } - }, query_schema['references']) - - def test_reference_wow_with_date(self): - self._reference_test_with_date(WoW('date')) - - def test_reference_wow_d_with_date(self): - self._reference_test_with_date(Delta(WoW('date'))) - - def test_reference_wow_p_with_date(self): - self._reference_test_with_date(DeltaPercentage(WoW('date'))) - - def test_reference_mom_with_date(self): - self._reference_test_with_date(MoM('date')) - - def test_reference_mom_d_with_date(self): - self._reference_test_with_date(Delta(MoM('date'))) - - def test_reference_mom_p_with_date(self): - self._reference_test_with_date(DeltaPercentage(MoM('date'))) - - def test_reference_qoq_with_date(self): - self._reference_test_with_date(QoQ('date')) - - def test_reference_qoq_d_with_date(self): - self._reference_test_with_date(Delta(QoQ('date'))) - - def test_reference_qoq_p_with_date(self): - self._reference_test_with_date(DeltaPercentage(QoQ('date'))) - - def test_reference_yoy_with_date(self): - self._reference_test_with_date(YoY('date')) - - def test_reference_yoy_d_with_date(self): - self._reference_test_with_date(Delta(YoY('date'))) - - def test_reference_yoy_p_with_date(self): - self._reference_test_with_date(DeltaPercentage(YoY('date'))) - - def test_reference_missing_dimension(self): - # Throws exception when reference key is not a dimension key - with self.assertRaises(SlicerException): - self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=[], - references=[WoW('blahblah')], - ) - - def test_reference_wrong_dimension_type(self): - # Reference dimension is required in order to use a reference with it - with self.assertRaises(SlicerException): - self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['locale'], - references=[WoW('locale')], - ) - - -class SlicerOperationSchemaTests(SlicerSchemaTests): - def test_totals_query_schema(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['date', 'locale', 'account'], - operations=[Totals('locale', 'account')], - ) - - self.assertTrue({'table', 'metrics', 'dimensions', 'rollup'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'date', 'locale', 'account', 'account_display'}, set(query_schema['dimensions'].keys())) - self.assertEqual('TRUNC("test"."dt",\'DD\')', str(query_schema['dimensions']['date'])) - - self.assertListEqual([['locale'], ['account', 'account_display']], query_schema['rollup']) - - def test_totals_operation_schema(self): - operation_schema = self.test_slicer.manager.operation_schema( - operations=[Totals('locale', 'account')], - ) - - self.assertListEqual([], operation_schema) - - def test_cumsum_operation_schema(self): - operation_schema = self.test_slicer.manager.operation_schema( - operations=[CumSum('foo')], - ) - - self.assertListEqual([{'key': 'cumsum', 'metric': 'foo'}], operation_schema) - - def test_cummean_operation_schema(self): - operation_schema = self.test_slicer.manager.operation_schema( - operations=[CumMean('foo')], - ) - - self.assertListEqual([{'key': 'cummean', 'metric': 'foo'}], operation_schema) - - def test_l1loss_operation_schema(self): - operation_schema = self.test_slicer.manager.operation_schema( - operations=[L1Loss('foo', 'bar')], - ) - - self.assertListEqual([{'key': 'l1loss', 'metric': 'foo', 'target': 'bar'}], operation_schema) - - def test_l2loss_operation_schema(self): - operation_schema = self.test_slicer.manager.operation_schema( - operations=[L2Loss('foo', 'bar')], - ) - - self.assertListEqual([{'key': 'l2loss', 'metric': 'foo', 'target': 'bar'}], operation_schema) - - -class SlicerSchemaJoinTests(SlicerSchemaTests): - def test_metric_with_join(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['piddle'], - ) - - self.assertTrue({'table', 'metrics', 'joins'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - join1 = query_schema['joins'][0] - self.assertEqual(self.test_join_table, join1[0]) - self.assertEqual('"test"."join_id"="join"."id"', str(join1[1])) - - self.assertSetEqual({'piddle'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("join"."piddle")', str(query_schema['metrics']['piddle'])) - - def test_metric_with_complex_join(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['paddle'], - ) - - self.assertTrue({'table', 'metrics', 'joins'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - join1 = query_schema['joins'][0] - self.assertEqual(self.test_join_table, join1[0]) - self.assertEqual('"test"."join_id"="join"."id"', str(join1[1])) - - self.assertSetEqual({'paddle'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("join"."paddle"+"test"."foo")', str(query_schema['metrics']['paddle'])) - - def test_dimension_with_join(self): - query_schema = self.test_slicer.manager.data_query_schema( - metrics=['foo'], - dimensions=['join_dimension'], - ) - - self.assertTrue({'table', 'metrics', 'dimensions', 'joins'}.issubset(query_schema.keys())) - self.assertEqual(self.test_table, query_schema['table']) - - self.assertSetEqual({'foo'}, set(query_schema['metrics'].keys())) - self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) - - self.assertSetEqual({'join_dimension', 'join_dimension_display'}, set(query_schema['dimensions'].keys())) - self.assertEqual('"join"."join_dimension"', str(query_schema['dimensions']['join_dimension'])) - self.assertEqual('"join"."join_dimension_display"', str(query_schema['dimensions']['join_dimension_display'])) - - -class SlicerDisplaySchemaTests(SlicerSchemaTests): - def test_metric_with_default_definition(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['foo'], - ) - - self.assertDictEqual( - { - 'metrics': {'foo': {'label': 'foo', 'axis': 0}}, - 'dimensions': {}, - 'references': {}, - }, - display_schema - ) - - def test_metric_with_custom_definition(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['bar'], - ) - - self.assertDictEqual( - { - 'metrics': {'bar': {'label': 'FizBuz', 'axis': 0}}, - 'dimensions': {}, - 'references': {}, - }, - display_schema - ) - - def test_date_dimension_default_interval(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['foo'], - dimensions=['date'], - ) - - self.assertDictEqual( - { - 'metrics': {'foo': {'label': 'foo', 'axis': 0}}, - 'dimensions': { - 'date': {'label': 'date'} - }, - 'references': {}, - }, - display_schema - ) - - def test_numeric_dimension_default_interval(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['foo'], - dimensions=['clicks'], - ) - - self.assertDictEqual( - { - 'metrics': {'foo': {'label': 'foo', 'axis': 0}}, - 'dimensions': { - 'clicks': {'label': 'My Clicks'} - }, - 'references': {}, - }, - display_schema - ) - - def test_categorical_dimension(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['foo'], - dimensions=['locale'], - ) - self.assertDictEqual( - { - 'metrics': {'foo': {'label': 'foo', 'axis': 0}}, - 'dimensions': { - 'locale': {'label': 'Locale', 'display_options': { - 'us': 'United States', 'de': 'Germany', np.nan: '', pd.NaT: '' - }}, - }, - 'references': {}, - }, - display_schema - ) - - def test_unique_dimension(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['foo'], - dimensions=['account'], - ) - self.assertDictEqual( - { - 'metrics': {'foo': {'label': 'foo', 'axis': 0}}, - 'dimensions': { - 'account': {'label': 'Account', 'display_field': 'account_display'}, - }, - 'references': {}, - }, - display_schema - ) - - def test_multiple_metrics_and_dimensions(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['foo', 'bar'], - dimensions=[('date', DatetimeDimension.month), ('clicks', 50, 100), 'locale', 'account'], - ) - - self.assertDictEqual( - { - 'metrics': { - 'foo': {'label': 'foo', 'axis': 0}, - 'bar': {'label': 'FizBuz', 'axis': 1}, - }, - 'dimensions': { - 'date': {'label': 'date'}, - 'clicks': {'label': 'My Clicks'}, - 'locale': {'label': 'Locale', 'display_options': { - 'us': 'United States', 'de': 'Germany', np.nan: '', pd.NaT: '' - }}, - 'account': {'label': 'Account', 'display_field': 'account_display'}, - }, - 'references': {}, - }, - display_schema - ) - - def test_reference_wow_with_date(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['foo'], - dimensions=['date'], - references=[WoW('date')], - ) - - self.assertDictEqual( - { - 'metrics': {'foo': {'label': 'foo', 'axis': 0}}, - 'dimensions': { - 'date': {'label': 'date'}, - }, - 'references': {'wow': 'WoW'}, - }, - display_schema - ) - - def test_cumsum_operation(self): - display_schema = self.test_slicer.manager.display_schema( - operations=[CumSum('foo')], - ) - - self.assertDictEqual( - { - 'metrics': {'foo_cumsum': {'label': 'foo cum. sum', 'axis': 0}}, - 'dimensions': {}, - 'references': {}, - }, - display_schema - ) - - def test_cummean_operation(self): - display_schema = self.test_slicer.manager.display_schema( - operations=[CumMean('foo')], - ) - - self.assertDictEqual( - { - 'metrics': {'foo_cummean': {'label': 'foo cum. mean', 'axis': 0}}, - 'dimensions': {}, - 'references': {}, - }, - display_schema - ) - - def test_l1loss_operation(self): - display_schema = self.test_slicer.manager.display_schema( - operations=[L1Loss('foo', 'bar')], - ) - - self.assertDictEqual( - { - 'metrics': {'foo_l1loss': {'label': 'foo L1 loss', 'axis': 0}}, - 'dimensions': {}, - 'references': {}, - }, - display_schema - ) - - def test_l2loss_operation(self): - display_schema = self.test_slicer.manager.display_schema( - operations=[L2Loss('foo', 'bar')], - ) - - self.assertDictEqual( - { - 'metrics': {'foo_l2loss': {'label': 'foo L2 loss', 'axis': 0}}, - 'dimensions': {}, - 'references': {}, - }, - display_schema - ) - - def test_operation_with_metric(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['bar'], - operations=[CumSum('foo')], - ) - - self.assertDictEqual( - { - 'metrics': { - 'bar': {'label': 'FizBuz', 'axis': 0}, - 'foo_cumsum': {'label': 'foo cum. sum', 'axis': 1} - }, - 'dimensions': {}, - 'references': {}, - }, - display_schema - ) - - def test_operation_with_same_metric(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['foo'], - operations=[CumSum('foo')], - ) - - self.assertDictEqual( - { - 'metrics': { - 'foo': {'label': 'foo', 'axis': 0}, - 'foo_cumsum': {'label': 'foo cum. sum', 'axis': 1} - }, - 'dimensions': {}, - 'references': {}, - }, - display_schema - ) - - def test_metric_with_rounding(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['decimal'], - dimensions=['date'], - ) - - self.assertDictEqual( - { - 'metrics': { - 'decimal': {'label': 'decimal', 'axis': 0, 'precision': 2}, - }, - 'dimensions': { - 'date': {'label': 'date'} - }, - 'references': {}, - }, - display_schema - ) - - def test_metric_with_prefix(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['dollar'], - dimensions=['date'], - ) - - self.assertDictEqual( - { - 'metrics': { - 'dollar': {'label': 'dollar', 'axis': 0, 'prefix': '$'}, - }, - 'dimensions': { - 'date': {'label': 'date'} - }, - 'references': {}, - }, - display_schema - ) - - def test_metric_with_suffix(self): - display_schema = self.test_slicer.manager.display_schema( - metrics=['euro'], - dimensions=['date'], - ) - - self.assertDictEqual( - { - 'metrics': { - 'euro': {'label': 'euro', 'axis': 0, 'suffix': '€'}, - }, - 'dimensions': { - 'date': {'label': 'date'} - }, - 'references': {}, - }, - display_schema - ) diff --git a/fireant/tests/slicer/transformers/__init__.py b/fireant/tests/slicer/transformers/__init__.py deleted file mode 100644 index 57d631c3..00000000 --- a/fireant/tests/slicer/transformers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# coding: utf-8 diff --git a/fireant/tests/slicer/transformers/base.py b/fireant/tests/slicer/transformers/base.py deleted file mode 100644 index 66836362..00000000 --- a/fireant/tests/slicer/transformers/base.py +++ /dev/null @@ -1,14 +0,0 @@ -# coding: utf-8 -import unittest - -import pandas as pd - -from fireant.slicer.transformers import Transformer - - -class TransformerTests(unittest.TestCase): - def test_transformer_api(self): - tx = Transformer() - - with self.assertRaises(NotImplementedError): - tx.transform(pd.DataFrame(), {}) diff --git a/fireant/tests/slicer/transformers/test_csv.py b/fireant/tests/slicer/transformers/test_csv.py deleted file mode 100644 index 74007aed..00000000 --- a/fireant/tests/slicer/transformers/test_csv.py +++ /dev/null @@ -1,262 +0,0 @@ -# coding: utf-8 -from unittest import TestCase - -from fireant.slicer.transformers import CSVRowIndexTransformer, CSVColumnIndexTransformer -from fireant.tests import mock_dataframes as mock_df - - -class CSVRowIndexTransformerTests(TestCase): - csv_tx = CSVRowIndexTransformer() - - def test_no_dims_single_metric(self): - df = mock_df.no_dims_multi_metric_df - - result = self.csv_tx.transform(df, mock_df.no_dims_multi_metric_schema) - - self.assertEqual('One,Two,Three,Four,Five,Six,Seven,Eight\n' - '0,1,2,3,4,5,6,7\n', result) - - def test_cont_dim_single_metric(self): - df = mock_df.cont_dim_single_metric_df - - result = self.csv_tx.transform(df, mock_df.cont_dim_single_metric_schema) - - self.assertEqual('Cont,One\n' - '0,0\n' - '1,1\n' - '2,2\n' - '3,3\n' - '4,4\n' - '5,5\n' - '6,6\n' - '7,7\n', result) - - def test_cont_dim_multi_metric(self): - df = mock_df.cont_dim_multi_metric_df - - result = self.csv_tx.transform(df, mock_df.cont_dim_multi_metric_schema) - - self.assertEqual('Cont,One,Two\n' - '0,0,0\n' - '1,1,2\n' - '2,2,4\n' - '3,3,6\n' - '4,4,8\n' - '5,5,10\n' - '6,6,12\n' - '7,7,14\n', result) - - def test_cont_cat_dim_single_metric(self): - df = mock_df.cont_cat_dims_single_metric_df - - result = self.csv_tx.transform(df, mock_df.cont_cat_dims_single_metric_schema) - - self.assertEqual('Cont,Cat1,One\n' - '0,A,0\n' - '0,B,1\n' - '1,A,2\n' - '1,B,3\n' - '2,A,4\n' - '2,B,5\n' - '3,A,6\n' - '3,B,7\n' - '4,A,8\n' - '4,B,9\n' - '5,A,10\n' - '5,B,11\n' - '6,A,12\n' - '6,B,13\n' - '7,A,14\n' - '7,B,15\n', result) - - def test_cont_cat_dim_multi_metric(self): - df = mock_df.cont_cat_dims_multi_metric_df - - result = self.csv_tx.transform(df, mock_df.cont_cat_dims_multi_metric_schema) - - self.assertEqual('Cont,Cat1,One,Two\n' - '0,A,0,0\n' - '0,B,1,2\n' - '1,A,2,4\n' - '1,B,3,6\n' - '2,A,4,8\n' - '2,B,5,10\n' - '3,A,6,12\n' - '3,B,7,14\n' - '4,A,8,16\n' - '4,B,9,18\n' - '5,A,10,20\n' - '5,B,11,22\n' - '6,A,12,24\n' - '6,B,13,26\n' - '7,A,14,28\n' - '7,B,15,30\n', result) - - def test_cont_cat_cat_dim_multi_metric(self): - df = mock_df.cont_cat_cat_dims_multi_metric_df - - result = self.csv_tx.transform(df, mock_df.cont_cat_cat_dims_multi_metric_schema) - - self.assertEqual('Cont,Cat1,Cat2,One,Two\n' - '0,A,Y,0,0\n' - '0,A,Z,1,2\n' - '0,B,Y,2,4\n' - '0,B,Z,3,6\n' - '1,A,Y,4,8\n' - '1,A,Z,5,10\n' - '1,B,Y,6,12\n' - '1,B,Z,7,14\n' - '2,A,Y,8,16\n' - '2,A,Z,9,18\n' - '2,B,Y,10,20\n' - '2,B,Z,11,22\n' - '3,A,Y,12,24\n' - '3,A,Z,13,26\n' - '3,B,Y,14,28\n' - '3,B,Z,15,30\n' - '4,A,Y,16,32\n' - '4,A,Z,17,34\n' - '4,B,Y,18,36\n' - '4,B,Z,19,38\n' - '5,A,Y,20,40\n' - '5,A,Z,21,42\n' - '5,B,Y,22,44\n' - '5,B,Z,23,46\n' - '6,A,Y,24,48\n' - '6,A,Z,25,50\n' - '6,B,Y,26,52\n' - '6,B,Z,27,54\n' - '7,A,Y,28,56\n' - '7,A,Z,29,58\n' - '7,B,Y,30,60\n' - '7,B,Z,31,62\n', result) - - def test_rollup_cont_cat_cat_dim_multi_metric(self): - df = mock_df.rollup_cont_cat_cat_dims_multi_metric_df - - result = self.csv_tx.transform(df, mock_df.rollup_cont_cat_cat_dims_multi_metric_schema) - - self.assertEqual('Cont,Cat1,Cat2,One,Two\n' - '0,Total,Total,12,24\n' - '0,A,Total,1,2\n' - '0,A,Y,0,0\n' - '0,A,Z,1,2\n' - '0,B,Total,5,10\n' - '0,B,Y,2,4\n' - '0,B,Z,3,6\n' - '1,Total,Total,44,88\n' - '1,A,Total,9,18\n' - '1,A,Y,4,8\n' - '1,A,Z,5,10\n' - '1,B,Total,13,26\n' - '1,B,Y,6,12\n' - '1,B,Z,7,14\n' - '2,Total,Total,76,152\n' - '2,A,Total,17,34\n' - '2,A,Y,8,16\n' - '2,A,Z,9,18\n' - '2,B,Total,21,42\n' - '2,B,Y,10,20\n' - '2,B,Z,11,22\n' - '3,Total,Total,108,216\n' - '3,A,Total,25,50\n' - '3,A,Y,12,24\n' - '3,A,Z,13,26\n' - '3,B,Total,29,58\n' - '3,B,Y,14,28\n' - '3,B,Z,15,30\n' - '4,Total,Total,140,280\n' - '4,A,Total,33,66\n' - '4,A,Y,16,32\n' - '4,A,Z,17,34\n' - '4,B,Total,37,74\n' - '4,B,Y,18,36\n' - '4,B,Z,19,38\n' - '5,Total,Total,172,344\n' - '5,A,Total,41,82\n' - '5,A,Y,20,40\n' - '5,A,Z,21,42\n' - '5,B,Total,45,90\n' - '5,B,Y,22,44\n' - '5,B,Z,23,46\n' - '6,Total,Total,204,408\n' - '6,A,Total,49,98\n' - '6,A,Y,24,48\n' - '6,A,Z,25,50\n' - '6,B,Total,53,106\n' - '6,B,Y,26,52\n' - '6,B,Z,27,54\n' - '7,Total,Total,236,472\n' - '7,A,Total,57,114\n' - '7,A,Y,28,56\n' - '7,A,Z,29,58\n' - '7,B,Total,61,122\n' - '7,B,Y,30,60\n' - '7,B,Z,31,62\n', result) - - -class CSVColumnIndexTransformerTests(CSVRowIndexTransformerTests): - csv_tx = CSVColumnIndexTransformer() - - def test_cont_cat_dim_single_metric(self): - df = mock_df.cont_cat_dims_single_metric_df - - result = self.csv_tx.transform(df, mock_df.cont_cat_dims_single_metric_schema) - - self.assertEqual('Cont,One (A),One (B)\n' - '0,0,1\n' - '1,2,3\n' - '2,4,5\n' - '3,6,7\n' - '4,8,9\n' - '5,10,11\n' - '6,12,13\n' - '7,14,15\n', result) - - def test_cont_cat_dim_multi_metric(self): - df = mock_df.cont_cat_dims_multi_metric_df - - result = self.csv_tx.transform(df, mock_df.cont_cat_dims_multi_metric_schema) - - self.assertEqual('Cont,One (A),One (B),Two (A),Two (B)\n' - '0,0,1,0,2\n' - '1,2,3,4,6\n' - '2,4,5,8,10\n' - '3,6,7,12,14\n' - '4,8,9,16,18\n' - '5,10,11,20,22\n' - '6,12,13,24,26\n' - '7,14,15,28,30\n', result) - - def test_cont_cat_cat_dim_multi_metric(self): - df = mock_df.cont_cat_cat_dims_multi_metric_df - - result = self.csv_tx.transform(df, mock_df.cont_cat_cat_dims_multi_metric_schema) - - self.assertEqual('Cont,"One (A, Y)","One (A, Z)","One (B, Y)","One (B, Z)",' - '"Two (A, Y)","Two (A, Z)","Two (B, Y)","Two (B, Z)"\n' - '0,0,1,2,3,0,2,4,6\n' - '1,4,5,6,7,8,10,12,14\n' - '2,8,9,10,11,16,18,20,22\n' - '3,12,13,14,15,24,26,28,30\n' - '4,16,17,18,19,32,34,36,38\n' - '5,20,21,22,23,40,42,44,46\n' - '6,24,25,26,27,48,50,52,54\n' - '7,28,29,30,31,56,58,60,62\n', result) - - def test_rollup_cont_cat_cat_dim_multi_metric(self): - df = mock_df.rollup_cont_cat_cat_dims_multi_metric_df - - result = self.csv_tx.transform(df, mock_df.rollup_cont_cat_cat_dims_multi_metric_schema) - - self.assertEqual('Cont,One,One (A),"One (A, Y)","One (A, Z)",One (B),' - '"One (B, Y)","One (B, Z)",Two,Two (A),"Two (A, Y)",' - '"Two (A, Z)",Two (B),"Two (B, Y)","Two (B, Z)"\n' - '0,12,1,0,1,5,2,3,24,2,0,2,10,4,6\n' - '1,44,9,4,5,13,6,7,88,18,8,10,26,12,14\n' - '2,76,17,8,9,21,10,11,152,34,16,18,42,20,22\n' - '3,108,25,12,13,29,14,15,216,50,24,26,58,28,30\n' - '4,140,33,16,17,37,18,19,280,66,32,34,74,36,38\n' - '5,172,41,20,21,45,22,23,344,82,40,42,90,44,46\n' - '6,204,49,24,25,53,26,27,408,98,48,50,106,52,54\n' - '7,236,57,28,29,61,30,31,472,114,56,58,122,60,62\n', result) diff --git a/fireant/tests/slicer/transformers/test_datatables.py b/fireant/tests/slicer/transformers/test_datatables.py deleted file mode 100644 index 977a9f99..00000000 --- a/fireant/tests/slicer/transformers/test_datatables.py +++ /dev/null @@ -1,1120 +0,0 @@ -# coding: utf-8 -import locale as lc -from collections import OrderedDict -from datetime import date, datetime -from unittest import TestCase - -import numpy as np -import pandas as pd -from fireant import settings -from fireant.slicer.operations import Totals -from fireant.slicer.transformers import DataTablesRowIndexTransformer, DataTablesColumnIndexTransformer -from fireant.slicer.transformers import datatables -from fireant.tests import mock_dataframes as mock_df - -lc.setlocale(lc.LC_ALL, 'C') - - -class DataTablesRowIndexTransformerTests(TestCase): - maxDiff = None - dt_tx = DataTablesRowIndexTransformer() - - def test_no_dims_multi_metric(self): - # Tests transformation of a single metric with a single continuous dimension - result = self.dt_tx.transform(mock_df.no_dims_multi_metric_df, mock_df.no_dims_multi_metric_schema) - self.assertDictEqual({ - 'columns': [{'title': 'One', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'data': 'one'}, - {'title': 'Two', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'data': 'two'}, - {'title': 'Three', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}, - 'data': 'three'}, - {'title': 'Four', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'data': 'four'}, - {'title': 'Five', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'data': 'five'}, - {'title': 'Six', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'data': 'six'}, - {'title': 'Seven', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}, - 'data': 'seven'}, - {'title': 'Eight', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}, - 'data': 'eight'}], - 'data': [{'six': {'value': 5, 'display': '5'}, 'seven': {'value': 6, 'display': '6'}, - 'three': {'value': 2, 'display': '2'}, 'one': {'value': 0, 'display': '0'}, - 'two': {'value': 1, 'display': '1'}, 'four': {'value': 3, 'display': '3'}, - 'eight': {'value': 7, 'display': '7'}, 'five': {'value': 4, 'display': '4'}}]}, result) - - def test_cont_dim_single_metric(self): - # Tests transformation of a single metric with a single continuous dimension - result = self.dt_tx.transform(mock_df.cont_dim_single_metric_df, mock_df.cont_dim_single_metric_schema) - self.assertDictEqual({ - 'columns': [{'data': 'cont', 'render': {'_': 'value', 'sort': 'value', 'type': 'value'}, 'title': 'Cont'}, - {'data': 'one', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'title': 'One'}], - 'data': [{'cont': {'value': i}, 'one': {'value': i, 'display': str(i)}} - for i in range(8)]}, result) - - def test_cont_dim_multi_metric(self): - # Tests transformation of two metrics with a single continuous dimension - result = self.dt_tx.transform(mock_df.cont_dim_multi_metric_df, mock_df.cont_dim_multi_metric_schema) - self.assertDictEqual({ - 'columns': [{'title': 'Cont', 'data': 'cont', 'render': {'_': 'value', 'sort': 'value', 'type': 'value'}}, - {'title': 'One', 'data': 'one', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}}, - {'title': 'Two', 'data': 'two', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}}], - 'data': [{ - 'cont': {'value': i}, - 'one': {'value': i, 'display': str(i)}, - 'two': {'value': 2 * i, 'display': str(2 * i)} - } for i in range(8)]}, result) - - def test_time_series_date(self): - # Tests transformation of a single-metric, single-dimension result - result = self.dt_tx.transform(mock_df.time_dim_single_metric_df, mock_df.time_dim_single_metric_schema) - self.assertDictEqual({ - 'columns': [{'render': {'_': 'value', 'sort': 'value', 'type': 'value'}, 'data': 'date', 'title': 'Date'}, - {'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'data': 'one', 'title': 'One'}], - 'data': [{'date': {'value': '2000-01-0%d' % (1 + i)}, 'one': {'value': i, 'display': str(i)}} - for i in range(8)]}, result) - - def test_time_series_date_with_ref(self): - # Tests transformation of a single-metric, single-dimension result using a WoW reference - result = self.dt_tx.transform(mock_df.time_dim_single_metric_ref_df, mock_df.time_dim_single_metric_ref_schema) - self.assertDictEqual({ - 'columns': [{'render': {'_': 'value', 'sort': 'value', 'type': 'value'}, 'data': 'date', 'title': 'Date'}, - {'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'data': 'one', 'title': 'One'}, - {'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'data': 'wow.one', - 'title': 'One WoW'}], - 'data': [{ - 'date': {'value': '2000-01-0%d' % (1 + i)}, - 'one': {'value': i, 'display': str(i)}, - 'wow': {'one': {'value': 2 * i, 'display': str(2 * i)}} - } for i in range(8)]}, result) - - def test_uni_dim_single_metric(self): - # Tests transformation of a metric with a unique dimension with one key and display - result = self.dt_tx.transform(mock_df.uni_dim_single_metric_df, mock_df.uni_dim_single_metric_schema) - self.assertDictEqual({ - 'columns': [ - {'data': 'uni', 'render': {'type': 'display', '_': 'display', 'sort': 'display'}, 'title': 'Uni'}, - {'data': 'one', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'title': 'One'}], - 'data': [{'one': {'value': 0, 'display': '0'}, 'uni': {'display': 'Aa', 'value': 1}}, - {'one': {'value': 1, 'display': '1'}, 'uni': {'display': 'Bb', 'value': 2}}, - {'one': {'value': 2, 'display': '2'}, 'uni': {'display': 'Cc', 'value': 3}}]}, result) - - def test_uni_dim_multi_metric(self): - # Tests transformation of a metric with a unique dimension with one key and display - result = self.dt_tx.transform(mock_df.uni_dim_multi_metric_df, mock_df.uni_dim_multi_metric_schema) - self.assertDictEqual({ - 'columns': [ - {'render': {'_': 'display', 'type': 'display', 'sort': 'display'}, 'title': 'Uni', 'data': 'uni'}, - {'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, 'title': 'One', 'data': 'one'}, - {'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, 'title': 'Two', 'data': 'two'}], - 'data': [{'one': {'value': 0, 'display': '0'}, 'two': {'value': 0, 'display': '0'}, - 'uni': {'value': 1, 'display': 'Aa'}}, - {'one': {'value': 1, 'display': '1'}, 'two': {'value': 2, 'display': '2'}, - 'uni': {'value': 2, 'display': 'Bb'}}, - {'one': {'value': 2, 'display': '2'}, 'two': {'value': 4, 'display': '4'}, - 'uni': {'value': 3, 'display': 'Cc'}}]}, result) - - def test_cat_cat_dim_single_metric(self): - # Tests transformation of a single metric with two categorical dimensions - result = self.dt_tx.transform(mock_df.cat_cat_dims_single_metric_df, mock_df.cat_cat_dims_single_metric_schema) - self.assertDictEqual({ - 'columns': [ - {'render': {'_': 'display', 'type': 'display', 'sort': 'display'}, 'title': 'Cat1', 'data': 'cat1'}, - {'render': {'_': 'display', 'type': 'display', 'sort': 'display'}, 'title': 'Cat2', 'data': 'cat2'}, - {'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, 'title': 'One', 'data': 'one'}], - 'data': [{'cat1': {'display': 'A', 'value': 'a'}, 'cat2': {'display': 'Y', 'value': 'y'}, - 'one': {'display': '0', 'value': 0}}, - {'cat1': {'display': 'A', 'value': 'a'}, 'cat2': {'display': 'Z', 'value': 'z'}, - 'one': {'display': '1', 'value': 1}}, - {'cat1': {'display': 'B', 'value': 'b'}, 'cat2': {'display': 'Y', 'value': 'y'}, - 'one': {'display': '2', 'value': 2}}, - {'cat1': {'display': 'B', 'value': 'b'}, 'cat2': {'display': 'Z', 'value': 'z'}, - 'one': {'display': '3', 'value': 3}}]}, result) - - def test_cat_cat_dim_multi_metric(self): - # Tests transformation of two metrics with two categorical dimensions - result = self.dt_tx.transform(mock_df.cat_cat_dims_multi_metric_df, mock_df.cat_cat_dims_multi_metric_schema) - - self.assertDictEqual({ - 'data': [{'one': {'value': 0, 'display': '0'}, 'two': {'value': 0, 'display': '0'}, - 'cat2': {'display': 'Y', 'value': 'y'}, 'cat1': {'display': 'A', 'value': 'a'}}, - {'one': {'value': 1, 'display': '1'}, 'two': {'value': 2, 'display': '2'}, - 'cat2': {'display': 'Z', 'value': 'z'}, 'cat1': {'display': 'A', 'value': 'a'}}, - {'one': {'value': 2, 'display': '2'}, 'two': {'value': 4, 'display': '4'}, - 'cat2': {'display': 'Y', 'value': 'y'}, 'cat1': {'display': 'B', 'value': 'b'}}, - {'one': {'value': 3, 'display': '3'}, 'two': {'value': 6, 'display': '6'}, - 'cat2': {'display': 'Z', 'value': 'z'}, 'cat1': {'display': 'B', 'value': 'b'}}], - 'columns': [ - {'title': 'Cat1', 'render': {'_': 'display', 'type': 'display', 'sort': 'display'}, 'data': 'cat1'}, - {'title': 'Cat2', 'render': {'_': 'display', 'type': 'display', 'sort': 'display'}, 'data': 'cat2'}, - {'title': 'One', 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, 'data': 'one'}, - {'title': 'Two', 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, 'data': 'two'}]}, - result) - - def test_rollup_cont_cat_uni_dims_multi_metric_df(self): - # Tests transformation of two metrics with a continuous, a categorical and a unique dimensions - result = self.dt_tx.transform(mock_df.rollup_cont_cat_uni_dims_multi_metric_df, - mock_df.rollup_cont_cat_uni_dims_multi_metric_schema) - - self.assertDictEqual({ - 'data': [{'cont': {'value': 0}, 'cat1': {'display': '_total', 'value': '_total'}, - 'one': {'display': '30', 'value': 30}, 'two': {'display': '1230', 'value': 1230}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 0}, 'cat1': {'display': 'A', 'value': 'a'}, 'one': {'display': '0', 'value': 0}, - 'two': {'display': '100', 'value': 100}, 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 0}, 'cat1': {'display': 'A', 'value': 'a'}, 'one': {'display': '1', 'value': 1}, - 'two': {'display': '101', 'value': 101}, 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 0}, 'cat1': {'display': 'A', 'value': 'a'}, 'one': {'display': '2', 'value': 2}, - 'two': {'display': '102', 'value': 102}, 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 0}, 'cat1': {'display': 'A', 'value': 'a'}, 'one': {'display': '3', 'value': 3}, - 'two': {'display': '303', 'value': 303}, 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 0}, 'cat1': {'display': 'B', 'value': 'b'}, 'one': {'display': '3', 'value': 3}, - 'two': {'display': '103', 'value': 103}, 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 0}, 'cat1': {'display': 'B', 'value': 'b'}, 'one': {'display': '4', 'value': 4}, - 'two': {'display': '104', 'value': 104}, 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 0}, 'cat1': {'display': 'B', 'value': 'b'}, 'one': {'display': '5', 'value': 5}, - 'two': {'display': '105', 'value': 105}, 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 0}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '12', 'value': 12}, 'two': {'display': '312', 'value': 312}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 1}, 'cat1': {'display': '_total', 'value': '_total'}, - 'one': {'display': '102', 'value': 102}, 'two': {'display': '1302', 'value': 1302}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 1}, 'cat1': {'display': 'A', 'value': 'a'}, 'one': {'display': '6', 'value': 6}, - 'two': {'display': '106', 'value': 106}, 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 1}, 'cat1': {'display': 'A', 'value': 'a'}, 'one': {'display': '7', 'value': 7}, - 'two': {'display': '107', 'value': 107}, 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 1}, 'cat1': {'display': 'A', 'value': 'a'}, 'one': {'display': '8', 'value': 8}, - 'two': {'display': '108', 'value': 108}, 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 1}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '21', 'value': 21}, 'two': {'display': '321', 'value': 321}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 1}, 'cat1': {'display': 'B', 'value': 'b'}, 'one': {'display': '9', 'value': 9}, - 'two': {'display': '109', 'value': 109}, 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 1}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '10', 'value': 10}, 'two': {'display': '110', 'value': 110}, - 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 1}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '11', 'value': 11}, 'two': {'display': '111', 'value': 111}, - 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 1}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '30', 'value': 30}, 'two': {'display': '330', 'value': 330}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 2}, 'cat1': {'display': '_total', 'value': '_total'}, - 'one': {'display': '174', 'value': 174}, 'two': {'display': '1374', 'value': 1374}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 2}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '12', 'value': 12}, 'two': {'display': '112', 'value': 112}, - 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 2}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '13', 'value': 13}, 'two': {'display': '113', 'value': 113}, - 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 2}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '14', 'value': 14}, 'two': {'display': '114', 'value': 114}, - 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 2}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '39', 'value': 39}, 'two': {'display': '339', 'value': 339}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 2}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '15', 'value': 15}, 'two': {'display': '115', 'value': 115}, - 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 2}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '16', 'value': 16}, 'two': {'display': '116', 'value': 116}, - 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 2}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '17', 'value': 17}, 'two': {'display': '117', 'value': 117}, - 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 2}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '48', 'value': 48}, 'two': {'display': '348', 'value': 348}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 3}, 'cat1': {'display': '_total', 'value': '_total'}, - 'one': {'display': '246', 'value': 246}, 'two': {'display': '1446', 'value': 1446}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 3}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '18', 'value': 18}, 'two': {'display': '118', 'value': 118}, - 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 3}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '19', 'value': 19}, 'two': {'display': '119', 'value': 119}, - 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 3}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '20', 'value': 20}, 'two': {'display': '120', 'value': 120}, - 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 3}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '57', 'value': 57}, 'two': {'display': '357', 'value': 357}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 3}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '21', 'value': 21}, 'two': {'display': '121', 'value': 121}, - 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 3}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '22', 'value': 22}, 'two': {'display': '122', 'value': 122}, - 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 3}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '23', 'value': 23}, 'two': {'display': '123', 'value': 123}, - 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 3}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '66', 'value': 66}, 'two': {'display': '366', 'value': 366}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 4}, 'cat1': {'display': '_total', 'value': '_total'}, - 'one': {'display': '318', 'value': 318}, 'two': {'display': '1518', 'value': 1518}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 4}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '24', 'value': 24}, 'two': {'display': '124', 'value': 124}, - 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 4}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '25', 'value': 25}, 'two': {'display': '125', 'value': 125}, - 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 4}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '26', 'value': 26}, 'two': {'display': '126', 'value': 126}, - 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 4}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '75', 'value': 75}, 'two': {'display': '375', 'value': 375}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 4}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '27', 'value': 27}, 'two': {'display': '127', 'value': 127}, - 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 4}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '28', 'value': 28}, 'two': {'display': '128', 'value': 128}, - 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 4}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '29', 'value': 29}, 'two': {'display': '129', 'value': 129}, - 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 4}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '84', 'value': 84}, 'two': {'display': '384', 'value': 384}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 5}, 'cat1': {'display': '_total', 'value': '_total'}, - 'one': {'display': '390', 'value': 390}, 'two': {'display': '1590', 'value': 1590}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 5}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '30', 'value': 30}, 'two': {'display': '130', 'value': 130}, - 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 5}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '31', 'value': 31}, 'two': {'display': '131', 'value': 131}, - 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 5}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '32', 'value': 32}, 'two': {'display': '132', 'value': 132}, - 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 5}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '93', 'value': 93}, 'two': {'display': '393', 'value': 393}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 5}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '33', 'value': 33}, 'two': {'display': '133', 'value': 133}, - 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 5}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '34', 'value': 34}, 'two': {'display': '134', 'value': 134}, - 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 5}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '35', 'value': 35}, 'two': {'display': '135', 'value': 135}, - 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 5}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '102', 'value': 102}, 'two': {'display': '402', 'value': 402}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 6}, 'cat1': {'display': '_total', 'value': '_total'}, - 'one': {'display': '462', 'value': 462}, 'two': {'display': '1662', 'value': 1662}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 6}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '36', 'value': 36}, 'two': {'display': '136', 'value': 136}, - 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 6}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '37', 'value': 37}, 'two': {'display': '137', 'value': 137}, - 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 6}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '38', 'value': 38}, 'two': {'display': '138', 'value': 138}, - 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 6}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '111', 'value': 111}, 'two': {'display': '411', 'value': 411}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 6}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '39', 'value': 39}, 'two': {'display': '139', 'value': 139}, - 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 6}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '40', 'value': 40}, 'two': {'display': '140', 'value': 140}, - 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 6}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '41', 'value': 41}, 'two': {'display': '141', 'value': 141}, - 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 6}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '120', 'value': 120}, 'two': {'display': '420', 'value': 420}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 7}, 'cat1': {'display': '_total', 'value': '_total'}, - 'one': {'display': '534', 'value': 534}, 'two': {'display': '1734', 'value': 1734}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 7}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '42', 'value': 42}, 'two': {'display': '142', 'value': 142}, - 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 7}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '43', 'value': 43}, 'two': {'display': '143', 'value': 143}, - 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 7}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '44', 'value': 44}, 'two': {'display': '144', 'value': 144}, - 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 7}, 'cat1': {'display': 'A', 'value': 'a'}, - 'one': {'display': '129', 'value': 129}, 'two': {'display': '429', 'value': 429}, - 'uni': {'display': '_total', 'value': '_total'}}, - {'cont': {'value': 7}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '45', 'value': 45}, 'two': {'display': '145', 'value': 145}, - 'uni': {'display': 'Aa', 'value': 1}}, - {'cont': {'value': 7}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '46', 'value': 46}, 'two': {'display': '146', 'value': 146}, - 'uni': {'display': 'Bb', 'value': 2}}, - {'cont': {'value': 7}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '47', 'value': 47}, 'two': {'display': '147', 'value': 147}, - 'uni': {'display': 'Cc', 'value': 3}}, - {'cont': {'value': 7}, 'cat1': {'display': 'B', 'value': 'b'}, - 'one': {'display': '138', 'value': 138}, 'two': {'display': '438', 'value': 438}, - 'uni': {'display': '_total', 'value': '_total'}}], - 'columns': [ - {'render': {'sort': 'value', '_': 'value', 'type': 'value'}, 'title': 'Cont', 'data': 'cont'}, - {'render': {'sort': 'display', '_': 'display', 'type': 'display'}, 'title': 'Cat1', 'data': 'cat1'}, - {'render': {'sort': 'display', '_': 'display', 'type': 'display'}, 'title': 'Uni', 'data': 'uni'}, - {'render': {'sort': 'value', '_': 'display', 'type': 'value'}, 'title': 'One', 'data': 'one'}, - {'render': {'sort': 'value', '_': 'display', 'type': 'value'}, 'title': 'Two', 'data': 'two'}]}, - result) - - def test_rollup_cont_cat_cat_dims_multi_metric_df(self): - # Tests transformation of two metrics with a continuous and two categorical dimensions - result = self.dt_tx.transform(mock_df.rollup_cont_cat_cat_dims_multi_metric_df, - mock_df.rollup_cont_cat_cat_dims_multi_metric_schema) - self.assertDictEqual({ - 'columns': [{'title': 'Cont', 'render': {'_': 'value', 'sort': 'value', 'type': 'value'}, 'data': 'cont'}, - {'title': 'Cat1', 'render': {'type': 'display', '_': 'display', 'sort': 'display'}, - 'data': 'cat1'}, - {'title': 'Cat2', 'render': {'type': 'display', '_': 'display', 'sort': 'display'}, - 'data': 'cat2'}, - {'title': 'One', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'data': 'one'}, - {'title': 'Two', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'data': 'two'}], - 'data': [ - {'one': {'value': 12, 'display': '12'}, 'two': {'value': 24, 'display': '24'}, 'cont': {'value': 0}, - 'cat1': {'value': Totals.key, 'display': Totals.label}, - 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 1, 'display': '1'}, 'two': {'value': 2, 'display': '2'}, 'cont': {'value': 0}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 0, 'display': '0'}, 'two': {'value': 0, 'display': '0'}, 'cont': {'value': 0}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 1, 'display': '1'}, 'two': {'value': 2, 'display': '2'}, 'cont': {'value': 0}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 5, 'display': '5'}, 'two': {'value': 10, 'display': '10'}, 'cont': {'value': 0}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 2, 'display': '2'}, 'two': {'value': 4, 'display': '4'}, 'cont': {'value': 0}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 3, 'display': '3'}, 'two': {'value': 6, 'display': '6'}, 'cont': {'value': 0}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 44, 'display': '44'}, 'two': {'value': 88, 'display': '88'}, 'cont': {'value': 1}, - 'cat1': {'value': Totals.key, 'display': Totals.label}, - 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 9, 'display': '9'}, 'two': {'value': 18, 'display': '18'}, 'cont': {'value': 1}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 4, 'display': '4'}, 'two': {'value': 8, 'display': '8'}, 'cont': {'value': 1}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 5, 'display': '5'}, 'two': {'value': 10, 'display': '10'}, 'cont': {'value': 1}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 13, 'display': '13'}, 'two': {'value': 26, 'display': '26'}, 'cont': {'value': 1}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 6, 'display': '6'}, 'two': {'value': 12, 'display': '12'}, 'cont': {'value': 1}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 7, 'display': '7'}, 'two': {'value': 14, 'display': '14'}, 'cont': {'value': 1}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 76, 'display': '76'}, 'two': {'value': 152, 'display': '152'}, 'cont': {'value': 2}, - 'cat1': {'value': Totals.key, 'display': Totals.label}, - 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 17, 'display': '17'}, 'two': {'value': 34, 'display': '34'}, 'cont': {'value': 2}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 8, 'display': '8'}, 'two': {'value': 16, 'display': '16'}, 'cont': {'value': 2}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 9, 'display': '9'}, 'two': {'value': 18, 'display': '18'}, 'cont': {'value': 2}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 21, 'display': '21'}, 'two': {'value': 42, 'display': '42'}, 'cont': {'value': 2}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 10, 'display': '10'}, 'two': {'value': 20, 'display': '20'}, 'cont': {'value': 2}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 11, 'display': '11'}, 'two': {'value': 22, 'display': '22'}, 'cont': {'value': 2}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 108, 'display': '108'}, 'two': {'value': 216, 'display': '216'}, 'cont': {'value': 3}, - 'cat1': {'value': Totals.key, 'display': Totals.label}, - 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 25, 'display': '25'}, 'two': {'value': 50, 'display': '50'}, 'cont': {'value': 3}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 12, 'display': '12'}, 'two': {'value': 24, 'display': '24'}, 'cont': {'value': 3}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 13, 'display': '13'}, 'two': {'value': 26, 'display': '26'}, 'cont': {'value': 3}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 29, 'display': '29'}, 'two': {'value': 58, 'display': '58'}, 'cont': {'value': 3}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 14, 'display': '14'}, 'two': {'value': 28, 'display': '28'}, 'cont': {'value': 3}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 15, 'display': '15'}, 'two': {'value': 30, 'display': '30'}, 'cont': {'value': 3}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 140, 'display': '140'}, 'two': {'value': 280, 'display': '280'}, 'cont': {'value': 4}, - 'cat1': {'value': Totals.key, 'display': Totals.label}, - 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 33, 'display': '33'}, 'two': {'value': 66, 'display': '66'}, 'cont': {'value': 4}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 16, 'display': '16'}, 'two': {'value': 32, 'display': '32'}, 'cont': {'value': 4}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 17, 'display': '17'}, 'two': {'value': 34, 'display': '34'}, 'cont': {'value': 4}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 37, 'display': '37'}, 'two': {'value': 74, 'display': '74'}, 'cont': {'value': 4}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 18, 'display': '18'}, 'two': {'value': 36, 'display': '36'}, 'cont': {'value': 4}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 19, 'display': '19'}, 'two': {'value': 38, 'display': '38'}, 'cont': {'value': 4}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 172, 'display': '172'}, 'two': {'value': 344, 'display': '344'}, 'cont': {'value': 5}, - 'cat1': {'value': Totals.key, 'display': Totals.label}, - 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 41, 'display': '41'}, 'two': {'value': 82, 'display': '82'}, 'cont': {'value': 5}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 20, 'display': '20'}, 'two': {'value': 40, 'display': '40'}, 'cont': {'value': 5}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 21, 'display': '21'}, 'two': {'value': 42, 'display': '42'}, 'cont': {'value': 5}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 45, 'display': '45'}, 'two': {'value': 90, 'display': '90'}, 'cont': {'value': 5}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 22, 'display': '22'}, 'two': {'value': 44, 'display': '44'}, 'cont': {'value': 5}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 23, 'display': '23'}, 'two': {'value': 46, 'display': '46'}, 'cont': {'value': 5}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 204, 'display': '204'}, 'two': {'value': 408, 'display': '408'}, 'cont': {'value': 6}, - 'cat1': {'value': Totals.key, 'display': Totals.label}, - 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 49, 'display': '49'}, 'two': {'value': 98, 'display': '98'}, 'cont': {'value': 6}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 24, 'display': '24'}, 'two': {'value': 48, 'display': '48'}, 'cont': {'value': 6}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 25, 'display': '25'}, 'two': {'value': 50, 'display': '50'}, 'cont': {'value': 6}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 53, 'display': '53'}, 'two': {'value': 106, 'display': '106'}, 'cont': {'value': 6}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 26, 'display': '26'}, 'two': {'value': 52, 'display': '52'}, 'cont': {'value': 6}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 27, 'display': '27'}, 'two': {'value': 54, 'display': '54'}, 'cont': {'value': 6}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 236, 'display': '236'}, 'two': {'value': 472, 'display': '472'}, 'cont': {'value': 7}, - 'cat1': {'value': Totals.key, 'display': Totals.label}, - 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 57, 'display': '57'}, 'two': {'value': 114, 'display': '114'}, 'cont': {'value': 7}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 28, 'display': '28'}, 'two': {'value': 56, 'display': '56'}, 'cont': {'value': 7}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 29, 'display': '29'}, 'two': {'value': 58, 'display': '58'}, 'cont': {'value': 7}, - 'cat1': {'value': 'a', 'display': 'A'}, 'cat2': {'value': 'z', 'display': 'Z'}}, - {'one': {'value': 61, 'display': '61'}, 'two': {'value': 122, 'display': '122'}, 'cont': {'value': 7}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': Totals.key, 'display': Totals.label}}, - {'one': {'value': 30, 'display': '30'}, 'two': {'value': 60, 'display': '60'}, 'cont': {'value': 7}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'y', 'display': 'Y'}}, - {'one': {'value': 31, 'display': '31'}, 'two': {'value': 62, 'display': '62'}, 'cont': {'value': 7}, - 'cat1': {'value': 'b', 'display': 'B'}, 'cat2': {'value': 'z', 'display': 'Z'}}]} - , result) - - -class DataTablesColumnIndexTransformerTests(TestCase): - maxDiff = None - dt_tx = DataTablesColumnIndexTransformer() - - def test_no_dims_multi_metric(self): - # Tests transformation of a single metric with a single continuous dimension - result = self.dt_tx.transform(mock_df.no_dims_multi_metric_df, mock_df.no_dims_multi_metric_schema) - self.assertDictEqual({ - 'data': [{'eight': {'value': 7, 'display': '7'}, 'one': {'value': 0, 'display': '0'}, - 'seven': {'value': 6, 'display': '6'}, 'four': {'value': 3, 'display': '3'}, - 'three': {'value': 2, 'display': '2'}, 'five': {'value': 4, 'display': '4'}, - 'six': {'value': 5, 'display': '5'}, 'two': {'value': 1, 'display': '1'}}], - 'columns': [{'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'title': 'One', 'data': 'one'}, - {'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'title': 'Two', 'data': 'two'}, - {'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'title': 'Three', - 'data': 'three'}, - {'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'title': 'Four', 'data': 'four'}, - {'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'title': 'Five', 'data': 'five'}, - {'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'title': 'Six', 'data': 'six'}, - {'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'title': 'Seven', - 'data': 'seven'}, - {'render': {'type': 'value', '_': 'display', 'sort': 'value'}, 'title': 'Eight', - 'data': 'eight'}]}, result) - - def test_cont_dim_single_metric(self): - # Tests transformation of a single metric with a single continuous dimension - result = self.dt_tx.transform(mock_df.cont_dim_single_metric_df, mock_df.cont_dim_single_metric_schema) - self.assertDictEqual({ - 'columns': [{'title': 'Cont', 'data': 'cont', 'render': {'_': 'value', 'sort': 'value', 'type': 'value'}}, - {'title': 'One', 'data': 'one', 'render': {'type': 'value', '_': 'display', 'sort': 'value'}}], - 'data': [{ - 'cont': {'value': i}, - 'one': {'value': i, 'display': str(i)}, - } for i in range(8)]}, result) - - def test_cont_dim_multi_metric(self): - # Tests transformation of two metrics with a single continuous dimension - result = self.dt_tx.transform(mock_df.cont_dim_multi_metric_df, mock_df.cont_dim_multi_metric_schema) - self.assertDictEqual({ - 'columns': [{'data': 'cont', 'render': {'_': 'value', 'sort': 'value', 'type': 'value'}, 'title': 'Cont'}, - {'data': 'one', 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, 'title': 'One'}, - {'data': 'two', 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, 'title': 'Two'}], - 'data': [{ - 'cont': {'value': i}, - 'one': {'value': i, 'display': str(i)}, - 'two': {'value': 2 * i, 'display': str(2 * i)}, - } for i in range(8)]}, result) - - def test_time_series_date_to_millis(self): - # Tests transformation of a single-metric, single-dimension result - result = self.dt_tx.transform(mock_df.time_dim_single_metric_df, mock_df.time_dim_single_metric_schema) - self.assertDictEqual({ - 'columns': [{'data': 'date', 'render': {'_': 'value', 'sort': 'value', 'type': 'value'}, 'title': 'Date'}, - {'data': 'one', 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, 'title': 'One'}], - 'data': [{ - 'date': {'value': '2000-01-0%d' % (i + 1)}, - 'one': {'value': i, 'display': str(i)} - } for i in range(8)]}, result) - - def test_time_series_date_with_ref(self): - # Tests transformation of a single-metric, single-dimension result using a WoW reference - result = self.dt_tx.transform(mock_df.time_dim_single_metric_ref_df, mock_df.time_dim_single_metric_ref_schema) - self.assertDictEqual({ - 'columns': [{'data': 'date', 'render': {'_': 'value', 'sort': 'value', 'type': 'value'}, 'title': 'Date'}, - {'data': 'one', 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, 'title': 'One'}, - {'data': 'wow.one', 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, - 'title': 'One WoW'}], - 'data': [{ - 'date': {'value': '2000-01-0%d' % (i + 1)}, - 'one': {'value': i, 'display': str(i)}, - 'wow': {'one': {'value': 2 * i, 'display': str(2 * i)}} - } for i in range(8)]}, result) - - def test_cont_cat_dim_single_metric(self): - # Tests transformation of a single metric with a continuous and a categorical dimension - result = self.dt_tx.transform(mock_df.cont_cat_dims_single_metric_df, - mock_df.cont_cat_dims_single_metric_schema) - self.assertDictEqual({ - 'columns': [{'data': 'cont', 'title': 'Cont', 'render': {'_': 'value', 'sort': 'value', 'type': 'value'}}, - {'data': 'a.one', 'title': 'One (A)', - 'render': {'type': 'value', '_': 'display', 'sort': 'value'}}, - {'data': 'b.one', 'title': 'One (B)', - 'render': {'type': 'value', '_': 'display', 'sort': 'value'}}], - 'data': [{'b': {'one': {'value': 1, 'display': '1'}}, 'a': {'one': {'value': 0, 'display': '0'}}, - 'cont': {'value': 0}}, - {'b': {'one': {'value': 3, 'display': '3'}}, 'a': {'one': {'value': 2, 'display': '2'}}, - 'cont': {'value': 1}}, - {'b': {'one': {'value': 5, 'display': '5'}}, 'a': {'one': {'value': 4, 'display': '4'}}, - 'cont': {'value': 2}}, - {'b': {'one': {'value': 7, 'display': '7'}}, 'a': {'one': {'value': 6, 'display': '6'}}, - 'cont': {'value': 3}}, - {'b': {'one': {'value': 9, 'display': '9'}}, 'a': {'one': {'value': 8, 'display': '8'}}, - 'cont': {'value': 4}}, - {'b': {'one': {'value': 11, 'display': '11'}}, 'a': {'one': {'value': 10, 'display': '10'}}, - 'cont': {'value': 5}}, - {'b': {'one': {'value': 13, 'display': '13'}}, 'a': {'one': {'value': 12, 'display': '12'}}, - 'cont': {'value': 6}}, - {'b': {'one': {'value': 15, 'display': '15'}}, 'a': {'one': {'value': 14, 'display': '14'}}, - 'cont': {'value': 7}}]} - , result) - - def test_cont_cat_dim_multi_metric(self): - # Tests transformation of two metrics with a continuous and a categorical dimension - result = self.dt_tx.transform(mock_df.cont_cat_dims_multi_metric_df, mock_df.cont_cat_dims_multi_metric_schema) - self.assertDictEqual({ - 'data': [{'b': {'one': {'display': '1', 'value': 1}, 'two': {'display': '2', 'value': 2}}, - 'a': {'one': {'display': '0', 'value': 0}, 'two': {'display': '0', 'value': 0}}, - 'cont': {'value': 0}}, - {'b': {'one': {'display': '3', 'value': 3}, 'two': {'display': '6', 'value': 6}}, - 'a': {'one': {'display': '2', 'value': 2}, 'two': {'display': '4', 'value': 4}}, - 'cont': {'value': 1}}, - {'b': {'one': {'display': '5', 'value': 5}, 'two': {'display': '10', 'value': 10}}, - 'a': {'one': {'display': '4', 'value': 4}, 'two': {'display': '8', 'value': 8}}, - 'cont': {'value': 2}}, - {'b': {'one': {'display': '7', 'value': 7}, 'two': {'display': '14', 'value': 14}}, - 'a': {'one': {'display': '6', 'value': 6}, 'two': {'display': '12', 'value': 12}}, - 'cont': {'value': 3}}, - {'b': {'one': {'display': '9', 'value': 9}, 'two': {'display': '18', 'value': 18}}, - 'a': {'one': {'display': '8', 'value': 8}, 'two': {'display': '16', 'value': 16}}, - 'cont': {'value': 4}}, - {'b': {'one': {'display': '11', 'value': 11}, 'two': {'display': '22', 'value': 22}}, - 'a': {'one': {'display': '10', 'value': 10}, 'two': {'display': '20', 'value': 20}}, - 'cont': {'value': 5}}, - {'b': {'one': {'display': '13', 'value': 13}, 'two': {'display': '26', 'value': 26}}, - 'a': {'one': {'display': '12', 'value': 12}, 'two': {'display': '24', 'value': 24}}, - 'cont': {'value': 6}}, - {'b': {'one': {'display': '15', 'value': 15}, 'two': {'display': '30', 'value': 30}}, - 'a': {'one': {'display': '14', 'value': 14}, 'two': {'display': '28', 'value': 28}}, - 'cont': {'value': 7}}], - 'columns': [{'title': 'Cont', 'render': {'_': 'value', 'sort': 'value', 'type': 'value'}, 'data': 'cont'}, - {'title': 'One (A)', 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, - 'data': 'a.one'}, - {'title': 'One (B)', 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, - 'data': 'b.one'}, - {'title': 'Two (A)', 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, - 'data': 'a.two'}, - {'title': 'Two (B)', 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}, - 'data': 'b.two'}]}, result) - - def test_cont_cat_cat_dim_multi_metric(self): - # Tests transformation of two metrics with a continuous and two categorical dimensions - result = self.dt_tx.transform(mock_df.cont_cat_cat_dims_multi_metric_df, - mock_df.cont_cat_cat_dims_multi_metric_schema) - self.assertDictEqual({ - 'data': [{'b': {'z': {'one': {'value': 3, 'display': '3'}, 'two': {'value': 6, 'display': '6'}}, - 'y': {'one': {'value': 2, 'display': '2'}, 'two': {'value': 4, 'display': '4'}}}, - 'a': {'z': {'one': {'value': 1, 'display': '1'}, 'two': {'value': 2, 'display': '2'}}, - 'y': {'one': {'value': 0, 'display': '0'}, 'two': {'value': 0, 'display': '0'}}}, - 'cont': {'value': 0}}, { - 'b': {'z': {'one': {'value': 7, 'display': '7'}, 'two': {'value': 14, 'display': '14'}}, - 'y': {'one': {'value': 6, 'display': '6'}, 'two': {'value': 12, 'display': '12'}}}, - 'a': {'z': {'one': {'value': 5, 'display': '5'}, 'two': {'value': 10, 'display': '10'}}, - 'y': {'one': {'value': 4, 'display': '4'}, 'two': {'value': 8, 'display': '8'}}}, - 'cont': {'value': 1}}, { - 'b': {'z': {'one': {'value': 11, 'display': '11'}, 'two': {'value': 22, 'display': '22'}}, - 'y': {'one': {'value': 10, 'display': '10'}, 'two': {'value': 20, 'display': '20'}}}, - 'a': {'z': {'one': {'value': 9, 'display': '9'}, 'two': {'value': 18, 'display': '18'}}, - 'y': {'one': {'value': 8, 'display': '8'}, 'two': {'value': 16, 'display': '16'}}}, - 'cont': {'value': 2}}, { - 'b': {'z': {'one': {'value': 15, 'display': '15'}, 'two': {'value': 30, 'display': '30'}}, - 'y': {'one': {'value': 14, 'display': '14'}, 'two': {'value': 28, 'display': '28'}}}, - 'a': {'z': {'one': {'value': 13, 'display': '13'}, 'two': {'value': 26, 'display': '26'}}, - 'y': {'one': {'value': 12, 'display': '12'}, 'two': {'value': 24, 'display': '24'}}}, - 'cont': {'value': 3}}, { - 'b': {'z': {'one': {'value': 19, 'display': '19'}, 'two': {'value': 38, 'display': '38'}}, - 'y': {'one': {'value': 18, 'display': '18'}, 'two': {'value': 36, 'display': '36'}}}, - 'a': {'z': {'one': {'value': 17, 'display': '17'}, 'two': {'value': 34, 'display': '34'}}, - 'y': {'one': {'value': 16, 'display': '16'}, 'two': {'value': 32, 'display': '32'}}}, - 'cont': {'value': 4}}, { - 'b': {'z': {'one': {'value': 23, 'display': '23'}, 'two': {'value': 46, 'display': '46'}}, - 'y': {'one': {'value': 22, 'display': '22'}, 'two': {'value': 44, 'display': '44'}}}, - 'a': {'z': {'one': {'value': 21, 'display': '21'}, 'two': {'value': 42, 'display': '42'}}, - 'y': {'one': {'value': 20, 'display': '20'}, 'two': {'value': 40, 'display': '40'}}}, - 'cont': {'value': 5}}, { - 'b': {'z': {'one': {'value': 27, 'display': '27'}, 'two': {'value': 54, 'display': '54'}}, - 'y': {'one': {'value': 26, 'display': '26'}, 'two': {'value': 52, 'display': '52'}}}, - 'a': {'z': {'one': {'value': 25, 'display': '25'}, 'two': {'value': 50, 'display': '50'}}, - 'y': {'one': {'value': 24, 'display': '24'}, 'two': {'value': 48, 'display': '48'}}}, - 'cont': {'value': 6}}, { - 'b': {'z': {'one': {'value': 31, 'display': '31'}, 'two': {'value': 62, 'display': '62'}}, - 'y': {'one': {'value': 30, 'display': '30'}, 'two': {'value': 60, 'display': '60'}}}, - 'a': {'z': {'one': {'value': 29, 'display': '29'}, 'two': {'value': 58, 'display': '58'}}, - 'y': {'one': {'value': 28, 'display': '28'}, 'two': {'value': 56, 'display': '56'}}}, - 'cont': {'value': 7}}], - 'columns': [{'data': 'cont', 'title': 'Cont', 'render': {'_': 'value', 'sort': 'value', 'type': 'value'}}, - {'data': 'a.y.one', 'title': 'One (A, Y)', - 'render': {'type': 'value', '_': 'display', 'sort': 'value'}}, - {'data': 'a.z.one', 'title': 'One (A, Z)', - 'render': {'type': 'value', '_': 'display', 'sort': 'value'}}, - {'data': 'b.y.one', 'title': 'One (B, Y)', - 'render': {'type': 'value', '_': 'display', 'sort': 'value'}}, - {'data': 'b.z.one', 'title': 'One (B, Z)', - 'render': {'type': 'value', '_': 'display', 'sort': 'value'}}, - {'data': 'a.y.two', 'title': 'Two (A, Y)', - 'render': {'type': 'value', '_': 'display', 'sort': 'value'}}, - {'data': 'a.z.two', 'title': 'Two (A, Z)', - 'render': {'type': 'value', '_': 'display', 'sort': 'value'}}, - {'data': 'b.y.two', 'title': 'Two (B, Y)', - 'render': {'type': 'value', '_': 'display', 'sort': 'value'}}, - {'data': 'b.z.two', 'title': 'Two (B, Z)', - 'render': {'type': 'value', '_': 'display', 'sort': 'value'}}]}, - result) - - def test_cont_cat_uni_dim_multi_metric(self): - # Tests transformation of two metrics with a continuous and two categorical dimensions - result = self.dt_tx.transform(mock_df.cont_cat_uni_dims_multi_metric_df, - mock_df.cont_cat_uni_dims_multi_metric_schema) - self.assertDictEqual({ - 'data': [{'cont': {'value': 0}, - 'b': {1: {'one': {'display': '3', 'value': 3}, 'two': {'display': '103', 'value': 103}}, - 2: {'one': {'display': '4', 'value': 4}, 'two': {'display': '104', 'value': 104}}, - 3: {'one': {'display': '5', 'value': 5}, 'two': {'display': '105', 'value': 105}}}, - 'a': {1: {'one': {'display': '0', 'value': 0}, 'two': {'display': '100', 'value': 100}}, - 2: {'one': {'display': '1', 'value': 1}, 'two': {'display': '101', 'value': 101}}, - 3: {'one': {'display': '2', 'value': 2}, 'two': {'display': '102', 'value': 102}}}}, - {'cont': {'value': 1}, - 'b': {1: {'one': {'display': '9', 'value': 9}, 'two': {'display': '109', 'value': 109}}, - 2: {'one': {'display': '10', 'value': 10}, 'two': {'display': '110', 'value': 110}}, - 3: {'one': {'display': '11', 'value': 11}, 'two': {'display': '111', 'value': 111}}}, - 'a': {1: {'one': {'display': '6', 'value': 6}, 'two': {'display': '106', 'value': 106}}, - 2: {'one': {'display': '7', 'value': 7}, 'two': {'display': '107', 'value': 107}}, - 3: {'one': {'display': '8', 'value': 8}, 'two': {'display': '108', 'value': 108}}}}, - {'cont': {'value': 2}, - 'b': {1: {'one': {'display': '15', 'value': 15}, 'two': {'display': '115', 'value': 115}}, - 2: {'one': {'display': '16', 'value': 16}, 'two': {'display': '116', 'value': 116}}, - 3: {'one': {'display': '17', 'value': 17}, 'two': {'display': '117', 'value': 117}}}, - 'a': {1: {'one': {'display': '12', 'value': 12}, 'two': {'display': '112', 'value': 112}}, - 2: {'one': {'display': '13', 'value': 13}, 'two': {'display': '113', 'value': 113}}, - 3: {'one': {'display': '14', 'value': 14}, 'two': {'display': '114', 'value': 114}}}}, - {'cont': {'value': 3}, - 'b': {1: {'one': {'display': '21', 'value': 21}, 'two': {'display': '121', 'value': 121}}, - 2: {'one': {'display': '22', 'value': 22}, 'two': {'display': '122', 'value': 122}}, - 3: {'one': {'display': '23', 'value': 23}, 'two': {'display': '123', 'value': 123}}}, - 'a': {1: {'one': {'display': '18', 'value': 18}, 'two': {'display': '118', 'value': 118}}, - 2: {'one': {'display': '19', 'value': 19}, 'two': {'display': '119', 'value': 119}}, - 3: {'one': {'display': '20', 'value': 20}, 'two': {'display': '120', 'value': 120}}}}, - {'cont': {'value': 4}, - 'b': {1: {'one': {'display': '27', 'value': 27}, 'two': {'display': '127', 'value': 127}}, - 2: {'one': {'display': '28', 'value': 28}, 'two': {'display': '128', 'value': 128}}, - 3: {'one': {'display': '29', 'value': 29}, 'two': {'display': '129', 'value': 129}}}, - 'a': {1: {'one': {'display': '24', 'value': 24}, 'two': {'display': '124', 'value': 124}}, - 2: {'one': {'display': '25', 'value': 25}, 'two': {'display': '125', 'value': 125}}, - 3: {'one': {'display': '26', 'value': 26}, 'two': {'display': '126', 'value': 126}}}}, - {'cont': {'value': 5}, - 'b': {1: {'one': {'display': '33', 'value': 33}, 'two': {'display': '133', 'value': 133}}, - 2: {'one': {'display': '34', 'value': 34}, 'two': {'display': '134', 'value': 134}}, - 3: {'one': {'display': '35', 'value': 35}, 'two': {'display': '135', 'value': 135}}}, - 'a': {1: {'one': {'display': '30', 'value': 30}, 'two': {'display': '130', 'value': 130}}, - 2: {'one': {'display': '31', 'value': 31}, 'two': {'display': '131', 'value': 131}}, - 3: {'one': {'display': '32', 'value': 32}, 'two': {'display': '132', 'value': 132}}}}, - {'cont': {'value': 6}, - 'b': {1: {'one': {'display': '39', 'value': 39}, 'two': {'display': '139', 'value': 139}}, - 2: {'one': {'display': '40', 'value': 40}, 'two': {'display': '140', 'value': 140}}, - 3: {'one': {'display': '41', 'value': 41}, 'two': {'display': '141', 'value': 141}}}, - 'a': {1: {'one': {'display': '36', 'value': 36}, 'two': {'display': '136', 'value': 136}}, - 2: {'one': {'display': '37', 'value': 37}, 'two': {'display': '137', 'value': 137}}, - 3: {'one': {'display': '38', 'value': 38}, 'two': {'display': '138', 'value': 138}}}}, - {'cont': {'value': 7}, - 'b': {1: {'one': {'display': '45', 'value': 45}, 'two': {'display': '145', 'value': 145}}, - 2: {'one': {'display': '46', 'value': 46}, 'two': {'display': '146', 'value': 146}}, - 3: {'one': {'display': '47', 'value': 47}, 'two': {'display': '147', 'value': 147}}}, - 'a': {1: {'one': {'display': '42', 'value': 42}, 'two': {'display': '142', 'value': 142}}, - 2: {'one': {'display': '43', 'value': 43}, 'two': {'display': '143', 'value': 143}}, - 3: {'one': {'display': '44', 'value': 44}, 'two': {'display': '144', 'value': 144}}}}], - 'columns': [{'data': 'cont', 'title': 'Cont', 'render': {'_': 'value', 'sort': 'value', 'type': 'value'}}, - {'data': 'a.1.one', 'title': 'One (A, Aa)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'a.2.one', 'title': 'One (A, Bb)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'a.3.one', 'title': 'One (A, Cc)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'b.1.one', 'title': 'One (B, Aa)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'b.2.one', 'title': 'One (B, Bb)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'b.3.one', 'title': 'One (B, Cc)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'a.1.two', 'title': 'Two (A, Aa)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'a.2.two', 'title': 'Two (A, Bb)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'a.3.two', 'title': 'Two (A, Cc)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'b.1.two', 'title': 'Two (B, Aa)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'b.2.two', 'title': 'Two (B, Bb)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'b.3.two', 'title': 'Two (B, Cc)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}]}, - result) - - def test_rollup_cont_cat_uni_dims_multi_metric_df(self): - # Tests transformation of two metrics with a continuous, a categorical and a unique dimensions - result = self.dt_tx.transform(mock_df.rollup_cont_cat_uni_dims_multi_metric_df, - mock_df.rollup_cont_cat_uni_dims_multi_metric_schema) - - self.assertDictEqual({'data': [ - {'cont': {'value': 0}, - '_total': {'_total': {'one': {'display': '30', 'value': 30}, 'two': {'display': '1230', 'value': 1230}}}, - 'b': {1: {'one': {'display': '3', 'value': 3}, 'two': {'display': '103', 'value': 103}}, - 2: {'one': {'display': '4', 'value': 4}, 'two': {'display': '104', 'value': 104}}, - 3: {'one': {'display': '5', 'value': 5}, 'two': {'display': '105', 'value': 105}}, - '_total': {'one': {'display': '12', 'value': 12}, 'two': {'display': '312', 'value': 312}}}, - 'a': {1: {'one': {'display': '0', 'value': 0}, 'two': {'display': '100', 'value': 100}}, - 2: {'one': {'display': '1', 'value': 1}, 'two': {'display': '101', 'value': 101}}, - 3: {'one': {'display': '2', 'value': 2}, 'two': {'display': '102', 'value': 102}}, - '_total': {'one': {'display': '3', 'value': 3}, 'two': {'display': '303', 'value': 303}}}}, - - {'cont': {'value': 1}, - '_total': {'_total': {'one': {'display': '102', 'value': 102}, 'two': {'display': '1302', 'value': 1302}}}, - 'b': {1: {'one': {'display': '9', 'value': 9}, 'two': {'display': '109', 'value': 109}}, - 2: {'one': {'display': '10', 'value': 10}, 'two': {'display': '110', 'value': 110}}, - 3: {'one': {'display': '11', 'value': 11}, 'two': {'display': '111', 'value': 111}}, - '_total': {'one': {'display': '30', 'value': 30}, 'two': {'display': '330', 'value': 330}}}, - 'a': {1: {'one': {'display': '6', 'value': 6}, 'two': {'display': '106', 'value': 106}}, - 2: {'one': {'display': '7', 'value': 7}, 'two': {'display': '107', 'value': 107}}, - 3: {'one': {'display': '8', 'value': 8}, 'two': {'display': '108', 'value': 108}}, - '_total': {'one': {'display': '21', 'value': 21}, 'two': {'display': '321', 'value': 321}}}}, - {'cont': {'value': 2}, - '_total': {'_total': {'one': {'display': '174', 'value': 174}, 'two': {'display': '1374', 'value': 1374}}}, - 'b': {1: {'one': {'display': '15', 'value': 15}, 'two': {'display': '115', 'value': 115}}, - 2: {'one': {'display': '16', 'value': 16}, 'two': {'display': '116', 'value': 116}}, - 3: {'one': {'display': '17', 'value': 17}, 'two': {'display': '117', 'value': 117}}, - '_total': {'one': {'display': '48', 'value': 48}, 'two': {'display': '348', 'value': 348}}}, - 'a': {1: {'one': {'display': '12', 'value': 12}, 'two': {'display': '112', 'value': 112}}, - 2: {'one': {'display': '13', 'value': 13}, 'two': {'display': '113', 'value': 113}}, - 3: {'one': {'display': '14', 'value': 14}, 'two': {'display': '114', 'value': 114}}, - '_total': {'one': {'display': '39', 'value': 39}, 'two': {'display': '339', 'value': 339}}}}, - {'cont': {'value': 3}, - '_total': {'_total': {'one': {'display': '246', 'value': 246}, 'two': {'display': '1446', 'value': 1446}}}, - 'b': {1: {'one': {'display': '21', 'value': 21}, 'two': {'display': '121', 'value': 121}}, - 2: {'one': {'display': '22', 'value': 22}, 'two': {'display': '122', 'value': 122}}, - 3: {'one': {'display': '23', 'value': 23}, 'two': {'display': '123', 'value': 123}}, - '_total': {'one': {'display': '66', 'value': 66}, 'two': {'display': '366', 'value': 366}}}, - 'a': {1: {'one': {'display': '18', 'value': 18}, 'two': {'display': '118', 'value': 118}}, - 2: {'one': {'display': '19', 'value': 19}, 'two': {'display': '119', 'value': 119}}, - 3: {'one': {'display': '20', 'value': 20}, 'two': {'display': '120', 'value': 120}}, - '_total': {'one': {'display': '57', 'value': 57}, 'two': {'display': '357', 'value': 357}}}}, - {'cont': {'value': 4}, - '_total': {'_total': {'one': {'display': '318', 'value': 318}, 'two': {'display': '1518', 'value': 1518}}}, - 'b': {1: {'one': {'display': '27', 'value': 27}, 'two': {'display': '127', 'value': 127}}, - 2: {'one': {'display': '28', 'value': 28}, 'two': {'display': '128', 'value': 128}}, - 3: {'one': {'display': '29', 'value': 29}, 'two': {'display': '129', 'value': 129}}, - '_total': {'one': {'display': '84', 'value': 84}, 'two': {'display': '384', 'value': 384}}}, - 'a': {1: {'one': {'display': '24', 'value': 24}, 'two': {'display': '124', 'value': 124}}, - 2: {'one': {'display': '25', 'value': 25}, 'two': {'display': '125', 'value': 125}}, - 3: {'one': {'display': '26', 'value': 26}, 'two': {'display': '126', 'value': 126}}, - '_total': {'one': {'display': '75', 'value': 75}, 'two': {'display': '375', 'value': 375}}}}, - {'cont': {'value': 5}, - '_total': {'_total': {'one': {'display': '390', 'value': 390}, 'two': {'display': '1590', 'value': 1590}}}, - 'b': {1: {'one': {'display': '33', 'value': 33}, 'two': {'display': '133', 'value': 133}}, - 2: {'one': {'display': '34', 'value': 34}, 'two': {'display': '134', 'value': 134}}, - 3: {'one': {'display': '35', 'value': 35}, 'two': {'display': '135', 'value': 135}}, - '_total': {'one': {'display': '102', 'value': 102}, 'two': {'display': '402', 'value': 402}}}, - 'a': {1: {'one': {'display': '30', 'value': 30}, 'two': {'display': '130', 'value': 130}}, - 2: {'one': {'display': '31', 'value': 31}, 'two': {'display': '131', 'value': 131}}, - 3: {'one': {'display': '32', 'value': 32}, 'two': {'display': '132', 'value': 132}}, - '_total': {'one': {'display': '93', 'value': 93}, 'two': {'display': '393', 'value': 393}}}}, - {'cont': {'value': 6}, - '_total': {'_total': {'one': {'display': '462', 'value': 462}, 'two': {'display': '1662', 'value': 1662}}}, - 'b': {1: {'one': {'display': '39', 'value': 39}, 'two': {'display': '139', 'value': 139}}, - 2: {'one': {'display': '40', 'value': 40}, 'two': {'display': '140', 'value': 140}}, - 3: {'one': {'display': '41', 'value': 41}, 'two': {'display': '141', 'value': 141}}, - '_total': {'one': {'display': '120', 'value': 120}, 'two': {'display': '420', 'value': 420}}}, - 'a': {1: {'one': {'display': '36', 'value': 36}, 'two': {'display': '136', 'value': 136}}, - 2: {'one': {'display': '37', 'value': 37}, 'two': {'display': '137', 'value': 137}}, - 3: {'one': {'display': '38', 'value': 38}, 'two': {'display': '138', 'value': 138}}, - '_total': {'one': {'display': '111', 'value': 111}, 'two': {'display': '411', 'value': 411}}}}, - {'cont': {'value': 7}, - '_total': {'_total': {'one': {'display': '534', 'value': 534}, 'two': {'display': '1734', 'value': 1734}}}, - 'b': {1: {'one': {'display': '45', 'value': 45}, 'two': {'display': '145', 'value': 145}}, - 2: {'one': {'display': '46', 'value': 46}, 'two': {'display': '146', 'value': 146}}, - 3: {'one': {'display': '47', 'value': 47}, 'two': {'display': '147', 'value': 147}}, - '_total': {'one': {'display': '138', 'value': 138}, 'two': {'display': '438', 'value': 438}}}, - 'a': {1: {'one': {'display': '42', 'value': 42}, 'two': {'display': '142', 'value': 142}}, - 2: {'one': {'display': '43', 'value': 43}, 'two': {'display': '143', 'value': 143}}, - 3: {'one': {'display': '44', 'value': 44}, 'two': {'display': '144', 'value': 144}}, - '_total': {'one': {'display': '129', 'value': 129}, 'two': {'display': '429', 'value': 429}}}}], - 'columns': [{'title': 'Cont', 'data': 'cont', 'render': {'_': 'value', 'type': 'value', 'sort': 'value'}}, - {'title': 'One', 'data': '_total._total.one', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'One (A, Aa)', 'data': 'a.1.one', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'One (A, Bb)', 'data': 'a.2.one', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'One (A, Cc)', 'data': 'a.3.one', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'One (A)', 'data': 'a._total.one', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'One (B, Aa)', 'data': 'b.1.one', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'One (B, Bb)', 'data': 'b.2.one', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'One (B, Cc)', 'data': 'b.3.one', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'One (B)', 'data': 'b._total.one', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'Two', 'data': '_total._total.two', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'Two (A, Aa)', 'data': 'a.1.two', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'Two (A, Bb)', 'data': 'a.2.two', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'Two (A, Cc)', 'data': 'a.3.two', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'Two (A)', 'data': 'a._total.two', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'Two (B, Aa)', 'data': 'b.1.two', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'Two (B, Bb)', 'data': 'b.2.two', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'Two (B, Cc)', 'data': 'b.3.two', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'title': 'Two (B)', 'data': 'b._total.two', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}]}, - result) - - def test_rollup_cont_cat_cat_dims_multi_metric_df(self): - # Tests transformation of two metrics with a continuous and two categorical dimensions - result = self.dt_tx.transform(mock_df.rollup_cont_cat_cat_dims_multi_metric_df, - mock_df.rollup_cont_cat_cat_dims_multi_metric_schema) - self.assertDictEqual({ - 'columns': [{'data': 'cont', 'title': 'Cont', - 'render': {'_': 'value', 'type': 'value', 'sort': 'value'}}, - {'data': '_total._total.one', 'title': 'One', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'a._total.one', 'title': 'One (A)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'a.y.one', 'title': 'One (A, Y)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'a.z.one', 'title': 'One (A, Z)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'b._total.one', 'title': 'One (B)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'b.y.one', 'title': 'One (B, Y)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'b.z.one', 'title': 'One (B, Z)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': '_total._total.two', 'title': 'Two', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'a._total.two', 'title': 'Two (A)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'a.y.two', 'title': 'Two (A, Y)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'a.z.two', 'title': 'Two (A, Z)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'b._total.two', 'title': 'Two (B)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'b.y.two', 'title': 'Two (B, Y)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}, - {'data': 'b.z.two', 'title': 'Two (B, Z)', - 'render': {'_': 'display', 'type': 'value', 'sort': 'value'}}], - 'data': [{'b': {'z': {'two': {'value': 6, 'display': '6'}, 'one': {'value': 3, 'display': '3'}}, - Totals.key: {'two': {'value': 10, 'display': '10'}, 'one': {'value': 5, 'display': '5'}}, - 'y': {'two': {'value': 4, 'display': '4'}, 'one': {'value': 2, 'display': '2'}}}, - 'a': {'z': {'two': {'value': 2, 'display': '2'}, 'one': {'value': 1, 'display': '1'}}, - Totals.key: {'two': {'value': 2, 'display': '2'}, 'one': {'value': 1, 'display': '1'}}, - 'y': {'two': {'value': 0, 'display': '0'}, 'one': {'value': 0, 'display': '0'}}}, - 'cont': {'value': 0}, - Totals.key: { - Totals.key: {'two': {'value': 24, 'display': '24'}, 'one': {'value': 12, 'display': '12'}}}}, - {'b': {'z': {'two': {'value': 14, 'display': '14'}, 'one': {'value': 7, 'display': '7'}}, - Totals.key: {'two': {'value': 26, 'display': '26'}, 'one': {'value': 13, 'display': '13'}}, - 'y': {'two': {'value': 12, 'display': '12'}, 'one': {'value': 6, 'display': '6'}}}, - 'a': {'z': {'two': {'value': 10, 'display': '10'}, 'one': {'value': 5, 'display': '5'}}, - Totals.key: {'two': {'value': 18, 'display': '18'}, 'one': {'value': 9, 'display': '9'}}, - 'y': {'two': {'value': 8, 'display': '8'}, 'one': {'value': 4, 'display': '4'}}}, - 'cont': {'value': 1}, - Totals.key: { - Totals.key: {'two': {'value': 88, 'display': '88'}, 'one': {'value': 44, 'display': '44'}}}}, - {'b': {'z': {'two': {'value': 22, 'display': '22'}, 'one': {'value': 11, 'display': '11'}}, - Totals.key: {'two': {'value': 42, 'display': '42'}, 'one': {'value': 21, 'display': '21'}}, - 'y': {'two': {'value': 20, 'display': '20'}, 'one': {'value': 10, 'display': '10'}}}, - 'a': {'z': {'two': {'value': 18, 'display': '18'}, 'one': {'value': 9, 'display': '9'}}, - Totals.key: {'two': {'value': 34, 'display': '34'}, 'one': {'value': 17, 'display': '17'}}, - 'y': {'two': {'value': 16, 'display': '16'}, 'one': {'value': 8, 'display': '8'}}}, - 'cont': {'value': 2}, - Totals.key: {Totals.key: {'two': {'value': 152, 'display': '152'}, - 'one': {'value': 76, 'display': '76'}}}}, - {'b': {'z': {'two': {'value': 30, 'display': '30'}, 'one': {'value': 15, 'display': '15'}}, - Totals.key: {'two': {'value': 58, 'display': '58'}, 'one': {'value': 29, 'display': '29'}}, - 'y': {'two': {'value': 28, 'display': '28'}, 'one': {'value': 14, 'display': '14'}}}, - 'a': {'z': {'two': {'value': 26, 'display': '26'}, 'one': {'value': 13, 'display': '13'}}, - Totals.key: {'two': {'value': 50, 'display': '50'}, 'one': {'value': 25, 'display': '25'}}, - 'y': {'two': {'value': 24, 'display': '24'}, 'one': {'value': 12, 'display': '12'}}}, - 'cont': {'value': 3}, - Totals.key: {Totals.key: {'two': {'value': 216, 'display': '216'}, - 'one': {'value': 108, 'display': '108'}}}}, - {'b': {'z': {'two': {'value': 38, 'display': '38'}, 'one': {'value': 19, 'display': '19'}}, - Totals.key: {'two': {'value': 74, 'display': '74'}, 'one': {'value': 37, 'display': '37'}}, - 'y': {'two': {'value': 36, 'display': '36'}, 'one': {'value': 18, 'display': '18'}}}, - 'a': {'z': {'two': {'value': 34, 'display': '34'}, 'one': {'value': 17, 'display': '17'}}, - Totals.key: {'two': {'value': 66, 'display': '66'}, 'one': {'value': 33, 'display': '33'}}, - 'y': {'two': {'value': 32, 'display': '32'}, 'one': {'value': 16, 'display': '16'}}}, - 'cont': {'value': 4}, - Totals.key: {Totals.key: {'two': {'value': 280, 'display': '280'}, - 'one': {'value': 140, 'display': '140'}}}}, - {'b': {'z': {'two': {'value': 46, 'display': '46'}, 'one': {'value': 23, 'display': '23'}}, - Totals.key: {'two': {'value': 90, 'display': '90'}, 'one': {'value': 45, 'display': '45'}}, - 'y': {'two': {'value': 44, 'display': '44'}, 'one': {'value': 22, 'display': '22'}}}, - 'a': {'z': {'two': {'value': 42, 'display': '42'}, 'one': {'value': 21, 'display': '21'}}, - Totals.key: {'two': {'value': 82, 'display': '82'}, 'one': {'value': 41, 'display': '41'}}, - 'y': {'two': {'value': 40, 'display': '40'}, 'one': {'value': 20, 'display': '20'}}}, - 'cont': {'value': 5}, - Totals.key: {Totals.key: {'two': {'value': 344, 'display': '344'}, - 'one': {'value': 172, 'display': '172'}}}}, - {'b': {'z': {'two': {'value': 54, 'display': '54'}, 'one': {'value': 27, 'display': '27'}}, - Totals.key: {'two': {'value': 106, 'display': '106'}, - 'one': {'value': 53, 'display': '53'}}, - 'y': {'two': {'value': 52, 'display': '52'}, 'one': {'value': 26, 'display': '26'}}}, - 'a': {'z': {'two': {'value': 50, 'display': '50'}, 'one': {'value': 25, 'display': '25'}}, - Totals.key: {'two': {'value': 98, 'display': '98'}, 'one': {'value': 49, 'display': '49'}}, - 'y': {'two': {'value': 48, 'display': '48'}, 'one': {'value': 24, 'display': '24'}}}, - 'cont': {'value': 6}, - Totals.key: {Totals.key: {'two': {'value': 408, 'display': '408'}, - 'one': {'value': 204, 'display': '204'}}}}, - {'b': {'z': {'two': {'value': 62, 'display': '62'}, 'one': {'value': 31, 'display': '31'}}, - Totals.key: {'two': {'value': 122, 'display': '122'}, - 'one': {'value': 61, 'display': '61'}}, - 'y': {'two': {'value': 60, 'display': '60'}, 'one': {'value': 30, 'display': '30'}}}, - 'a': {'z': {'two': {'value': 58, 'display': '58'}, 'one': {'value': 29, 'display': '29'}}, - Totals.key: {'two': {'value': 114, 'display': '114'}, - 'one': {'value': 57, 'display': '57'}}, - 'y': {'two': {'value': 56, 'display': '56'}, 'one': {'value': 28, 'display': '28'}}}, - 'cont': {'value': 7}, - Totals.key: {Totals.key: {'two': {'value': 472, 'display': '472'}, - 'one': {'value': 236, 'display': '236'}}}} - ]}, - result) - - def test_max_cols(self, ): - settings.datatables_maxcols = 24 - - df = mock_df.cont_cat_cat_dims_multi_metric_df.reorder_levels([1, 2, 0]) - schema = mock_df.cont_cat_cat_dims_multi_metric_schema.copy() - schema['dimensions'] = OrderedDict([(k, schema['dimensions'][k]) - for k in ['cat1', 'cat2', 'cont']]) - - result = self.dt_tx.transform(df, schema) - - self.assertEqual(24, len(result['columns'])) - - for column in result['columns']: - data_location = column['data'] - levels = data_location.split('.') - - data = result['data'][0] - for i, level in enumerate(levels): - self.assertIn(level, data, msg='Missing data level %d [%s] in %s.' % (i, level, data_location)) - data = data[level] - - -class DatatablesUtilityTests(TestCase): - def test_nan_data_point(self): - # np.nan is converted to None - result = datatables._safe(np.nan) - self.assertIsNone(result) - - def test_str_data_point(self): - result = datatables._safe(u'abc') - self.assertEqual('abc', result) - - def test_int64_data_point(self): - # Needs to be cast to python int - result = datatables._safe(np.int64(1)) - self.assertEqual(int(1), result) - - def test_date_data_point(self): - # Needs to be converted to milliseconds - result = datatables._safe(pd.Timestamp(date(2000, 1, 1))) - self.assertEqual('2000-01-01', result) - - def test_datetime_data_point(self): - # Needs to be converted to milliseconds - result = datatables._safe(pd.Timestamp(datetime(2000, 1, 1, 1))) - self.assertEqual('2000-01-01T01:00:00', result) - - def test_no_precision_with_zero(self): - result = datatables._pretty(0.0, {}) - self.assertEqual('0', result) - - def test_no_precision_with_non_zero(self): - result = datatables._pretty(0.01, {}) - self.assertEqual('0.01', result) - - def test_precision(self): - result = datatables._pretty(0.123456789, {'precision': 2}) - self.assertEqual('0.12', result) - - def test_zero_precision(self): - result = datatables._pretty(1784.0, {'precision': 0}) - self.assertEqual('1784', result) - - def test_prefix(self): - result = datatables._pretty(0.12, {'prefix': '$'}) - self.assertEqual('$0.12', result) - - def test_suffix(self): - result = datatables._pretty(0.12, {'suffix': '€'}) - self.assertEqual('0.12€', result) - - def test_format_value_does_not_prettify_none_string(self): - value = datatables._format_value('None', {}) - self.assertDictEqual(value, {'value': 'None', 'display': 'None'}) - - def test_format_value_does_prettify_non_none_strings(self): - value = datatables._format_value('abcde', {}) - self.assertDictEqual(value, {'value': 'abcde', 'display': 'abcde'}) - - def test_format_value_does_prettify_int_values(self): - value = datatables._format_value(123, {}) - self.assertDictEqual(value, {'value': 123, 'display': '123'}) - - def test_format_value_does_prettify_pandas_date_objects(self): - value = datatables._format_value(pd.Timestamp(date(2016, 5, 10)), {}) - self.assertDictEqual(value, {'value': '2016-05-10', 'display': '2016-05-10'}) diff --git a/fireant/tests/slicer/transformers/test_highcharts.py b/fireant/tests/slicer/transformers/test_highcharts.py deleted file mode 100644 index aaed989d..00000000 --- a/fireant/tests/slicer/transformers/test_highcharts.py +++ /dev/null @@ -1,715 +0,0 @@ -# coding: utf-8 -from datetime import date -from unittest import TestCase - -import numpy as np -import pandas as pd -from pypika import Table - -from fireant.slicer import ( - BooleanDimension, - CategoricalDimension, - ContinuousDimension, - DatetimeDimension, - Metric, - Slicer, - UniqueDimension, -) -from fireant.slicer.transformers import ( - HighchartsAreaPercentageTransformer, - HighchartsAreaTransformer, - HighchartsBarTransformer, - HighchartsColumnTransformer, - HighchartsLineTransformer, - HighchartsPieTransformer, - HighchartsStackedBarTransformer, - HighchartsStackedColumnTransformer, - TransformationException, - highcharts, -) -from fireant.tests import mock_dataframes as mock_df -from fireant.tests.database.mock_database import TestDatabase - - -class BaseHighchartsTransformerTests(TestCase): - """ - Test methods that are common to all Highcharts Transformers - """ - - def evaluate_tooltip_options(self, series, prefix=None, suffix=None, precision=None): - self.assertIn('tooltip', series) - - tooltip = series['tooltip'] - if prefix is not None: - self.assertIn('valuePrefix', tooltip) - self.assertEqual(prefix, tooltip['valuePrefix']) - if suffix is not None: - self.assertIn('valueSuffix', tooltip) - self.assertEqual(suffix, tooltip['valueSuffix']) - if precision is not None: - self.assertIn('valueDecimals', tooltip) - self.assertEqual(precision, tooltip['valueDecimals']) - - else: - self.assertSetEqual({'type'}, set(series['xAxis'].keys())) - - -class HighchartsLineTransformerTests(BaseHighchartsTransformerTests): - """ - Line charts work with the following requests: - - 1-cont-dim, *-metric - 1-cont-dim, *-dim, *-metric - """ - chart_type = HighchartsLineTransformer.chart_type - - @classmethod - def setUpClass(cls): - cls.hc_tx = HighchartsLineTransformer() - - test_table = Table('test_table') - test_db = TestDatabase() - cls.test_slicer = Slicer( - table=test_table, - database=test_db, - - dimensions=[ - BooleanDimension('bool', definition=test_table.clicks < 10), - CategoricalDimension('cat', definition=test_table.cat), - ContinuousDimension('cont', definition=test_table.clicks), - DatetimeDimension('date', definition=test_table.date), - UniqueDimension('uni', definition=test_table.uni_id, display_field=test_table.uni_name), - ], - metrics=[Metric('foo')], - ) - - def evaluate_chart_options(self, result, num_series=1, xaxis_type='linear', dash_style='Solid'): - self.assertSetEqual({'title', 'legend', 'series', 'chart', 'plotOptions', 'tooltip', 'xAxis', 'yAxis'}, - set(result.keys())) - self.assertEqual(num_series, len(result['series'])) - - self.assertSetEqual({'text'}, set(result['title'].keys())) - self.assertIsNone(result['title']['text']) - - self.assertEqual(self.chart_type, result['chart']['type']) - - self.assertSetEqual({'type'}, set(result['xAxis'].keys())) - self.assertEqual(xaxis_type, result['xAxis']['type']) - - for series in result['series']: - self.assertSetEqual({'name', 'data', 'tooltip', 'yAxis', 'color', 'dashStyle'}, set(series.keys())) - - def evaluate_result(self, df, result): - result_data = [series['data'] for series in result['series']] - - for data, (_, row) in zip(result_data, df.iteritems()): - self.assertListEqual(list(row.iteritems()), data) - - def test_require_dimensions(self): - with self.assertRaises(TransformationException): - self.hc_tx.prevalidate_request(self.test_slicer, [], [], [], [], [], []) - - def test_require_continuous_first_dimension(self): - # A ContinuousDimension type is required for the first dimension - self.hc_tx.prevalidate_request(self.test_slicer, [], ['cont'], [], [], [], []) - self.hc_tx.prevalidate_request(self.test_slicer, [], ['date'], [], [], [], []) - - with self.assertRaises(TransformationException): - self.hc_tx.prevalidate_request(self.test_slicer, [], ['cat'], [], [], [], []) - with self.assertRaises(TransformationException): - self.hc_tx.prevalidate_request(self.test_slicer, [], ['uni'], [], [], [], []) - - def test_series_single_metric(self): - # Tests transformation of a single-metric, single-dimension result - df = mock_df.cont_dim_single_metric_df - - result = self.hc_tx.transform(df, mock_df.cont_dim_single_metric_schema) - - self.evaluate_chart_options(result) - - self.assertSetEqual( - {'One'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df, result) - - def test_series_multi_metric(self): - # Tests transformation of a multi-metric, single-dimension result - df = mock_df.cont_dim_multi_metric_df - - result = self.hc_tx.transform(df, mock_df.cont_dim_multi_metric_schema) - - self.evaluate_chart_options(result, num_series=2) - - self.assertSetEqual( - {'One', 'Two'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df, result) - - def test_time_series_date_to_millis(self): - # Tests transformation of a single-metric, single-dimension result - df = mock_df.time_dim_single_metric_df - - result = self.hc_tx.transform(df, mock_df.time_dim_single_metric_schema) - - self.evaluate_chart_options(result, xaxis_type='datetime') - - self.assertSetEqual( - {'One'}, - {series['name'] for series in result['series']} - ) - - df2 = df.copy() - df2.index = df2.index.astype(int) // int(1e6) - self.evaluate_result(df2, result) - - def test_time_series_date_with_ref(self): - # Tests transformation of a single-metric, single-dimension result using a WoW reference - df = mock_df.time_dim_single_metric_ref_df - - result = self.hc_tx.transform(df, mock_df.time_dim_single_metric_ref_schema) - - self.evaluate_chart_options(result, num_series=2, xaxis_type='datetime') - - self.assertSetEqual( - {'One', 'One WoW'}, - {series['name'] for series in result['series']} - ) - - df2 = df.copy() - df2.index = df2.index.astype(int) // int(1e6) - self.evaluate_result(df2, result) - - def test_cont_uni_dim_single_metric(self): - # Tests transformation of a metric and a unique dimension - df = mock_df.cont_uni_dims_single_metric_df - - result = self.hc_tx.transform(df, mock_df.cont_uni_dims_single_metric_schema) - - self.evaluate_chart_options(result, num_series=3) - - self.assertSetEqual( - {'One (Aa)', 'One (Bb)', 'One (Cc)'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df.unstack(level=[1, 2]), result) - - def test_cont_uni_dim_multi_metric(self): - # Tests transformation of two metrics and a unique dimension - df = mock_df.cont_uni_dims_multi_metric_df - - result = self.hc_tx.transform(df, mock_df.cont_uni_dims_multi_metric_schema) - - self.evaluate_chart_options(result, num_series=6) - - self.assertSetEqual( - {'One (Aa)', 'One (Bb)', 'One (Cc)', 'Two (Aa)', 'Two (Bb)', 'Two (Cc)'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df.unstack(level=[1, 2]), result) - - def test_cont_bool_dim_single_metric(self): - # Tests transformation of a metric and a boolean dimension - df = mock_df.cont_bool_dims_single_metric_df - - result = self.hc_tx.transform(df, mock_df.cont_bool_dims_single_metric_schema) - - self.evaluate_chart_options(result, num_series=3) - - self.assertSetEqual( - {'One (Null)', 'One (False)', 'One (True)'}, - {series['name'] for series in result['series']} - ) - self.evaluate_result(df.unstack(level=1), result) - - def test_cont_bool_dim_multi_metric(self): - # Tests transformation of two metrics and a boolean dimension - df = mock_df.cont_bool_dims_multi_metric_df - - result = self.hc_tx.transform(df, mock_df.cont_bool_dims_multi_metric_schema) - - self.evaluate_chart_options(result, num_series=6) - - self.assertSetEqual( - {'One (Null)', 'One (False)', 'One (True)', 'Two (Null)', 'Two (False)', 'Two (True)'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df.unstack(level=[1]), result) - - def test_double_dimension_single_metric(self): - # Tests transformation of a single-metric, double-dimension result - df = mock_df.cont_cat_dims_single_metric_df - - result = self.hc_tx.transform(df, mock_df.cont_cat_dims_single_metric_schema) - - self.evaluate_chart_options(result, num_series=2) - - self.assertSetEqual( - {'One (A)', 'One (B)'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df.unstack(level=1), result) - - def test_double_dimension_multi_metric(self): - # Tests transformation of a multi-metric, double-dimension result - df = mock_df.cont_cat_dims_multi_metric_df - - result = self.hc_tx.transform(df, mock_df.cont_cat_dims_multi_metric_schema) - - self.evaluate_chart_options(result, num_series=4) - - self.assertSetEqual( - {'One (A)', 'One (B)', 'Two (A)', 'Two (B)'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df.unstack(level=1), result) - - def test_triple_dimension_multi_metric(self): - # Tests transformation of a multi-metric, double-dimension result - df = mock_df.cont_cat_cat_dims_multi_metric_df - - result = self.hc_tx.transform(df, mock_df.cont_cat_cat_dims_multi_metric_schema) - - self.evaluate_chart_options(result, num_series=8) - - self.assertSetEqual( - {'One (A, Y)', 'One (A, Z)', 'One (B, Y)', 'One (B, Z)', - 'Two (A, Y)', 'Two (A, Z)', 'Two (B, Y)', 'Two (B, Z)'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df.unstack(level=[1, 2]), result) - - def test_rollup_triple_dimension_multi_metric(self): - # Tests transformation of a multi-metric, double-dimension result - df = mock_df.rollup_cont_cat_cat_dims_multi_metric_df - - result = self.hc_tx.transform(df, mock_df.rollup_cont_cat_cat_dims_multi_metric_schema) - - self.evaluate_chart_options(result, num_series=14) - - self.assertSetEqual( - {'One', 'One (A)', 'One (A, Y)', 'One (A, Z)', 'One (B)', 'One (B, Y)', 'One (B, Z)', - 'Two', 'Two (A)', 'Two (A, Y)', 'Two (A, Z)', 'Two (B)', 'Two (B, Y)', 'Two (B, Z)'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df.unstack(level=[1, 2]), result) - - def test_cont_dim_pretty(self): - # Tests transformation of two metrics and a unique dimension - df = mock_df.cont_dim_pretty_df - - result = self.hc_tx.transform(df, mock_df.cont_dim_pretty_schema) - - self.evaluate_chart_options(result) - - self.assertSetEqual( - {'One'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_tooltip_options(result['series'][0], prefix='!', suffix='~', precision=1) - self.evaluate_result(df, result) - - -class HighchartsAreaTransformerTests(HighchartsLineTransformerTests): - chart_type = HighchartsAreaTransformer.chart_type - - @classmethod - def setUpClass(cls): - super(HighchartsAreaTransformerTests, cls).setUpClass() - cls.hc_tx = HighchartsAreaTransformer() - - def evaluate_chart_options(self, result, num_series=1, xaxis_type='linear', dash_style='Solid'): - self.assertSetEqual({'title', 'series', 'chart', 'plotOptions', 'legend', 'tooltip', 'xAxis', 'yAxis'}, - set(result.keys())) - self.assertEqual(num_series, len(result['series'])) - - self.assertSetEqual({'text'}, set(result['title'].keys())) - self.assertIsNone(result['title']['text']) - - self.assertEqual(self.chart_type, result['chart']['type']) - - self.assertSetEqual({'type'}, set(result['xAxis'].keys())) - self.assertEqual(xaxis_type, result['xAxis']['type']) - - for series in result['series']: - self.assertSetEqual({'name', 'data', 'tooltip', 'color', 'dashStyle'}, set(series.keys())) - - -class HighchartsAreaPercentageTransformerTests(HighchartsAreaTransformerTests): - chart_type = HighchartsAreaPercentageTransformer.chart_type - - @classmethod - def setUpClass(cls): - super(HighchartsAreaPercentageTransformerTests, cls).setUpClass() - cls.hc_tx = HighchartsAreaPercentageTransformer() - - -class HighchartsColumnTransformerTests(TestCase): - """ - Bar and Column charts work with the following requests: - - 1-dim, *-metric - 2-dim, 1-metric - """ - chart_type = HighchartsColumnTransformer.chart_type - - @classmethod - def setUpClass(cls): - cls.hc_tx = HighchartsColumnTransformer() - - def evaluate_chart_options(self, result, num_results=1, categories=None): - self.assertSetEqual({'title', 'series', 'chart', 'legend', 'tooltip', 'xAxis', 'yAxis', 'plotOptions'}, - set(result.keys())) - self.assertEqual(num_results, len(result['series'])) - - self.assertSetEqual({'text'}, set(result['title'].keys())) - self.assertIsNone(result['title']['text']) - - self.assertEqual(self.chart_type, result['chart']['type']) - self.assertEqual('categorical', result['xAxis']['type']) - - if categories: - self.assertSetEqual({'type', 'categories'}, set(result['xAxis'].keys())) - - for series in result['series']: - self.assertSetEqual({'name', 'data', 'yAxis', 'color', 'tooltip'}, set(series.keys())) - - def evaluate_tooltip_options(self, series, prefix=None, suffix=None, precision=None): - self.assertIn('tooltip', series) - - tooltip = series['tooltip'] - if prefix is not None: - self.assertIn('valuePrefix', tooltip) - self.assertEqual(prefix, tooltip['valuePrefix']) - if suffix is not None: - self.assertIn('valueSuffix', tooltip) - self.assertEqual(suffix, tooltip['valueSuffix']) - if precision is not None: - self.assertIn('valueDecimals', tooltip) - self.assertEqual(precision, tooltip['valueDecimals']) - - else: - self.assertSetEqual({'type'}, set(series['xAxis'].keys())) - - def evaluate_result(self, df, result): - result_data = [series['data'] for series in result['series']] - - for data, (_, item) in zip(result_data, df.iteritems()): - self.assertListEqual(list(item.iteritems()), data) - - def test_no_dims_multi_metric(self): - df = mock_df.no_dims_multi_metric_df - - result = self.hc_tx.transform(df, mock_df.no_dims_multi_metric_schema) - - self.evaluate_chart_options(result, num_results=8) - - self.assertSetEqual( - {'One', 'Two', 'Three', 'Four', 'Five', 'Six', 'Seven', 'Eight'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df, result) - - def test_cat_dim_single_metric(self): - # Tests transformation of a single-metric, single-dimension result - df = mock_df.cat_dim_single_metric_df - - result = self.hc_tx.transform(df, mock_df.cat_dim_single_metric_schema) - - self.evaluate_chart_options(result, categories=['A', 'B']) - - self.assertSetEqual( - {'One'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df, result) - - def test_cat_dim_multi_metric(self): - # Tests transformation of a single-metric, single-dimension result - df = mock_df.cat_dim_multi_metric_df - - result = self.hc_tx.transform(df, mock_df.cat_dim_multi_metric_schema) - - self.evaluate_chart_options(result, num_results=2, categories=['A', 'B']) - - self.assertSetEqual( - {'One', 'Two'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df, result) - - def test_cat_cat_dim_single_metric(self): - # Tests transformation of a multi-metric, single-dimension result - df = mock_df.cat_cat_dims_single_metric_df - - result = self.hc_tx.transform(df, mock_df.cat_cat_dims_single_metric_schema) - - self.evaluate_chart_options(result, num_results=2, categories=['A', 'B']) - - self.assertSetEqual( - {'One (Y)', 'One (Z)'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df.unstack(), result) - - def test_cat_dim_multi_metric(self): - # Tests transformation of a single-metric, single-dimension result - df = mock_df.cat_dim_multi_metric_df - - result = self.hc_tx.transform(df, mock_df.cat_dim_multi_metric_schema) - - self.evaluate_chart_options(result, num_results=2, categories=['A', 'B']) - - self.assertSetEqual( - {'One', 'Two'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df, result) - - def test_cat_cat_dim_single_metric(self): - # Tests transformation of a multi-metric, single-dimension result - df = mock_df.cat_cat_dims_single_metric_df - - result = self.hc_tx.transform(df, mock_df.cat_cat_dims_single_metric_schema) - - self.evaluate_chart_options(result, num_results=2, categories=['A', 'B']) - - self.assertSetEqual( - {'One (Y)', 'One (Z)'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df.unstack(), result) - - def test_uni_dim_single_metric(self): - # Tests transformation of a metric and a unique dimension - df = mock_df.uni_dim_single_metric_df - - result = self.hc_tx.transform(df, mock_df.uni_dim_single_metric_schema) - - self.evaluate_chart_options(result, categories=['Uni_1', 'Uni_2', 'Uni_3']) - - self.assertSetEqual( - {'One'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df, result) - - def test_uni_dim_multi_metric(self): - # Tests transformation of two metrics and a unique dimension - df = mock_df.uni_dim_multi_metric_df - - result = self.hc_tx.transform(df, mock_df.uni_dim_multi_metric_schema) - - self.evaluate_chart_options(result, num_results=2, categories=['Uni_1', 'Uni_2', 'Uni_3']) - - self.assertSetEqual( - {'One', 'Two'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_result(df, result) - - def test_cont_dim_pretty(self): - # Tests transformation of two metrics and a unique dimension - df = mock_df.cont_dim_pretty_df - - result = self.hc_tx.transform(df, mock_df.cont_dim_pretty_schema) - - self.evaluate_chart_options(result) - - self.assertSetEqual( - {'One'}, - {series['name'] for series in result['series']} - ) - - self.evaluate_tooltip_options(result['series'][0], prefix='!', suffix='~', precision=1) - self.evaluate_result(df, result) - - -class HighchartsStackedColumnTransformerTests(HighchartsColumnTransformerTests): - chart_type = HighchartsStackedColumnTransformer.chart_type - - @classmethod - def setUpClass(cls): - cls.hc_tx = HighchartsStackedColumnTransformer() - - def evaluate_chart_options(self, result, num_results=1, categories=None): - self.assertSetEqual({'title', 'series', 'chart', 'legend', 'tooltip', 'xAxis', 'yAxis', 'plotOptions'}, - set(result.keys())) - self.assertEqual(num_results, len(result['series'])) - - self.assertSetEqual({'text'}, set(result['title'].keys())) - self.assertIsNone(result['title']['text']) - - self.assertEqual(self.chart_type, result['chart']['type']) - self.assertEqual('categorical', result['xAxis']['type']) - - if categories: - self.assertSetEqual({'type', 'categories'}, set(result['xAxis'].keys())) - - for series in result['series']: - self.assertSetEqual({'name', 'data', 'color', 'tooltip'}, set(series.keys())) - - -class HighchartsBarTransformerTests(HighchartsColumnTransformerTests): - chart_type = HighchartsBarTransformer.chart_type - - @classmethod - def setUpClass(cls): - cls.hc_tx = HighchartsBarTransformer() - - -class HighchartsStackedBarTransformerTests(HighchartsStackedColumnTransformerTests): - chart_type = HighchartsStackedBarTransformer.chart_type - - @classmethod - def setUpClass(cls): - cls.hc_tx = HighchartsStackedBarTransformer() - - -class HighchartsUtilityTests(TestCase): - def test_str_data_point(self): - result = highcharts._format_data_point('abc') - self.assertEqual('abc', result) - - def test_int64_data_point(self): - # Needs to be cast to python int - result = highcharts._format_data_point(np.int64(1)) - self.assertEqual(int(1), result) - - def test_datetime_data_point(self): - # Needs to be converted to milliseconds - result = highcharts._format_data_point(pd.Timestamp(date(2000, 1, 1))) - self.assertEqual(946684800000, result) - - def test_nan_data_point(self): - # Needs to be cast to python int - result = highcharts._format_data_point(np.nan) - self.assertIsNone(result) - - -class HighChartsPieChartTests(BaseHighchartsTransformerTests): - """ - Pie charts work with the following requests: - - 1-metric, *-dim - """ - type = HighchartsPieTransformer.chart_type - - @classmethod - def setUpClass(cls): - cls.hc_tx = HighchartsPieTransformer() - - def evaluate_chart_options(self, result, num_results=1): - self.assertSetEqual({'title', 'series', 'legend', 'chart', 'tooltip', 'plotOptions'}, set(result.keys())) - self.assertEqual(num_results, len(result['series'])) - - self.assertSetEqual({'text'}, set(result['title'].keys())) - self.assertIsNone(result['title']['text']) - - self.assertEqual(self.type, result['chart']['type']) - - for series in result['series']: - self.assertSetEqual({'name', 'data'}, set(series.keys())) - - def test_no_dims_single_metric(self): - # Tests transformation of a single-metric, no-dimension result - df = mock_df.no_dims_single_metric_df - - result = self.hc_tx.transform(df, mock_df.no_dims_single_metric_schema) - self.evaluate_chart_options(result, num_results=1) - self.assertEqual(result['series'][0], {'name': 'One', 'data': [('', 0.0)]}) - - def test_cat_dim_single_metric(self): - # Tests transformation of a single-metric, single-dimension result - df = mock_df.cat_dim_single_metric_df - result = self.hc_tx.transform(df, mock_df.cat_dim_single_metric_schema) - result_series = result['series'][0] - self.assertEqual(result_series, {'data': [('A', 0.0), ('B', 1.0)], 'name': 'One'}) - - def test_uni_dim_single_metric(self): - # Tests transformation of a single metric and a unique dimension - df = mock_df.uni_dim_single_metric_df - result = self.hc_tx.transform(df, mock_df.uni_dim_single_metric_schema) - result_series = result['series'][0] - self.assertEqual(result_series, {'data': [('Aa', 0.0), ('Bb', 1.0), ('Cc', 2.0)], 'name': 'One'}) - - def test_date_dim_single_metric(self): - # Tests transformation of a single metric and a datetime dimension - df = mock_df.time_dim_single_metric_df - result = self.hc_tx.transform(df, mock_df.time_dim_single_metric_schema) - result_series = result['series'][0] - self.assertEqual(result_series, { - 'data': [ - ('2000-01-01 00:00:00', 0), - ('2000-01-02 00:00:00', 1), - ('2000-01-03 00:00:00', 2), - ('2000-01-04 00:00:00', 3), - ('2000-01-05 00:00:00', 4), - ('2000-01-06 00:00:00', 5), - ('2000-01-07 00:00:00', 6), - ('2000-01-08 00:00:00', 7), - ], 'name': 'One' - }) - - def test_cat_cat_and_a_single_metric(self): - # Tests transformation of two categorical dimensions with a single metric - df = mock_df.cat_cat_dims_single_metric_df - result = self.hc_tx.transform(df, mock_df.cat_cat_dims_single_metric_schema) - result_series = result['series'][0] - self.assertEqual(result_series, - {'data': [('(A, Y)', 0.0), ('(A, Z)', 1.0), ('(B, Y)', 2.0), ('(B, Z)', 3.0)], 'name': 'One'}) - - def test_cont_cat_uni_and_a_single_metric(self): - # Tests transformation of a categorical and unique dimensions with a single metric - df = mock_df.cont_cat_uni_dims_multi_metric_df - result = self.hc_tx.transform(df, mock_df.cont_cat_uni_dims_multi_metric_schema) - result_series = result['series'][0] - self.assertEqual(result_series, - { - 'data': [('(0, A, Aa)', 0.0), ('(0, A, Bb)', 1.0), ('(0, A, Cc)', 2.0), - ('(0, B, Aa)', 3.0), - ('(0, B, Bb)', 4.0), ('(0, B, Cc)', 5.0), ('(1, A, Aa)', 6.0), - ('(1, A, Bb)', 7.0), - ('(1, A, Cc)', 8.0), ('(1, B, Aa)', 9.0), ('(1, B, Bb)', 10.0), - ('(1, B, Cc)', 11.0), - ('(2, A, Aa)', 12.0), ('(2, A, Bb)', 13.0), ('(2, A, Cc)', 14.0), - ('(2, B, Aa)', 15.0), ('(2, B, Bb)', 16.0), ('(2, B, Cc)', 17.0), - ('(3, A, Aa)', 18.0), ('(3, A, Bb)', 19.0), ('(3, A, Cc)', 20.0), - ('(3, B, Aa)', 21.0), ('(3, B, Bb)', 22.0), ('(3, B, Cc)', 23.0), - ('(4, A, Aa)', 24.0), ('(4, A, Bb)', 25.0), ('(4, A, Cc)', 26.0), - ('(4, B, Aa)', 27.0), ('(4, B, Bb)', 28.0), ('(4, B, Cc)', 29.0), - ('(5, A, Aa)', 30.0), ('(5, A, Bb)', 31.0), ('(5, A, Cc)', 32.0), - ('(5, B, Aa)', 33.0), ('(5, B, Bb)', 34.0), ('(5, B, Cc)', 35.0), - ('(6, A, Aa)', 36.0), ('(6, A, Bb)', 37.0), ('(6, A, Cc)', 38.0), - ('(6, B, Aa)', 39.0), ('(6, B, Bb)', 40.0), ('(6, B, Cc)', 41.0), - ('(7, A, Aa)', 42.0), ('(7, A, Bb)', 43.0), ('(7, A, Cc)', 44.0), - ('(7, B, Aa)', 45.0), ('(7, B, Bb)', 46.0), ('(7, B, Cc)', 47.0)], - 'name': 'One' - }) - - def test_unique_dim_single_metric_pretty_tooltip(self): - # Tests transformation of a single metrics and a unique dimension with correct tooltip - df = mock_df.uni_dim_pretty_df - result = self.hc_tx.transform(df, mock_df.uni_dim_pretty_schema) - result_series = result['series'][0] - self.assertEqual(result_series, {'data': [('Aa', 0.0), ('Bb', 1.0), ('Cc', 2.0)], 'name': 'One'}) - self.evaluate_tooltip_options(result, prefix='!', suffix='~', precision=1) diff --git a/fireant/tests/slicer/transformers/test_notebooks.py b/fireant/tests/slicer/transformers/test_notebooks.py deleted file mode 100644 index 0fa9e2a1..00000000 --- a/fireant/tests/slicer/transformers/test_notebooks.py +++ /dev/null @@ -1,480 +0,0 @@ -# coding: utf-8 -from datetime import datetime -from unittest import TestCase - -from mock import ( - ANY, - MagicMock, - call, - patch, -) - -from fireant.slicer.operations import Totals -from fireant.slicer.transformers import ( - PandasRowIndexTransformer, - TransformationException, -) -from fireant.tests import mock_dataframes as mock_df - - -class PandasRowIndexTransformerTests(TestCase): - pd_tx = PandasRowIndexTransformer() - - def test_no_dims_multi_metric(self): - # Tests transformation of a single metric with a single continuous dimension - result = self.pd_tx.transform(mock_df.no_dims_multi_metric_df, mock_df.no_dims_multi_metric_schema) - - self.assertListEqual([None], list(result.index.names)) - self.assertListEqual([0], list(result.index)) - - self.assertListEqual(['One', 'Two', 'Three', 'Four', 'Five', 'Six', 'Seven', 'Eight'], list(result.columns)) - self.assertListEqual([a for a in range(8)], result.values[0].tolist()) - - def test_cont_dim_single_metric(self): - # Tests transformation of a single metric with a single continuous dimension - result = self.pd_tx.transform(mock_df.cont_dim_single_metric_df, mock_df.cont_dim_single_metric_schema) - - self.assertListEqual(['Cont'], list(result.index.names)) - self.assertListEqual([a for a in range(8)], list(result.index)) - - self.assertListEqual(['One'], list(result.columns)) - self.assertListEqual([a for a in range(8)], list(result['One'])) - - def test_cont_dim_multi_metric(self): - # Tests transformation of two metrics with a single continuous dimension - result = self.pd_tx.transform(mock_df.cont_dim_multi_metric_df, mock_df.cont_dim_multi_metric_schema) - - self.assertListEqual(['Cont'], list(result.index.names)) - self.assertListEqual([a for a in range(8)], list(result.index)) - - self.assertListEqual(['One', 'Two'], list(result.columns)) - self.assertListEqual([a for a in range(8)], list(result['One'])) - self.assertListEqual([2 * a for a in range(8)], list(result['Two'])) - - def test_time_series_date(self): - # Tests transformation of a single-metric, single-dimension result - result = self.pd_tx.transform(mock_df.time_dim_single_metric_df, mock_df.time_dim_single_metric_schema) - - self.assertListEqual(['Date'], list(result.index.names)) - self.assertListEqual([datetime(2000, 1, i) for i in range(1, 9)], list(result.index)) - - self.assertListEqual(['One'], list(result.columns)) - self.assertListEqual([a for a in range(8)], list(result['One'])) - - def test_time_series_date_with_ref(self): - # Tests transformation of a single-metric, single-dimension result using a WoW reference - result = self.pd_tx.transform(mock_df.time_dim_single_metric_ref_df, mock_df.time_dim_single_metric_ref_schema) - - self.assertListEqual(['Date'], list(result.index.names)) - self.assertListEqual([datetime(2000, 1, i) for i in range(1, 9)], list(result.index)) - - self.assertListEqual([('One', None), ('One', 'WoW')], list(result.columns)) - self.assertListEqual([a for a in range(8)], list(result[('One', None)])) - self.assertListEqual([2 * a for a in range(8)], list(result[('One', 'WoW')])) - - def test_uni_dim_single_metric(self): - # Tests transformation of a metric with a unique dimension with one key and display - result = self.pd_tx.transform(mock_df.uni_dim_single_metric_df, mock_df.uni_dim_single_metric_schema) - - self.assertListEqual(['Uni'], list(result.index.names)) - self.assertListEqual(['One'], list(result.columns)) - - def test_uni_dim_multi_metric(self): - # Tests transformation of a metric with a unique dimension with one key and display - result = self.pd_tx.transform(mock_df.uni_dim_multi_metric_df, mock_df.uni_dim_multi_metric_schema) - - self.assertListEqual(['Uni'], list(result.index.names)) - self.assertListEqual(['Aa', 'Bb', 'Cc'], list(result.index)) - - self.assertListEqual(['One', 'Two'], list(result.columns)) - self.assertListEqual([a for a in range(3)], list(result['One'])) - self.assertListEqual([2 * a for a in range(3)], list(result['Two'])) - - def test_cat_cat_dim_single_metric(self): - # Tests transformation of a single metric with two categorical dimensions - result = self.pd_tx.transform(mock_df.cat_cat_dims_single_metric_df, mock_df.cat_cat_dims_single_metric_schema) - - self.assertListEqual(['Cat1', 'Cat2'], list(result.index.names)) - self.assertListEqual(['A', 'B'], list(result.index.levels[0])) - self.assertListEqual(['Y', 'Z'], list(result.index.levels[1])) - - self.assertListEqual(['One'], list(result.columns)) - self.assertListEqual([a for a in range(4)], list(result['One'])) - - def test_cat_cat_dim_multi_metric(self): - # Tests transformation of two metrics with two categorical dimensions - result = self.pd_tx.transform(mock_df.cat_cat_dims_multi_metric_df, mock_df.cat_cat_dims_multi_metric_schema) - - self.assertListEqual(['Cat1', 'Cat2'], list(result.index.names)) - self.assertListEqual(['A', 'B'], list(result.index.levels[0])) - self.assertListEqual(['Y', 'Z'], list(result.index.levels[1])) - - self.assertListEqual(['One', 'Two'], list(result.columns)) - self.assertListEqual([a for a in range(4)], list(result['One'])) - self.assertListEqual([2 * a for a in range(4)], list(result['Two'])) - - def test_rollup_cont_cat_cat_dim_multi_metric(self): - # Tests transformation of two metrics with two categorical dimensions - df = mock_df.rollup_cont_cat_cat_dims_multi_metric_df - - result = self.pd_tx.transform(df, mock_df.rollup_cont_cat_cat_dims_multi_metric_schema) - - self.assertListEqual(['Cont', 'Cat1', 'Cat2'], list(result.index.names)) - self.assertListEqual([0, 1, 2, 3, 4, 5, 6, 7], list(result.index.levels[0])) - self.assertListEqual([Totals.label, 'A', 'B'], list(result.index.levels[1])) - self.assertListEqual([Totals.label, 'Y', 'Z'], list(result.index.levels[2])) - - self.assertListEqual(['One', 'Two'], list(result.columns)) - self.assertListEqual([12, 1, 0, 1, 5, 2, 3, - 44, 9, 4, 5, 13, 6, 7, - 76, 17, 8, 9, 21, 10, 11, - 108, 25, 12, 13, 29, 14, 15, - 140, 33, 16, 17, 37, 18, 19, - 172, 41, 20, 21, 45, 22, 23, - 204, 49, 24, 25, 53, 26, 27, - 236, 57, 28, 29, 61, 30, 31], list(result['One'])) - self.assertListEqual([24, 2, 0, 2, 10, 4, 6, - 88, 18, 8, 10, 26, 12, 14, - 152, 34, 16, 18, 42, 20, 22, - 216, 50, 24, 26, 58, 28, 30, - 280, 66, 32, 34, 74, 36, 38, - 344, 82, 40, 42, 90, 44, 46, - 408, 98, 48, 50, 106, 52, 54, - 472, 114, 56, 58, 122, 60, 62], list(result['Two'])) - - def test_empty_dataframe_is_handled(self): - df = mock_df.cat_cat_dims_single_metric_empty_df - - result = self.pd_tx.transform(df, mock_df.cat_cat_dims_single_metric_schema) - - self.assertTrue(result.empty) - self.assertEqual(result.index.names, ['Cat1', 'Cat2']) - self.assertEqual(list(result.index.levels[0]), []) - self.assertEqual(list(result.index.levels[1]), []) - self.assertEqual(list(result.columns), ['One']) - - -class PandasColumnIndexTransformerTests(TestCase): - pd_tx = PandasRowIndexTransformer() - - def test_no_dims_multi_metric(self): - # Tests transformation of a single metric with a single continuous dimension - result = self.pd_tx.transform(mock_df.no_dims_multi_metric_df, mock_df.no_dims_multi_metric_schema) - - self.assertListEqual([None], list(result.index.names)) - self.assertListEqual([0], list(result.index)) - - self.assertListEqual(['One', 'Two', 'Three', 'Four', 'Five', 'Six', 'Seven', 'Eight'], list(result.columns)) - self.assertListEqual([a for a in range(8)], result.values[0].tolist()) - - def test_cont_dim_single_metric(self): - # Tests transformation of a single metric with a single continuous dimension - result = self.pd_tx.transform(mock_df.cont_dim_single_metric_df, mock_df.cont_dim_single_metric_schema) - - self.assertListEqual(['Cont'], list(result.index.names)) - self.assertListEqual([a for a in range(8)], list(result.index)) - - self.assertListEqual(['One'], list(result.columns)) - self.assertListEqual([a for a in range(8)], list(result['One'])) - - def test_cont_dim_multi_metric(self): - # Tests transformation of two metrics with a single continuous dimension - result = self.pd_tx.transform(mock_df.cont_dim_multi_metric_df, mock_df.cont_dim_multi_metric_schema) - - self.assertListEqual(['Cont'], list(result.index.names)) - self.assertListEqual([a for a in range(8)], list(result.index)) - - self.assertListEqual(['One', 'Two'], list(result.columns)) - self.assertListEqual([a for a in range(8)], list(result['One'])) - self.assertListEqual([2 * a for a in range(8)], list(result['Two'])) - - def test_time_series_date_to_millis(self): - # Tests transformation of a single-metric, single-dimension result - result = self.pd_tx.transform(mock_df.time_dim_single_metric_df, mock_df.time_dim_single_metric_schema) - - self.assertListEqual(['Date'], list(result.index.names)) - self.assertListEqual([datetime(2000, 1, i) for i in range(1, 9)], list(result.index)) - - self.assertListEqual(['One'], list(result.columns)) - self.assertListEqual([a for a in range(8)], list(result['One'])) - - def test_time_series_date_with_ref(self): - # Tests transformation of a single-metric, single-dimension result using a WoW reference - result = self.pd_tx.transform(mock_df.time_dim_single_metric_ref_df, mock_df.time_dim_single_metric_ref_schema) - - self.assertListEqual(['Date'], list(result.index.names)) - self.assertListEqual([datetime(2000, 1, i) for i in range(1, 9)], list(result.index)) - - self.assertListEqual([('One', None), ('One', 'WoW')], list(result.columns)) - self.assertListEqual([a for a in range(8)], list(result[('One', None)])) - self.assertListEqual([2 * a for a in range(8)], list(result[('One', 'WoW')])) - - def test_cont_cat_dim_single_metric(self): - # Tests transformation of a single metric with a continuous and a categorical dimension - result = self.pd_tx.transform(mock_df.cont_cat_dims_single_metric_df, - mock_df.cont_cat_dims_single_metric_schema) - - self.assertListEqual(['Cont', 'Cat1'], list(result.index.names)) - self.assertListEqual([(a, b) - for a in range(8) - for b in ['A', 'B']], list(result.index)) - - self.assertListEqual(['One'], list(result.columns)) - self.assertListEqual([a for a in range(16)], list(result['One'])) - - def test_cont_cat_dim_multi_metric(self): - # Tests transformation of two metrics with a continuous and a categorical dimension - result = self.pd_tx.transform(mock_df.cont_cat_dims_multi_metric_df, mock_df.cont_cat_dims_multi_metric_schema) - - self.assertListEqual(['Cont', 'Cat1'], list(result.index.names)) - self.assertListEqual([(a, b) - for a in range(8) - for b in ['A', 'B']], list(result.index)) - - self.assertListEqual(['One', 'Two'], list(result.columns)) - self.assertListEqual([a for a in range(16)], list(result['One'])) - self.assertListEqual([2 * a for a in range(16)], list(result['Two'])) - - def test_cont_cat_cat_dim_multi_metric(self): - # Tests transformation of two metrics with a continuous and two categorical dimensions - result = self.pd_tx.transform(mock_df.cont_cat_cat_dims_multi_metric_df, - mock_df.cont_cat_cat_dims_multi_metric_schema) - - self.assertListEqual(['Cont', 'Cat1', 'Cat2'], list(result.index.names)) - self.assertListEqual([(a, b, c) - for a in range(8) - for b in ['A', 'B'] - for c in ['Y', 'Z']], list(result.index)) - - self.assertListEqual(['One', 'Two'], list(result.columns)) - self.assertListEqual([a for a in range(32)], list(result['One'])) - self.assertListEqual([2 * a for a in range(32)], list(result['Two'])) - - def test_cont_cat_uni_dim_multi_metric(self): - # Tests transformation of two metrics with a continuous and two categorical dimensions - result = self.pd_tx.transform(mock_df.cont_cat_uni_dims_multi_metric_df, - mock_df.cont_cat_uni_dims_multi_metric_schema) - - self.assertListEqual(['Cont', 'Cat1', 'Uni'], list(result.index.names)) - self.assertListEqual([(a, b, c) - for a in range(8) - for b in ['A', 'B'] - for c in ['Aa', 'Bb', 'Cc']], list(result.index)) - - self.assertListEqual(['One', 'Two'], list(result.columns)) - self.assertListEqual([a for a in range(48)], list(result['One'])) - self.assertListEqual([a for a in range(100, 148)], list(result['Two'])) - - def test_rollup_cont_cat_cat_dims_multi_metric_df(self): - # Tests transformation of two metrics with a continuous and two categorical dimensions - result = self.pd_tx.transform(mock_df.rollup_cont_cat_cat_dims_multi_metric_df, - mock_df.rollup_cont_cat_cat_dims_multi_metric_schema) - - self.assertListEqual(['Cont', 'Cat1', 'Cat2'], list(result.index.names)) - self.assertListEqual([0, 1, 2, 3, 4, 5, 6, 7], list(result.index.levels[0])) - self.assertListEqual([Totals.label, 'A', 'B'], list(result.index.levels[1])) - self.assertListEqual([Totals.label, 'Y', 'Z'], list(result.index.levels[2])) - - self.assertListEqual(['One', 'Two'], list(result.columns)) - self.assertListEqual([12, 1, 0, 1, 5, 2, 3, - 44, 9, 4, 5, 13, 6, 7, - 76, 17, 8, 9, 21, 10, 11, - 108, 25, 12, 13, 29, 14, 15, - 140, 33, 16, 17, 37, 18, 19, - 172, 41, 20, 21, 45, 22, 23, - 204, 49, 24, 25, 53, 26, 27, - 236, 57, 28, 29, 61, 30, 31], list(result['One'])) - self.assertListEqual([24, 2, 0, 2, 10, 4, 6, - 88, 18, 8, 10, 26, 12, 14, - 152, 34, 16, 18, 42, 20, 22, - 216, 50, 24, 26, 58, 28, 30, - 280, 66, 32, 34, 74, 36, 38, - 344, 82, 40, 42, 90, 44, 46, - 408, 98, 48, 50, 106, 52, 54, - 472, 114, 56, 58, 122, 60, 62], list(result['Two'])) - - -class MatplotlibLineChartTransformerTests(TestCase): - mock_matplotlib = MagicMock(name='mock_matplotlib') - patcher = patch.dict('sys.modules', { - 'matplotlib': mock_matplotlib, - 'matplotlib.pyplot': mock_matplotlib.pyplot, - }) - - def setUp(self): - self.patcher.start() - - mock_axes = MagicMock() - mock_axes.__iter__.return_value = iter([0, 1, 2, 3]) - self.mock_matplotlib.pyplot.subplots.return_value = MagicMock(), mock_axes - - def tearDown(self): - self.patcher.stop() - - from fireant.slicer.transformers import MatplotlibLineChartTransformer - plt_tx = MatplotlibLineChartTransformer() - - def _assert_matplotlib_calls(self, mock_plot, metrics): - if 1 == len(metrics): - calls = [call(figsize=(14, 5)), - call().legend(loc='center left', bbox_to_anchor=(1, 0.5)), - call().legend().set_title(metrics[0])] - else: - calls = [c - for metric in metrics - for c in [call(ax=ANY), - call().legend(loc='center left', bbox_to_anchor=(1, 0.5)), - call().legend().set_title(metric)]] - - mock_plot.line.assert_has_calls(calls) - - @patch('pandas.DataFrame.plot') - def test_series_single_metric(self, mock_plot): - # Tests transformation of a single-metric, single-dimension result - result = self.plt_tx.transform(mock_df.cont_dim_single_metric_df, mock_df.cont_dim_single_metric_schema) - - self._assert_matplotlib_calls(mock_plot, ['One']) - - @patch('pandas.Series.plot') - def test_series_multi_metric(self, mock_plot): - # Tests transformation of a multi-metric, single-dimension result - result = self.plt_tx.transform(mock_df.cont_dim_multi_metric_df, mock_df.cont_dim_multi_metric_schema) - - self._assert_matplotlib_calls(mock_plot, ['One', 'Two']) - - @patch('pandas.DataFrame.plot') - def test_time_series_date_to_millis(self, mock_plot): - # Tests transformation of a single-metric, single-dimension result - result = self.plt_tx.transform(mock_df.time_dim_single_metric_df, mock_df.time_dim_single_metric_schema) - - self._assert_matplotlib_calls(mock_plot, ['One']) - - @patch('pandas.DataFrame.plot') - def test_time_series_date_with_ref(self, mock_plot): - # Tests transformation of a single-metric, single-dimension result using a WoW reference - result = self.plt_tx.transform(mock_df.time_dim_single_metric_ref_df, mock_df.time_dim_single_metric_ref_schema) - - self._assert_matplotlib_calls(mock_plot, ['One']) - - @patch('pandas.DataFrame.plot') - def test_cont_uni_dim_single_metric(self, mock_plot): - # Tests transformation of a metric and a unique dimension - result = self.plt_tx.transform(mock_df.cont_uni_dims_single_metric_df, - mock_df.cont_uni_dims_single_metric_schema) - - self._assert_matplotlib_calls(mock_plot, ['One']) - - @patch('pandas.DataFrame.plot') - def test_cont_uni_dim_multi_metric(self, mock_plot): - # Tests transformation of two metrics and a unique dimension - result = self.plt_tx.transform(mock_df.cont_uni_dims_multi_metric_df, mock_df.cont_uni_dims_multi_metric_schema) - - self._assert_matplotlib_calls(mock_plot, ['One', 'Two']) - - @patch('pandas.DataFrame.plot') - def test_double_dimension_single_metric(self, mock_plot): - # Tests transformation of a single-metric, double-dimension result - result = self.plt_tx.transform(mock_df.cont_cat_dims_single_metric_df, - mock_df.cont_cat_dims_single_metric_schema) - - self._assert_matplotlib_calls(mock_plot, ['One']) - - @patch('pandas.DataFrame.plot') - def test_double_dimension_multi_metric(self, mock_plot): - # Tests transformation of a multi-metric, double-dimension result - result = self.plt_tx.transform(mock_df.cont_cat_dims_multi_metric_df, mock_df.cont_cat_dims_multi_metric_schema) - - self._assert_matplotlib_calls(mock_plot, ['One', 'Two']) - - @patch('pandas.DataFrame.plot') - def test_triple_dimension_multi_metric(self, mock_plot): - # Tests transformation of a multi-metric, double-dimension result - df = mock_df.cont_cat_cat_dims_multi_metric_df - - result = self.plt_tx.transform(df, mock_df.cont_cat_cat_dims_multi_metric_schema) - - self._assert_matplotlib_calls(mock_plot, ['One', 'Two']) - - @patch('pandas.DataFrame.plot') - def test_rollup_triple_dimension_multi_metric(self, mock_plot): - # Tests transformation of a multi-metric, double-dimension result - df = mock_df.rollup_cont_cat_cat_dims_multi_metric_df - - result = self.plt_tx.transform(df, mock_df.rollup_cont_cat_cat_dims_multi_metric_schema) - - self._assert_matplotlib_calls(mock_plot, ['One', 'Two']) - - def test_require_at_least_one_dimension(self): - df = mock_df.no_dims_multi_metric_df - - with self.assertRaises(TransformationException): - self.plt_tx.transform(df, mock_df.no_dims_multi_metric_schema) - - -class MatplotlibBarChartTransformerTests(TestCase): - mock_matplotlib = MagicMock(name='mock_matplotlib') - patcher = patch.dict('sys.modules', { - 'matplotlib': mock_matplotlib, - 'matplotlib.pyplot': mock_matplotlib.pyplot, - }) - - def setUp(self): - self.patcher.start() - - mock_axes = MagicMock() - mock_axes.__iter__.return_value = iter([0, 1, 2, 3]) - self.mock_matplotlib.pyplot.subplots.return_value = MagicMock(), mock_axes - - def tearDown(self): - self.patcher.stop() - - from fireant.slicer.transformers import MatplotlibBarChartTransformer - plt_tx = MatplotlibBarChartTransformer() - - def _assert_matplotlib_calls(self, mock_plot): - mock_plot.bar.assert_has_calls([call(figsize=(14, 5)), - call().legend(loc='center left', bbox_to_anchor=(1, 0.5))]) - - @patch('pandas.DataFrame.plot') - def test_series_single_metric(self, mock_plot): - # Tests transformation of a single-metric, single-dimension result - result = self.plt_tx.transform(mock_df.cont_dim_single_metric_df, mock_df.cont_dim_single_metric_schema) - - self._assert_matplotlib_calls(mock_plot) - - @patch('pandas.DataFrame.plot') - def test_series_multi_metric(self, mock_plot): - # Tests transformation of a multi-metric, single-dimension result - result = self.plt_tx.transform(mock_df.cont_dim_multi_metric_df, mock_df.cont_dim_multi_metric_schema) - - self._assert_matplotlib_calls(mock_plot) - - @patch('pandas.DataFrame.plot') - def test_time_series_date_to_millis(self, mock_plot): - # Tests transformation of a single-metric, single-dimension result - result = self.plt_tx.transform(mock_df.time_dim_single_metric_df, mock_df.time_dim_single_metric_schema) - - self._assert_matplotlib_calls(mock_plot) - - @patch('pandas.DataFrame.plot') - def test_time_series_date_with_ref(self, mock_plot): - # Tests transformation of a single-metric, single-dimension result using a WoW reference - result = self.plt_tx.transform(mock_df.time_dim_single_metric_ref_df, mock_df.time_dim_single_metric_ref_schema) - - self._assert_matplotlib_calls(mock_plot) - - @patch('pandas.DataFrame.plot') - def test_cont_uni_dim_single_metric(self, mock_plot): - # Tests transformation of a metric and a unique dimension - result = self.plt_tx.transform(mock_df.cont_uni_dims_single_metric_df, - mock_df.cont_uni_dims_single_metric_schema) - - self._assert_matplotlib_calls(mock_plot) - - @patch('pandas.DataFrame.plot') - def test_double_dimension_single_metric(self, mock_plot): - # Tests transformation of a single-metric, double-dimension result - result = self.plt_tx.transform(mock_df.cont_cat_dims_single_metric_df, - mock_df.cont_cat_dims_single_metric_schema) - - self._assert_matplotlib_calls(mock_plot) diff --git a/fireant/tests/slicer/widgets/__init__.py b/fireant/tests/slicer/widgets/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py new file mode 100644 index 00000000..f452be7d --- /dev/null +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -0,0 +1,480 @@ +import locale as lc +from unittest import ( + TestCase, +) + +import pandas as pd +from datetime import date +from mock import Mock + +from fireant.slicer.widgets.datatables import ( + DataTablesJS, + _format_metric_cell, +) +from fireant.tests.slicer.mocks import ( + mock_politics_database as mock_df, + slicer, +) + +lc.setlocale(lc.LC_ALL, 'C') + + +class DataTablesTransformerTests(TestCase): + maxDiff = None + + @classmethod + def setUpClass(cls): + cls.single_metric_df = pd.DataFrame(mock_df[['votes']] + .sum()).T + + cls.multi_metric_df = pd.DataFrame(mock_df[['votes', 'wins']] + .sum()).T + + cls.cont_dim_df = mock_df[['timestamp', 'votes', 'wins']] \ + .groupby('timestamp') \ + .sum() + + cls.cat_dim_df = mock_df[['political_party', 'votes', 'wins']] \ + .groupby('political_party') \ + .sum() + + cls.uni_dim_df = mock_df[['candidate', 'candidate_display', 'votes', 'wins']] \ + .groupby(['candidate', 'candidate_display']) \ + .sum() \ + .reset_index('candidate_display') + + cls.cont_cat_dim_df = mock_df[['timestamp', 'political_party', 'votes', 'wins']] \ + .groupby(['timestamp', 'political_party']) \ + .sum() + + cls.cont_uni_dim_df = mock_df[['timestamp', 'state', 'state_display', 'votes', 'wins']] \ + .groupby(['timestamp', 'state', 'state_display']) \ + .sum() \ + .reset_index('state_display') + + def test_single_metric(self): + result = DataTablesJS(metrics=[slicer.metrics.votes]) \ + .transform(self.single_metric_df, slicer) + + self.assertEqual({ + 'columns': [{ + 'data': 'votes', + 'title': 'Votes', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'votes': {'value': 111674336, 'display': '111674336'} + }], + }, result) + + def test_single_metric_with_dataframe_containing_more(self): + result = DataTablesJS(metrics=[slicer.metrics.votes]) \ + .transform(self.multi_metric_df, slicer) + + self.assertEqual({ + 'columns': [{ + 'data': 'votes', + 'title': 'Votes', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'votes': {'value': 111674336, 'display': '111674336'} + }], + }, result) + + def test_multiple_metrics(self): + result = DataTablesJS(metrics=[slicer.metrics.votes, slicer.metrics.wins]) \ + .transform(self.multi_metric_df, slicer) + + self.assertEqual({ + 'columns': [{ + 'data': 'votes', + 'title': 'Votes', + 'render': {'_': 'value', 'display': 'display'}, + }, { + 'data': 'wins', + 'title': 'Wins', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'votes': {'value': 111674336, 'display': '111674336'}, + 'wins': {'value': 12, 'display': '12'}, + }], + }, result) + + def test_multiple_metrics_reversed(self): + result = DataTablesJS(metrics=[slicer.metrics.wins, slicer.metrics.votes]) \ + .transform(self.multi_metric_df, slicer) + + self.assertEqual({ + 'columns': [{ + 'data': 'wins', + 'title': 'Wins', + 'render': {'_': 'value', 'display': 'display'}, + }, { + 'data': 'votes', + 'title': 'Votes', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'wins': {'value': 12, 'display': '12'}, + 'votes': {'value': 111674336, 'display': '111674336'}, + }], + }, result) + + def test_time_series_dim(self): + result = DataTablesJS(metrics=[slicer.metrics.wins]) \ + .transform(self.cont_dim_df, slicer) + + self.assertEqual({ + 'columns': [{ + 'data': 'timestamp', + 'title': 'Timestamp', + 'render': {'_': 'value'}, + }, { + 'data': 'wins', + 'title': 'Wins', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'timestamp': {'value': '1996-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'timestamp': {'value': '2000-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'timestamp': {'value': '2004-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'timestamp': {'value': '2008-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'timestamp': {'value': '2012-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'timestamp': {'value': '2016-01-01'}, + 'wins': {'display': '2', 'value': 2} + }], + }, result) + + def test_cat_dim(self): + result = DataTablesJS(metrics=[slicer.metrics.wins]) \ + .transform(self.cat_dim_df, slicer) + + self.assertEqual({ + 'columns': [{ + 'data': 'political_party', + 'title': 'Party', + 'render': {'_': 'value', 'display': 'display'}, + }, { + 'data': 'wins', + 'title': 'Wins', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'political_party': {'display': 'Democrat', 'value': 'd'}, + 'wins': {'display': '6', 'value': 6} + }, { + 'political_party': {'display': 'Independent', 'value': 'i'}, + 'wins': {'display': '0', 'value': 0} + }, { + 'political_party': {'display': 'Republican', 'value': 'r'}, + 'wins': {'display': '6', 'value': 6} + }], + }, result) + + def test_uni_dim(self): + result = DataTablesJS(metrics=[slicer.metrics.wins]) \ + .transform(self.uni_dim_df, slicer) + + print(result['data'][0]) + + self.assertEqual({ + 'columns': [{ + 'data': 'candidate', + 'render': {'_': 'value', 'display': 'display'}, + 'title': 'Candidate' + }, { + 'data': 'wins', + 'render': {'_': 'value', 'display': 'display'}, + 'title': 'Wins' + }], + 'data': [{ + 'candidate': {'display': 'Bill Clinton', 'value': 1}, + 'wins': {'display': '2', 'value': 2} + }, { + 'candidate': {'display': 'Bob Dole', 'value': 2}, + 'wins': {'display': '0', 'value': 0} + }, { + 'candidate': {'display': 'Ross Perot', 'value': 3}, + 'wins': {'display': '0', 'value': 0} + }, { + 'candidate': {'display': 'George Bush', 'value': 4}, + 'wins': {'display': '4', 'value': 4} + }, { + 'candidate': {'display': 'Al Gore', 'value': 5}, + 'wins': {'display': '0', 'value': 0} + }, { + 'candidate': {'display': 'John Kerry', 'value': 6}, + 'wins': {'display': '0', 'value': 0} + }, { + 'candidate': {'display': 'Barrack Obama', 'value': 7}, + 'wins': {'display': '4', 'value': 4} + }, { + 'candidate': {'display': 'John McCain', 'value': 8}, + 'wins': {'display': '0', 'value': 0} + }, { + 'candidate': {'display': 'Mitt Romney', 'value': 9}, + 'wins': {'display': '0', 'value': 0} + }, { + 'candidate': {'display': 'Donald Trump', 'value': 10}, + 'wins': {'display': '2', 'value': 2} + }, { + 'candidate': {'display': 'Hillary Clinton', 'value': 11}, + 'wins': {'display': '0', 'value': 0} + }], + }, result) + + def test_multi_dims_time_series_and_uni(self): + result = DataTablesJS(metrics=[slicer.metrics.wins]) \ + .transform(self.cont_uni_dim_df, slicer) + + self.assertEqual({ + 'columns': [{ + 'data': 'timestamp', + 'title': 'Timestamp', + 'render': {'_': 'value'}, + }, { + 'data': 'state', + 'render': {'_': 'value', 'display': 'display'}, + 'title': 'State' + }, { + 'data': 'wins', + 'title': 'Wins', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'timestamp': {'value': '1996-01-01'}, + 'state': {'display': 'Texas', 'value': 1}, + 'wins': {'display': '1', 'value': 1} + }, { + 'timestamp': {'value': '1996-01-01'}, + 'state': {'display': 'California', 'value': 2}, + 'wins': {'display': '1', 'value': 1} + }, { + 'timestamp': {'value': '2000-01-01'}, + 'state': {'display': 'Texas', 'value': 1}, + 'wins': {'display': '1', 'value': 1} + }, { + 'timestamp': {'value': '2000-01-01'}, + 'state': {'display': 'California', 'value': 2}, + 'wins': {'display': '1', 'value': 1} + }, { + 'timestamp': {'value': '2004-01-01'}, + 'state': {'display': 'Texas', 'value': 1}, + 'wins': {'display': '1', 'value': 1} + }, { + 'timestamp': {'value': '2004-01-01'}, + 'state': {'display': 'California', 'value': 2}, + 'wins': {'display': '1', 'value': 1} + }, { + 'timestamp': {'value': '2008-01-01'}, + 'state': {'display': 'Texas', 'value': 1}, + 'wins': {'display': '1', 'value': 1} + }, { + 'timestamp': {'value': '2008-01-01'}, + 'state': {'display': 'California', 'value': 2}, + 'wins': {'display': '1', 'value': 1} + }, { + 'timestamp': {'value': '2012-01-01'}, + 'state': {'display': 'Texas', 'value': 1}, + 'wins': {'display': '1', 'value': 1} + }, { + 'timestamp': {'value': '2012-01-01'}, + 'state': {'display': 'California', 'value': 2}, + 'wins': {'display': '1', 'value': 1} + }, { + 'timestamp': {'value': '2016-01-01'}, + 'state': {'display': 'Texas', 'value': 1}, + 'wins': {'display': '1', 'value': 1} + }, { + 'timestamp': {'value': '2016-01-01'}, + 'state': {'display': 'California', 'value': 2}, + 'wins': {'display': '1', 'value': 1} + }], + }, result) + + def test_pivoted_single_dimension_no_effect(self): + result = DataTablesJS(metrics=[slicer.metrics.wins], pivot=True) \ + .transform(self.cat_dim_df, slicer) + + self.assertEqual({ + 'columns': [{ + 'data': 'political_party', + 'title': 'Party', + 'render': {'_': 'value', 'display': 'display'}, + }, { + 'data': 'wins', + 'title': 'Wins', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'political_party': {'display': 'Democrat', 'value': 'd'}, + 'wins': {'display': '6', 'value': 6} + }, { + 'political_party': {'display': 'Independent', 'value': 'i'}, + 'wins': {'display': '0', 'value': 0} + }, { + 'political_party': {'display': 'Republican', 'value': 'r'}, + 'wins': {'display': '6', 'value': 6} + }], + }, result) + + def test_pivoted_multi_dims_time_series_and_cat(self): + result = DataTablesJS(metrics=[slicer.metrics.wins], pivot=True) \ + .transform(self.cont_cat_dim_df, slicer) + + self.assertEqual({ + 'columns': [{ + 'data': 'timestamp', + 'title': 'Timestamp', + 'render': {'_': 'value'}, + }, { + 'data': 'wins.d', + 'title': 'Wins (Democrat)', + 'render': {'_': 'value', 'display': 'display'}, + }, { + 'data': 'wins.i', + 'title': 'Wins (Independent)', + 'render': {'_': 'value', 'display': 'display'}, + }, { + 'data': 'wins.r', + 'title': 'Wins (Republican)', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'timestamp': {'value': '1996-01-01'}, + 'wins': { + 'd': {'display': '2', 'value': 2.0}, + 'i': {'display': '0', 'value': 0.0}, + 'r': {'display': '0', 'value': 0.0} + } + }, { + 'timestamp': {'value': '2000-01-01'}, + 'wins': { + 'd': {'display': '0', 'value': 0.0}, + 'i': {'display': '0', 'value': 0.0}, + 'r': {'display': '2', 'value': 2.0} + } + }, { + 'timestamp': {'value': '2004-01-01'}, + 'wins': { + 'd': {'display': '0', 'value': 0.0}, + 'i': {'display': '0', 'value': 0.0}, + 'r': {'display': '2', 'value': 2.0} + } + }, { + 'timestamp': {'value': '2008-01-01'}, + 'wins': { + 'd': {'display': '2', 'value': 2.0}, + 'i': {'display': '0', 'value': 0.0}, + 'r': {'display': '0', 'value': 0.0} + } + }, { + 'timestamp': {'value': '2012-01-01'}, + 'wins': { + 'd': {'display': '2', 'value': 2.0}, + 'i': {'display': '0', 'value': 0.0}, + 'r': {'display': '0', 'value': 0.0} + } + }, { + 'timestamp': {'value': '2016-01-01'}, + 'wins': { + 'd': {'display': '0', 'value': 0.0}, + 'i': {'display': '0', 'value': 0.0}, + 'r': {'display': '2', 'value': 2.0} + } + }], + }, result) + + def test_pivoted_multi_dims_time_series_and_uni(self): + result = DataTablesJS(metrics=[slicer.metrics.votes], pivot=True) \ + .transform(self.cont_uni_dim_df, slicer) + + self.assertEqual({ + 'columns': [{ + 'data': 'timestamp', + 'title': 'Timestamp', + 'render': {'_': 'value'}, + }, { + 'data': 'votes.1', + 'title': 'Votes (Texas)', + 'render': {'_': 'value', 'display': 'display'}, + }, { + 'data': 'votes.2', + 'title': 'Votes (California)', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'timestamp': {'value': '1996-01-01'}, + 'votes': { + 1: {'display': '5574387', 'value': 5574387}, + 2: {'display': '9646062', 'value': 9646062} + } + }, { + 'timestamp': {'value': '2000-01-01'}, + 'votes': { + 1: {'display': '6233385', 'value': 6233385}, + 2: {'display': '10428632', 'value': 10428632} + } + }, { + 'timestamp': {'value': '2004-01-01'}, + 'votes': { + 1: {'display': '7359621', 'value': 7359621}, + 2: {'display': '12255311', 'value': 12255311} + } + }, { + 'timestamp': {'value': '2008-01-01'}, + 'votes': { + 1: {'display': '8007961', 'value': 8007961}, + 2: {'display': '13286254', 'value': 13286254} + } + }, { + 'timestamp': {'value': '2012-01-01'}, + 'votes': { + 1: {'display': '7877967', 'value': 7877967}, + 2: {'display': '12694243', 'value': 12694243} + } + }, { + 'timestamp': {'value': '2016-01-01'}, + 'votes': { + 1: {'display': '5072915', 'value': 5072915}, + 2: {'display': '13237598', 'value': 13237598} + } + }], + }, result) + + +class MetricCellFormatTests(TestCase): + def _mock_metric(self, prefix=None, suffix=None, precision=None): + mock_metric = Mock() + mock_metric.prefix = prefix + mock_metric.suffix = suffix + mock_metric.precision = precision + return mock_metric + + def test_does_not_prettify_none_string(self): + value = _format_metric_cell('None', self._mock_metric()) + self.assertDictEqual(value, {'value': 'None', 'display': 'None'}) + + def test_does_prettify_non_none_strings(self): + value = _format_metric_cell('abcde', self._mock_metric()) + self.assertDictEqual(value, {'value': 'abcde', 'display': 'abcde'}) + + def test_does_prettify_int_values(self): + value = _format_metric_cell(123, self._mock_metric()) + self.assertDictEqual(value, {'value': 123, 'display': '123'}) + + def test_does_prettify_pandas_date_objects(self): + value = _format_metric_cell(pd.Timestamp(date(2016, 5, 10)), self._mock_metric()) + self.assertDictEqual(value, {'value': '2016-05-10', 'display': '2016-05-10'}) diff --git a/fireant/tests/slicer/widgets/test_formats.py b/fireant/tests/slicer/widgets/test_formats.py new file mode 100644 index 00000000..3c241f71 --- /dev/null +++ b/fireant/tests/slicer/widgets/test_formats.py @@ -0,0 +1,78 @@ +from unittest import ( + TestCase, +) + +import numpy as np +import pandas as pd +from datetime import ( + date, + datetime, +) + +from fireant.slicer.widgets import formats + + +class SafeValueTests(TestCase): + def test_nan_data_point(self): + # np.nan is converted to None + result = formats.safe(np.nan) + self.assertIsNone(result) + + def test_str_data_point(self): + result = formats.safe(u'abc') + self.assertEqual('abc', result) + + def test_int64_data_point(self): + # Needs to be cast to python int + result = formats.safe(np.int64(1)) + self.assertEqual(int(1), result) + + def test_date_data_point(self): + # Needs to be converted to milliseconds + result = formats.safe(date(2000, 1, 1)) + self.assertEqual('2000-01-01', result) + + def test_datetime_data_point(self): + # Needs to be converted to milliseconds + result = formats.safe(datetime(2000, 1, 1, 1)) + self.assertEqual('2000-01-01T01:00:00', result) + + def test_ts_date_data_point(self): + # Needs to be converted to milliseconds + result = formats.safe(pd.Timestamp(date(2000, 1, 1))) + self.assertEqual('2000-01-01', result) + + def test_ts_datetime_data_point(self): + # Needs to be converted to milliseconds + result = formats.safe(pd.Timestamp(datetime(2000, 1, 1, 1))) + self.assertEqual('2000-01-01T01:00:00', result) + + +class DisplayValueTests(TestCase): + def test_precision_default(self): + result = formats.display(0.123456789) + self.assertEqual('0.123457', result) + + def test_zero_precision(self): + result = formats.display(0.123456789, precision=0) + self.assertEqual('0', result) + + def test_precision(self): + result = formats.display(0.123456789, precision=2) + self.assertEqual('0.12', result) + + def test_precision_zero(self): + result = formats.display(0.0) + self.assertEqual('0', result) + + def test_precision_trim_trailing_zeros(self): + result = formats.display(1.01) + self.assertEqual('1.01', result) + + def test_prefix(self): + result = formats.display(0.12, prefix='$') + self.assertEqual('$0.12', result) + + def test_suffix(self): + result = formats.display(0.12, suffix='€') + self.assertEqual('0.12€', result) diff --git a/fireant/utils.py b/fireant/utils.py index de5e34a6..eaecb35c 100644 --- a/fireant/utils.py +++ b/fireant/utils.py @@ -1,10 +1,17 @@ -# coding: utf-8 def wrap_list(value): return value if isinstance(value, (tuple, list)) else [value] +def deep_get(d, keys, default=None): + d_level = d + for key in keys: + if key not in d_level: + return default + d_level = d_level[key] + return d_level + def flatten(items): return [item for level in items for item in wrap_list(level)] @@ -40,3 +47,42 @@ def merge_dicts(*dict_args): for dictionary in dict_args: result.update(dictionary) return result + + +def immutable(func): + """ + Decorator for wrapper "builder" functions. These are functions on the Query class or other classes used for + building queries which mutate the query and return self. To make the build functions immutable, this decorator is + used which will deepcopy the current instance. This decorator will return the return value of the inner function + or the new copy of the instance. The inner function does not need to return self. + """ + import copy + + def _copy(self, *args, **kwargs): + self_copy = copy.deepcopy(self) + result = func(self_copy, *args, **kwargs) + + # Return self if the inner function returns None. This way the inner function can return something + # different (for example when creating joins, a different builder is returned). + if result is None: + return self_copy + + return result + + return _copy + + +def ordered_distinct_list(l): + seen = set() + return [x + for x in l + if not x in seen + and not seen.add(x)] + + +def ordered_distinct_list_by_attr(l, attr='key'): + seen = set() + return [x + for x in l + if not getattr(x, attr) in seen + and not seen.add(getattr(x, attr))] diff --git a/requirements.txt b/requirements.txt index 7d61441a..51350847 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,10 @@ six -pandas<0.20 -pypika==0.9.4 -pymysql>=0.7.11 -vertica-python>=0.6 -psycopg2>=2.7.3.1 +pandas==0.22.0 +pypika==0.10.3 +pymysql==0.8.0 +vertica-python==0.7.3 +psycopg2==2.7.3.2 +toposort==1.5 matplotlib mock +typing==3.6.2 \ No newline at end of file diff --git a/setup.py b/setup.py index 8aba3e81..ad98037c 100644 --- a/setup.py +++ b/setup.py @@ -15,9 +15,9 @@ packages=['fireant', 'fireant.database', - 'fireant.dashboards', 'fireant.slicer', - 'fireant.slicer.transformers'], + 'fireant.slicer.queries', + 'fireant.slicer.widgets'], # Include additional files into the package include_package_data=True, diff --git a/tox.ini b/tox.ini index daa9bc9b..5834d3eb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py33,py34,py35,py36 +envlist = py33,py34,py35,py36 [testenv] deps = pypika commands = setup.py build test \ No newline at end of file From 6bc009e72dbff0343c3778ca75969681fc979ce5 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 23 Jan 2018 18:50:36 +0100 Subject: [PATCH 002/123] Added highcharts transformer --- fireant/__init__.py | 2 +- fireant/slicer/dimensions.py | 21 +- fireant/slicer/filters.py | 4 +- fireant/slicer/queries.py | 422 ---------- fireant/slicer/queries/builder.py | 20 +- fireant/slicer/queries/logger.py | 2 +- fireant/slicer/references.py | 21 +- fireant/slicer/widgets/base.py | 13 +- fireant/slicer/widgets/datatables.py | 126 +-- fireant/slicer/widgets/formats.py | 1 - fireant/slicer/widgets/helpers.py | 79 ++ fireant/slicer/widgets/highcharts.py | 317 +++++++- fireant/slicer/widgets/matplotlib.py | 3 +- fireant/slicer/widgets/pandas.py | 3 +- fireant/tests/slicer/mocks.py | 68 +- .../tests/slicer/widgets/test_datatables.py | 150 ++-- .../tests/slicer/widgets/test_highcharts.py | 765 ++++++++++++++++++ setup.py | 39 +- 18 files changed, 1468 insertions(+), 588 deletions(-) delete mode 100644 fireant/slicer/queries.py create mode 100644 fireant/slicer/widgets/helpers.py create mode 100644 fireant/tests/slicer/widgets/test_highcharts.py diff --git a/fireant/__init__.py b/fireant/__init__.py index c261f00a..575178df 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -10,4 +10,4 @@ Pandas, ) -__version__ = '{major}.{minor}.{patch}'.format(major=1, minor=0, patch=0) +__version__ = '1.0.0' diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index fd1a6eaa..58bfcc23 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -1,4 +1,5 @@ from fireant.utils import immutable + from .base import SlicerElement from .exceptions import QueryException from .filters import ( @@ -30,7 +31,9 @@ class BooleanDimension(Dimension): """ def __init__(self, key, label=None, definition=None): - super(BooleanDimension, self).__init__(key=key, label=label, definition=definition) + super(BooleanDimension, self).__init__(key=key, + label=label, + definition=definition) def is_(self, value): return BooleanFilter(self.definition, value) @@ -43,7 +46,9 @@ class CategoricalDimension(Dimension): """ def __init__(self, key, label=None, definition=None, display_values=()): - super(CategoricalDimension, self).__init__(key=key, label=label, definition=definition) + super(CategoricalDimension, self).__init__(key=key, + label=label, + definition=definition) self.display_values = dict(display_values) def isin(self, values): @@ -89,17 +94,23 @@ class ContinuousDimension(Dimension): """ def __init__(self, key, label=None, definition=None, default_interval=NumericInterval(1, 0)): - super(ContinuousDimension, self).__init__(key=key, label=label, definition=definition) + super(ContinuousDimension, self).__init__(key=key, + label=label, + definition=definition) self.interval = default_interval class DatetimeDimension(ContinuousDimension): """ - WRITEME + A subclass of ContinuousDimension which reflects a date/time data type. Intervals are replaced with time intervals + such as daily, weekly, annually, etc. A reference can be used to show a comparison over time such as + week-over-week or month-over-month. """ def __init__(self, key, label=None, definition=None, default_interval=daily): - super(DatetimeDimension, self).__init__(key=key, label=label, definition=definition, + super(DatetimeDimension, self).__init__(key=key, + label=label, + definition=definition, default_interval=default_interval) self.references = [] diff --git a/fireant/slicer/filters.py b/fireant/slicer/filters.py index a3d52172..dab88ff6 100644 --- a/fireant/slicer/filters.py +++ b/fireant/slicer/filters.py @@ -50,13 +50,13 @@ def __init__(self, element_key, value): class ContainsFilter(DimensionFilter): def __init__(self, dimension_definition, values): definition = dimension_definition.isin(values) - super(DimensionFilter, self).__init__(definition) + super(ContainsFilter, self).__init__(definition) class ExcludesFilter(DimensionFilter): def __init__(self, dimension_definition, values): definition = dimension_definition.notin(values) - super(DimensionFilter, self).__init__(definition) + super(ExcludesFilter, self).__init__(definition) class RangeFilter(DimensionFilter): diff --git a/fireant/slicer/queries.py b/fireant/slicer/queries.py deleted file mode 100644 index 91ab769f..00000000 --- a/fireant/slicer/queries.py +++ /dev/null @@ -1,422 +0,0 @@ -# coding: utf-8 -import copy -import logging -import time -from functools import partial -from itertools import chain - -import pandas as pd - -from fireant import utils -from fireant.slicer.references import ( - Delta, - DeltaPercentage, - YoY, -) -from pypika import ( - JoinType, - MySQLQuery, - PostgreSQLQuery, - RedshiftQuery, -) - -query_logger = logging.getLogger('fireant.query_log$') - - -class QueryNotSupportedError(Exception): - pass - - -class QueryManager(object): - def __init__(self, database): - # Get the correct pypika database query class - self.query_cls = database.query_cls - - def query_data(self, database, table, joins=None, - metrics=None, dimensions=None, - mfilters=None, dfilters=None, - references=None, rollup=None, pagination=None): - """ - Loads a pandas data frame given a table and a description of the request. - - :param database: - The database interface to use to execute the connection - - :param table: - Type: pypika.Table - (Required) The primary table to select data from. In SQL, this is the table in the FROM clause. - - :param joins: - Type: list[tuple[pypika.Table, pypika.Criterion, pypika.JoinType]] - (Optional) A list of tables to join in the query. If a metric should be selected from a table besides the - table parameter, the additional table *must* be joined. - - The tuple includes in the following order: Join table, Join criterion, Join type - - :param metrics: - Type: dict[str: pypika.Field]] - A dict containing metrics fields to query. This value is required to contain at least one entry. The key - used will match the corresponding column in the returned pd.DataFrame. - - :param dimensions: - Type: OrderedDict[str: pypika.Field]] - (Optional) An OrderedDict containing indices to query. If empty, a pd.Series will be returned containing - one value per metric. If given one entry, a pd.DataFrame with a single index will be returned. If more - than one value is given, then a pd.DataFrame with a multi-index will be returned. The key used will match - the corresponding index level in the returned DataFrame. - - If a dict is used instead of an OrderedDict, then the index levels cannot be guaranteed, so in most cases an - OrderedDict should be used. - - :param mfilters: - Type: list[pypika.Criterion] - (Optional) A list containing criterions to use in the HAVING clause of the query. This will reduce the data - to instances where metrics meet the criteria. - - :param dfilters: - Type: list[pypika.Criterion] - (Optional) A list containing criterions to use in the WHERE clause of the query. This will reduce the data - to instances where dimensions meet the criteria. - - :param references: - Type: dict[str: str]] - (Optional) A dict containing comparison operations. The key must match a valid reference. The value must - match the key of a provided dimension. The value must match a dimensions of the appropriate type, usually a - Time Series dimension. - - For each value passed, the query will be duplicated containing values for a comparison. - - References can also modify the comparison field to show values such as change. The format is as follows: - - {key}_{modifier} - - The modifier type is optional and the underscore must be omitted when one is not used. - - Valid references are as follows: - - - *wow*: Week over Week - duplicates the query for values one week prior. - - *mom*: Month over Month - duplicates the query for values 4 weeks prior. - - *qoq*: Quarter over Quarter - duplicates the query for values one quarter prior. - - *yoy*: Year over Year - duplicates the query for values 52 weeks prior. - - Valid modifier types are as follows: - - - *d*: Delta - difference between previous value and current - - *p*: Delta Percentage - difference between previous value and current as a percentage of the previous - value. - - Examples: - - "wow": Week over Week - "wow_d": delta Week over Week - - :param rollup: - Type: list[str] - (Optional) Lists the dimensions to include for ROLLUP which calculates the totals across groups. This list - must be a subset of the keys of the dimensions parameter dict. - - .. note:: - - When using rollup for less than all of the dimensions, the dimensions included in the ROLLUP will be - moved after the non-ROLLUP dimensions. - - :param pagination: - Type: ``fireant.slicer.pagination.Paginator`` - (Optional) A Paginator class defining the limit, offset and order by statements for the query - - :return: - A pd.DataFrame indexed by the provided dimensions parameters containing columns for each metrics parameter. - """ - if rollup and issubclass(database.query_cls, (MySQLQuery, PostgreSQLQuery, RedshiftQuery)): - # MySQL, postgreSQL and Redshift doesn't support query rollups in the same way as Vertica, Oracle etc. - # We therefore don't support it for now. - raise QueryNotSupportedError("This database type currently doesn't support ROLLUP operations!") - - query = self._build_data_query( - database, table, joins, metrics, dimensions, dfilters, mfilters, references, rollup, pagination - ) - - dataframe = self._get_dataframe_from_query(database, query) - dataframe.columns = [col.decode('utf-8') if isinstance(col, bytes) else col - for col in dataframe.columns] - - for column in dataframe.columns: - # Only replace NaNs for columns of type object. Column types other than that tend to be checked - # against in the transformers. Which would be a problem when replacing NaNs with a string - # because that alters the type of the column. - if dataframe[column].dtype == object: - dataframe[column] = dataframe[column].fillna('') - - if dimensions: - dataframe = dataframe.set_index( - # Removed the reference keys for now - list(dimensions.keys()) # + ['{1}_{0}'.format(*ref) for ref in references.items()] - ) - - if references: - dataframe.columns = pd.MultiIndex.from_product([[''] + list(references.keys()), list(metrics.keys())]) - - return dataframe.fillna(0) - - def _get_dataframe_from_query(self, database, query): - """ - Returns a Pandas Dataframe built from the result of the query. - The query is also logged along with its duration. - - :param database: Database object - :param query: PyPika query object - :return: Pandas Dataframe built from the result of the query - """ - start_time = time.time() - query_string = str(query) - query_logger.debug(query_string) - - dataframe = database.fetch_dataframe(query_string) - - query_logger.info('[duration: {duration} seconds]: {query}'.format( - duration=round(time.time() - start_time, 4), - query=query_string) - ) - - return dataframe - - def query_dimension_options(self, database, table, joins=None, dimensions=None, filters=None, limit=None): - """ - Builds and executes a query to retrieve possible dimension options given a set of filters. - - :param database: - The database interface to use to execute the connection - - :param table: - See above - - :param joins: - See above - - :param dimensions: - See above - - :param filters: - See above - - :param limit: - An optional limit to the number of results returned. - - :return: - """ - query = self._build_dimension_query(table, joins, dimensions, filters, limit) - results = database.fetch(str(query)) - return [{k: v for k, v in zip(dimensions.keys(), result)} - for result in results] - - def _build_data_query(self, database, table, joins, metrics, dimensions, - dfilters, mfilters, references, rollup, pagination): - args = (table, joins or dict(), metrics or dict(), dimensions or dict(), - dfilters or dict(), mfilters or dict(), rollup or list(), pagination or None) - - query = self._build_query_inner(table, joins, metrics, dimensions, dfilters, mfilters, rollup) - - if references: - return self._build_reference_query(query, database, references, *args) - - if pagination: - return self._add_pagination(query, pagination) - - return self._add_sorting(query, [query.field(dkey).as_(alias) - for alias, dkey in dimensions.items()]) - - def _build_reference_query(self, query, database, references, table, joins, metrics, dimensions, dfilters, - mfilters, rollup, pagination): - wrapper_query = self.query_cls.from_(query).select(*[query.field(key).as_(key) - for key in list(dimensions.keys()) + list(metrics.keys())]) - - for reference_key, schema in references.items(): - dimension_key = schema['dimension'] - - date_add = partial(database.date_add, date_part=schema['time_unit'], interval=schema['interval']) - - # The interval term from pypika does not take into account leap years, therefore the interval - # needs to be replaced with a database specific one when appropriate. - yoy_keys = [YoY.key, Delta.generate_key(YoY.key), DeltaPercentage.generate_key(YoY.key)] - if reference_key in yoy_keys: - # If YoY reference is used with the week interval, the dates will fail to match in the join - # as the first day of the ISO year is different. Therefore, we truncate the reference date by adding - # a year, truncating it and removing a year so the dates match. - from fireant.slicer import DatetimeDimension - week = DatetimeDimension.week - dim = dimensions[dimension_key] - # week is the default date format. Vertica uses 'IW'. - if hasattr(dim, 'date_format') and dim.date_format in [week, 'IW']: - trunc_and_add = database.trunc_date(database.date_add('year', 1, dim.field), 'week') - dimensions[dimension_key] = database.date_add('year', -1, field=trunc_and_add) - - # Don't reuse the dfilters arg otherwise intervals will be aggregated on each iteration - replaced_dfilters = self._replace_filters_for_ref(dfilters, schema['definition'], date_add) - ref_query = self._build_query_inner(table, joins, metrics, dimensions, - replaced_dfilters, mfilters, rollup) - join_criteria = self._build_reference_join_criteria(dimension_key, dimensions, date_add, query, ref_query) - - # Optional modifier function to modify the metric in the reference query. This is for delta and delta - # percentage references. It is None for normal references and this default should be used - modifier = schema.get('modifier') - - # Join the reference query and select the reference dimension and all metrics - # This ignores additional dimensions since they are identical to the primary results - if join_criteria: - wrapper_query = wrapper_query.join(ref_query, JoinType.left).on(join_criteria) - else: - wrapper_query = wrapper_query.from_(ref_query) - - if modifier: - get_reference_field = lambda key: modifier(query.field(key), ref_query.field(key)) - - else: - get_reference_field = lambda key: ref_query.field(key) - - wrapper_query = wrapper_query.select( - *[get_reference_field(key).as_(self._suffix(key, reference_key)) - for key in metrics.keys()] - ) - - if pagination: - return self._add_pagination(wrapper_query, pagination) - - return self._add_sorting(wrapper_query, [query.field(dkey).as_(alias) for alias, dkey in dimensions.items()]) - - def _build_dimension_query(self, table, joins, dimensions, filters, limit=None): - query = self.query_cls.from_(table).distinct() - query = self._add_joins(joins, query) - query = self._add_filters(query, filters, []) - - for key, dimension in dimensions.items(): - query = query.select(dimension.as_(key)) - - if limit: - query = query[:limit] - - return query - - def _build_query_inner(self, table, joins, metrics, dimensions, dfilters, mfilters, rollup): - query = self.query_cls.from_(table) - query = self._add_joins(joins, query) - query = self._select_dimensions(query, dimensions, rollup) - query = self._select_metrics(query, metrics) - return self._add_filters(query, dfilters, mfilters) - - @staticmethod - def _add_joins(joins, query): - for join_table, criterion, join_type in joins: - query = query.join(join_table, how=join_type).on(criterion) - return query - - @staticmethod - def _select_dimensions(query, dimensions, rollup): - dims = [dimension.as_(key) - for key, dimension in dimensions.items() - if key not in chain(*rollup)] - if dims: - query = query.select(*dims).groupby(*dims) - - # Rollup is passed in as a list of lists so that multiple columns can be rolled up together (such as for - # Unique dimensions) - rollup_dims = [[dimension.as_(dimension_key) - for dimension_key, dimension in dimensions.items() - if dimension_key in keys] - for keys in rollup] - - # Remove entry levels - flattened_rollup_dims = utils.flatten(rollup_dims) - - if flattened_rollup_dims: - query = query.select(*flattened_rollup_dims).rollup(*rollup_dims) - - return query - - @staticmethod - def _select_metrics(query, metrics): - return query.select(*[metric.as_(key) - for key, metric in metrics.items()]) - - @staticmethod - def _add_filters(query, dfilters, mfilters): - if dfilters: - for dfx in dfilters: - query = query.where(dfx) - - if mfilters: - for mfx in mfilters: - query = query.having(mfx) - - return query - - @staticmethod - def _add_sorting(query, dimensions): - if dimensions: - return query.orderby(*dimensions) - return query - - @staticmethod - def _add_pagination(query, pagination): - """ Add offset, limit and order pagination to the query """ - query = query[pagination.offset: pagination.limit] - select_mapping = {s.alias:s for s in query._selects} - for key, order in pagination.order: - pagination_field = select_mapping[key] - if not pagination_field: - raise Exception("Invalid metric or dimension key used for pagination: {key}" - .format(key=key)) - query = query.orderby(pagination_field, order=order) - return query - - @staticmethod - def _suffix(key, suffix): - return '%s_%s' % (key, suffix) if suffix else key - - @staticmethod - def _replace_filters_for_ref(dfilters, target_dimension, date_add): - """ - Replaces the dimension used by a reference in the dimension schema and dimension filter schema. - - We do this in order to build the same query with a shifted date instead of the actual date. - """ - new_dfilters = [] - for dfilter in dfilters: - try: - # FIXME this is a bit hacky. Casts the fields to string to see if the filter uses this dimensinon - # TODO provide a utility in pypika for checking if these are the same - filter_fields = ''.join(str(f) for f in dfilter.term.fields()) - target_fields = ''.join(str(f) for f in target_dimension.fields()) - if filter_fields == target_fields: - dfilter = copy.deepcopy(dfilter) - # Note: date_add is only passed the field as it is a partial/curried function - dfilter.term = date_add(field=dfilter.term) - - except (AttributeError, IndexError): - pass # If the above if-expression cannot be evaluated, then its not the filter we are looking for - - new_dfilters.append(dfilter) - - return new_dfilters - - @staticmethod - def _build_reference_join_criteria(dimension_key, dimensions, date_add, query, ref_query): - ref_join_criteria = [] - if dimension_key in dimensions: - ref_join_criteria.append( - # Note: date_add is only passed the field as it is a partial/curried function - query.field(dimension_key) == date_add(field=ref_query.field(dimension_key)) - ) - - # The below set is sorted to ensure that the ON part of the join is always consistently ordered - ref_join_criteria += [query.field(dkey) == ref_query.field(dkey) - for dkey in sorted(set(dimensions.keys()) - {dimension_key})] - - # If there are no selected dimensions, this will be an empty list. - if not ref_join_criteria: - return None - - ref_criteria = ref_join_criteria.pop(0) - for criteria in ref_join_criteria: - ref_criteria &= criteria - - return ref_criteria diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index fe2e5111..de90d949 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -1,15 +1,15 @@ from collections import defaultdict -from toposort import ( - CircularDependencyError, - toposort_flatten, -) - from fireant.utils import ( immutable, ordered_distinct_list, ordered_distinct_list_by_attr, ) +from toposort import ( + CircularDependencyError, + toposort_flatten, +) + from .database import fetch_data from .references import join_reference from ..exceptions import ( @@ -54,7 +54,6 @@ def dimension(self, dimension): """ :param dimension: - :param rollup: :return: """ self._dimensions.append(dimension) @@ -62,7 +61,6 @@ def dimension(self, dimension): @immutable def filter(self, filter): """ - :param filter: :return: """ @@ -213,12 +211,14 @@ def render(self): :return: """ - dataframe = fetch_data(self.slicer.database, self.query, index_levels=[dimension.key - for dimension in self._dimensions]) + data_frame = fetch_data(self.slicer.database, + self.query, + index_levels=[dimension.key + for dimension in self._dimensions]) # Apply operations ... # Apply transformations - return [widget.transform(dataframe, self.slicer) + return [widget.transform(data_frame, self.slicer) for widget in self._widgets] diff --git a/fireant/slicer/queries/logger.py b/fireant/slicer/queries/logger.py index c1944a63..75d9f56b 100644 --- a/fireant/slicer/queries/logger.py +++ b/fireant/slicer/queries/logger.py @@ -1,3 +1,3 @@ import logging -logger = logging.getLogger('fireant.query_log$') \ No newline at end of file +logger = logging.getLogger('fireant.query_log$') diff --git a/fireant/slicer/references.py b/fireant/slicer/references.py index 3ac32a12..df013ae0 100644 --- a/fireant/slicer/references.py +++ b/fireant/slicer/references.py @@ -1,10 +1,7 @@ class Reference(object): - def __init__(self, key, time_unit, interval, delta=False, percent=False): + def __init__(self, key, label, time_unit, interval, delta=False, percent=False): self.key = key - if delta: - self.key += '_delta' - if percent: - self.key += '_percent' + self.label = label self.time_unit = time_unit self.interval = interval @@ -13,11 +10,13 @@ def __init__(self, key, time_unit, interval, delta=False, percent=False): self.is_percent = percent def delta(self, percent=False): - return Reference(self.key, self.time_unit, self.interval, delta=True, percent=percent) + key = self.key + '_delta_percent' if percent else self.key + '_delta' + label = self.label + ' Δ%' if percent else self.label + ' Δ' + return Reference(key, label, self.time_unit, self.interval, delta=True, percent=percent) -DayOverDay = Reference('dod', 'day', 1) -WeekOverWeek = Reference('wow', 'week', 1) -MonthOverMonth = Reference('mom', 'month', 1) -QuarterOverQuarter = Reference('qoq', 'quarter', 1) -YearOverYear = Reference('yoy', 'year', 1) +DayOverDay = Reference('dod', 'DoD', 'day', 1) +WeekOverWeek = Reference('wow', 'WoW', 'week', 1) +MonthOverMonth = Reference('mom', 'MoM', 'month', 1) +QuarterOverQuarter = Reference('qoq', 'QoQ', 'quarter', 1) +YearOverYear = Reference('yoy', 'YoY', 'year', 1) diff --git a/fireant/slicer/widgets/base.py b/fireant/slicer/widgets/base.py index 6e11aa5c..388af41b 100644 --- a/fireant/slicer/widgets/base.py +++ b/fireant/slicer/widgets/base.py @@ -1,7 +1,12 @@ from fireant.utils import immutable -class Widget(object): +class Widget: + def transform(self, data_frame, slicer, dimensions): + raise NotImplementedError() + + +class MetricsWidget(Widget): def __init__(self, metrics=()): self._metrics = list(metrics) @@ -13,7 +18,7 @@ def metric(self, metric): def metrics(self): return [metric for group in self._metrics - for metric in (group.metrics if hasattr(group, 'metrics') else [group])] + for metric in getattr(group, 'metrics', [group])] - def transform(self, data_frame, slicer): - raise NotImplementedError() + def transform(self, data_frame, slicer, dimensions): + super(MetricsWidget, self).transform(data_frame, slicer, dimensions) diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index ac5a3996..6b9010e0 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -1,13 +1,19 @@ import itertools import pandas as pd - from fireant import ( ContinuousDimension, utils, ) + from . import formats -from .base import Widget +from .base import MetricsWidget +from .helpers import ( + dimensional_metric_label, + extract_display_values, + reference_key, + reference_label, +) def _format_dimension_cell(dimension_value, display_values): @@ -49,7 +55,7 @@ def _format_metric_cell(value, metric): HARD_MAX_COLUMNS = 24 -class DataTablesJS(Widget): +class DataTablesJS(MetricsWidget): def __init__(self, metrics=(), pivot=False, max_columns=None): super(DataTablesJS, self).__init__(metrics) self.pivot = pivot @@ -57,19 +63,23 @@ def __init__(self, metrics=(), pivot=False, max_columns=None): if max_columns is not None \ else HARD_MAX_COLUMNS - def transform(self, data_frame, slicer): + def transform(self, data_frame, slicer, dimensions): """ :param data_frame: :param slicer: + :param dimensions: :return: """ - dimension_keys = list(filter(None, data_frame.index.names)) - dimensions = [getattr(slicer.dimensions, key) - for key in dimension_keys] - dimension_display_values = self._dimension_display_values(dimensions, data_frame) + dimension_display_values = extract_display_values(dimensions, data_frame) - metric_keys = [metric.key for metric in self.metrics] + references = [reference + for dimension in dimensions + for reference in getattr(dimension, 'references', ())] + + metric_keys = [reference_key(metric, reference) + for metric in self.metrics + for reference in [None] + references] data_frame = data_frame[metric_keys] pivot_index_to_columns = self.pivot and isinstance(data_frame.index, pd.MultiIndex) @@ -79,34 +89,33 @@ def transform(self, data_frame, slicer): .unstack(level=levels) \ .fillna(value=0) - columns = self._dimension_columns(dimensions[:1]) \ - + self._metric_columns_pivoted(data_frame.columns, dimension_display_values) + dimension_columns = self._dimension_columns(dimensions[:1]) + + render_column_label = dimensional_metric_label(dimensions, dimension_display_values) + metric_columns = self._metric_columns_pivoted(references, + data_frame.columns, + render_column_label) - else: - columns = self._dimension_columns(dimensions) + self._metric_columns() - data = [self._data_row(dimensions, dimension_values, dimension_display_values, row_data) + else: + dimension_columns = self._dimension_columns(dimensions) + metric_columns = self._metric_columns(references) + + columns = (dimension_columns + metric_columns)[:self.max_columns] + data = [self._data_row(dimensions, + dimension_values, + dimension_display_values, + references, + row_data) for dimension_values, row_data in data_frame.iterrows()] return dict(columns=columns, data=data) - def _dimension_display_values(self, dimensions, data_frame): - dv_by_dimension = {} - - for dimension in dimensions: - dkey = dimension.key - if hasattr(dimension, 'display_values'): - dv_by_dimension[dkey] = dimension.display_values - elif hasattr(dimension, 'display_key'): - dv_by_dimension[dkey] = data_frame[dimension.display_key].groupby(level=dkey).first() - - return dv_by_dimension - - def _dimension_columns(self, dimensions): + @staticmethod + def _dimension_columns(dimensions): """ - :param data_frame: - :param slicer: + :param dimensions: :return: """ columns = [] @@ -122,48 +131,67 @@ def _dimension_columns(self, dimensions): return columns - def _metric_columns(self): + def _metric_columns(self, references): """ :return: """ columns = [] for metric in self.metrics: - columns.append(dict(title=metric.label or metric.key, - data=metric.key, - render=dict(_='value', display='display'))) + for reference in [None] + references: + title = reference_label(metric, reference) + data = reference_key(metric, reference) + + columns.append(dict(title=title, + data=data, + render=dict(_='value', display='display'))) return columns - def _metric_columns_pivoted(self, df_columns, display_values): + def _metric_columns_pivoted(self, references, df_columns, render_column_label): """ - :param index_levels: - :param display_values: + :param references: + :param df_columns: + :param render_column_label: :return: """ columns = [] for metric in self.metrics: - dimension_keys = df_columns.names[1:] - for level_values in itertools.product(*map(list, df_columns.levels[1:])): - level_display_values = [utils.deep_get(display_values, [key, raw_value], raw_value) - for key, raw_value in zip(dimension_keys, level_values)] - - columns.append(dict(title="{metric} ({dimensions})".format(metric=metric.label or metric.key, - dimensions=", ".join(level_display_values)), - data='.'.join([metric.key] + [str(x) for x in level_values]), - render=dict(_='value', display='display'))) + dimension_value_sets = [list(level) + for level in df_columns.levels[1:]] + + for dimension_values in itertools.product(*dimension_value_sets): + for reference in [None] + references: + key = reference_key(metric, reference) + title = render_column_label(metric, reference, dimension_values) + data = '.'.join([key] + [str(x) for x in dimension_values]) + + columns.append(dict(title=title, + data=data, + render=dict(_='value', display='display'))) return columns - def _data_row(self, dimensions, dimension_values, dimension_display_values, row_data): + def _data_row(self, dimensions, dimension_values, dimension_display_values, references, row_data): + """ + + :param dimensions: + :param dimension_values: + :param dimension_display_values: + :param row_data: + :return: + """ row = {} for dimension, dimension_value in zip(dimensions, utils.wrap_list(dimension_values)): row[dimension.key] = _format_dimension_cell(dimension_value, dimension_display_values.get(dimension.key)) for metric in self.metrics: - row[metric.key] = _format_dimensional_metric_cell(row_data, metric) \ - if isinstance(row_data.index, pd.MultiIndex) \ - else _format_metric_cell(row_data[metric.key], metric) + for reference in [None] + references: + key = reference_key(metric, reference) + + row[key] = _format_dimensional_metric_cell(row_data, metric) \ + if isinstance(row_data.index, pd.MultiIndex) \ + else _format_metric_cell(row_data[key], metric) return row diff --git a/fireant/slicer/widgets/formats.py b/fireant/slicer/widgets/formats.py index 42ad941c..c6383f04 100644 --- a/fireant/slicer/widgets/formats.py +++ b/fireant/slicer/widgets/formats.py @@ -4,7 +4,6 @@ import pandas as pd from datetime import ( date, - datetime, time, ) diff --git a/fireant/slicer/widgets/helpers.py b/fireant/slicer/widgets/helpers.py new file mode 100644 index 00000000..bccfce2f --- /dev/null +++ b/fireant/slicer/widgets/helpers.py @@ -0,0 +1,79 @@ +from fireant import utils + + +def extract_display_values(dimensions, data_frame): + """ + + :param dimensions: + :param data_frame: + :return: + """ + dv_by_dimension = {} + + for dimension in dimensions: + dkey = dimension.key + + if hasattr(dimension, 'display_values'): + dv_by_dimension[dkey] = dimension.display_values + + elif hasattr(dimension, 'display_key'): + dv_by_dimension[dkey] = data_frame[dimension.display_key].groupby(level=dkey).first() + + return dv_by_dimension + + +def reference_key(metric, reference): + key = metric.key + + if reference is not None: + return '{}_{}'.format(key, reference.key) + + return key + + +def reference_label(metric, reference): + label = metric.label or metric.key + + if reference is not None: + return '{} ({})'.format(label, reference.label) + + return label + + +def dimensional_metric_label(dimensions, dimension_display_values): + """ + Creates a callback function for rendering series labels. + + :param dimensions: + A list of fireant.Dimension which is being rendered. + + :param dimension_display_values: + A dictionary containing key-value pairs for each dimension. + :return: + a callback function which renders a label for a metric, reference, and list of dimension values. + """ + + def render_series_label(metric, reference, dimension_values): + """ + Returns a string label for a metric, reference, and set of values for zero or more dimensions. + + :param metric: + an instance of fireant.Metric + :param reference: + an instance of fireant.Reference + :param dimension_values: + a tuple of dimension values. Can be zero-length or longer. + :return: + """ + dimension_labels = [utils.deep_get(dimension_display_values, + [dimension.key, dimension_value], + dimension_value) + for dimension, dimension_value in zip(dimensions[1:], dimension_values)] + + if dimension_labels: + return '{} ({})'.format(reference_label(metric, reference), + ', '.join(dimension_labels)) + + return metric.label + + return render_series_label diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 704905ca..a38cc062 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -1,34 +1,323 @@ +import itertools + +import numpy as np +import pandas as pd +from datetime import datetime +from fireant import utils from fireant.utils import immutable -from .base import Widget + +from .base import ( + MetricsWidget, + Widget, +) +from .helpers import ( + dimensional_metric_label, + extract_display_values, + reference_key, +) + +DEFAULT_COLORS = ( + "#DDDF0D", + "#55BF3B", + "#DF5353", + "#7798BF", + "#AAEEEE", + "#FF0066", + "#EEAAEE", + "#DF5353", + "#7798BF", + "#AAEEEE", +) + +DASH_STYLES = ( + 'Solid', + 'ShortDash', + 'ShortDot', + 'ShortDashDot', + 'ShortDashDotDot', + 'Dot', + 'Dash', + 'LongDash', + 'DashDot', + 'LongDashDot', + 'LongDashDotDot', +) + +MARKER_SYMBOLS = ( + "circle", + "square", + "diamond", + "triangle", + "triangle-down", +) + + +class ChartWidget(MetricsWidget): + type = None + needs_marker = False + stacked = False + + def __init__(self, name=None, metrics=(), stacked=False): + super(ChartWidget, self).__init__(metrics=metrics) + self.name = name + self.stacked = self.stacked or stacked class HighCharts(Widget): - class PieChart(Widget): - pass + class PieChart(ChartWidget): + type = 'pie' - class LineChart(Widget): - pass + class LineChart(ChartWidget): + type = 'line' + needs_marker = True - class ColumnChart(Widget): - pass + class AreaChart(ChartWidget): + type = 'area' + needs_marker = True - class BarChart(Widget): - pass + class BarChart(ChartWidget): + type = 'bar' - def __init__(self, axes=()): - self.axes = list(axes) + class ColumnChart(ChartWidget): + type = 'column' + + class StackedBarChart(BarChart): + stacked = True - def metric(self, metric): - raise NotImplementedError() + class StackedColumnChart(ColumnChart): + stacked = True + + class AreaPercentageChart(AreaChart): + stacked = True + + def __init__(self, title=None, axes=(), colors=None): + self.title = title + self.axes = list(axes) + self.colors = colors or DEFAULT_COLORS @immutable - def axis(self, axis): + def axis(self, axis: ChartWidget): + """ + (Immutable) Adds an axis to the Chart. + + :param axis: + :return: + """ + self.axes.append(axis) @property def metrics(self): + """ + :return: + A set of metrics used in this chart. This collects all metrics across all axes. + """ seen = set() return [metric for axis in self.axes for metric in axis.metrics if not (metric.key in seen or seen.add(metric.key))] + + def transform(self, data_frame, slicer, dimensions): + """ + - Main entry point - + + Transforms a data frame into highcharts JSON format. + + See https://api.highcharts.com/highcharts/ + + :param data_frame: + The data frame containing the data. Index must match the dimensions parameter. + :param slicer: + The slicer that is in use. + :param dimensions: + A list of dimensions that are being rendered. + :return: + A dict meant to be dumped as JSON. + """ + colors = itertools.cycle(self.colors) + + levels = list(range(1, len(dimensions))) + groups = list(data_frame.groupby(level=levels)) \ + if levels \ + else [([], data_frame)] + + dimension_display_values = extract_display_values(dimensions, data_frame) + render_series_label = dimensional_metric_label(dimensions, dimension_display_values) + + references = [reference + for dimension in dimensions + for reference in getattr(dimension, 'references', ())] + + y_axes, series = [], [] + for axis_idx, axis in enumerate(self.axes): + colors, series_colors = itertools.tee(colors) + axis_color = next(colors) if 1 < len(self.metrics) else None + + # prepend axes, append series, this keeps everything ordered left-to-right + y_axes[0:0] = self._render_y_axis(axis_idx, + axis_color, + references) + series += self._render_series(axis, + axis_idx, + axis_color, + series_colors, + groups, + render_series_label, + references) + + x_axis = self._render_x_axis(data_frame, dimension_display_values) + plot_options = self._render_plot_options(data_frame) + + return { + "title": {"text": self.title}, + "xAxis": x_axis, + "yAxis": y_axes, + "plotOptions": plot_options, + "series": series, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + } + + + def _render_x_axis(self, data_frame, dimension_display_values): + """ + Renders the xAxis configuraiton. + + https://api.highcharts.com/highcharts/yAxis + + :param data_frame: + :param dimension_display_values: + :return: + """ + first_level = data_frame.index.levels[0] \ + if isinstance(data_frame.index, pd.MultiIndex) \ + else data_frame.index + + if isinstance(first_level, pd.DatetimeIndex): + return {"type": "datetime"} + + categories = ["All"] \ + if isinstance(first_level, pd.RangeIndex) \ + else [utils.deep_get(dimension_display_values, + [first_level.name, dimension_value], + dimension_value) + for dimension_value in first_level] + + return { + "type": "category", + "categories": categories, + } + + def _render_y_axis(self, axis_idx, color, references): + """ + Renders the yAxis configuraiton. + + https://api.highcharts.com/highcharts/yAxis + + :param axis_idx: + :param color: + :param references: + :return: + """ + y_axes = [{ + "id": str(axis_idx), + "title": {"text": None}, + "labels": {"style": {"color": color}} + }] + + y_axes += [{ + "id": "{}_{}".format(axis_idx, reference.key), + "title": {"text": reference.label}, + "opposite": True, + "labels": {"style": {"color": color}} + } + for reference in references + if reference.is_delta] + + return y_axes + + def _render_plot_options(self, data_frame): + """ + Renders the plotOptions configuration + + https://api.highcharts.com/highcharts/plotOptions + + :param data_frame: + :return: + """ + first_level = data_frame.index.levels[0] \ + if isinstance(data_frame.index, pd.MultiIndex) \ + else data_frame.index + + if isinstance(first_level, pd.DatetimeIndex): + epoch = np.datetime64(datetime.utcfromtimestamp(0)) + ms = np.timedelta64(1, 'ms') + millis = [d / ms + for d in first_level.values[:2] - epoch] + + return { + "series": { + "pointStart": millis[0], + "pointInterval": np.diff(millis)[0] + } + } + + return {} + + def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, render_series_label, references): + """ + Renders the series configuration. + + https://api.highcharts.com/highcharts/series + + :param axis: + :param axis_idx: + :param axis_color: + :param colors: + :param data_frame_groups: + :param render_series_label: + :param references: + :return: + """ + has_multi_metric = 1 < len(axis.metrics) + + series = [] + for metric in axis.metrics: + visible = True + symbols = itertools.cycle(MARKER_SYMBOLS) + series_color = next(colors) if has_multi_metric else None + + for (dimension_values, group_df), symbol in zip(data_frame_groups, symbols): + if not has_multi_metric: + series_color = next(colors) + + for reference, dash_style in zip([None] + references, itertools.cycle(DASH_STYLES)): + key = reference_key(metric, reference) + + series.append({ + "type": axis.type, + "color": series_color, + "dashStyle": dash_style, + "visible": visible, + + "name": render_series_label(metric, reference, utils.wrap_list(dimension_values)), + + "data": [float(x) for x in group_df[key].values], + + "yAxis": ("{}_{}".format(axis_idx, reference.key) + if reference is not None and reference.is_delta + else str(axis_idx)), + + "marker": {"symbol": symbol, "fillColor": axis_color or series_color} \ + if axis.needs_marker \ + else {}, + + "stacking": "normal" \ + if axis.stacked \ + else None, + }) + + visible = False # Only display the first in each group + + return series diff --git a/fireant/slicer/widgets/matplotlib.py b/fireant/slicer/widgets/matplotlib.py index 4e746133..c3c61adb 100644 --- a/fireant/slicer/widgets/matplotlib.py +++ b/fireant/slicer/widgets/matplotlib.py @@ -2,4 +2,5 @@ class Matplotlib(Widget): - pass + def transform(self, data_frame, slicer, dimensions): + raise NotImplementedError() diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index f01f4b17..11f8e14d 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -2,4 +2,5 @@ class Pandas(Widget): - pass + def transform(self, data_frame, slicer, dimensions): + raise NotImplementedError() diff --git a/fireant/tests/slicer/mocks.py b/fireant/tests/slicer/mocks.py index 0a2e4919..00f48890 100644 --- a/fireant/tests/slicer/mocks.py +++ b/fireant/tests/slicer/mocks.py @@ -5,10 +5,12 @@ from unittest.mock import Mock import pandas as pd -from datetime import date - +from datetime import ( + datetime, +) from fireant import * from fireant import VerticaDatabase + from pypika import ( JoinType, Table, @@ -213,7 +215,7 @@ class TestDatabase(VerticaDatabase): election_year = elections[election_id] winner = election_candidate_wins[(election_id, candidate_id)] records.append(PoliticsRow( - timestamp=date(int(election_year), 1, 1), + timestamp=datetime(int(election_year), 1, 1), candidate=candidate_id, candidate_display=candidates[candidate_id], political_party=candidate_parties[candidate_id], election=election_id, election_display=elections[election_id], @@ -224,3 +226,63 @@ class TestDatabase(VerticaDatabase): )) mock_politics_database = pd.DataFrame.from_records(records, columns=columns) + +single_metric_df = pd.DataFrame(mock_politics_database[['votes']] + .sum()).T + +multi_metric_df = pd.DataFrame(mock_politics_database[['votes', 'wins']] + .sum()).T + +cont_dim_df = mock_politics_database[['timestamp', 'votes', 'wins']] \ + .groupby('timestamp') \ + .sum() + +cat_dim_df = mock_politics_database[['political_party', 'votes', 'wins']] \ + .groupby('political_party') \ + .sum() + +uni_dim_df = mock_politics_database[['candidate', 'candidate_display', 'votes', 'wins']] \ + .groupby(['candidate', 'candidate_display']) \ + .sum() \ + .reset_index('candidate_display') + +cont_cat_dim_df = mock_politics_database[['timestamp', 'political_party', 'votes', 'wins']] \ + .groupby(['timestamp', 'political_party']) \ + .sum() + +cont_uni_dim_df = mock_politics_database[['timestamp', 'state', 'state_display', 'votes', 'wins']] \ + .groupby(['timestamp', 'state', 'state_display']) \ + .sum() \ + .reset_index('state_display') + + +def ref(data_frame, columns): + ref_cols = {column: '%s_eoe' % column + for column in columns} + + ref_df = cont_uni_dim_df \ + .shift(2) \ + .rename(columns=ref_cols)[list(ref_cols.values())] + + return (cont_uni_dim_df + .copy() + .join(ref_df) + .iloc[2:]) + + +def ref_delta(ref_data_frame, columns): + ref_columns = ['%s_eoe' % column for column in columns] + + delta_data_frame = pd.DataFrame( + data=ref_data_frame[ref_columns].values - ref_data_frame[columns].values, + columns=['%s_eoe_delta' % column for column in columns], + index=ref_data_frame.index + ) + return ref_data_frame.join(delta_data_frame) + + +_columns = ['votes', 'wins'] +cont_uni_dim_ref_df = ref(cont_uni_dim_df, _columns) +cont_uni_dim_ref_delta_df = ref_delta(cont_uni_dim_ref_df, _columns) + +ElectionOverElection = Reference('eoe', 'EoE', 'year', 4) diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py index f452be7d..04c27a90 100644 --- a/fireant/tests/slicer/widgets/test_datatables.py +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -1,60 +1,32 @@ -import locale as lc -from unittest import ( - TestCase, -) +from unittest import TestCase import pandas as pd from datetime import date -from mock import Mock - from fireant.slicer.widgets.datatables import ( DataTablesJS, _format_metric_cell, ) from fireant.tests.slicer.mocks import ( - mock_politics_database as mock_df, + ElectionOverElection, + cat_dim_df, + cont_cat_dim_df, + cont_dim_df, + cont_uni_dim_df, + cont_uni_dim_ref_df, + multi_metric_df, + single_metric_df, slicer, + uni_dim_df, ) - -lc.setlocale(lc.LC_ALL, 'C') +from mock import Mock class DataTablesTransformerTests(TestCase): maxDiff = None - @classmethod - def setUpClass(cls): - cls.single_metric_df = pd.DataFrame(mock_df[['votes']] - .sum()).T - - cls.multi_metric_df = pd.DataFrame(mock_df[['votes', 'wins']] - .sum()).T - - cls.cont_dim_df = mock_df[['timestamp', 'votes', 'wins']] \ - .groupby('timestamp') \ - .sum() - - cls.cat_dim_df = mock_df[['political_party', 'votes', 'wins']] \ - .groupby('political_party') \ - .sum() - - cls.uni_dim_df = mock_df[['candidate', 'candidate_display', 'votes', 'wins']] \ - .groupby(['candidate', 'candidate_display']) \ - .sum() \ - .reset_index('candidate_display') - - cls.cont_cat_dim_df = mock_df[['timestamp', 'political_party', 'votes', 'wins']] \ - .groupby(['timestamp', 'political_party']) \ - .sum() - - cls.cont_uni_dim_df = mock_df[['timestamp', 'state', 'state_display', 'votes', 'wins']] \ - .groupby(['timestamp', 'state', 'state_display']) \ - .sum() \ - .reset_index('state_display') - def test_single_metric(self): result = DataTablesJS(metrics=[slicer.metrics.votes]) \ - .transform(self.single_metric_df, slicer) + .transform(single_metric_df, slicer, []) self.assertEqual({ 'columns': [{ @@ -69,7 +41,7 @@ def test_single_metric(self): def test_single_metric_with_dataframe_containing_more(self): result = DataTablesJS(metrics=[slicer.metrics.votes]) \ - .transform(self.multi_metric_df, slicer) + .transform(multi_metric_df, slicer, []) self.assertEqual({ 'columns': [{ @@ -84,7 +56,7 @@ def test_single_metric_with_dataframe_containing_more(self): def test_multiple_metrics(self): result = DataTablesJS(metrics=[slicer.metrics.votes, slicer.metrics.wins]) \ - .transform(self.multi_metric_df, slicer) + .transform(multi_metric_df, slicer, []) self.assertEqual({ 'columns': [{ @@ -104,7 +76,7 @@ def test_multiple_metrics(self): def test_multiple_metrics_reversed(self): result = DataTablesJS(metrics=[slicer.metrics.wins, slicer.metrics.votes]) \ - .transform(self.multi_metric_df, slicer) + .transform(multi_metric_df, slicer, []) self.assertEqual({ 'columns': [{ @@ -124,7 +96,7 @@ def test_multiple_metrics_reversed(self): def test_time_series_dim(self): result = DataTablesJS(metrics=[slicer.metrics.wins]) \ - .transform(self.cont_dim_df, slicer) + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) self.assertEqual({ 'columns': [{ @@ -159,7 +131,7 @@ def test_time_series_dim(self): def test_cat_dim(self): result = DataTablesJS(metrics=[slicer.metrics.wins]) \ - .transform(self.cat_dim_df, slicer) + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) self.assertEqual({ 'columns': [{ @@ -185,9 +157,7 @@ def test_cat_dim(self): def test_uni_dim(self): result = DataTablesJS(metrics=[slicer.metrics.wins]) \ - .transform(self.uni_dim_df, slicer) - - print(result['data'][0]) + .transform(uni_dim_df, slicer, [slicer.dimensions.candidate]) self.assertEqual({ 'columns': [{ @@ -237,7 +207,7 @@ def test_uni_dim(self): def test_multi_dims_time_series_and_uni(self): result = DataTablesJS(metrics=[slicer.metrics.wins]) \ - .transform(self.cont_uni_dim_df, slicer) + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) self.assertEqual({ 'columns': [{ @@ -306,7 +276,7 @@ def test_multi_dims_time_series_and_uni(self): def test_pivoted_single_dimension_no_effect(self): result = DataTablesJS(metrics=[slicer.metrics.wins], pivot=True) \ - .transform(self.cat_dim_df, slicer) + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) self.assertEqual({ 'columns': [{ @@ -332,7 +302,7 @@ def test_pivoted_single_dimension_no_effect(self): def test_pivoted_multi_dims_time_series_and_cat(self): result = DataTablesJS(metrics=[slicer.metrics.wins], pivot=True) \ - .transform(self.cont_cat_dim_df, slicer) + .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party]) self.assertEqual({ 'columns': [{ @@ -399,7 +369,7 @@ def test_pivoted_multi_dims_time_series_and_cat(self): def test_pivoted_multi_dims_time_series_and_uni(self): result = DataTablesJS(metrics=[slicer.metrics.votes], pivot=True) \ - .transform(self.cont_uni_dim_df, slicer) + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) self.assertEqual({ 'columns': [{ @@ -454,6 +424,82 @@ def test_pivoted_multi_dims_time_series_and_uni(self): }], }, result) + def test_time_series_ref(self): + result = DataTablesJS(metrics=[slicer.metrics.votes]) \ + .transform(cont_uni_dim_ref_df, slicer, [slicer.dimensions.timestamp.reference(ElectionOverElection), + slicer.dimensions.state]) + + self.assertEqual({ + 'columns': [{ + 'data': 'timestamp', + 'title': 'Timestamp', + 'render': {'_': 'value'}, + }, { + 'data': 'state', + 'render': {'_': 'value', 'display': 'display'}, + 'title': 'State' + }, { + 'data': 'votes', + 'title': 'Votes', + 'render': {'_': 'value', 'display': 'display'}, + }, { + 'data': 'votes_eoe', + 'title': 'Votes (EoE)', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'timestamp': {'value': '2000-01-01'}, + 'state': {'display': 'Texas', 'value': 1}, + 'votes': {'display': '6233385', 'value': 6233385.}, + 'votes_eoe': {'display': '5574387', 'value': 5574387.}, + }, { + 'timestamp': {'value': '2000-01-01'}, + 'state': {'display': 'California', 'value': 2}, + 'votes': {'display': '10428632', 'value': 10428632.}, + 'votes_eoe': {'display': '9646062', 'value': 9646062.}, + }, { + 'timestamp': {'value': '2004-01-01'}, + 'state': {'display': 'Texas', 'value': 1}, + 'votes': {'display': '7359621', 'value': 7359621.}, + 'votes_eoe': {'display': '6233385', 'value': 6233385.}, + }, { + 'timestamp': {'value': '2004-01-01'}, + 'state': {'display': 'California', 'value': 2}, + 'votes': {'display': '12255311', 'value': 12255311.}, + 'votes_eoe': {'display': '10428632', 'value': 10428632.}, + }, { + 'timestamp': {'value': '2008-01-01'}, + 'state': {'display': 'Texas', 'value': 1}, + 'votes': {'display': '8007961', 'value': 8007961.}, + 'votes_eoe': {'display': '7359621', 'value': 7359621.}, + }, { + 'timestamp': {'value': '2008-01-01'}, + 'state': {'display': 'California', 'value': 2}, + 'votes': {'display': '13286254', 'value': 13286254.}, + 'votes_eoe': {'display': '12255311', 'value': 12255311.}, + }, { + 'timestamp': {'value': '2012-01-01'}, + 'state': {'display': 'Texas', 'value': 1}, + 'votes': {'display': '7877967', 'value': 7877967.}, + 'votes_eoe': {'display': '8007961', 'value': 8007961.}, + }, { + 'timestamp': {'value': '2012-01-01'}, + 'state': {'display': 'California', 'value': 2}, + 'votes': {'display': '12694243', 'value': 12694243.}, + 'votes_eoe': {'display': '13286254', 'value': 13286254.}, + }, { + 'timestamp': {'value': '2016-01-01'}, + 'state': {'display': 'Texas', 'value': 1}, + 'votes': {'display': '5072915', 'value': 5072915.}, + 'votes_eoe': {'display': '7877967', 'value': 7877967.}, + }, { + 'timestamp': {'value': '2016-01-01'}, + 'state': {'display': 'California', 'value': 2}, + 'votes': {'display': '13237598', 'value': 13237598.}, + 'votes_eoe': {'display': '12694243', 'value': 12694243.}, + }], + }, result) + class MetricCellFormatTests(TestCase): def _mock_metric(self, prefix=None, suffix=None, precision=None): diff --git a/fireant/tests/slicer/widgets/test_highcharts.py b/fireant/tests/slicer/widgets/test_highcharts.py new file mode 100644 index 00000000..12306497 --- /dev/null +++ b/fireant/tests/slicer/widgets/test_highcharts.py @@ -0,0 +1,765 @@ +from unittest import TestCase + +from fireant.slicer.widgets.highcharts import HighCharts +from fireant.tests.slicer.mocks import ( + ElectionOverElection, + cat_dim_df, + cont_dim_df, + cont_uni_dim_df, + cont_uni_dim_ref_delta_df, + cont_uni_dim_ref_df, + multi_metric_df, + single_metric_df, + slicer, +) + + +class HighchartsLineChartTransformerTests(TestCase): + maxDiff = None + + chart_class = HighCharts.LineChart + chart_type = 'line' + stacking = None + + def test_single_metric_line_chart(self): + result = HighCharts("Time Series, Single Metric", axes=[ + self.chart_class(metrics=[slicer.metrics.votes]) + ]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) + + self.assertEqual({ + "title": {"text": "Time Series, Single Metric"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}} + }], + "plotOptions": { + "series": { + "pointInterval": 126230400000.0, + "pointStart": 820454400000.0 + } + }, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes", + "yAxis": "0", + "data": [15220449.0, 16662017.0, 19614932.0, 21294215.0, 20572210.0, 18310513.0], + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": True, + }] + }, result) + + def test_single_metric_with_uni_dim_line_chart(self): + result = HighCharts("Time Series with Unique Dimension and Single Metric", axes=[ + self.chart_class(metrics=[slicer.metrics.votes]) + ]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, + slicer.dimensions.state]) + + self.assertEqual({ + "title": {"text": "Time Series with Unique Dimension and Single Metric"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}} + }], + "plotOptions": { + "series": { + "pointInterval": 126230400000.0, + "pointStart": 820454400000.0 + } + }, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes (Texas)", + "yAxis": "0", + "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Votes (California)", + "yAxis": "0", + "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "color": "#55BF3B", + "marker": {"symbol": "square", "fillColor": "#55BF3B"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": False, + }] + }, result) + + def test_multi_metrics_single_axis_line_chart(self): + result = HighCharts("Time Series with Unique Dimension and Multiple Metrics", axes=[ + self.chart_class(metrics=[slicer.metrics.votes, + slicer.metrics.wins]) + ]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, + slicer.dimensions.state]) + + self.assertEqual({ + "title": {"text": "Time Series with Unique Dimension and Multiple Metrics"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": "#DDDF0D"}} + }], + "plotOptions": { + "series": { + "pointInterval": 126230400000.0, + "pointStart": 820454400000.0 + } + }, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes (Texas)", + "yAxis": "0", + "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Votes (California)", + "yAxis": "0", + "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "color": "#DDDF0D", + "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": False, + }, { + "type": self.chart_type, + "name": "Wins (Texas)", + "yAxis": "0", + "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "color": "#55BF3B", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Wins (California)", + "yAxis": "0", + "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "color": "#55BF3B", + "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": False, + }] + }, result) + + def test_multi_metrics_multi_axis_line_chart(self): + result = HighCharts("Time Series with Unique Dimension and Multiple Metrics, Multi-Axis", axes=[ + self.chart_class(metrics=[slicer.metrics.votes]), + self.chart_class(metrics=[slicer.metrics.wins]), + ]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, + slicer.dimensions.state]) + + self.assertEqual({ + "title": {"text": "Time Series with Unique Dimension and Multiple Metrics, Multi-Axis"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "1", + "title": {"text": None}, + "labels": {"style": {"color": "#55BF3B"}} + }, { + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": "#DDDF0D"}} + }], + "plotOptions": { + "series": { + "pointInterval": 126230400000.0, + "pointStart": 820454400000.0 + } + }, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes (Texas)", + "yAxis": "0", + "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Votes (California)", + "yAxis": "0", + "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "color": "#55BF3B", + "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": False, + }, { + "type": self.chart_type, + "name": "Wins (Texas)", + "yAxis": "1", + "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "color": "#55BF3B", + "marker": {"symbol": "circle", "fillColor": "#55BF3B"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Wins (California)", + "yAxis": "1", + "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "color": "#DF5353", + "marker": {"symbol": "square", "fillColor": "#55BF3B"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": False, + }] + }, result) + + def test_uni_dim_with_ref_line_chart(self): + result = HighCharts("Time Series with Unique Dimension and Reference", axes=[ + self.chart_class(metrics=[slicer.metrics.votes]) + ]) \ + .transform(cont_uni_dim_ref_df, slicer, [slicer.dimensions.timestamp.reference(ElectionOverElection), + slicer.dimensions.state]) + + self.assertEqual({ + "title": {"text": "Time Series with Unique Dimension and Reference"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}} + }], + "plotOptions": { + "series": { + "pointInterval": 126230400000.0, + "pointStart": 820454400000.0 + } + }, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes (Texas)", + "yAxis": "0", + "data": [6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Votes (EoE) (Texas)", + "yAxis": "0", + "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0], + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "ShortDash", + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Votes (California)", + "yAxis": "0", + "data": [10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "color": "#55BF3B", + "marker": {"symbol": "square", "fillColor": "#55BF3B"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": False, + }, { + "type": self.chart_type, + "name": "Votes (EoE) (California)", + "yAxis": "0", + "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0], + "color": "#55BF3B", + "marker": {"symbol": "square", "fillColor": "#55BF3B"}, + "dashStyle": "ShortDash", + "stacking": self.stacking, + "visible": False, + }] + }, result) + + def test_uni_dim_with_ref_delta_line_chart(self): + result = HighCharts("Time Series with Unique Dimension and Delta Reference", axes=[ + self.chart_class(metrics=[slicer.metrics.votes]) + ]) \ + .transform(cont_uni_dim_ref_delta_df, + slicer, + [slicer.dimensions.timestamp.reference(ElectionOverElection.delta()), + slicer.dimensions.state]) + + self.assertEqual({ + "title": {"text": "Time Series with Unique Dimension and Delta Reference"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}} + }, { + "id": "0_eoe_delta", + "title": {"text": "EoE Δ"}, + "opposite": True, + "labels": {"style": {"color": None}} + }], + "plotOptions": { + "series": { + "pointInterval": 126230400000.0, + "pointStart": 820454400000.0 + } + }, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes (Texas)", + "yAxis": "0", + "data": [6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Votes (EoE Δ) (Texas)", + "yAxis": "0_eoe_delta", + "data": [-658998.0, -1126236.0, -648340.0, 129994.0, 2805052.0], + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "ShortDash", + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Votes (California)", + "yAxis": "0", + "data": [10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "color": "#55BF3B", + "marker": {"symbol": "square", "fillColor": "#55BF3B"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": False, + }, { + "type": self.chart_type, + "name": "Votes (EoE Δ) (California)", + "yAxis": "0_eoe_delta", + "data": [-782570.0, -1826679.0, -1030943.0, 592011.0, -543355.0], + "color": "#55BF3B", + "marker": {"symbol": "square", "fillColor": "#55BF3B"}, + "dashStyle": "ShortDash", + "stacking": self.stacking, + "visible": False, + }] + }, result) + + +class HighchartsBarChartTransformerTests(TestCase): + maxDiff = None + + chart_class = HighCharts.BarChart + chart_type = 'bar' + stacking = None + + def test_single_metric_bar_chart(self): + result = HighCharts("All Votes", axes=[ + self.chart_class(metrics=[slicer.metrics.votes]) + ]) \ + .transform(single_metric_df, slicer, []) + + self.assertEqual({ + "title": {"text": "All Votes"}, + "xAxis": { + "type": "category", + "categories": ["All"] + }, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}} + }], + "plotOptions": {}, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes", + "yAxis": "0", + "data": [111674336.0], + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": True, + }] + }, result) + + def test_multi_metric_bar_chart(self): + result = HighCharts("Votes and Wins", axes=[ + self.chart_class(metrics=[slicer.metrics.votes, + slicer.metrics.wins]) + ]) \ + .transform(multi_metric_df, slicer, []) + + self.assertEqual({ + "title": {"text": "Votes and Wins"}, + "xAxis": { + "type": "category", + "categories": ["All"] + }, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": "#DDDF0D"}} + + }], + "plotOptions": {}, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes", + "yAxis": "0", + "data": [111674336.0], + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Wins", + "yAxis": "0", + "data": [12.0], + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": True, + }] + }, result) + + def test_cat_dim_single_metric_bar_chart(self): + result = HighCharts("Votes and Wins", axes=[ + self.chart_class(metrics=[slicer.metrics.votes]) + ]) \ + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + + self.assertEqual({ + "title": {"text": "Votes and Wins"}, + "xAxis": { + "type": "category", + "categories": ["Democrat", "Independent", "Republican"] + }, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}} + + }], + "plotOptions": {}, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes", + "yAxis": "0", + "data": [54551568.0, 1076384.0, 56046384.0], + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": True, + }] + }, result) + + def test_cat_dim_multi_metric_bar_chart(self): + result = HighCharts("Votes and Wins", axes=[ + self.chart_class(metrics=[slicer.metrics.votes, + slicer.metrics.wins]) + ]) \ + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + + self.assertEqual({ + "title": {"text": "Votes and Wins"}, + "xAxis": { + "type": "category", + "categories": ["Democrat", "Independent", "Republican"] + }, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": "#DDDF0D"}} + + }], + "plotOptions": {}, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes", + "yAxis": "0", + "data": [54551568.0, 1076384.0, 56046384.0], + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Wins", + "yAxis": "0", + "data": [6.0, 0.0, 6.0], + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": True, + }] + }, result) + + def test_cont_uni_dims_single_metric_bar_chart(self): + result = HighCharts("Election Votes by State", axes=[ + self.chart_class(metrics=[slicer.metrics.votes]) + ]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + + self.assertEqual({ + "title": {"text": "Election Votes by State"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}} + + }], + "plotOptions": { + "series": { + "pointInterval": 126230400000.0, + "pointStart": 820454400000.0 + } + }, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes (Texas)", + "yAxis": "0", + "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Votes (California)", + "yAxis": "0", + "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": False, + }] + }, result) + + def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): + result = HighCharts("Election Votes by State", axes=[ + self.chart_class(metrics=[slicer.metrics.votes, + slicer.metrics.wins]), + ]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + + self.assertEqual({ + "title": {"text": "Election Votes by State"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": "#DDDF0D"}} + + }], + "plotOptions": { + "series": { + "pointInterval": 126230400000.0, + "pointStart": 820454400000.0 + } + }, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes (Texas)", + "yAxis": "0", + "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Votes (California)", + "yAxis": "0", + "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": False, + }, { + "type": self.chart_type, + "name": "Wins (Texas)", + "yAxis": "0", + "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Wins (California)", + "yAxis": "0", + "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": False, + }] + }, result) + + def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): + result = HighCharts("Election Votes by State", axes=[ + self.chart_class(metrics=[slicer.metrics.votes]), + self.chart_class(metrics=[slicer.metrics.wins]), + ]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + + self.assertEqual({ + "title": {"text": "Election Votes by State"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "1", + "title": {"text": None}, + "labels": {"style": {"color": "#55BF3B"}} + + }, { + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": "#DDDF0D"}} + + }], + "plotOptions": { + "series": { + "pointInterval": 126230400000.0, + "pointStart": 820454400000.0 + } + }, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes (Texas)", + "yAxis": "0", + "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Votes (California)", + "yAxis": "0", + "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": False, + }, { + "type": self.chart_type, + "name": "Wins (Texas)", + "yAxis": "1", + "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": True, + }, { + "type": self.chart_type, + "name": "Wins (California)", + "yAxis": "1", + "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "color": "#DF5353", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + "visible": False, + }] + }, result) + + +class HighchartsColumnChartTransformerTests(HighchartsBarChartTransformerTests): + chart_class = HighCharts.ColumnChart + chart_type = 'column' + + +class HighchartsStackedBarChartTransformerTests(HighchartsBarChartTransformerTests): + maxDiff = None + + chart_class = HighCharts.StackedBarChart + chart_type = 'bar' + stacking = "normal" + + +class HighchartsStackedColumnChartTransformerTests(HighchartsBarChartTransformerTests): + chart_class = HighCharts.StackedColumnChart + chart_type = 'column' + stacking = "normal" + + +class HighchartsAreaChartTransformerTests(HighchartsLineChartTransformerTests): + chart_class = HighCharts.AreaChart + chart_type = 'area' + + +class HighchartsAreaPercentChartTransformerTests(HighchartsLineChartTransformerTests): + chart_class = HighCharts.AreaPercentageChart + chart_type = 'area' + stacking = "normal" + + +class HighchartsPieChartTransformerTests(TestCase): + def test_single_metric_pie_chart(self): + pass diff --git a/setup.py b/setup.py index ad98037c..1c894b76 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,28 @@ -# coding: utf8 +import codecs +import os +import re -from fireant import __version__ from setuptools import setup +here = os.path.abspath(os.path.dirname(__file__)) + + +def read(*parts): + with codecs.open(os.path.join(here, *parts), 'r') as fp: + return fp.read() + + +def find_version(*file_paths): + version_file = read(*file_paths) + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", + version_file, re.M) + if version_match: + return version_match.group(1) + raise RuntimeError("Unable to find version string.") + setup( name='fireant', - version=__version__, + version=find_version("fireant", "__init__.py"), author='KAYAK, GmbH', author_email='bandit@kayak.com', @@ -56,19 +73,19 @@ install_requires=[ 'six', - 'pandas<0.20', - 'pypika>=0.8.0', - 'vertica-python>=0.6', - 'pymysql>=0.7.11' + 'pandas==0.22.0', + 'pypika==0.10.3', + 'toposort==1.5', + 'typing==3.6.2', ], tests_require=[ 'mock' ], extras_require={ - 'vertica': ['vertica-python>=0.6'], - 'mysql': ['pymysql>=0.7.11'], - 'redshift': ['psycopg2>=2.7.3.1'], - 'postgresql': ['psycopg2>=2.7.3.1'], + 'vertica': ['vertica-python==0.7.3'], + 'mysql': ['pymysql==0.8.0'], + 'redshift': ['psycopg2==2.7.3.2'], + 'postgresql': ['psycopg2==2.7.3.2'], 'matplotlib': ['matplotlib'], }, From d590d18105adfa6f8395108f153e423c9bb23efb Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 24 Jan 2018 13:35:13 +0100 Subject: [PATCH 003/123] Added functionality for operations, added CumSum, CumAvg, and CumProd --- fireant/database/base.py | 2 +- fireant/database/mysql.py | 2 +- fireant/database/postgresql.py | 2 +- fireant/slicer/__init__.py | 6 +- fireant/slicer/base.py | 6 - fireant/slicer/exceptions.py | 1 + fireant/slicer/filters.py | 6 - fireant/slicer/intervals.py | 2 +- fireant/slicer/operations.py | 60 +++++-- fireant/slicer/queries/builder.py | 29 ++- fireant/slicer/queries/database.py | 16 +- fireant/slicer/slicers.py | 18 ++ fireant/tests/database/test_databases.py | 4 +- fireant/tests/database/test_mysql.py | 2 +- fireant/tests/database/test_postgresql.py | 2 +- fireant/tests/database/test_redshift.py | 2 +- fireant/tests/database/test_vertica.py | 2 +- fireant/tests/slicer/matchers.py | 28 +++ fireant/tests/slicer/mocks.py | 5 +- .../__init__.py} | 0 .../test_builder.py} | 167 ++++++++++++++---- fireant/tests/slicer/queries/test_database.py | 21 +++ fireant/tests/slicer/test_operations.py | 104 +++++++++++ .../tests/slicer/widgets/test_datatables.py | 2 +- 24 files changed, 393 insertions(+), 96 deletions(-) create mode 100644 fireant/tests/slicer/matchers.py rename fireant/tests/slicer/{test_fetch_data.py => queries/__init__.py} (100%) rename fireant/tests/slicer/{test_querybuilder.py => queries/test_builder.py} (90%) create mode 100644 fireant/tests/slicer/queries/test_database.py create mode 100644 fireant/tests/slicer/test_operations.py diff --git a/fireant/database/base.py b/fireant/database/base.py index d0334818..4fcc0352 100644 --- a/fireant/database/base.py +++ b/fireant/database/base.py @@ -29,6 +29,6 @@ def fetch(self, query): cursor.execute(query) return cursor.fetchall() - def fetch_dataframe(self, query): + def fetch_data(self, query): with self.connect() as connection: return pd.read_sql(query, connection) diff --git a/fireant/database/mysql.py b/fireant/database/mysql.py index d0b06bc3..b59bdb36 100644 --- a/fireant/database/mysql.py +++ b/fireant/database/mysql.py @@ -60,7 +60,7 @@ def fetch(self, query): cursor.execute(query) return cursor.fetchall() - def fetch_dataframe(self, query): + def fetch_data(self, query): return pd.read_sql(query, self.connect()) def trunc_date(self, field, interval): diff --git a/fireant/database/postgresql.py b/fireant/database/postgresql.py index 9027974f..a5027338 100644 --- a/fireant/database/postgresql.py +++ b/fireant/database/postgresql.py @@ -47,7 +47,7 @@ def fetch(self, query): cursor.execute(query) return cursor.fetchall() - def fetch_dataframe(self, query): + def fetch_data(self, query): return pd.read_sql(query, self.connect()) def trunc_date(self, field, interval): diff --git a/fireant/slicer/__init__.py b/fireant/slicer/__init__.py index e97e3b7f..722309f8 100644 --- a/fireant/slicer/__init__.py +++ b/fireant/slicer/__init__.py @@ -24,10 +24,10 @@ from .joins import Join from .metrics import Metric from .operations import ( - CumAvg, + CumMean, + CumProd, CumSum, - L1Loss, - L2Loss, + Operation, ) from .references import ( DayOverDay, diff --git a/fireant/slicer/base.py b/fireant/slicer/base.py index 6f99f693..9dd50a1a 100644 --- a/fireant/slicer/base.py +++ b/fireant/slicer/base.py @@ -18,11 +18,5 @@ def __init__(self, key, label=None, definition=None): self.label = label or key self.definition = definition - def __unicode__(self): - return self.key - def __repr__(self): return self.key - - def __str__(self): - return self.key diff --git a/fireant/slicer/exceptions.py b/fireant/slicer/exceptions.py index 32a03ffc..5bcf1dcf 100644 --- a/fireant/slicer/exceptions.py +++ b/fireant/slicer/exceptions.py @@ -9,5 +9,6 @@ class QueryException(SlicerException): class MissingTableJoinException(SlicerException): pass + class CircularJoinsException(SlicerException): pass diff --git a/fireant/slicer/filters.py b/fireant/slicer/filters.py index dab88ff6..385f3a12 100644 --- a/fireant/slicer/filters.py +++ b/fireant/slicer/filters.py @@ -9,15 +9,9 @@ def __eq__(self, other): return isinstance(other, self.__class__) \ and str(self.definition) == str(other.definition) - def __str__(self): - return str(self.definition) - def __repr__(self): return str(self.definition) - def __unicode__(self): - return str(self.definition) - class DimensionFilter(Filter): pass diff --git a/fireant/slicer/intervals.py b/fireant/slicer/intervals.py index a8c86630..9177a172 100644 --- a/fireant/slicer/intervals.py +++ b/fireant/slicer/intervals.py @@ -11,7 +11,7 @@ def __eq__(self, other): def __hash__(self): return hash('NumericInterval %d %d' % (self.size, self.offset)) - def __str__(self): + def __repr__(self): return 'NumericInterval(size=%d,offset=%d)' % (self.size, self.offset) diff --git a/fireant/slicer/operations.py b/fireant/slicer/operations.py index 5c5c3f4d..411e683a 100644 --- a/fireant/slicer/operations.py +++ b/fireant/slicer/operations.py @@ -1,43 +1,67 @@ +import numpy as np +import pandas as pd + from .metrics import Metric +def _extract_key_or_arg(data_frame, key): + return data_frame[key] \ + if key in data_frame \ + else key + + class Operation(object): """ The `Operation` class represents an operation in the `Slicer` API. """ - pass + + def apply(self, data_frame): + raise NotImplementedError() -class _Loss(Operation): - def __init__(self, expected, actual): - self.expected = expected - self.actual = actual +class _Cumulative(Operation): + def __init__(self, arg): + self.arg = arg @property def metrics(self): return [metric - for metric in [self.expected, self.actual] + for metric in [self.arg] if isinstance(metric, Metric)] + def apply(self, data_frame): + raise NotImplementedError() -class L1Loss(_Loss): pass +class CumSum(_Cumulative): + def apply(self, data_frame): + if isinstance(data_frame.index, pd.MultiIndex): + return data_frame[self.arg.key] \ + .groupby(level=data_frame.index.names[1:]) \ + .cumsum() -class L2Loss(_Loss): pass + return data_frame[self.arg.key].cumsum() -class _Cumulative(Operation): - def __init__(self, arg): - self.arg = arg +class CumProd(_Cumulative): + def apply(self, data_frame): + if isinstance(data_frame.index, pd.MultiIndex): + return data_frame[self.arg.key] \ + .groupby(level=data_frame.index.names[1:]) \ + .cumprod() - @property - def metrics(self): - return [metric - for metric in [self.arg] - if isinstance(metric, Metric)] + return data_frame[self.arg.key].cumprod() -class CumSum(_Cumulative): pass +class CumMean(_Cumulative): + @staticmethod + def cummean(x): + return x.cumsum() / np.arange(1, len(x) + 1) + def apply(self, data_frame): + if isinstance(data_frame.index, pd.MultiIndex): + return data_frame[self.arg.key] \ + .groupby(level=data_frame.index.names[1:]) \ + .apply(self.cummean) -class CumAvg(_Cumulative): pass + return self.cummean(data_frame[self.arg.key]) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index de90d949..0ad572ac 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -18,6 +18,7 @@ ) from ..filters import DimensionFilter from ..intervals import DatetimeInterval +from ..operations import Operation def _build_dimension_definition(dimension, interval_func): @@ -41,30 +42,30 @@ def __init__(self, slicer): self._orders = [] @immutable - def widget(self, widget): + def widget(self, *widgets): """ :param widget: :return: """ - self._widgets.append(widget) + self._widgets += widgets @immutable - def dimension(self, dimension): + def dimension(self, *dimensions): """ :param dimension: :return: """ - self._dimensions.append(dimension) + self._dimensions += dimensions @immutable - def filter(self, filter): + def filter(self, *filters): """ :param filter: :return: """ - self._filters.append(filter) + self._filters += filters @property def tables(self): @@ -90,6 +91,17 @@ def metrics(self): for widget in self._widgets for metric in widget.metrics]) + @property + def operations(self): + """ + :return: + an ordered, distinct list of metrics used in all widgets as part of this query. + """ + return ordered_distinct_list_by_attr([metric + for widget in self._widgets + for metric in widget.metrics + if isinstance(metric, Operation)]) + @property def joins(self): """ @@ -217,8 +229,9 @@ def render(self): for dimension in self._dimensions]) # Apply operations - ... + for operation in self.operations: + data_frame[operation.key] = operation.apply(data_frame) # Apply transformations - return [widget.transform(data_frame, self.slicer) + return [widget.transform(data_frame, self.slicer, self._dimensions) for widget in self._widgets] diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index 3403a433..7902f3e9 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -1,21 +1,27 @@ import time +from ...database.base import Database +from typing import Iterable + from .logger import logger -def fetch_data(database, query, index_levels): +def fetch_data(database: Database, query: str, index_levels: Iterable[str]): """ Returns a Pandas Dataframe built from the result of the query. The query is also logged along with its duration. - :param database: Database object - :param query: PyPika query object - :return: Pandas Dataframe built from the result of the query + :param database: + instance of fireant.Database, database middleware + :param query: Query string + :param index_levels: A list of dimension keys, used for setting the index on the result data frame. + + :return: pd.DataFrame constructed from the result of the query """ start_time = time.time() logger.debug(query) - dataframe = database.fetch_dataframe(query) + dataframe = database.fetch_data(str(query)) logger.info('[{duration} seconds]: {query}' .format(duration=round(time.time() - start_time, 4), diff --git a/fireant/slicer/slicers.py b/fireant/slicer/slicers.py index 2796e16a..58cf64fe 100644 --- a/fireant/slicer/slicers.py +++ b/fireant/slicer/slicers.py @@ -1,6 +1,14 @@ +import itertools + from .queries import QueryBuilder +def _match_items(left, right, key): + return all([a is not None + and b is not None + and getattr(a, key) == getattr(b, key) + for a, b in itertools.zip_longest(left, right)]) + class _Container(object): def __init__(self, items): self._items = items @@ -60,3 +68,13 @@ def query(self): WRITEME """ return QueryBuilder(self) + + def __eq__(self, other): + return isinstance(other, Slicer) \ + and _match_items(self.metrics, other.metrics, 'key') \ + and _match_items(self.dimensions, other.dimensions, 'key') + + def __repr__(self): + return 'Slicer(metrics=[{}],dimensions=[{}])' \ + .format(','.join([m.key for m in self.metrics]), + ','.join([d.key for d in self.dimensions])) diff --git a/fireant/tests/database/test_databases.py b/fireant/tests/database/test_databases.py index d7d17028..644f7f08 100644 --- a/fireant/tests/database/test_databases.py +++ b/fireant/tests/database/test_databases.py @@ -1,6 +1,6 @@ from unittest import TestCase -from mock import patch, MagicMock +from unittest.mock import patch, MagicMock from fireant.database import Database from pypika import Field @@ -26,7 +26,7 @@ def test_fetch_dataframe(self, mock_connect, mock_read_sql): query = 'SELECT 1' mock_read_sql.return_value = 'OK' - result = Database().fetch_dataframe(query) + result = Database().fetch_data(query) self.assertEqual(mock_read_sql.return_value, result) diff --git a/fireant/tests/database/test_mysql.py b/fireant/tests/database/test_mysql.py index b85f1054..fb0cf679 100644 --- a/fireant/tests/database/test_mysql.py +++ b/fireant/tests/database/test_mysql.py @@ -1,6 +1,6 @@ from unittest import TestCase -from mock import ( +from unittest.mock import ( ANY, Mock, patch, diff --git a/fireant/tests/database/test_postgresql.py b/fireant/tests/database/test_postgresql.py index 7ff5889e..0f498c4e 100644 --- a/fireant/tests/database/test_postgresql.py +++ b/fireant/tests/database/test_postgresql.py @@ -1,6 +1,6 @@ from unittest import TestCase -from mock import ( +from unittest.mock import ( Mock, patch, ) diff --git a/fireant/tests/database/test_redshift.py b/fireant/tests/database/test_redshift.py index eb3f8e59..be69d3ce 100644 --- a/fireant/tests/database/test_redshift.py +++ b/fireant/tests/database/test_redshift.py @@ -1,5 +1,5 @@ -from mock import ( +from unittest.mock import ( Mock, patch, ) diff --git a/fireant/tests/database/test_vertica.py b/fireant/tests/database/test_vertica.py index 06d27690..8400e368 100644 --- a/fireant/tests/database/test_vertica.py +++ b/fireant/tests/database/test_vertica.py @@ -1,6 +1,6 @@ from unittest import TestCase -from mock import patch, Mock +from unittest.mock import patch, Mock from fireant import ( hourly, diff --git a/fireant/tests/slicer/matchers.py b/fireant/tests/slicer/matchers.py new file mode 100644 index 00000000..372f2921 --- /dev/null +++ b/fireant/tests/slicer/matchers.py @@ -0,0 +1,28 @@ +from fireant import ( + Dimension, + Metric, +) + + +class _ElementsMatcher: + expected_class = None + + def __init__(self, *elements): + self.elements = elements + + def __eq__(self, other): + return all([isinstance(actual, self.expected_class) + and expected.key == actual.key + for expected, actual in zip(self.elements, other)]) + + def __repr__(self): + return '[{}]'.format(','.join(str(element) + for element in self.elements)) + + +class MetricMatcher(_ElementsMatcher): + expected_class = Metric + + +class DimensionMatcher(_ElementsMatcher): + expected_class = Dimension diff --git a/fireant/tests/slicer/mocks.py b/fireant/tests/slicer/mocks.py index 00f48890..b10fefe1 100644 --- a/fireant/tests/slicer/mocks.py +++ b/fireant/tests/slicer/mocks.py @@ -2,7 +2,6 @@ OrderedDict, namedtuple, ) -from unittest.mock import Mock import pandas as pd from datetime import ( @@ -10,6 +9,7 @@ ) from fireant import * from fireant import VerticaDatabase +from unittest.mock import Mock from pypika import ( JoinType, @@ -23,6 +23,9 @@ class TestDatabase(VerticaDatabase): connect = Mock() + def __eq__(self, other): + return isinstance(other, TestDatabase) + test_database = TestDatabase() politicians_table = Table('politician', schema='politics') diff --git a/fireant/tests/slicer/test_fetch_data.py b/fireant/tests/slicer/queries/__init__.py similarity index 100% rename from fireant/tests/slicer/test_fetch_data.py rename to fireant/tests/slicer/queries/__init__.py diff --git a/fireant/tests/slicer/test_querybuilder.py b/fireant/tests/slicer/queries/test_builder.py similarity index 90% rename from fireant/tests/slicer/test_querybuilder.py rename to fireant/tests/slicer/queries/test_builder.py index 58a8f2a0..926bb700 100644 --- a/fireant/tests/slicer/test_querybuilder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -1,9 +1,15 @@ from unittest import TestCase +from unittest.mock import ( + ANY, + Mock, + patch, +) +import fireant as f from datetime import date -import fireant as f -from .mocks import slicer +from ..matchers import DimensionMatcher +from ..mocks import slicer class SlicerShortcutTests(TestCase): @@ -537,9 +543,9 @@ def test_build_query_with_cumsum_operation(self): 'GROUP BY "timestamp" ' 'ORDER BY "timestamp"', query) - def test_build_query_with_cumavg_operation(self): + def test_build_query_with_cummean_operation(self): query = slicer.query() \ - .widget(f.DataTablesJS([f.CumAvg(slicer.metrics.votes)])) \ + .widget(f.DataTablesJS([f.CumMean(slicer.metrics.votes)])) \ .dimension(slicer.dimensions.timestamp) \ .query @@ -550,40 +556,6 @@ def test_build_query_with_cumavg_operation(self): 'GROUP BY "timestamp" ' 'ORDER BY "timestamp"', query) - def test_build_query_with_l1loss_operation_constant(self): - query = slicer.query() \ - .widget(f.DataTablesJS([f.L1Loss(slicer.metrics.turnout, 1)])) \ - .dimension(slicer.dimensions.timestamp) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("politician"."timestamp",\'DD\') "timestamp",' - 'SUM("politician"."votes")/COUNT("voter"."id") "turnout" ' - 'FROM "politics"."politician" ' - 'OUTER JOIN "locations"."district" ' - 'ON "politician"."district_id"="district"."id" ' - 'JOIN "politics"."voter" ' - 'ON "district"."id"="voter"."district_id" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', query) - - def test_build_query_with_l2loss_operation_constant(self): - query = slicer.query() \ - .widget(f.DataTablesJS([f.L2Loss(slicer.metrics.turnout, 1)])) \ - .dimension(slicer.dimensions.timestamp) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("politician"."timestamp",\'DD\') "timestamp",' - 'SUM("politician"."votes")/COUNT("voter"."id") "turnout" ' - 'FROM "politics"."politician" ' - 'OUTER JOIN "locations"."district" ' - 'ON "politician"."district_id"="district"."id" ' - 'JOIN "politics"."voter" ' - 'ON "district"."id"="voter"."district_id" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', query) - # noinspection SqlDialectInspection,SqlNoDataSourceInspection class QueryBuilderDatetimeReferenceTests(TestCase): @@ -1159,3 +1131,122 @@ class QueryBuilderValidationTests(TestCase): def test_query_requires_at_least_one_metric(self): pass + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +@patch('fireant.slicer.queries.builder.fetch_data') +class QueryBuilderRenderTests(TestCase): + def test_pass_slicer_database_as_arg(self, mock_fetch_data: Mock): + mock_widget = Mock(name='mock_widget') + mock_widget.metrics = [slicer.metrics.votes] + + slicer.query() \ + .widget(mock_widget) \ + .render() + + mock_fetch_data.assert_called_once_with(slicer.database, + ANY, + index_levels=ANY) + + def test_pass_query_from_builder_as_arg(self, mock_fetch_data: Mock): + mock_widget = Mock(name='mock_widget') + mock_widget.metrics = [slicer.metrics.votes] + + slicer.query() \ + .widget(mock_widget) \ + .render() + + mock_fetch_data.assert_called_once_with(ANY, + 'SELECT SUM("votes") "votes" FROM "politics"."politician"', + index_levels=ANY) + + def test_builder_dimensions_as_arg_with_zero_dimensions(self, mock_fetch_data: Mock): + mock_widget = Mock(name='mock_widget') + mock_widget.metrics = [slicer.metrics.votes] + + slicer.query() \ + .widget(mock_widget) \ + .render() + + mock_fetch_data.assert_called_once_with(ANY, ANY, index_levels=[]) + + def test_builder_dimensions_as_arg_with_one_dimension(self, mock_fetch_data: Mock): + mock_widget = Mock(name='mock_widget') + mock_widget.metrics = [slicer.metrics.votes] + + slicer.query() \ + .widget(mock_widget) \ + .dimension(slicer.dimensions.state) \ + .render() + + mock_fetch_data.assert_called_once_with(ANY, ANY, index_levels=['state']) + + def test_builder_dimensions_as_arg_with_multiple_dimensions(self, mock_fetch_data: Mock): + mock_widget = Mock(name='mock_widget') + mock_widget.metrics = [slicer.metrics.votes] + + slicer.query() \ + .widget(mock_widget) \ + .dimension(slicer.dimensions.timestamp, slicer.dimensions.state, slicer.dimensions.political_party) \ + .render() + + mock_fetch_data.assert_called_once_with(ANY, ANY, index_levels=['timestamp', 'state', 'political_party']) + + def test_call_transform_on_widget(self, mock_fetch_data: Mock): + mock_widget = Mock(name='mock_widget') + mock_widget.metrics = [slicer.metrics.votes] + + # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work + slicer.query() \ + .dimension(slicer.dimensions.timestamp) \ + .widget(mock_widget) \ + .render() + + mock_widget.transform.assert_called_once_with(mock_fetch_data.return_value, + slicer, + DimensionMatcher(slicer.dimensions.timestamp)) + + def test_returns_results_from_widget_transform(self, mock_fetch_data: Mock): + mock_widget = Mock(name='mock_widget') + mock_widget.metrics = [slicer.metrics.votes] + + # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work + result = slicer.query() \ + .dimension(slicer.dimensions.timestamp) \ + .widget(mock_widget) \ + .render() + + self.assertListEqual(result, [mock_widget.transform.return_value]) + + def test_operations_evaluated(self, mock_fetch_data: Mock): + mock_widget = Mock(name='mock_widget') + mock_operation = Mock(name='mock_operation ', spec=f.Operation) + mock_operation.key, mock_operation.definition = 'mock_operation', slicer.table.abc + mock_widget.metrics = [mock_operation] + mock_df = {} + mock_fetch_data.return_value = mock_df + + # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work + slicer.query() \ + .dimension(slicer.dimensions.timestamp) \ + .widget(mock_widget) \ + .render() + + mock_operation.apply.assert_called_once_with(mock_df) + + def test_operations_results_stored_in_data_frame(self, mock_fetch_data: Mock): + mock_widget = Mock(name='mock_widget') + mock_operation = Mock(name='mock_operation ', spec=f.Operation) + mock_operation.key, mock_operation.definition = 'mock_operation', slicer.table.abc + mock_widget.metrics = [mock_operation] + mock_df = {} + mock_fetch_data.return_value = mock_df + + # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work + slicer.query() \ + .dimension(slicer.dimensions.timestamp) \ + .widget(mock_widget) \ + .render() + + self.assertIn(mock_operation.key, mock_df) + self.assertEqual(mock_df[mock_operation.key], mock_operation.apply.return_value) diff --git a/fireant/tests/slicer/queries/test_database.py b/fireant/tests/slicer/queries/test_database.py new file mode 100644 index 00000000..1ba175e0 --- /dev/null +++ b/fireant/tests/slicer/queries/test_database.py @@ -0,0 +1,21 @@ +from unittest import TestCase + +from fireant.slicer.queries.database import fetch_data +from unittest.mock import Mock + + +class FetchDataTests(TestCase): + @classmethod + def setUpClass(cls): + cls.mock_database = Mock(name='database') + cls.mock_data_frame = cls.mock_database.fetch_data.return_value = Mock(name='data_frame') + cls.mock_query = 'SELECT *' + cls.mock_index_levels = ['a', 'b'] + + cls.result = fetch_data(cls.mock_database, cls.mock_query, cls.mock_index_levels) + + def test_fetch_data_called_on_database(self): + self.mock_database.fetch_data.assert_called_once_with(self.mock_query) + + def test_index_set_on_data_frame_result(self): + self.mock_data_frame.set_index.assert_called_once_with(self.mock_index_levels) diff --git a/fireant/tests/slicer/test_operations.py b/fireant/tests/slicer/test_operations.py new file mode 100644 index 00000000..772e8c4b --- /dev/null +++ b/fireant/tests/slicer/test_operations.py @@ -0,0 +1,104 @@ +from unittest import TestCase + +import pandas as pd +import pandas.testing +from fireant import ( + CumMean, + CumProd, + CumSum, +) +from fireant.tests.slicer.mocks import ( + cont_dim_df, + cont_uni_dim_df, + cont_uni_dim_ref_df, + slicer, +) + + +class CumSumTests(TestCase): + def test_apply_to_timeseries(self): + cumsum = CumSum(slicer.metrics.wins) + result = cumsum.apply(cont_dim_df) + + expected = pd.Series([2, 4, 6, 8, 10, 12], + name='wins', + index=cont_dim_df.index) + pandas.testing.assert_series_equal(result, expected) + + def test_apply_to_timeseries_with_uni_dim(self): + cumsum = CumSum(slicer.metrics.wins) + result = cumsum.apply(cont_uni_dim_df) + + expected = pd.Series([1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6], + name='wins', + index=cont_uni_dim_df.index) + pandas.testing.assert_series_equal(result, expected) + + def test_apply_to_timeseries_with_uni_dim_and_ref(self): + cumsum = CumSum(slicer.metrics.wins) + result = cumsum.apply(cont_uni_dim_ref_df) + + expected = pd.Series([1, 1, 2, 2, 3, 3, 4, 4, 5, 5], + name='wins', + index=cont_uni_dim_ref_df.index) + pandas.testing.assert_series_equal(result, expected) + + +class CumProdTests(TestCase): + def test_apply_to_timeseries(self): + cumprod = CumProd(slicer.metrics.wins) + result = cumprod.apply(cont_dim_df) + + expected = pd.Series([2, 4, 8, 16, 32, 64], + name='wins', + index=cont_dim_df.index) + pandas.testing.assert_series_equal(result, expected) + + def test_apply_to_timeseries_with_uni_dim(self): + cumprod = CumProd(slicer.metrics.wins) + result = cumprod.apply(cont_uni_dim_df) + + expected = pd.Series([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + name='wins', + index=cont_uni_dim_df.index) + pandas.testing.assert_series_equal(result, expected) + + def test_apply_to_timeseries_with_uni_dim_and_ref(self): + cumprod = CumProd(slicer.metrics.wins) + result = cumprod.apply(cont_uni_dim_ref_df) + + expected = pd.Series([1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + name='wins', + index=cont_uni_dim_ref_df.index) + pandas.testing.assert_series_equal(result, expected) + + +class CumMeanTests(TestCase): + def test_apply_to_timeseries(self): + cummean = CumMean(slicer.metrics.votes) + result = cummean.apply(cont_dim_df) + + expected = pd.Series([15220449.0, 15941233.0, 17165799.3, 18197903.25, 18672764.6, 18612389.3], + name='votes', + index=cont_dim_df.index) + pandas.testing.assert_series_equal(result, expected) + + def test_apply_to_timeseries_with_uni_dim(self): + cummean = CumMean(slicer.metrics.votes) + result = cummean.apply(cont_uni_dim_df) + + expected = pd.Series([5574387.0, 9646062.0, 5903886.0, 10037347.0, 6389131.0, 10776668.3, + 6793838.5, 11404064.75, 7010664.2, 11662100.4, 6687706.0, 11924683.3], + name='votes', + index=cont_uni_dim_df.index) + pandas.testing.assert_series_equal(result, expected) + + def test_apply_to_timeseries_with_uni_dim_and_ref(self): + cummean = CumMean(slicer.metrics.votes) + result = cummean.apply(cont_uni_dim_ref_df) + + expected = pd.Series([6233385.0, 10428632.0, 6796503.0, 11341971.5, 7200322.3, + 11990065.6, 7369733.5, 12166110.0, 6910369.8, 12380407.6], + name='votes', + index=cont_uni_dim_ref_df.index) + pandas.testing.assert_series_equal(result, expected) diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py index 04c27a90..d80616a2 100644 --- a/fireant/tests/slicer/widgets/test_datatables.py +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -1,4 +1,5 @@ from unittest import TestCase +from unittest.mock import Mock import pandas as pd from datetime import date @@ -18,7 +19,6 @@ slicer, uni_dim_df, ) -from mock import Mock class DataTablesTransformerTests(TestCase): From 7f2dc5a607913fa9ea74ecde90a7efcdfad8ac5d Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 26 Jan 2018 15:24:42 +0100 Subject: [PATCH 004/123] Added support for Totals for MySQL and Vertica backends --- fireant/database/base.py | 5 +- fireant/database/mysql.py | 3 + fireant/database/vertica.py | 3 + fireant/slicer/dimensions.py | 22 +++- fireant/slicer/exceptions.py | 7 ++ fireant/slicer/queries/builder.py | 93 +++++++++++--- fireant/slicer/queries/database.py | 58 +++++++-- fireant/slicer/widgets/base.py | 8 ++ fireant/slicer/widgets/datatables.py | 11 +- fireant/slicer/widgets/formats.py | 23 +++- fireant/slicer/widgets/highcharts.py | 29 +++-- fireant/tests/database/test_databases.py | 2 +- fireant/tests/slicer/mocks.py | 57 ++++++--- fireant/tests/slicer/queries/test_builder.py | 108 ++++++++++++++-- fireant/tests/slicer/queries/test_database.py | 119 +++++++++++++++++- .../tests/slicer/widgets/test_datatables.py | 24 ++-- fireant/tests/slicer/widgets/test_formats.py | 17 ++- 17 files changed, 493 insertions(+), 96 deletions(-) diff --git a/fireant/database/base.py b/fireant/database/base.py index 4fcc0352..c3df135d 100644 --- a/fireant/database/base.py +++ b/fireant/database/base.py @@ -31,4 +31,7 @@ def fetch(self, query): def fetch_data(self, query): with self.connect() as connection: - return pd.read_sql(query, connection) + return pd.read_sql(query, connection, coerce_float=True, parse_dates=True) + + def totals(self, query, terms): + raise NotImplementedError diff --git a/fireant/database/mysql.py b/fireant/database/mysql.py index b59bdb36..c507a2fe 100644 --- a/fireant/database/mysql.py +++ b/fireant/database/mysql.py @@ -70,3 +70,6 @@ def date_add(self, field, date_part, interval): # adding an extra 's' as MySQL's interval doesn't work with 'year', 'week' etc, it expects a plural interval_term = terms.Interval(**{'{}s'.format(str(date_part)): interval, 'dialect': Dialects.MYSQL}) return DateAdd(field, interval_term) + + def totals(self, query, *terms): + raise NotImplementedError diff --git a/fireant/database/vertica.py b/fireant/database/vertica.py index 419c7bf6..5e855437 100644 --- a/fireant/database/vertica.py +++ b/fireant/database/vertica.py @@ -65,3 +65,6 @@ def trunc_date(self, field, interval): def date_add(self, field, date_part, interval): return fn.TimestampAdd(date_part, interval, field) + + def totals(self, query, terms): + return query.rollup(*terms) diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 58bfcc23..2db0e00d 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -24,7 +24,23 @@ def __init__(self, key, label=None, definition=None): super(Dimension, self).__init__(key, label, definition) -class BooleanDimension(Dimension): +class RollupDimension(Dimension): + """ + This represents a dimension which can be rolled up to display the totals across the dimension in addition to the + break-down by dimension. Rollup returns `NULL` for the dimension value in most databases and therefore it is not + safe to use roll up in combination with dimension definitions that can return `NULL`. + """ + + def __init__(self, key, label=None, definition=None): + super(Dimension, self).__init__(key, label, definition) + self.is_rollup = False + + @immutable + def rollup(self): + self.is_rollup = True + + +class BooleanDimension(RollupDimension): """ This is a dimension that represents a boolean true/false value. The expression should always result in a boolean value. @@ -39,7 +55,7 @@ def is_(self, value): return BooleanFilter(self.definition, value) -class CategoricalDimension(Dimension): +class CategoricalDimension(RollupDimension): """ This is a dimension that represents an enum-like database field, with a finite list of options to chose from. It provides support for configuring a display value for each of the possible values. @@ -58,7 +74,7 @@ def notin(self, values): return ExcludesFilter(self.definition, values) -class UniqueDimension(Dimension): +class UniqueDimension(RollupDimension): """ This is a dimension that represents a field in a database which is a unique identifier, such as a primary/foreign key. It provides support for a display value field which is selected and used in the results. diff --git a/fireant/slicer/exceptions.py b/fireant/slicer/exceptions.py index 5bcf1dcf..f93ebefd 100644 --- a/fireant/slicer/exceptions.py +++ b/fireant/slicer/exceptions.py @@ -12,3 +12,10 @@ class MissingTableJoinException(SlicerException): class CircularJoinsException(SlicerException): pass + + +class RollupException(SlicerException): + pass + +class MissingMetricsException(SlicerException): + pass diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 0ad572ac..605ddabe 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -1,20 +1,22 @@ from collections import defaultdict -from fireant.utils import ( - immutable, - ordered_distinct_list, - ordered_distinct_list_by_attr, -) from toposort import ( CircularDependencyError, toposort_flatten, ) +from fireant.utils import ( + immutable, + ordered_distinct_list, + ordered_distinct_list_by_attr, +) from .database import fetch_data from .references import join_reference +from ..dimensions import RollupDimension from ..exceptions import ( CircularJoinsException, MissingTableJoinException, + RollupException, ) from ..filters import DimensionFilter from ..intervals import DatetimeInterval @@ -29,6 +31,55 @@ def _build_dimension_definition(dimension, interval_func): return dimension.definition.as_(dimension.key) +def _select_groups(terms, query, rollup, database): + query = query.select(*terms) + + if not rollup: + return query.groupby(*terms) + + if 1 < len(terms): + # This step packs multiple terms together so that they are rolled up together. This is needed for unique + # dimensions to keep the display field grouped together with the definition field. + terms = [terms] + + return database.totals(query, terms) + + +def is_rolling_up(dimension, rolling_up): + if rolling_up: + if not isinstance(dimension, RollupDimension): + raise RollupException('Cannot roll up dimension {}'.format(dimension)) + return True + return getattr(dimension, "is_rollup", False) + + +def clean(index): + import pandas as pd + + if isinstance(index, (pd.DatetimeIndex, pd.RangeIndex)): + return index + + return index.astype('str') + + +def clean_index(data_frame): + import pandas as pd + + if hasattr(data_frame.index, 'levels'): + data_frame.index = pd.MultiIndex( + levels=[level.astype('str') + if not isinstance(level, (pd.DatetimeIndex, pd.RangeIndex)) + else level + for level in data_frame.index.levels], + labels=data_frame.index.labels + ) + + elif not isinstance(data_frame.index, (pd.DatetimeIndex, pd.RangeIndex)): + data_frame.index = data_frame.index.astype('str') + + return data_frame + + class QueryBuilder(object): """ @@ -45,7 +96,7 @@ def __init__(self, slicer): def widget(self, *widgets): """ - :param widget: + :param widgets: :return: """ self._widgets += widgets @@ -54,7 +105,7 @@ def widget(self, *widgets): def dimension(self, *dimensions): """ - :param dimension: + :param dimensions: :return: """ self._dimensions += dimensions @@ -62,7 +113,7 @@ def dimension(self, *dimensions): @immutable def filter(self, *filters): """ - :param filter: + :param filters: :return: """ self._filters += filters @@ -153,14 +204,21 @@ def query(self): query = query.join(join.table, how=join.join_type).on(join.criterion) # Add dimensions + rolling_up = False for dimension in self._dimensions: + rolling_up = is_rolling_up(dimension, rolling_up) + dimension_definition = _build_dimension_definition(dimension, self.slicer.database.trunc_date) - query = query.select(dimension_definition).groupby(dimension_definition) - # Add display definition field if hasattr(dimension, 'display_definition'): + # Add display definition field dimension_display_definition = dimension.display_definition.as_(dimension.display_key) - query = query.select(dimension_display_definition).groupby(dimension_display_definition) + fields = [dimension_definition, dimension_display_definition] + + else: + fields = [dimension_definition] + + query = _select_groups(fields, query, rolling_up, self.slicer.database) # Add metrics query = query.select(*[metric.definition.as_(metric.key) @@ -223,10 +281,11 @@ def render(self): :return: """ + query = self.query + data_frame = fetch_data(self.slicer.database, - self.query, - index_levels=[dimension.key - for dimension in self._dimensions]) + query, + dimensions=self._dimensions) # Apply operations for operation in self.operations: @@ -235,3 +294,9 @@ def render(self): # Apply transformations return [widget.transform(data_frame, self.slicer, self._dimensions) for widget in self._widgets] + + def __str__(self): + return self.query + + def __iter__(self): + return iter(self.render()) diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index 7902f3e9..d3041dfc 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -1,30 +1,70 @@ import time -from ...database.base import Database +import pandas as pd from typing import Iterable +from fireant.database.base import Database from .logger import logger +from ..dimensions import ( + Dimension, + RollupDimension, +) +CONTINUOUS_DIMS = (pd.DatetimeIndex, pd.RangeIndex) -def fetch_data(database: Database, query: str, index_levels: Iterable[str]): + +def fetch_data(database: Database, query: str, dimensions: Iterable[Dimension]): """ - Returns a Pandas Dataframe built from the result of the query. - The query is also logged along with its duration. + Executes a query to fetch data from database middleware and builds/cleans the data as a data frame. The query + execution is logged with its duration. :param database: - instance of fireant.Database, database middleware + instance of `fireant.Database`, database middleware :param query: Query string - :param index_levels: A list of dimension keys, used for setting the index on the result data frame. + :param dimensions: A list of dimensions, used for setting the index on the result data frame. - :return: pd.DataFrame constructed from the result of the query + :return: `pd.DataFrame` constructed from the result of the query """ start_time = time.time() logger.debug(query) - dataframe = database.fetch_data(str(query)) + data_frame = database.fetch_data(str(query)) logger.info('[{duration} seconds]: {query}' .format(duration=round(time.time() - start_time, 4), query=query)) - return dataframe.set_index(index_levels) + return clean_and_apply_index(data_frame, dimensions) + + +def clean_and_apply_index(data_frame: pd.DataFrame, dimensions: Iterable[Dimension]): + """ + + :param data_frame: + :param dimensions: + :return: + """ + dimension_keys = [d.key for d in dimensions] + + for i, dimension in enumerate(dimensions): + if not isinstance(dimension, RollupDimension): + continue + + level = dimension.key + if dimension.is_rollup: + # Rename the first instance of NaN to totals for each dimension value + # If there are multiple dimensions, we need to group by the preceding dimensions for each dimension + data_frame[level] = ( + data_frame + .groupby(dimension_keys[:i])[level] + .fillna('Totals', limit=1) + ) if 0 < i else ( + data_frame[level].fillna('Totals', limit=1) + ) + + data_frame[level] = data_frame[level] \ + .fillna('') \ + .astype('str') + + # Set index on dimension columns + return data_frame.set_index(dimension_keys) diff --git a/fireant/slicer/widgets/base.py b/fireant/slicer/widgets/base.py index 388af41b..fa0968b7 100644 --- a/fireant/slicer/widgets/base.py +++ b/fireant/slicer/widgets/base.py @@ -1,3 +1,4 @@ +from fireant.slicer.exceptions import MissingMetricsException from fireant.utils import immutable @@ -16,9 +17,16 @@ def metric(self, metric): @property def metrics(self): + if 0 == len(self._metrics): + raise MissingMetricsException(str(self)) + return [metric for group in self._metrics for metric in getattr(group, 'metrics', [group])] def transform(self, data_frame, slicer, dimensions): super(MetricsWidget, self).transform(data_frame, slicer, dimensions) + + def __repr__(self): + return '{}(metrics={})'.format(self.__class__.__name__, + ','.join(str(m) for m in self._metrics)) diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index 6b9010e0..45ef2c7e 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -17,7 +17,7 @@ def _format_dimension_cell(dimension_value, display_values): - dimension_cell = {'value': formats.safe(dimension_value)} + dimension_cell = {'value': formats.format_dimension_value(dimension_value)} if display_values is not None: dimension_cell['display'] = display_values.get(dimension_value, dimension_value) @@ -30,9 +30,9 @@ def _format_dimensional_metric_cell(row_data, metric): for key, next_row in row_data.groupby(level=-1): next_row.reset_index(level=-1, drop=True, inplace=True) - safe_key = formats.safe(key) + safe_key = formats.format_dimension_value(key) - level[safe_key] = _format_dimensional_metric_cell(next_row, metric) \ + level[key] = _format_dimensional_metric_cell(next_row, metric) \ if isinstance(next_row.index, pd.MultiIndex) \ else _format_metric_cell(next_row[metric.key], metric) @@ -40,7 +40,7 @@ def _format_dimensional_metric_cell(row_data, metric): def _format_metric_cell(value, metric): - raw_value = formats.safe(value) + raw_value = formats.format_metric_value(value) return { 'value': raw_value, 'display': formats.display(raw_value, @@ -84,7 +84,7 @@ def transform(self, data_frame, slicer, dimensions): pivot_index_to_columns = self.pivot and isinstance(data_frame.index, pd.MultiIndex) if pivot_index_to_columns: - levels = list(range(1, len(dimensions))) + levels = data_frame.index.names[1:] data_frame = data_frame \ .unstack(level=levels) \ .fillna(value=0) @@ -96,7 +96,6 @@ def transform(self, data_frame, slicer, dimensions): data_frame.columns, render_column_label) - else: dimension_columns = self._dimension_columns(dimensions) metric_columns = self._metric_columns(references) diff --git a/fireant/slicer/widgets/formats.py b/fireant/slicer/widgets/formats.py index c6383f04..c553a0e1 100644 --- a/fireant/slicer/widgets/formats.py +++ b/fireant/slicer/widgets/formats.py @@ -10,7 +10,28 @@ NO_TIME = time(0) -def safe(value): +def format_dimension_value(value): + if isinstance(value, date): + if not hasattr(value, 'time') or value.time() == NO_TIME: + return value.strftime('%Y-%m-%d') + else: + return value.strftime('%Y-%m-%dT%H:%M:%S') + + for type_cast in (int, float): + try: + return type_cast(value) + except: + pass + + if 'True' is value: + return True + if 'False' is value: + return False + + return value + + +def format_metric_value(value): if isinstance(value, date): if not hasattr(value, 'time') or value.time() == NO_TIME: return value.strftime('%Y-%m-%d') diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index a38cc062..1230079b 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -57,7 +57,7 @@ class ChartWidget(MetricsWidget): needs_marker = False stacked = False - def __init__(self, name=None, metrics=(), stacked=False): + def __init__(self, metrics=(), name=None, stacked=False): super(ChartWidget, self).__init__(metrics=metrics) self.name = name self.stacked = self.stacked or stacked @@ -137,7 +137,7 @@ def transform(self, data_frame, slicer, dimensions): """ colors = itertools.cycle(self.colors) - levels = list(range(1, len(dimensions))) + levels = data_frame.index.names[1:] groups = list(data_frame.groupby(level=levels)) \ if levels \ else [([], data_frame)] @@ -179,8 +179,8 @@ def transform(self, data_frame, slicer, dimensions): "legend": {"useHTML": True}, } - - def _render_x_axis(self, data_frame, dimension_display_values): + @staticmethod + def _render_x_axis(data_frame, dimension_display_values): """ Renders the xAxis configuraiton. @@ -209,7 +209,8 @@ def _render_x_axis(self, data_frame, dimension_display_values): "categories": categories, } - def _render_y_axis(self, axis_idx, color, references): + @staticmethod + def _render_y_axis(axis_idx, color, references): """ Renders the yAxis configuraiton. @@ -237,7 +238,8 @@ def _render_y_axis(self, axis_idx, color, references): return y_axes - def _render_plot_options(self, data_frame): + @staticmethod + def _render_plot_options(data_frame): """ Renders the plotOptions configuration @@ -265,7 +267,8 @@ def _render_plot_options(self, data_frame): return {} - def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, render_series_label, references): + @staticmethod + def _render_series(axis, axis_idx, axis_color, colors, data_frame_groups, render_series_label, references): """ Renders the series configuration. @@ -309,13 +312,13 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, if reference is not None and reference.is_delta else str(axis_idx)), - "marker": {"symbol": symbol, "fillColor": axis_color or series_color} \ - if axis.needs_marker \ - else {}, + "marker": ({"symbol": symbol, "fillColor": axis_color or series_color} + if axis.needs_marker + else {}), - "stacking": "normal" \ - if axis.stacked \ - else None, + "stacking": ("normal" + if axis.stacked + else None), }) visible = False # Only display the first in each group diff --git a/fireant/tests/database/test_databases.py b/fireant/tests/database/test_databases.py index 644f7f08..a21898c5 100644 --- a/fireant/tests/database/test_databases.py +++ b/fireant/tests/database/test_databases.py @@ -30,7 +30,7 @@ def test_fetch_dataframe(self, mock_connect, mock_read_sql): self.assertEqual(mock_read_sql.return_value, result) - mock_read_sql.assert_called_once_with(query, mock_connect().__enter__()) + mock_read_sql.assert_called_once_with(query, mock_connect().__enter__(), coerce_float=True, parse_dates=True) def test_database_api(self): db = Database() diff --git a/fireant/tests/slicer/mocks.py b/fireant/tests/slicer/mocks.py index b10fefe1..15e4452a 100644 --- a/fireant/tests/slicer/mocks.py +++ b/fireant/tests/slicer/mocks.py @@ -2,15 +2,15 @@ OrderedDict, namedtuple, ) +from unittest.mock import Mock import pandas as pd from datetime import ( datetime, ) + from fireant import * from fireant import VerticaDatabase -from unittest.mock import Mock - from pypika import ( JoinType, Table, @@ -203,15 +203,15 @@ def __eq__(self, other): (6, 11): False, } -columns = ['timestamp', - 'candidate', 'candidate_display', - 'political_party', - 'election', 'election_display', - 'state', 'state_display', - 'winner', - 'votes', - 'wins'] -PoliticsRow = namedtuple('PoliticsRow', columns) +df_columns = ['timestamp', + 'candidate', 'candidate_display', + 'political_party', + 'election', 'election_display', + 'state', 'state_display', + 'winner', + 'votes', + 'wins'] +PoliticsRow = namedtuple('PoliticsRow', df_columns) records = [] for (election_id, candidate_id, state_id), votes in election_candidate_state_votes.items(): @@ -228,7 +228,7 @@ def __eq__(self, other): wins=(1 if winner else 0), )) -mock_politics_database = pd.DataFrame.from_records(records, columns=columns) +mock_politics_database = pd.DataFrame.from_records(records, columns=df_columns) single_metric_df = pd.DataFrame(mock_politics_database[['votes']] .sum()).T @@ -263,7 +263,7 @@ def ref(data_frame, columns): ref_cols = {column: '%s_eoe' % column for column in columns} - ref_df = cont_uni_dim_df \ + ref_df = data_frame \ .shift(2) \ .rename(columns=ref_cols)[list(ref_cols.values())] @@ -288,4 +288,33 @@ def ref_delta(ref_data_frame, columns): cont_uni_dim_ref_df = ref(cont_uni_dim_df, _columns) cont_uni_dim_ref_delta_df = ref_delta(cont_uni_dim_ref_df, _columns) -ElectionOverElection = Reference('eoe', 'EoE', 'year', 4) + +def totals(data_frame, dimensions, columns): + """ + Computes the totals across a dimension and adds the total as an extra row. + """ + dfx = data_frame.unstack(level=dimensions) + for c in columns: + dfx[(c, 'Total')] = dfx[c].sum(axis=1) + + return dfx.stack(level=dimensions) + + +# Convert all index values to string +for l in list(locals().values()): + if not isinstance(l, pd.DataFrame): + continue + + if hasattr(l.index, 'levels'): + l.index = pd.MultiIndex(levels=[level.astype('str') + if not isinstance(level, (pd.DatetimeIndex, pd.RangeIndex)) + else level + for level in l.index.levels], + labels=l.index.labels) + elif not isinstance(l.index, (pd.DatetimeIndex, pd.RangeIndex)): + l.index = l.index.astype('str') + +cont_cat_dim_totals_df = totals(cont_cat_dim_df, ['political_party'], _columns) +cont_uni_dim_totals_df = totals(cont_uni_dim_df, ['state'], _columns) + +ElectionOverElection = Reference('eoe', 'EoE', 'year', 4) \ No newline at end of file diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 926bb700..aeacc77c 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -5,9 +5,10 @@ patch, ) -import fireant as f from datetime import date +import fireant as f +from fireant.slicer.exceptions import RollupException from ..matchers import DimensionMatcher from ..mocks import slicer @@ -256,6 +257,93 @@ def test_build_query_with_multiple_dimensions_and_visualizations(self): 'ORDER BY "timestamp","political_party"', query) +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderDimensionRollupTests(TestCase): + maxDiff = None + + def test_build_query_with_rollup_cat_dimension(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.political_party.rollup()) \ + .query + + self.assertEqual('SELECT ' + '"political_party" "political_party",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY ROLLUP("political_party") ' + 'ORDER BY "political_party"', query) + + def test_build_query_with_rollup_uni_dimension(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.candidate.rollup()) \ + .query + + self.assertEqual('SELECT ' + '"candidate_id" "candidate",' + '"candidate_name" "candidate_display",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY ROLLUP(("candidate_id","candidate_name")) ' + 'ORDER BY "candidate"', query) + + def test_rollup_following_non_rolled_up_dimensions(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp, + slicer.dimensions.candidate.rollup()) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + '"candidate_id" "candidate",' + '"candidate_name" "candidate_display",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp",ROLLUP(("candidate_id","candidate_name")) ' + 'ORDER BY "timestamp","candidate"', query) + + def test_force_all_dimensions_following_rollup_to_be_rolled_up(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.political_party.rollup(), + slicer.dimensions.candidate) \ + .query + + self.assertEqual('SELECT ' + '"political_party" "political_party",' + '"candidate_id" "candidate",' + '"candidate_name" "candidate_display",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY ROLLUP("political_party",("candidate_id","candidate_name")) ' + 'ORDER BY "political_party","candidate"', query) + + def test_force_all_dimensions_following_rollup_to_be_rolled_up_with_split_dimension_calls(self): + query = slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.political_party.rollup()) \ + .dimension(slicer.dimensions.candidate) \ + .query + + self.assertEqual('SELECT ' + '"political_party" "political_party",' + '"candidate_id" "candidate",' + '"candidate_name" "candidate_display",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY ROLLUP("political_party",("candidate_id","candidate_name")) ' + 'ORDER BY "political_party","candidate"', query) + + def test_raise_exception_when_trying_to_rollup_continuous_dimension(self): + with self.assertRaises(RollupException): + slicer.query() \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.political_party.rollup(), + slicer.dimensions.timestamp) \ + .query + # noinspection SqlDialectInspection,SqlNoDataSourceInspection class QueryBuilderDimensionFilterTests(TestCase): maxDiff = None @@ -1146,7 +1234,7 @@ def test_pass_slicer_database_as_arg(self, mock_fetch_data: Mock): mock_fetch_data.assert_called_once_with(slicer.database, ANY, - index_levels=ANY) + dimensions=ANY) def test_pass_query_from_builder_as_arg(self, mock_fetch_data: Mock): mock_widget = Mock(name='mock_widget') @@ -1158,7 +1246,7 @@ def test_pass_query_from_builder_as_arg(self, mock_fetch_data: Mock): mock_fetch_data.assert_called_once_with(ANY, 'SELECT SUM("votes") "votes" FROM "politics"."politician"', - index_levels=ANY) + dimensions=ANY) def test_builder_dimensions_as_arg_with_zero_dimensions(self, mock_fetch_data: Mock): mock_widget = Mock(name='mock_widget') @@ -1168,29 +1256,33 @@ def test_builder_dimensions_as_arg_with_zero_dimensions(self, mock_fetch_data: M .widget(mock_widget) \ .render() - mock_fetch_data.assert_called_once_with(ANY, ANY, index_levels=[]) + mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=[]) def test_builder_dimensions_as_arg_with_one_dimension(self, mock_fetch_data: Mock): mock_widget = Mock(name='mock_widget') mock_widget.metrics = [slicer.metrics.votes] + dimensions = [slicer.dimensions.state] + slicer.query() \ .widget(mock_widget) \ - .dimension(slicer.dimensions.state) \ + .dimension(*dimensions) \ .render() - mock_fetch_data.assert_called_once_with(ANY, ANY, index_levels=['state']) + mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=DimensionMatcher(*dimensions)) def test_builder_dimensions_as_arg_with_multiple_dimensions(self, mock_fetch_data: Mock): mock_widget = Mock(name='mock_widget') mock_widget.metrics = [slicer.metrics.votes] + dimensions = slicer.dimensions.timestamp, slicer.dimensions.state, slicer.dimensions.political_party + slicer.query() \ .widget(mock_widget) \ - .dimension(slicer.dimensions.timestamp, slicer.dimensions.state, slicer.dimensions.political_party) \ + .dimension(*dimensions) \ .render() - mock_fetch_data.assert_called_once_with(ANY, ANY, index_levels=['timestamp', 'state', 'political_party']) + mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=DimensionMatcher(*dimensions)) def test_call_transform_on_widget(self, mock_fetch_data: Mock): mock_widget = Mock(name='mock_widget') diff --git a/fireant/tests/slicer/queries/test_database.py b/fireant/tests/slicer/queries/test_database.py index 1ba175e0..c1fd51f7 100644 --- a/fireant/tests/slicer/queries/test_database.py +++ b/fireant/tests/slicer/queries/test_database.py @@ -1,8 +1,21 @@ from unittest import TestCase - -from fireant.slicer.queries.database import fetch_data from unittest.mock import Mock +import numpy as np +import pandas as pd + +from fireant.slicer.queries.database import ( + clean_and_apply_index, + fetch_data, +) +from fireant.tests.slicer.mocks import ( + cat_dim_df, + cont_dim_df, + cont_uni_dim_df, + slicer, + uni_dim_df, +) + class FetchDataTests(TestCase): @classmethod @@ -10,12 +23,108 @@ def setUpClass(cls): cls.mock_database = Mock(name='database') cls.mock_data_frame = cls.mock_database.fetch_data.return_value = Mock(name='data_frame') cls.mock_query = 'SELECT *' - cls.mock_index_levels = ['a', 'b'] + cls.mock_dimensions = [Mock(), Mock()] - cls.result = fetch_data(cls.mock_database, cls.mock_query, cls.mock_index_levels) + cls.result = fetch_data(cls.mock_database, cls.mock_query, cls.mock_dimensions) def test_fetch_data_called_on_database(self): self.mock_database.fetch_data.assert_called_once_with(self.mock_query) def test_index_set_on_data_frame_result(self): - self.mock_data_frame.set_index.assert_called_once_with(self.mock_index_levels) + self.mock_data_frame.set_index.assert_called_once_with([d.key for d in self.mock_dimensions]) + + +cat_dim_nans_df = cat_dim_df.append( + pd.DataFrame([[300, 2]], + columns=cat_dim_df.columns, + index=pd.Index([None], + name=cat_dim_df.index.name))) +uni_dim_nans_df = uni_dim_df.append( + pd.DataFrame([[None, 300, 2]], + columns=uni_dim_df.columns, + index=pd.Index([None], + name=uni_dim_df.index.name))) + + +def add_nans(df): + return pd.DataFrame([[None, 300, 2]], + columns=df.columns, + index=pd.Index([None], name=df.index.names[1])) + + +cont_uni_dim_nans_df = cont_uni_dim_df \ + .append(cont_uni_dim_df.groupby(level='timestamp').apply(add_nans)) \ + .sort_index() + + +# print(cont_uni_dim_nans_df) + + +def totals(df): + return pd.DataFrame([[None] + list(df.sum())], + columns=df.columns, + index=pd.Index([None], name=df.index.names[1])) + + +cont_uni_dim_nans_totals_df = cont_uni_dim_nans_df \ + .append(cont_uni_dim_nans_df.groupby(level='timestamp').apply(totals)) \ + .sort_index() \ + .sortlevel(level=[0, 1], ascending=False) # This sorts the DF so that the first instance of NaN is the totals + + +# print(cont_uni_dim_nans_totals_df) + + +class FetchDataCleanIndexTests(TestCase): + def test_set_time_series_index_level(self): + result = clean_and_apply_index(cont_dim_df.reset_index(), + [slicer.dimensions.timestamp]) + + self.assertListEqual(result.index.names, cont_dim_df.index.names) + + def test_set_cat_dim_index(self): + result = clean_and_apply_index(cat_dim_df.reset_index(), + [slicer.dimensions.political_party]) + + self.assertListEqual(list(result.index), ['d', 'i', 'r']) + + def test_set_cat_dim_index_with_nan_converted_to_empty_str(self): + result = clean_and_apply_index(cat_dim_nans_df.reset_index(), + [slicer.dimensions.political_party]) + + self.assertListEqual(list(result.index), ['d', 'i', 'r', '']) + + def test_convert_cat_totals(self): + result = clean_and_apply_index(cat_dim_nans_df.reset_index(), + [slicer.dimensions.political_party.rollup()]) + + self.assertListEqual(list(result.index), ['d', 'i', 'r', 'Totals']) + + def test_convert_numeric_values_to_string(self): + result = clean_and_apply_index(uni_dim_df.reset_index(), [slicer.dimensions.candidate]) + self.assertEqual(result.index.dtype.type, np.object_) + + def test_set_uni_dim_index(self): + result = clean_and_apply_index(uni_dim_df.reset_index(), + [slicer.dimensions.candidate]) + + self.assertListEqual(list(result.index), [str(x + 1) for x in range(11)]) + + def test_set_uni_dim_index_with_nan_converted_to_empty_str(self): + result = clean_and_apply_index(uni_dim_nans_df.reset_index(), + [slicer.dimensions.candidate]) + + self.assertListEqual(list(result.index), [str(x + 1) for x in range(11)] + ['']) + + def test_convert_uni_totals(self): + result = clean_and_apply_index(uni_dim_nans_df.reset_index(), + [slicer.dimensions.candidate.rollup()]) + + self.assertListEqual(list(result.index), [str(x + 1) for x in range(11)] + ['Totals']) + + def test_set_index_for_multiindex_with_nans_and_totals(self): + result = clean_and_apply_index(cont_uni_dim_nans_totals_df.reset_index(), + [slicer.dimensions.timestamp, slicer.dimensions.state.rollup()]) + + print(result) + self.assertListEqual(list(result.index.levels[1]), ['', '1', '2', 'Totals']) diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py index d80616a2..afadc4e5 100644 --- a/fireant/tests/slicer/widgets/test_datatables.py +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -388,38 +388,38 @@ def test_pivoted_multi_dims_time_series_and_uni(self): 'data': [{ 'timestamp': {'value': '1996-01-01'}, 'votes': { - 1: {'display': '5574387', 'value': 5574387}, - 2: {'display': '9646062', 'value': 9646062} + '1': {'display': '5574387', 'value': 5574387}, + '2': {'display': '9646062', 'value': 9646062} } }, { 'timestamp': {'value': '2000-01-01'}, 'votes': { - 1: {'display': '6233385', 'value': 6233385}, - 2: {'display': '10428632', 'value': 10428632} + '1': {'display': '6233385', 'value': 6233385}, + '2': {'display': '10428632', 'value': 10428632} } }, { 'timestamp': {'value': '2004-01-01'}, 'votes': { - 1: {'display': '7359621', 'value': 7359621}, - 2: {'display': '12255311', 'value': 12255311} + '1': {'display': '7359621', 'value': 7359621}, + '2': {'display': '12255311', 'value': 12255311} } }, { 'timestamp': {'value': '2008-01-01'}, 'votes': { - 1: {'display': '8007961', 'value': 8007961}, - 2: {'display': '13286254', 'value': 13286254} + '1': {'display': '8007961', 'value': 8007961}, + '2': {'display': '13286254', 'value': 13286254} } }, { 'timestamp': {'value': '2012-01-01'}, 'votes': { - 1: {'display': '7877967', 'value': 7877967}, - 2: {'display': '12694243', 'value': 12694243} + '1': {'display': '7877967', 'value': 7877967}, + '2': {'display': '12694243', 'value': 12694243} } }, { 'timestamp': {'value': '2016-01-01'}, 'votes': { - 1: {'display': '5072915', 'value': 5072915}, - 2: {'display': '13237598', 'value': 13237598} + '1': {'display': '5072915', 'value': 5072915}, + '2': {'display': '13237598', 'value': 13237598} } }], }, result) diff --git a/fireant/tests/slicer/widgets/test_formats.py b/fireant/tests/slicer/widgets/test_formats.py index 3c241f71..3cde2f5e 100644 --- a/fireant/tests/slicer/widgets/test_formats.py +++ b/fireant/tests/slicer/widgets/test_formats.py @@ -8,43 +8,42 @@ date, datetime, ) - from fireant.slicer.widgets import formats -class SafeValueTests(TestCase): +class FormatMetricValueTests(TestCase): def test_nan_data_point(self): # np.nan is converted to None - result = formats.safe(np.nan) + result = formats.format_metric_value(np.nan) self.assertIsNone(result) def test_str_data_point(self): - result = formats.safe(u'abc') + result = formats.format_metric_value(u'abc') self.assertEqual('abc', result) def test_int64_data_point(self): # Needs to be cast to python int - result = formats.safe(np.int64(1)) + result = formats.format_metric_value(np.int64(1)) self.assertEqual(int(1), result) def test_date_data_point(self): # Needs to be converted to milliseconds - result = formats.safe(date(2000, 1, 1)) + result = formats.format_metric_value(date(2000, 1, 1)) self.assertEqual('2000-01-01', result) def test_datetime_data_point(self): # Needs to be converted to milliseconds - result = formats.safe(datetime(2000, 1, 1, 1)) + result = formats.format_metric_value(datetime(2000, 1, 1, 1)) self.assertEqual('2000-01-01T01:00:00', result) def test_ts_date_data_point(self): # Needs to be converted to milliseconds - result = formats.safe(pd.Timestamp(date(2000, 1, 1))) + result = formats.format_metric_value(pd.Timestamp(date(2000, 1, 1))) self.assertEqual('2000-01-01', result) def test_ts_datetime_data_point(self): # Needs to be converted to milliseconds - result = formats.safe(pd.Timestamp(datetime(2000, 1, 1, 1))) + result = formats.format_metric_value(pd.Timestamp(datetime(2000, 1, 1, 1))) self.assertEqual('2000-01-01T01:00:00', result) From 32d2e20ed66da0c8e412c961255044039d02983b Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 26 Jan 2018 16:40:19 +0100 Subject: [PATCH 005/123] Added fix for aligning weekdays through leap years when querying YoY references using a weekly interval on Vertica --- fireant/database/base.py | 2 +- fireant/database/mysql.py | 2 +- fireant/database/postgresql.py | 2 +- fireant/database/vertica.py | 10 +- fireant/slicer/intervals.py | 9 ++ fireant/slicer/queries/builder.py | 27 ----- fireant/slicer/queries/references.py | 23 ++-- fireant/slicer/references.py | 2 +- fireant/slicer/widgets/highcharts.py | 16 +-- fireant/tests/slicer/queries/test_builder.py | 106 ++++++++++++++++++- 10 files changed, 150 insertions(+), 49 deletions(-) diff --git a/fireant/database/base.py b/fireant/database/base.py index c3df135d..d9edd45b 100644 --- a/fireant/database/base.py +++ b/fireant/database/base.py @@ -19,7 +19,7 @@ def connect(self): def trunc_date(self, field, interval): raise NotImplementedError - def date_add(self, field: terms.Term, date_part: str, interval: int): + def date_add(self, field: terms.Term, date_part: str, interval: int, align_weekday=False): """ Database specific function for adding or subtracting dates """ raise NotImplementedError diff --git a/fireant/database/mysql.py b/fireant/database/mysql.py index c507a2fe..74f8bebb 100644 --- a/fireant/database/mysql.py +++ b/fireant/database/mysql.py @@ -66,7 +66,7 @@ def fetch_data(self, query): def trunc_date(self, field, interval): return Trunc(field, interval) - def date_add(self, field, date_part, interval): + def date_add(self, field, date_part, interval, align_weekday=False): # adding an extra 's' as MySQL's interval doesn't work with 'year', 'week' etc, it expects a plural interval_term = terms.Interval(**{'{}s'.format(str(date_part)): interval, 'dialect': Dialects.MYSQL}) return DateAdd(field, interval_term) diff --git a/fireant/database/postgresql.py b/fireant/database/postgresql.py index a5027338..a02ab238 100644 --- a/fireant/database/postgresql.py +++ b/fireant/database/postgresql.py @@ -53,5 +53,5 @@ def fetch_data(self, query): def trunc_date(self, field, interval): return Trunc(field, interval) - def date_add(self, field, date_part, interval): + def date_add(self, field, date_part, interval, align_weekday=False): return fn.DateAdd(date_part, interval, field) diff --git a/fireant/database/vertica.py b/fireant/database/vertica.py index 5e855437..9238d252 100644 --- a/fireant/database/vertica.py +++ b/fireant/database/vertica.py @@ -63,8 +63,14 @@ def trunc_date(self, field, interval): trunc_date_interval = self.DATETIME_INTERVALS.get(interval, 'DD') return Trunc(field, trunc_date_interval) - def date_add(self, field, date_part, interval): - return fn.TimestampAdd(date_part, interval, field) + def date_add(self, field, date_part, interval, align_weekday=False): + shifted_date = fn.TimestampAdd(date_part, interval, field) + + if align_weekday: + truncated = self.trunc_date(shifted_date, weekly) + return fn.TimestampAdd(date_part, -interval, truncated) + + return shifted_date def totals(self, query, terms): return query.rollup(*terms) diff --git a/fireant/slicer/intervals.py b/fireant/slicer/intervals.py index 9177a172..3aa0af00 100644 --- a/fireant/slicer/intervals.py +++ b/fireant/slicer/intervals.py @@ -14,6 +14,11 @@ def __hash__(self): def __repr__(self): return 'NumericInterval(size=%d,offset=%d)' % (self.size, self.offset) + def __eq__(self, other): + return isinstance(other, NumericInterval) \ + and all([self.size == other.size, + self.offset == other.offset]) + class DatetimeInterval(object): def __init__(self, key): @@ -22,6 +27,10 @@ def __init__(self, key): def __hash__(self): return hash(self.key) + def __eq__(self, other): + return isinstance(other, DatetimeInterval) \ + and self.key == other.key + hourly = DatetimeInterval('hourly') daily = DatetimeInterval('daily') diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 605ddabe..93fc05c7 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -53,33 +53,6 @@ def is_rolling_up(dimension, rolling_up): return getattr(dimension, "is_rollup", False) -def clean(index): - import pandas as pd - - if isinstance(index, (pd.DatetimeIndex, pd.RangeIndex)): - return index - - return index.astype('str') - - -def clean_index(data_frame): - import pandas as pd - - if hasattr(data_frame.index, 'levels'): - data_frame.index = pd.MultiIndex( - levels=[level.astype('str') - if not isinstance(level, (pd.DatetimeIndex, pd.RangeIndex)) - else level - for level in data_frame.index.levels], - labels=data_frame.index.labels - ) - - elif not isinstance(data_frame.index, (pd.DatetimeIndex, pd.RangeIndex)): - data_frame.index = data_frame.index.astype('str') - - return data_frame - - class QueryBuilder(object): """ diff --git a/fireant/slicer/queries/references.py b/fireant/slicer/queries/references.py index 89013b5b..2158829e 100644 --- a/fireant/slicer/queries/references.py +++ b/fireant/slicer/queries/references.py @@ -16,7 +16,11 @@ Criterion, Term, ) -from ..dimensions import Dimension +from ..dimensions import ( + DatetimeDimension, + Dimension, +) +from ..intervals import weekly from ..metrics import Metric from ..references import Reference @@ -24,12 +28,16 @@ def join_reference(reference: Reference, metrics: Iterator[Metric], dimensions: Iterator[Dimension], - ref_dimension: Dimension, + ref_dimension: DatetimeDimension, date_add: Callable, original_query, outer_query: QueryBuilder): ref_query = original_query.as_(reference.key) - date_add = partial(date_add, date_part=reference.time_unit, interval=reference.interval) + + date_add = partial(date_add, + date_part=reference.time_unit, + interval=reference.interval, + align_weekday=weekly == ref_dimension.interval) # FIXME this is a bit hacky, need to replace the ref dimension term in all of the filters with the offset if ref_query._wheres: @@ -43,14 +51,17 @@ def join_reference(reference: Reference, original_query, ref_query, date_add) - outer_query = outer_query.join(ref_query, JoinType.left).on(join_criterion) + outer_query = outer_query \ + .join(ref_query, JoinType.left) \ + .on(join_criterion) + # Add metrics ref_metric = _reference_metric(reference, original_query, ref_query) - outer_query = outer_query.select(*[ref_metric(metric).as_("{}_{}".format(metric.key, reference.key)) + + return outer_query.select(*[ref_metric(metric).as_("{}_{}".format(metric.key, reference.key)) for metric in metrics]) - return outer_query def _apply_to_term_in_criterion(target: Term, diff --git a/fireant/slicer/references.py b/fireant/slicer/references.py index df013ae0..f8106cb5 100644 --- a/fireant/slicer/references.py +++ b/fireant/slicer/references.py @@ -1,5 +1,5 @@ class Reference(object): - def __init__(self, key, label, time_unit, interval, delta=False, percent=False): + def __init__(self, key, label, time_unit: str, interval: int, delta=False, percent=False): self.key = key self.label = label diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 1230079b..8ebc7aa0 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -3,9 +3,9 @@ import numpy as np import pandas as pd from datetime import datetime + from fireant import utils from fireant.utils import immutable - from .base import ( MetricsWidget, Widget, @@ -228,13 +228,13 @@ def _render_y_axis(axis_idx, color, references): }] y_axes += [{ - "id": "{}_{}".format(axis_idx, reference.key), - "title": {"text": reference.label}, - "opposite": True, - "labels": {"style": {"color": color}} - } - for reference in references - if reference.is_delta] + "id": "{}_{}".format(axis_idx, reference.key), + "title": {"text": reference.label}, + "opposite": True, + "labels": {"style": {"color": color}} + } + for reference in references + if reference.is_delta] return y_axes diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index aeacc77c..e8cb2a15 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -1094,8 +1094,110 @@ def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_fil 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' 'ORDER BY "timestamp"', query) - def test_references_adapt_for_leap_year(self): - pass + def test_adapt_dow_for_leap_year_for_yoy_reference(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp(f.weekly) + .reference(f.YearOverYear)) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_yoy" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',-1,' + 'TRUNC(TIMESTAMPADD(\'year\',1,"sq1"."timestamp"),\'IW\')) ' + 'ORDER BY "timestamp"', query) + + def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp(f.weekly) + .reference(f.YearOverYear.delta())) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"base"."votes"-"sq1"."votes" "votes_yoy_delta" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',-1,' + 'TRUNC(TIMESTAMPADD(\'year\',1,"sq1"."timestamp"),\'IW\')) ' + 'ORDER BY "timestamp"', query) + + def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): + query = slicer.query() \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + metrics=[slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp(f.weekly) + .reference(f.YearOverYear.delta(True))) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '("base"."votes"-"sq1"."votes")*100/NULLIF("sq1"."votes",0) "votes_yoy_delta_percent" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',-1,' + 'TRUNC(TIMESTAMPADD(\'year\',1,"sq1"."timestamp"),\'IW\')) ' + 'ORDER BY "timestamp"', query) # noinspection SqlDialectInspection,SqlNoDataSourceInspection From 13fff75c674c7162c8a2a0c067f95c2a06da6813 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 26 Jan 2018 18:53:10 +0100 Subject: [PATCH 006/123] Cleaned up, addressed PR comments, and changed the highcharts transformer to no longer use plotOptions for the datetime index, since there might be breaks in the data. --- fireant/slicer/dimensions.py | 109 +++++++- fireant/slicer/operations.py | 22 +- fireant/slicer/queries/builder.py | 13 +- fireant/slicer/slicers.py | 33 ++- fireant/slicer/widgets/datatables.py | 68 ++++- fireant/slicer/widgets/formats.py | 44 ++- fireant/slicer/widgets/highcharts.py | 64 ++--- fireant/tests/slicer/queries/test_database.py | 7 - fireant/tests/slicer/widgets/test_formats.py | 28 +- .../tests/slicer/widgets/test_highcharts.py | 253 ++++++++++++------ 10 files changed, 460 insertions(+), 181 deletions(-) diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 2db0e00d..3ca1657b 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -1,5 +1,6 @@ -from fireant.utils import immutable +from typing import Iterable +from fireant.utils import immutable from .base import SlicerElement from .exceptions import QueryException from .filters import ( @@ -37,6 +38,11 @@ def __init__(self, key, label=None, definition=None): @immutable def rollup(self): + """ + Configures this dimension and all subsequent dimensions in a slicer query to be rolled up to provide the totals. + This will include an extra value for each pair of dimensions labeled `Totals`. which will include the totals for + the group. + """ self.is_rollup = True @@ -51,7 +57,15 @@ def __init__(self, key, label=None, definition=None): label=label, definition=definition) - def is_(self, value): + def is_(self, value: bool): + """ + Creates a filter to filter a slicer query. + + :param value: + True or False + :return: + A slicer query filter used to filter a slicer query to results where this dimension is True or False. + """ return BooleanFilter(self.definition, value) @@ -67,10 +81,30 @@ def __init__(self, key, label=None, definition=None, display_values=()): definition=definition) self.display_values = dict(display_values) - def isin(self, values): + def isin(self, values: Iterable): + """ + Creates a filter to filter a slicer query. + + :param values: + An iterable of value to constrain the slicer query results by. + + :return: + A slicer query filter used to filter a slicer query to results where this dimension is one of a set of + values. Opposite of #notin. + """ return ContainsFilter(self.definition, values) def notin(self, values): + """ + Creates a filter to filter a slicer query. + + :param values: + An iterable of value to constrain the slicer query results by. + + :return: + A slicer query filter used to filter a slicer query to results where this dimension is *not* one of a set of + values. Opposite of #isin. + """ return ExcludesFilter(self.definition, values) @@ -86,18 +120,53 @@ def __init__(self, key, label=None, definition=None, display_definition=None): self.display_definition = display_definition def isin(self, values, use_display=False): + """ + Creates a filter to filter a slicer query. + + :param values: + An iterable of value to constrain the slicer query results by. + :param use_display: + When True, the filter will be applied to the Dimesnion's display definition instead of the definition. + + :return: + A slicer query filter used to filter a slicer query to results where this dimension is one of a set of + values. Opposite of #notin. + """ if use_display and self.display_definition is None: raise QueryException('No value set for display_definition.') filter_field = self.display_definition if use_display else self.definition return ContainsFilter(filter_field, values) def notin(self, values, use_display=False): + """ + Creates a filter to filter a slicer query. + + :param values: + An iterable of value to constrain the slicer query results by. + :param use_display: + When True, the filter will be applied to the Dimesnion's display definition instead of the definition. + + :return: + A slicer query filter used to filter a slicer query to results where this dimension is *not* one of a set of + values. Opposite of #isin. + """ if use_display and self.display_definition is None: raise QueryException('No value set for display_definition.') filter_field = self.display_definition if use_display else self.definition return ExcludesFilter(filter_field, values) def wildcard(self, pattern): + """ + Creates a filter to filter a slicer query. + + :param pattern: + A pattern to match against the dimension's display definition. This pattern is used in the SQL query as + the + `LIKE` expression. + :return: + A slicer query filter used to filter a slicer query to results where this dimension's display definition + matches the pattern. + """ if self.display_definition is None: raise QueryException('No value set for display_definition.') return WildcardFilter(self.display_definition, pattern) @@ -132,13 +201,47 @@ def __init__(self, key, label=None, definition=None, default_interval=daily): @immutable def __call__(self, interval): + """ + When calling a datetime dimension an interval can be supplied: + + ``` + from fireant import weekly + + my_slicer.dimensions.date # Daily interval used as default + my_slicer.dimensions.date(weekly) # Daily interval used as default + ``` + + :param interval: + An interval to use with the dimension. See `fireant.intervals`. + :return: + A copy of the dimension with the interval set. + """ self.interval = interval @immutable def reference(self, reference): + """ + Add a reference to this dimension when building a slicer query. + + :param reference: + A reference to add to the query + :return: + A copy of the dimension with the reference added. + """ self.references.append(reference) def between(self, start, stop): + """ + Creates a filter to filter a slicer query. + + :param start: + The start time of the filter. This is the beginning of the window for which results should be included. + :param stop: + The stop time of the filter. This is the end of the window for which results should be included. + :return: + A slicer query filter used to filter a slicer query to results where this dimension is between the values + start and stop. + """ return RangeFilter(self.definition, start, stop) diff --git a/fireant/slicer/operations.py b/fireant/slicer/operations.py index 411e683a..99bb0d12 100644 --- a/fireant/slicer/operations.py +++ b/fireant/slicer/operations.py @@ -23,6 +23,16 @@ class _Cumulative(Operation): def __init__(self, arg): self.arg = arg + def _group_levels(self, index): + """ + Get the index levels that need to be grouped. This is to avoid apply the cumulative function across separate + dimensions. Only the first dimension should be accumulated across. + + :param index: + :return: + """ + return index.names[1:] + @property def metrics(self): return [metric @@ -36,8 +46,10 @@ def apply(self, data_frame): class CumSum(_Cumulative): def apply(self, data_frame): if isinstance(data_frame.index, pd.MultiIndex): + levels = self._group_levels(data_frame.index) + return data_frame[self.arg.key] \ - .groupby(level=data_frame.index.names[1:]) \ + .groupby(level=levels) \ .cumsum() return data_frame[self.arg.key].cumsum() @@ -46,8 +58,10 @@ def apply(self, data_frame): class CumProd(_Cumulative): def apply(self, data_frame): if isinstance(data_frame.index, pd.MultiIndex): + levels = self._group_levels(data_frame.index) + return data_frame[self.arg.key] \ - .groupby(level=data_frame.index.names[1:]) \ + .groupby(level=levels) \ .cumprod() return data_frame[self.arg.key].cumprod() @@ -60,8 +74,10 @@ def cummean(x): def apply(self, data_frame): if isinstance(data_frame.index, pd.MultiIndex): + levels = self._group_levels(data_frame.index) + return data_frame[self.arg.key] \ - .groupby(level=data_frame.index.names[1:]) \ + .groupby(level=levels) \ .apply(self.cummean) return self.cummean(data_frame[self.arg.key]) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 93fc05c7..c2faf944 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -6,6 +6,7 @@ ) from fireant.utils import ( + flatten, immutable, ordered_distinct_list, ordered_distinct_list_by_attr, @@ -94,14 +95,20 @@ def filter(self, *filters): @property def tables(self): """ + Collect all the tables from all of the definitions of all of the elements in the slicer query. This looks + through the metrics, dimensions, and filter included in this slicer query. It also checks both the definition + field of each element as well as the display definition for Unique Dimensions. + :return: A collection of tables required to execute a query, """ + return ordered_distinct_list([table - for group in [self.metrics, self._dimensions, self._filters] - for element in group - for attr in [getattr(element, 'definition', None), + for element in flatten([self.metrics, self._dimensions, self._filters]) + # Need extra for-loop to incl. the `display_definition` from `UniqueDimension` + for attr in [element.definition, getattr(element, 'display_definition', None)] + # ... but then filter Nones since most elements do not have `display_definition` if attr is not None for table in attr.tables_]) diff --git a/fireant/slicer/slicers.py b/fireant/slicer/slicers.py index 58cf64fe..4926a521 100644 --- a/fireant/slicer/slicers.py +++ b/fireant/slicer/slicers.py @@ -3,13 +3,20 @@ from .queries import QueryBuilder -def _match_items(left, right, key): - return all([a is not None - and b is not None - and getattr(a, key) == getattr(b, key) - for a, b in itertools.zip_longest(left, right)]) - class _Container(object): + """ + This is a list of slicer elements, metrics or dimensions, used for accessing an element by key with a dot syntax. + + For Example: + ``` + slicer = Slicer( + dimensions=[ + Dimension(key='my_dimension1') + ] + ) + slicer.dimensions.my_dimension1 + ``` + """ def __init__(self, items): self._items = items for item in items: @@ -18,6 +25,16 @@ def __init__(self, items): def __iter__(self): return iter(self._items) + def __eq__(self, other): + """ + Checks if the other object is an instance of _Container and has the same number of items with matching keys. + """ + return isinstance(other, _Container) \ + and all([a is not None + and b is not None + and a.key == b.key + for a, b in itertools.zip_longest(self._items, other._items)]) + class Slicer(object): """ @@ -71,8 +88,8 @@ def query(self): def __eq__(self, other): return isinstance(other, Slicer) \ - and _match_items(self.metrics, other.metrics, 'key') \ - and _match_items(self.dimensions, other.dimensions, 'key') + and self.metrics == other.metrics \ + and self.dimensions == other.dimensions def __repr__(self): return 'Slicer(metrics=[{}],dimensions=[{}])' \ diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index 45ef2c7e..b39bbeb3 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -1,11 +1,12 @@ import itertools import pandas as pd + from fireant import ( ContinuousDimension, + Metric, utils, ) - from . import formats from .base import MetricsWidget from .helpers import ( @@ -16,8 +17,20 @@ ) -def _format_dimension_cell(dimension_value, display_values): - dimension_cell = {'value': formats.format_dimension_value(dimension_value)} +def _render_dimension_cell(dimension_value: str, display_values: dict): + """ + Renders a table cell in a dimension column. + + :param dimension_value: + The raw value for the table cell. + + :param display_values: + The display value mapped from the raw value. + + :return: + A dict with the keys value and possible display. + """ + dimension_cell = {'value': formats.dimension_value(dimension_value)} if display_values is not None: dimension_cell['display'] = display_values.get(dimension_value, dimension_value) @@ -25,14 +38,30 @@ def _format_dimension_cell(dimension_value, display_values): return dimension_cell -def _format_dimensional_metric_cell(row_data, metric): +def _render_dimensional_metric_cell(row_data: pd.Series, metric: Metric): + """ + Renders a table cell in a metric column for pivoted tables where there are two or more dimensions. This function + is recursive to traverse multi-dimensional indices. + + :param row_data: + A series containing the value for the metric and it's index (for the dimension values). + + :param metric: + A reference to the slicer metric to access the display formatting. + + :return: + A deep dict in a tree structure with keys matching each dimension level. The top level will have keys matching + the first level of dimension values, and the next level will contain the next level of dimension values, for as + many index levels as there are. The last level will contain the return value of `_format_metric_cell`. + """ level = {} + + # Group by the last dimension, drop it, and fill the dict with either the raw metric values or the next level of + # dicts. for key, next_row in row_data.groupby(level=-1): next_row.reset_index(level=-1, drop=True, inplace=True) - safe_key = formats.format_dimension_value(key) - - level[key] = _format_dimensional_metric_cell(next_row, metric) \ + level[key] = _render_dimensional_metric_cell(next_row, metric) \ if isinstance(next_row.index, pd.MultiIndex) \ else _format_metric_cell(next_row[metric.key], metric) @@ -40,13 +69,24 @@ def _format_dimensional_metric_cell(row_data, metric): def _format_metric_cell(value, metric): - raw_value = formats.format_metric_value(value) + """ + Renders a table cell in a metric column for non-pivoted tables. + + :param value: + The raw value of the metric. + + :param metric: + A reference to the slicer metric to access the display formatting. + :return: + A dict containing the keys value and display with the raw and display metric values. + """ + raw_value = formats.metric_value(value) return { 'value': raw_value, - 'display': formats.display(raw_value, - prefix=metric.prefix, - suffix=metric.suffix, - precision=metric.precision) + 'display': formats.metric_display(raw_value, + prefix=metric.prefix, + suffix=metric.suffix, + precision=metric.precision) if raw_value is not None else None } @@ -183,13 +223,13 @@ def _data_row(self, dimensions, dimension_values, dimension_display_values, refe row = {} for dimension, dimension_value in zip(dimensions, utils.wrap_list(dimension_values)): - row[dimension.key] = _format_dimension_cell(dimension_value, dimension_display_values.get(dimension.key)) + row[dimension.key] = _render_dimension_cell(dimension_value, dimension_display_values.get(dimension.key)) for metric in self.metrics: for reference in [None] + references: key = reference_key(metric, reference) - row[key] = _format_dimensional_metric_cell(row_data, metric) \ + row[key] = _render_dimensional_metric_cell(row_data, metric) \ if isinstance(row_data.index, pd.MultiIndex) \ else _format_metric_cell(row_data[key], metric) diff --git a/fireant/slicer/widgets/formats.py b/fireant/slicer/widgets/formats.py index c553a0e1..2169a42f 100644 --- a/fireant/slicer/widgets/formats.py +++ b/fireant/slicer/widgets/formats.py @@ -4,14 +4,31 @@ import pandas as pd from datetime import ( date, + datetime, time, ) NO_TIME = time(0) +epoch = np.datetime64(datetime.utcfromtimestamp(0)) +milliseconds = np.timedelta64(1, 'ms') -def format_dimension_value(value): + +def dimension_value(value, str_date=True): + """ + Format a dimension value. This will coerce the raw string or date values into a proper primitive value like a + string, float, or int. + + :param value: + The raw str or datetime value + :param str_date: + When True, dates and datetimes will be converted to ISO strings. The time is omitted for dates. When False, the + datetime will be converted to a POSIX timestamp (millis-since-epoch). + """ if isinstance(value, date): + if not str_date: + return 1000 * value.timestamp() + if not hasattr(value, 'time') or value.time() == NO_TIME: return value.strftime('%Y-%m-%d') else: @@ -31,7 +48,14 @@ def format_dimension_value(value): return value -def format_metric_value(value): +def metric_value(value): + """ + Converts a raw metric value into a safe type. This will change dates into strings, NaNs into Nones, and np types + into their corresponding python types. + + :param value: + The raw metric value. + """ if isinstance(value, date): if not hasattr(value, 'time') or value.time() == NO_TIME: return value.strftime('%Y-%m-%d') @@ -51,7 +75,21 @@ def format_metric_value(value): return value -def display(value, prefix=None, suffix=None, precision=None): +def metric_display(value, prefix=None, suffix=None, precision=None): + """ + Converts a metric value into the display value by applying formatting. + + :param value: + The raw metric value. + :param prefix: + An optional prefix. + :param suffix: + An optional suffix. + :param precision: + The decimal precision, the number of decimal places to round to. + :return: + A formatted string containing the display value for the metric. + """ if isinstance(value, float): if precision is not None: float_format = '%d' if precision == 0 else '%.{}f'.format(precision) diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 8ebc7aa0..94a18401 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -1,11 +1,13 @@ import itertools -import numpy as np import pandas as pd -from datetime import datetime from fireant import utils from fireant.utils import immutable +from slicer.widgets.formats import ( + dimension_value, + metric_value, +) from .base import ( MetricsWidget, Widget, @@ -167,13 +169,11 @@ def transform(self, data_frame, slicer, dimensions): references) x_axis = self._render_x_axis(data_frame, dimension_display_values) - plot_options = self._render_plot_options(data_frame) return { "title": {"text": self.title}, "xAxis": x_axis, "yAxis": y_axes, - "plotOptions": plot_options, "series": series, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, @@ -212,7 +212,7 @@ def _render_x_axis(data_frame, dimension_display_values): @staticmethod def _render_y_axis(axis_idx, color, references): """ - Renders the yAxis configuraiton. + Renders the yAxis configuration. https://api.highcharts.com/highcharts/yAxis @@ -238,37 +238,7 @@ def _render_y_axis(axis_idx, color, references): return y_axes - @staticmethod - def _render_plot_options(data_frame): - """ - Renders the plotOptions configuration - - https://api.highcharts.com/highcharts/plotOptions - - :param data_frame: - :return: - """ - first_level = data_frame.index.levels[0] \ - if isinstance(data_frame.index, pd.MultiIndex) \ - else data_frame.index - - if isinstance(first_level, pd.DatetimeIndex): - epoch = np.datetime64(datetime.utcfromtimestamp(0)) - ms = np.timedelta64(1, 'ms') - millis = [d / ms - for d in first_level.values[:2] - epoch] - - return { - "series": { - "pointStart": millis[0], - "pointInterval": np.diff(millis)[0] - } - } - - return {} - - @staticmethod - def _render_series(axis, axis_idx, axis_color, colors, data_frame_groups, render_series_label, references): + def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, render_series_label, references): """ Renders the series configuration. @@ -292,11 +262,17 @@ def _render_series(axis, axis_idx, axis_color, colors, data_frame_groups, render series_color = next(colors) if has_multi_metric else None for (dimension_values, group_df), symbol in zip(data_frame_groups, symbols): + dimension_values = utils.wrap_list(dimension_values) + + is_timeseries = isinstance(group_df.index.levels[0] + if isinstance(group_df.index, pd.MultiIndex) + else group_df.index, pd.DatetimeIndex) + if not has_multi_metric: series_color = next(colors) for reference, dash_style in zip([None] + references, itertools.cycle(DASH_STYLES)): - key = reference_key(metric, reference) + metric_key = reference_key(metric, reference) series.append({ "type": axis.type, @@ -304,9 +280,9 @@ def _render_series(axis, axis_idx, axis_color, colors, data_frame_groups, render "dashStyle": dash_style, "visible": visible, - "name": render_series_label(metric, reference, utils.wrap_list(dimension_values)), + "name": render_series_label(metric, reference, dimension_values), - "data": [float(x) for x in group_df[key].values], + "data": self._render_data(group_df, metric_key, is_timeseries), "yAxis": ("{}_{}".format(axis_idx, reference.key) if reference is not None and reference.is_delta @@ -324,3 +300,13 @@ def _render_series(axis, axis_idx, axis_color, colors, data_frame_groups, render visible = False # Only display the first in each group return series + + def _render_data(self, group_df, metric_key, is_timeseries): + if is_timeseries: + return [[dimension_value(utils.wrap_list(dimension_values)[0], + str_date=False), + metric_value(y)] + for dimension_values, y in group_df[metric_key].iteritems()] + + return [metric_value(y) + for y in group_df[metric_key].values] diff --git a/fireant/tests/slicer/queries/test_database.py b/fireant/tests/slicer/queries/test_database.py index c1fd51f7..15f640ba 100644 --- a/fireant/tests/slicer/queries/test_database.py +++ b/fireant/tests/slicer/queries/test_database.py @@ -57,9 +57,6 @@ def add_nans(df): .sort_index() -# print(cont_uni_dim_nans_df) - - def totals(df): return pd.DataFrame([[None] + list(df.sum())], columns=df.columns, @@ -72,9 +69,6 @@ def totals(df): .sortlevel(level=[0, 1], ascending=False) # This sorts the DF so that the first instance of NaN is the totals -# print(cont_uni_dim_nans_totals_df) - - class FetchDataCleanIndexTests(TestCase): def test_set_time_series_index_level(self): result = clean_and_apply_index(cont_dim_df.reset_index(), @@ -126,5 +120,4 @@ def test_set_index_for_multiindex_with_nans_and_totals(self): result = clean_and_apply_index(cont_uni_dim_nans_totals_df.reset_index(), [slicer.dimensions.timestamp, slicer.dimensions.state.rollup()]) - print(result) self.assertListEqual(list(result.index.levels[1]), ['', '1', '2', 'Totals']) diff --git a/fireant/tests/slicer/widgets/test_formats.py b/fireant/tests/slicer/widgets/test_formats.py index 3cde2f5e..fcc5d9e2 100644 --- a/fireant/tests/slicer/widgets/test_formats.py +++ b/fireant/tests/slicer/widgets/test_formats.py @@ -14,64 +14,64 @@ class FormatMetricValueTests(TestCase): def test_nan_data_point(self): # np.nan is converted to None - result = formats.format_metric_value(np.nan) + result = formats.metric_value(np.nan) self.assertIsNone(result) def test_str_data_point(self): - result = formats.format_metric_value(u'abc') + result = formats.metric_value(u'abc') self.assertEqual('abc', result) def test_int64_data_point(self): # Needs to be cast to python int - result = formats.format_metric_value(np.int64(1)) + result = formats.metric_value(np.int64(1)) self.assertEqual(int(1), result) def test_date_data_point(self): # Needs to be converted to milliseconds - result = formats.format_metric_value(date(2000, 1, 1)) + result = formats.metric_value(date(2000, 1, 1)) self.assertEqual('2000-01-01', result) def test_datetime_data_point(self): # Needs to be converted to milliseconds - result = formats.format_metric_value(datetime(2000, 1, 1, 1)) + result = formats.metric_value(datetime(2000, 1, 1, 1)) self.assertEqual('2000-01-01T01:00:00', result) def test_ts_date_data_point(self): # Needs to be converted to milliseconds - result = formats.format_metric_value(pd.Timestamp(date(2000, 1, 1))) + result = formats.metric_value(pd.Timestamp(date(2000, 1, 1))) self.assertEqual('2000-01-01', result) def test_ts_datetime_data_point(self): # Needs to be converted to milliseconds - result = formats.format_metric_value(pd.Timestamp(datetime(2000, 1, 1, 1))) + result = formats.metric_value(pd.Timestamp(datetime(2000, 1, 1, 1))) self.assertEqual('2000-01-01T01:00:00', result) class DisplayValueTests(TestCase): def test_precision_default(self): - result = formats.display(0.123456789) + result = formats.metric_display(0.123456789) self.assertEqual('0.123457', result) def test_zero_precision(self): - result = formats.display(0.123456789, precision=0) + result = formats.metric_display(0.123456789, precision=0) self.assertEqual('0', result) def test_precision(self): - result = formats.display(0.123456789, precision=2) + result = formats.metric_display(0.123456789, precision=2) self.assertEqual('0.12', result) def test_precision_zero(self): - result = formats.display(0.0) + result = formats.metric_display(0.0) self.assertEqual('0', result) def test_precision_trim_trailing_zeros(self): - result = formats.display(1.01) + result = formats.metric_display(1.01) self.assertEqual('1.01', result) def test_prefix(self): - result = formats.display(0.12, prefix='$') + result = formats.metric_display(0.12, prefix='$') self.assertEqual('$0.12', result) def test_suffix(self): - result = formats.display(0.12, suffix='€') + result = formats.metric_display(0.12, suffix='€') self.assertEqual('0.12€', result) diff --git a/fireant/tests/slicer/widgets/test_highcharts.py b/fireant/tests/slicer/widgets/test_highcharts.py index 12306497..22724ff2 100644 --- a/fireant/tests/slicer/widgets/test_highcharts.py +++ b/fireant/tests/slicer/widgets/test_highcharts.py @@ -35,19 +35,18 @@ def test_single_metric_line_chart(self): "title": {"text": None}, "labels": {"style": {"color": None}} }], - "plotOptions": { - "series": { - "pointInterval": 126230400000.0, - "pointStart": 820454400000.0 - } - }, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, "name": "Votes", "yAxis": "0", - "data": [15220449.0, 16662017.0, 19614932.0, 21294215.0, 20572210.0, 18310513.0], + "data": [[820454400000.0, 15220449], + [946684800000.0, 16662017], + [1072915200000.0, 19614932], + [1199145600000.0, 21294215], + [1325376000000.0, 20572210], + [1451606400000.0, 18310513]], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -71,19 +70,18 @@ def test_single_metric_with_uni_dim_line_chart(self): "title": {"text": None}, "labels": {"style": {"color": None}} }], - "plotOptions": { - "series": { - "pointInterval": 126230400000.0, - "pointStart": 820454400000.0 - } - }, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "data": [[820454400000.0, 5574387], + [946684800000.0, 6233385], + [1072915200000.0, 7359621], + [1199145600000.0, 8007961], + [1325376000000.0, 7877967], + [1451606400000.0, 5072915]], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -93,7 +91,12 @@ def test_single_metric_with_uni_dim_line_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "data": [[820454400000.0, 9646062], + [946684800000.0, 10428632], + [1072915200000.0, 12255311], + [1199145600000.0, 13286254], + [1325376000000.0, 12694243], + [1451606400000.0, 13237598]], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -118,19 +121,18 @@ def test_multi_metrics_single_axis_line_chart(self): "title": {"text": None}, "labels": {"style": {"color": "#DDDF0D"}} }], - "plotOptions": { - "series": { - "pointInterval": 126230400000.0, - "pointStart": 820454400000.0 - } - }, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "data": [[820454400000.0, 5574387], + [946684800000.0, 6233385], + [1072915200000.0, 7359621], + [1199145600000.0, 8007961], + [1325376000000.0, 7877967], + [1451606400000.0, 5072915]], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -140,7 +142,12 @@ def test_multi_metrics_single_axis_line_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "data": [[820454400000.0, 9646062], + [946684800000.0, 10428632], + [1072915200000.0, 12255311], + [1199145600000.0, 13286254], + [1325376000000.0, 12694243], + [1451606400000.0, 13237598]], "color": "#DDDF0D", "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -150,7 +157,12 @@ def test_multi_metrics_single_axis_line_chart(self): "type": self.chart_type, "name": "Wins (Texas)", "yAxis": "0", - "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "data": [[820454400000.0, 1], + [946684800000.0, 1], + [1072915200000.0, 1], + [1199145600000.0, 1], + [1325376000000.0, 1], + [1451606400000.0, 1]], "color": "#55BF3B", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -160,7 +172,12 @@ def test_multi_metrics_single_axis_line_chart(self): "type": self.chart_type, "name": "Wins (California)", "yAxis": "0", - "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "data": [[820454400000.0, 1], + [946684800000.0, 1], + [1072915200000.0, 1], + [1199145600000.0, 1], + [1325376000000.0, 1], + [1451606400000.0, 1]], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -189,19 +206,18 @@ def test_multi_metrics_multi_axis_line_chart(self): "title": {"text": None}, "labels": {"style": {"color": "#DDDF0D"}} }], - "plotOptions": { - "series": { - "pointInterval": 126230400000.0, - "pointStart": 820454400000.0 - } - }, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "data": [[820454400000.0, 5574387], + [946684800000.0, 6233385], + [1072915200000.0, 7359621], + [1199145600000.0, 8007961], + [1325376000000.0, 7877967], + [1451606400000.0, 5072915]], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -211,7 +227,12 @@ def test_multi_metrics_multi_axis_line_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "data": [[820454400000.0, 9646062], + [946684800000.0, 10428632], + [1072915200000.0, 12255311], + [1199145600000.0, 13286254], + [1325376000000.0, 12694243], + [1451606400000.0, 13237598]], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -221,7 +242,12 @@ def test_multi_metrics_multi_axis_line_chart(self): "type": self.chart_type, "name": "Wins (Texas)", "yAxis": "1", - "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "data": [[820454400000.0, 1], + [946684800000.0, 1], + [1072915200000.0, 1], + [1199145600000.0, 1], + [1325376000000.0, 1], + [1451606400000.0, 1]], "color": "#55BF3B", "marker": {"symbol": "circle", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -231,7 +257,12 @@ def test_multi_metrics_multi_axis_line_chart(self): "type": self.chart_type, "name": "Wins (California)", "yAxis": "1", - "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "data": [[820454400000.0, 1], + [946684800000.0, 1], + [1072915200000.0, 1], + [1199145600000.0, 1], + [1325376000000.0, 1], + [1451606400000.0, 1]], "color": "#DF5353", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -255,19 +286,17 @@ def test_uni_dim_with_ref_line_chart(self): "title": {"text": None}, "labels": {"style": {"color": None}} }], - "plotOptions": { - "series": { - "pointInterval": 126230400000.0, - "pointStart": 820454400000.0 - } - }, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "data": [[946684800000.0, 6233385], + [1072915200000.0, 7359621], + [1199145600000.0, 8007961], + [1325376000000.0, 7877967], + [1451606400000.0, 5072915]], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -277,7 +306,11 @@ def test_uni_dim_with_ref_line_chart(self): "type": self.chart_type, "name": "Votes (EoE) (Texas)", "yAxis": "0", - "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0], + "data": [[946684800000.0, 5574387.0], + [1072915200000.0, 6233385.0], + [1199145600000.0, 7359621.0], + [1325376000000.0, 8007961.0], + [1451606400000.0, 7877967.0]], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "ShortDash", @@ -287,7 +320,11 @@ def test_uni_dim_with_ref_line_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "data": [[946684800000.0, 10428632], + [1072915200000.0, 12255311], + [1199145600000.0, 13286254], + [1325376000000.0, 12694243], + [1451606400000.0, 13237598]], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -297,7 +334,11 @@ def test_uni_dim_with_ref_line_chart(self): "type": self.chart_type, "name": "Votes (EoE) (California)", "yAxis": "0", - "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0], + "data": [[946684800000.0, 9646062.0], + [1072915200000.0, 10428632.0], + [1199145600000.0, 12255311.0], + [1325376000000.0, 13286254.0], + [1451606400000.0, 12694243.0]], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "ShortDash", @@ -328,19 +369,17 @@ def test_uni_dim_with_ref_delta_line_chart(self): "opposite": True, "labels": {"style": {"color": None}} }], - "plotOptions": { - "series": { - "pointInterval": 126230400000.0, - "pointStart": 820454400000.0 - } - }, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "data": [[946684800000.0, 6233385], + [1072915200000.0, 7359621], + [1199145600000.0, 8007961], + [1325376000000.0, 7877967], + [1451606400000.0, 5072915]], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -350,7 +389,11 @@ def test_uni_dim_with_ref_delta_line_chart(self): "type": self.chart_type, "name": "Votes (EoE Δ) (Texas)", "yAxis": "0_eoe_delta", - "data": [-658998.0, -1126236.0, -648340.0, 129994.0, 2805052.0], + "data": [[946684800000.0, -658998.0], + [1072915200000.0, -1126236.0], + [1199145600000.0, -648340.0], + [1325376000000.0, 129994.0], + [1451606400000.0, 2805052.0]], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "ShortDash", @@ -360,7 +403,11 @@ def test_uni_dim_with_ref_delta_line_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "data": [[946684800000.0, 10428632], + [1072915200000.0, 12255311], + [1199145600000.0, 13286254], + [1325376000000.0, 12694243], + [1451606400000.0, 13237598]], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -370,7 +417,11 @@ def test_uni_dim_with_ref_delta_line_chart(self): "type": self.chart_type, "name": "Votes (EoE Δ) (California)", "yAxis": "0_eoe_delta", - "data": [-782570.0, -1826679.0, -1030943.0, 592011.0, -543355.0], + "data": [[946684800000.0, -782570.0], + [1072915200000.0, -1826679.0], + [1199145600000.0, -1030943.0], + [1325376000000.0, 592011.0], + [1451606400000.0, -543355.0]], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "ShortDash", @@ -404,7 +455,6 @@ def test_single_metric_bar_chart(self): "title": {"text": None}, "labels": {"style": {"color": None}} }], - "plotOptions": {}, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ @@ -439,7 +489,6 @@ def test_multi_metric_bar_chart(self): "labels": {"style": {"color": "#DDDF0D"}} }], - "plotOptions": {}, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ @@ -483,7 +532,6 @@ def test_cat_dim_single_metric_bar_chart(self): "labels": {"style": {"color": None}} }], - "plotOptions": {}, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ @@ -518,7 +566,6 @@ def test_cat_dim_multi_metric_bar_chart(self): "labels": {"style": {"color": "#DDDF0D"}} }], - "plotOptions": {}, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ @@ -559,19 +606,18 @@ def test_cont_uni_dims_single_metric_bar_chart(self): "labels": {"style": {"color": None}} }], - "plotOptions": { - "series": { - "pointInterval": 126230400000.0, - "pointStart": 820454400000.0 - } - }, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "data": [[820454400000.0, 5574387], + [946684800000.0, 6233385], + [1072915200000.0, 7359621], + [1199145600000.0, 8007961], + [1325376000000.0, 7877967], + [1451606400000.0, 5072915]], "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -581,7 +627,12 @@ def test_cont_uni_dims_single_metric_bar_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "data": [[820454400000.0, 9646062], + [946684800000.0, 10428632], + [1072915200000.0, 12255311], + [1199145600000.0, 13286254], + [1325376000000.0, 12694243], + [1451606400000.0, 13237598]], "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -606,19 +657,18 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): "labels": {"style": {"color": "#DDDF0D"}} }], - "plotOptions": { - "series": { - "pointInterval": 126230400000.0, - "pointStart": 820454400000.0 - } - }, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "data": [[820454400000.0, 5574387], + [946684800000.0, 6233385], + [1072915200000.0, 7359621], + [1199145600000.0, 8007961], + [1325376000000.0, 7877967], + [1451606400000.0, 5072915]], "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -628,7 +678,12 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "data": [[820454400000.0, 9646062], + [946684800000.0, 10428632], + [1072915200000.0, 12255311], + [1199145600000.0, 13286254], + [1325376000000.0, 12694243], + [1451606400000.0, 13237598]], "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -638,7 +693,12 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): "type": self.chart_type, "name": "Wins (Texas)", "yAxis": "0", - "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "data": [[820454400000.0, 1], + [946684800000.0, 1], + [1072915200000.0, 1], + [1199145600000.0, 1], + [1325376000000.0, 1], + [1451606400000.0, 1]], "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -648,7 +708,12 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): "type": self.chart_type, "name": "Wins (California)", "yAxis": "0", - "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "data": [[820454400000.0, 1], + [946684800000.0, 1], + [1072915200000.0, 1], + [1199145600000.0, 1], + [1325376000000.0, 1], + [1451606400000.0, 1]], "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -678,19 +743,18 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "labels": {"style": {"color": "#DDDF0D"}} }], - "plotOptions": { - "series": { - "pointInterval": 126230400000.0, - "pointStart": 820454400000.0 - } - }, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [5574387.0, 6233385.0, 7359621.0, 8007961.0, 7877967.0, 5072915.0], + "data": [[820454400000.0, 5574387], + [946684800000.0, 6233385], + [1072915200000.0, 7359621], + [1199145600000.0, 8007961], + [1325376000000.0, 7877967], + [1451606400000.0, 5072915]], "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -700,7 +764,12 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [9646062.0, 10428632.0, 12255311.0, 13286254.0, 12694243.0, 13237598.0], + "data": [[820454400000.0, 9646062], + [946684800000.0, 10428632], + [1072915200000.0, 12255311], + [1199145600000.0, 13286254], + [1325376000000.0, 12694243], + [1451606400000.0, 13237598]], "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -710,7 +779,12 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "type": self.chart_type, "name": "Wins (Texas)", "yAxis": "1", - "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "data": [[820454400000.0, 1], + [946684800000.0, 1], + [1072915200000.0, 1], + [1199145600000.0, 1], + [1325376000000.0, 1], + [1451606400000.0, 1]], "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -720,7 +794,12 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "type": self.chart_type, "name": "Wins (California)", "yAxis": "1", - "data": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + "data": [[820454400000.0, 1], + [946684800000.0, 1], + [1072915200000.0, 1], + [1199145600000.0, 1], + [1325376000000.0, 1], + [1451606400000.0, 1]], "color": "#DF5353", "dashStyle": "Solid", "marker": {}, From f1b1c7fde966461cd25527255033fcccff880b69 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 29 Jan 2018 13:21:32 +0100 Subject: [PATCH 007/123] Added an entry point for querying dimension options --- fireant/slicer/queries/__init__.py | 5 +- fireant/slicer/queries/builder.py | 162 ++++++++++------ fireant/slicer/slicers.py | 28 ++- fireant/slicer/widgets/helpers.py | 35 +++- fireant/slicer/widgets/highcharts.py | 8 +- fireant/tests/slicer/queries/test_builder.py | 182 +++++++++--------- fireant/tests/slicer/queries/test_database.py | 4 +- .../slicer/queries/test_dimension_options.py | 67 +++++++ 8 files changed, 318 insertions(+), 173 deletions(-) create mode 100644 fireant/tests/slicer/queries/test_dimension_options.py diff --git a/fireant/slicer/queries/__init__.py b/fireant/slicer/queries/__init__.py index f53ef456..dd8b109a 100644 --- a/fireant/slicer/queries/__init__.py +++ b/fireant/slicer/queries/__init__.py @@ -1 +1,4 @@ -from .builder import QueryBuilder \ No newline at end of file +from .builder import ( + DimensionOptionQueryBuilder, + SlicerQueryBuilder, +) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index c2faf944..4a0746df 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -55,34 +55,11 @@ def is_rolling_up(dimension, rolling_up): class QueryBuilder(object): - """ - - """ - - def __init__(self, slicer): + def __init__(self, slicer, table): self.slicer = slicer - self._widgets = [] + self.table = table self._dimensions = [] self._filters = [] - self._orders = [] - - @immutable - def widget(self, *widgets): - """ - - :param widgets: - :return: - """ - self._widgets += widgets - - @immutable - def dimension(self, *dimensions): - """ - - :param dimensions: - :return: - """ - self._dimensions += dimensions @immutable def filter(self, *filters): @@ -93,7 +70,11 @@ def filter(self, *filters): self._filters += filters @property - def tables(self): + def _elements(self): + return flatten([self._dimensions, self._filters]) + + @property + def _tables(self): """ Collect all the tables from all of the definitions of all of the elements in the slicer query. This looks through the metrics, dimensions, and filter included in this slicer query. It also checks both the definition @@ -104,7 +85,7 @@ def tables(self): """ return ordered_distinct_list([table - for element in flatten([self.metrics, self._dimensions, self._filters]) + for element in self._elements # Need extra for-loop to incl. the `display_definition` from `UniqueDimension` for attr in [element.definition, getattr(element, 'display_definition', None)] @@ -113,28 +94,7 @@ def tables(self): for table in attr.tables_]) @property - def metrics(self): - """ - :return: - an ordered, distinct list of metrics used in all widgets as part of this query. - """ - return ordered_distinct_list_by_attr([metric - for widget in self._widgets - for metric in widget.metrics]) - - @property - def operations(self): - """ - :return: - an ordered, distinct list of metrics used in all widgets as part of this query. - """ - return ordered_distinct_list_by_attr([metric - for widget in self._widgets - for metric in widget.metrics - if isinstance(metric, Operation)]) - - @property - def joins(self): + def _joins(self): """ Given a set of tables required for a slicer query, this function finds the joins required for the query and sorts them topologically. @@ -148,12 +108,12 @@ def joins(self): dependencies = defaultdict(set) slicer_joins = {join.table: join for join in self.slicer.joins} - tables_to_eval = list(self.tables) + tables_to_eval = list(self._tables) while tables_to_eval: table = tables_to_eval.pop() - if self.slicer.table == table: + if self.table == table: continue if table not in slicer_joins: @@ -161,7 +121,7 @@ def joins(self): .format(table=str(table))) join = slicer_joins[table] - tables_required_for_join = set(join.criterion.tables_) - {self.slicer.table, join.table} + tables_required_for_join = set(join.criterion.tables_) - {self.table, join.table} dependencies[join] |= {slicer_joins[table] for table in tables_required_for_join} @@ -177,10 +137,10 @@ def query(self): """ WRITEME """ - query = self.slicer.database.query_cls.from_(self.slicer.table) + query = self.slicer.database.query_cls.from_(self.table) # Add joins - for join in self.joins: + for join in self._joins: query = query.join(join.table, how=join.join_type).on(join.criterion) # Add dimensions @@ -190,7 +150,7 @@ def query(self): dimension_definition = _build_dimension_definition(dimension, self.slicer.database.trunc_date) - if hasattr(dimension, 'display_definition'): + if hasattr(dimension, 'display_definition') and dimension.display_definition is not None: # Add display definition field dimension_display_definition = dimension.display_definition.as_(dimension.display_key) fields = [dimension_definition, dimension_display_definition] @@ -200,16 +160,79 @@ def query(self): query = _select_groups(fields, query, rolling_up, self.slicer.database) - # Add metrics - query = query.select(*[metric.definition.as_(metric.key) - for metric in self.metrics]) - # Add filters for filter_ in self._filters: query = query.where(filter_.definition) \ if isinstance(filter_, DimensionFilter) \ else query.having(filter_.definition) + return query + + +class SlicerQueryBuilder(QueryBuilder): + """ + WRITEME + """ + + def __init__(self, slicer): + super(SlicerQueryBuilder, self).__init__(slicer, slicer.table) + self._widgets = [] + self._orders = [] + + @immutable + def widget(self, *widgets): + """ + + :param widgets: + :return: + """ + self._widgets += widgets + + @immutable + def dimension(self, *dimensions): + """ + + :param dimensions: + :return: + """ + self._dimensions += dimensions + + @property + def metrics(self): + """ + :return: + an ordered, distinct list of metrics used in all widgets as part of this query. + """ + return ordered_distinct_list_by_attr([metric + for widget in self._widgets + for metric in widget.metrics]) + + @property + def operations(self): + """ + :return: + an ordered, distinct list of metrics used in all widgets as part of this query. + """ + return ordered_distinct_list_by_attr([metric + for widget in self._widgets + for metric in widget.metrics + if isinstance(metric, Operation)]) + + @property + def _elements(self): + return flatten([self.metrics, self._dimensions, self._filters]) + + @property + def query(self): + """ + WRITEME + """ + query = super(SlicerQueryBuilder, self).query + + # Add metrics + query = query.select(*[metric.definition.as_(metric.key) + for metric in self.metrics]) + # Add references references = [(reference, dimension) for dimension in self._dimensions @@ -220,7 +243,9 @@ def query(self): # Add ordering order = self._orders if self._orders else self._dimensions - query = query.orderby(*[element.definition.as_(element.key) + query = query.orderby(*[element.display_definition.as_(element.display_key) + if hasattr(element, 'display_definition') + else element.definition.as_(element.key) for element in order]) return str(query) @@ -280,3 +305,20 @@ def __str__(self): def __iter__(self): return iter(self.render()) + + +class DimensionOptionQueryBuilder(QueryBuilder): + def __init__(self, slicer, dimension): + super(DimensionOptionQueryBuilder, self).__init__(slicer, slicer.hint_table or slicer.table) + self._dimensions.append(dimension) + + @property + def query(self): + query = super(DimensionOptionQueryBuilder, self).query + + # Add ordering + query = query.orderby(*[element.display_definition.as_(element.display_key) + if hasattr(element, 'display_definition') + else element.definition.as_(element.key) + for element in self._dimensions]) + return str(query.distinct()) diff --git a/fireant/slicer/slicers.py b/fireant/slicer/slicers.py index 4926a521..7c927599 100644 --- a/fireant/slicer/slicers.py +++ b/fireant/slicer/slicers.py @@ -1,6 +1,13 @@ import itertools -from .queries import QueryBuilder +from .dimensions import ( + CategoricalDimension, + UniqueDimension, +) +from .queries import ( + DimensionOptionQueryBuilder, + SlicerQueryBuilder, +) class _Container(object): @@ -52,7 +59,7 @@ def __init__(self, table, database, joins=(), dimensions=(), metrics=(), hint_ta Constructor for a slicer. Contains all the fields to initialize the slicer. :param table: (Required) - A Pypika Table reference. The primary table that this slicer will retrieve data from. + A pypika Table reference. The primary table that this slicer will retrieve data from. :param database: (Required) A Database reference. Holds the connection details used by this slicer to execute queries. @@ -71,20 +78,23 @@ def __init__(self, table, database, joins=(), dimensions=(), metrics=(), hint_ta :param hint_table: (Optional) A hint table used for querying dimension options. If not present, the table will be used. The hint_table must have the same definition as the table omitting dimensions which do not have a set of options (such as - datetime dimensions) and the metrics. This is provided to more efficiently query dimension options. + datetime or boolean dimensions) and the metrics. This is provided to more efficiently query dimension + options. """ self.table = table self.database = database self.joins = joins + + self.hint_table = hint_table self.dimensions = Slicer.Dimensions(dimensions) self.metrics = Slicer.Metrics(metrics) - self.hint_table = hint_table - def query(self): - """ - WRITEME - """ - return QueryBuilder(self) + # add query builder entry points + self.data = SlicerQueryBuilder(self) + for dimension in dimensions: + if not isinstance(dimension, (UniqueDimension, CategoricalDimension)): + continue + dimension.options = DimensionOptionQueryBuilder(self, dimension) def __eq__(self, other): return isinstance(other, Slicer) \ diff --git a/fireant/slicer/widgets/helpers.py b/fireant/slicer/widgets/helpers.py index bccfce2f..6186f8b5 100644 --- a/fireant/slicer/widgets/helpers.py +++ b/fireant/slicer/widgets/helpers.py @@ -3,26 +3,43 @@ def extract_display_values(dimensions, data_frame): """ + Retrieves the display values for each dimension. + + For UniqueDimension, this will retrieve the display values from the data frame containing the data from the slicer + query. For CategoricalDimension, the values are retrieved from the set of display values configured in the slicer. :param dimensions: + A list of dimensions present in a slicer query. :param data_frame: + The data frame containing the data result of the slicer query. :return: + A dict containing keys for dimensions with display values (If there are no display values then the + dimension's key will not be present). The value of the dict will be either a dict or a data frame where the + display value can be accessed using the display value as the key. """ - dv_by_dimension = {} + display_values = {} for dimension in dimensions: - dkey = dimension.key + key = dimension.key if hasattr(dimension, 'display_values'): - dv_by_dimension[dkey] = dimension.display_values + display_values[key] = dimension.display_values - elif hasattr(dimension, 'display_key'): - dv_by_dimension[dkey] = data_frame[dimension.display_key].groupby(level=dkey).first() + elif hasattr(dimension, 'display_key') and dimension.display_key is not None: + display_values[key] = data_frame[dimension.display_key] \ + .groupby(level=key) \ + .first() - return dv_by_dimension + return display_values def reference_key(metric, reference): + """ + Format a metric key for a reference. + + :return: + A string that is used as the key for a reference metric. + """ key = metric.key if reference is not None: @@ -32,6 +49,12 @@ def reference_key(metric, reference): def reference_label(metric, reference): + """ + Format a metric label for a reference. + + :return: + A string that is used as the display value for a reference metric. + """ label = metric.label or metric.key if reference is not None: diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 94a18401..e25db233 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -4,14 +4,14 @@ from fireant import utils from fireant.utils import immutable -from slicer.widgets.formats import ( - dimension_value, - metric_value, -) from .base import ( MetricsWidget, Widget, ) +from .formats import ( + dimension_value, + metric_value, +) from .helpers import ( dimensional_metric_label, extract_display_values, diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index e8cb2a15..18e49453 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -30,7 +30,7 @@ class QueryBuilderMetricTests(TestCase): maxDiff = None def test_build_query_with_single_metric(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .query @@ -39,7 +39,7 @@ def test_build_query_with_single_metric(self): 'FROM "politics"."politician"', query) def test_build_query_with_multiple_metrics(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes, slicer.metrics.wins])) \ .query @@ -49,7 +49,7 @@ def test_build_query_with_multiple_metrics(self): 'FROM "politics"."politician"', query) def test_build_query_with_multiple_visualizations(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .widget(f.DataTablesJS([slicer.metrics.wins])) \ .query @@ -60,7 +60,7 @@ def test_build_query_with_multiple_visualizations(self): 'FROM "politics"."politician"', query) def test_build_query_for_chart_visualization_with_single_axis(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[ f.HighCharts.PieChart(metrics=[slicer.metrics.votes]) @@ -72,7 +72,7 @@ def test_build_query_for_chart_visualization_with_single_axis(self): 'FROM "politics"."politician"', query) def test_build_query_for_chart_visualization_with_multiple_axes(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts() .axis(f.HighCharts.PieChart(metrics=[slicer.metrics.votes])) .axis(f.HighCharts.PieChart(metrics=[slicer.metrics.wins]))) \ @@ -89,7 +89,7 @@ class QueryBuilderDimensionTests(TestCase): maxDiff = None def test_build_query_with_datetime_dimension(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp) \ .query @@ -102,7 +102,7 @@ def test_build_query_with_datetime_dimension(self): 'ORDER BY "timestamp"', query) def test_build_query_with_datetime_dimension_hourly(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp(f.hourly)) \ .query @@ -115,7 +115,7 @@ def test_build_query_with_datetime_dimension_hourly(self): 'ORDER BY "timestamp"', query) def test_build_query_with_datetime_dimension_daily(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp(f.daily)) \ .query @@ -128,7 +128,7 @@ def test_build_query_with_datetime_dimension_daily(self): 'ORDER BY "timestamp"', query) def test_build_query_with_datetime_dimension_weekly(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp(f.weekly)) \ .query @@ -141,7 +141,7 @@ def test_build_query_with_datetime_dimension_weekly(self): 'ORDER BY "timestamp"', query) def test_build_query_with_datetime_dimension_monthly(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp(f.monthly)) \ .query @@ -154,7 +154,7 @@ def test_build_query_with_datetime_dimension_monthly(self): 'ORDER BY "timestamp"', query) def test_build_query_with_datetime_dimension_quarterly(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp(f.quarterly)) \ .query @@ -167,7 +167,7 @@ def test_build_query_with_datetime_dimension_quarterly(self): 'ORDER BY "timestamp"', query) def test_build_query_with_datetime_dimension_annually(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp(f.annually)) \ .query @@ -180,7 +180,7 @@ def test_build_query_with_datetime_dimension_annually(self): 'ORDER BY "timestamp"', query) def test_build_query_with_boolean_dimension(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.winner) \ .query @@ -193,7 +193,7 @@ def test_build_query_with_boolean_dimension(self): 'ORDER BY "winner"', query) def test_build_query_with_categorical_dimension(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.political_party) \ .query @@ -206,7 +206,7 @@ def test_build_query_with_categorical_dimension(self): 'ORDER BY "political_party"', query) def test_build_query_with_unique_dimension(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.election) \ .query @@ -217,10 +217,10 @@ def test_build_query_with_unique_dimension(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "election","election_display" ' - 'ORDER BY "election"', query) + 'ORDER BY "election_display"', query) def test_build_query_with_multiple_dimensions(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.candidate) \ @@ -233,10 +233,10 @@ def test_build_query_with_multiple_dimensions(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp","candidate","candidate_display" ' - 'ORDER BY "timestamp","candidate"', query) + 'ORDER BY "timestamp","candidate_display"', query) def test_build_query_with_multiple_dimensions_and_visualizations(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes, slicer.metrics.wins])) \ .widget(f.HighCharts( axes=[ @@ -262,7 +262,7 @@ class QueryBuilderDimensionRollupTests(TestCase): maxDiff = None def test_build_query_with_rollup_cat_dimension(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.political_party.rollup()) \ .query @@ -275,7 +275,7 @@ def test_build_query_with_rollup_cat_dimension(self): 'ORDER BY "political_party"', query) def test_build_query_with_rollup_uni_dimension(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.candidate.rollup()) \ .query @@ -286,10 +286,10 @@ def test_build_query_with_rollup_uni_dimension(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY ROLLUP(("candidate_id","candidate_name")) ' - 'ORDER BY "candidate"', query) + 'ORDER BY "candidate_display"', query) def test_rollup_following_non_rolled_up_dimensions(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp, slicer.dimensions.candidate.rollup()) \ @@ -302,10 +302,10 @@ def test_rollup_following_non_rolled_up_dimensions(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp",ROLLUP(("candidate_id","candidate_name")) ' - 'ORDER BY "timestamp","candidate"', query) + 'ORDER BY "timestamp","candidate_display"', query) def test_force_all_dimensions_following_rollup_to_be_rolled_up(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.political_party.rollup(), slicer.dimensions.candidate) \ @@ -318,10 +318,10 @@ def test_force_all_dimensions_following_rollup_to_be_rolled_up(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY ROLLUP("political_party",("candidate_id","candidate_name")) ' - 'ORDER BY "political_party","candidate"', query) + 'ORDER BY "political_party","candidate_display"', query) def test_force_all_dimensions_following_rollup_to_be_rolled_up_with_split_dimension_calls(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.political_party.rollup()) \ .dimension(slicer.dimensions.candidate) \ @@ -334,11 +334,11 @@ def test_force_all_dimensions_following_rollup_to_be_rolled_up_with_split_dimens 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY ROLLUP("political_party",("candidate_id","candidate_name")) ' - 'ORDER BY "political_party","candidate"', query) + 'ORDER BY "political_party","candidate_display"', query) def test_raise_exception_when_trying_to_rollup_continuous_dimension(self): with self.assertRaises(RollupException): - slicer.query() \ + slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.political_party.rollup(), slicer.dimensions.timestamp) \ @@ -349,7 +349,7 @@ class QueryBuilderDimensionFilterTests(TestCase): maxDiff = None def test_build_query_with_filter_isin_categorical_dim(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.political_party.isin(['d'])) \ .query @@ -360,7 +360,7 @@ def test_build_query_with_filter_isin_categorical_dim(self): 'WHERE "political_party" IN (\'d\')', query) def test_build_query_with_filter_notin_categorical_dim(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.political_party.notin(['d'])) \ .query @@ -371,7 +371,7 @@ def test_build_query_with_filter_notin_categorical_dim(self): 'WHERE "political_party" NOT IN (\'d\')', query) def test_build_query_with_filter_isin_unique_dim(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.candidate.isin([1])) \ .query @@ -382,7 +382,7 @@ def test_build_query_with_filter_isin_unique_dim(self): 'WHERE "candidate_id" IN (1)', query) def test_build_query_with_filter_notin_unique_dim(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.candidate.notin([1])) \ .query @@ -393,7 +393,7 @@ def test_build_query_with_filter_notin_unique_dim(self): 'WHERE "candidate_id" NOT IN (1)', query) def test_build_query_with_filter_isin_unique_dim_display(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.candidate.isin(['Donald Trump'], use_display=True)) \ .query @@ -404,7 +404,7 @@ def test_build_query_with_filter_isin_unique_dim_display(self): 'WHERE "candidate_name" IN (\'Donald Trump\')', query) def test_build_query_with_filter_notin_unique_dim_display(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.candidate.notin(['Donald Trump'], use_display=True)) \ .query @@ -415,7 +415,7 @@ def test_build_query_with_filter_notin_unique_dim_display(self): 'WHERE "candidate_name" NOT IN (\'Donald Trump\')', query) def test_build_query_with_filter_wildcard_unique_dim(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.candidate.wildcard('%Trump')) \ .query @@ -427,24 +427,24 @@ def test_build_query_with_filter_wildcard_unique_dim(self): def test_build_query_with_filter_isin_raise_exception_when_display_definition_undefined(self): with self.assertRaises(f.QueryException): - slicer.query() \ + slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.deepjoin.isin([1], use_display=True)) def test_build_query_with_filter_notin_raise_exception_when_display_definition_undefined(self): with self.assertRaises(f.QueryException): - slicer.query() \ + slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.deepjoin.notin([1], use_display=True)) def test_build_query_with_filter_wildcard_raise_exception_when_display_definition_undefined(self): with self.assertRaises(f.QueryException): - slicer.query() \ + slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.deepjoin.wildcard('test')) def test_build_query_with_filter_range_datetime_dimension(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.timestamp.between(date(2009, 1, 20), date(2017, 1, 20))) \ .query @@ -455,7 +455,7 @@ def test_build_query_with_filter_range_datetime_dimension(self): 'WHERE "timestamp" BETWEEN \'2009-01-20\' AND \'2017-01-20\'', query) def test_build_query_with_filter_boolean_true(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.winner.is_(True)) \ .query @@ -466,7 +466,7 @@ def test_build_query_with_filter_boolean_true(self): 'WHERE "is_winner"', query) def test_build_query_with_filter_boolean_false(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.winner.is_(False)) \ .query @@ -482,7 +482,7 @@ class QueryBuilderMetricFilterTests(TestCase): maxDiff = None def test_build_query_with_metric_filter_eq(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.metrics.votes == 5) \ .query @@ -493,7 +493,7 @@ def test_build_query_with_metric_filter_eq(self): 'HAVING SUM("votes")=5', query) def test_build_query_with_metric_filter_eq_left(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(5 == slicer.metrics.votes) \ .query @@ -504,7 +504,7 @@ def test_build_query_with_metric_filter_eq_left(self): 'HAVING SUM("votes")=5', query) def test_build_query_with_metric_filter_ne(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.metrics.votes != 5) \ .query @@ -515,7 +515,7 @@ def test_build_query_with_metric_filter_ne(self): 'HAVING SUM("votes")<>5', query) def test_build_query_with_metric_filter_ne_left(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(5 != slicer.metrics.votes) \ .query @@ -526,7 +526,7 @@ def test_build_query_with_metric_filter_ne_left(self): 'HAVING SUM("votes")<>5', query) def test_build_query_with_metric_filter_gt(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.metrics.votes > 5) \ .query @@ -537,7 +537,7 @@ def test_build_query_with_metric_filter_gt(self): 'HAVING SUM("votes")>5', query) def test_build_query_with_metric_filter_gt_left(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(5 < slicer.metrics.votes) \ .query @@ -548,7 +548,7 @@ def test_build_query_with_metric_filter_gt_left(self): 'HAVING SUM("votes")>5', query) def test_build_query_with_metric_filter_gte(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.metrics.votes >= 5) \ .query @@ -559,7 +559,7 @@ def test_build_query_with_metric_filter_gte(self): 'HAVING SUM("votes")>=5', query) def test_build_query_with_metric_filter_gte_left(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(5 <= slicer.metrics.votes) \ .query @@ -570,7 +570,7 @@ def test_build_query_with_metric_filter_gte_left(self): 'HAVING SUM("votes")>=5', query) def test_build_query_with_metric_filter_lt(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.metrics.votes < 5) \ .query @@ -581,7 +581,7 @@ def test_build_query_with_metric_filter_lt(self): 'HAVING SUM("votes")<5', query) def test_build_query_with_metric_filter_lt_left(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(5 > slicer.metrics.votes) \ .query @@ -592,7 +592,7 @@ def test_build_query_with_metric_filter_lt_left(self): 'HAVING SUM("votes")<5', query) def test_build_query_with_metric_filter_lte(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.metrics.votes <= 5) \ .query @@ -603,7 +603,7 @@ def test_build_query_with_metric_filter_lte(self): 'HAVING SUM("votes")<=5', query) def test_build_query_with_metric_filter_lte_left(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(5 >= slicer.metrics.votes) \ .query @@ -619,7 +619,7 @@ class QueryBuilderOperationTests(TestCase): maxDiff = None def test_build_query_with_cumsum_operation(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([f.CumSum(slicer.metrics.votes)])) \ .dimension(slicer.dimensions.timestamp) \ .query @@ -632,7 +632,7 @@ def test_build_query_with_cumsum_operation(self): 'ORDER BY "timestamp"', query) def test_build_query_with_cummean_operation(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([f.CumMean(slicer.metrics.votes)])) \ .dimension(slicer.dimensions.timestamp) \ .query @@ -650,7 +650,7 @@ class QueryBuilderDatetimeReferenceTests(TestCase): maxDiff = None def test_dimension_with_single_reference_dod(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -684,7 +684,7 @@ def test_dimension_with_single_reference_dod(self): 'ORDER BY "timestamp"', query) def test_dimension_with_single_reference_wow(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -718,7 +718,7 @@ def test_dimension_with_single_reference_wow(self): 'ORDER BY "timestamp"', query) def test_dimension_with_single_reference_mom(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -752,7 +752,7 @@ def test_dimension_with_single_reference_mom(self): 'ORDER BY "timestamp"', query) def test_dimension_with_single_reference_qoq(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -786,7 +786,7 @@ def test_dimension_with_single_reference_qoq(self): 'ORDER BY "timestamp"', query) def test_dimension_with_single_reference_yoy(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -820,7 +820,7 @@ def test_dimension_with_single_reference_yoy(self): 'ORDER BY "timestamp"', query) def test_dimension_with_single_reference_as_a_delta(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -854,7 +854,7 @@ def test_dimension_with_single_reference_as_a_delta(self): 'ORDER BY "timestamp"', query) def test_dimension_with_single_reference_as_a_delta_percentage(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -888,7 +888,7 @@ def test_dimension_with_single_reference_as_a_delta_percentage(self): 'ORDER BY "timestamp"', query) def test_dimension_with_multiple_references(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -934,7 +934,7 @@ def test_dimension_with_multiple_references(self): 'ORDER BY "timestamp"', query) def test_reference_joins_nested_query_on_dimensions(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -973,7 +973,7 @@ def test_reference_joins_nested_query_on_dimensions(self): 'ORDER BY "timestamp","political_party"', query) def test_reference_with_unique_dimension_includes_display_definition(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -1012,10 +1012,10 @@ def test_reference_with_unique_dimension_includes_display_definition(self): 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' 'AND "base"."candidate"="sq1"."candidate" ' - 'ORDER BY "timestamp","candidate"', query) + 'ORDER BY "timestamp","candidate_display"', query) def test_adjust_reference_dimension_filters_in_reference_query(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -1053,7 +1053,7 @@ def test_adjust_reference_dimension_filters_in_reference_query(self): 'ORDER BY "timestamp"', query) def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_filters(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -1095,7 +1095,7 @@ def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_fil 'ORDER BY "timestamp"', query) def test_adapt_dow_for_leap_year_for_yoy_reference(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -1130,7 +1130,7 @@ def test_adapt_dow_for_leap_year_for_yoy_reference(self): 'ORDER BY "timestamp"', query) def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -1165,7 +1165,7 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): 'ORDER BY "timestamp"', query) def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( metrics=[slicer.metrics.votes])])) \ @@ -1205,7 +1205,7 @@ class QueryBuilderJoinTests(TestCase): maxDiff = None def test_dimension_with_join_includes_join_in_query(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.district) \ @@ -1220,10 +1220,10 @@ def test_dimension_with_join_includes_join_in_query(self): 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' 'GROUP BY "timestamp","district","district_display" ' - 'ORDER BY "timestamp","district"', query) + 'ORDER BY "timestamp","district_display"', query) def test_dimension_with_recursive_join_joins_all_join_tables(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.state) \ @@ -1240,10 +1240,10 @@ def test_dimension_with_recursive_join_joins_all_join_tables(self): 'JOIN "locations"."state" ' 'ON "district"."state_id"="state"."id" ' 'GROUP BY "timestamp","state","state_display" ' - 'ORDER BY "timestamp","state"', query) + 'ORDER BY "timestamp","state_display"', query) def test_metric_with_join_includes_join_in_query(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.voters])) \ .dimension(slicer.dimensions.district) \ .query @@ -1258,10 +1258,10 @@ def test_metric_with_join_includes_join_in_query(self): 'JOIN "politics"."voter" ' 'ON "district"."id"="voter"."district_id" ' 'GROUP BY "district","district_display" ' - 'ORDER BY "district"', query) + 'ORDER BY "district_display"', query) def test_dimension_filter_with_join_on_display_definition_does_not_include_join_in_query(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.district.isin([1])) \ .query @@ -1272,7 +1272,7 @@ def test_dimension_filter_with_join_on_display_definition_does_not_include_join_ 'WHERE "district_id" IN (1)', query) def test_dimension_filter_display_field_with_join_includes_join_in_query(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.district.isin(['District 4'], use_display=True)) \ .query @@ -1285,7 +1285,7 @@ def test_dimension_filter_display_field_with_join_includes_join_in_query(self): 'WHERE "district"."district_name" IN (\'District 4\')', query) def test_dimension_filter_with_recursive_join_includes_join_in_query(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.state.isin([1])) \ .query @@ -1298,7 +1298,7 @@ def test_dimension_filter_with_recursive_join_includes_join_in_query(self): 'WHERE "district"."state_id" IN (1)', query) def test_dimension_filter_with_deep_recursive_join_includes_joins_in_query(self): - query = slicer.query() \ + query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.deepjoin.isin([1])) \ .query @@ -1330,7 +1330,7 @@ def test_pass_slicer_database_as_arg(self, mock_fetch_data: Mock): mock_widget = Mock(name='mock_widget') mock_widget.metrics = [slicer.metrics.votes] - slicer.query() \ + slicer.data \ .widget(mock_widget) \ .render() @@ -1342,7 +1342,7 @@ def test_pass_query_from_builder_as_arg(self, mock_fetch_data: Mock): mock_widget = Mock(name='mock_widget') mock_widget.metrics = [slicer.metrics.votes] - slicer.query() \ + slicer.data \ .widget(mock_widget) \ .render() @@ -1354,7 +1354,7 @@ def test_builder_dimensions_as_arg_with_zero_dimensions(self, mock_fetch_data: M mock_widget = Mock(name='mock_widget') mock_widget.metrics = [slicer.metrics.votes] - slicer.query() \ + slicer.data \ .widget(mock_widget) \ .render() @@ -1366,7 +1366,7 @@ def test_builder_dimensions_as_arg_with_one_dimension(self, mock_fetch_data: Moc dimensions = [slicer.dimensions.state] - slicer.query() \ + slicer.data \ .widget(mock_widget) \ .dimension(*dimensions) \ .render() @@ -1379,7 +1379,7 @@ def test_builder_dimensions_as_arg_with_multiple_dimensions(self, mock_fetch_dat dimensions = slicer.dimensions.timestamp, slicer.dimensions.state, slicer.dimensions.political_party - slicer.query() \ + slicer.data \ .widget(mock_widget) \ .dimension(*dimensions) \ .render() @@ -1391,7 +1391,7 @@ def test_call_transform_on_widget(self, mock_fetch_data: Mock): mock_widget.metrics = [slicer.metrics.votes] # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work - slicer.query() \ + slicer.data \ .dimension(slicer.dimensions.timestamp) \ .widget(mock_widget) \ .render() @@ -1405,7 +1405,7 @@ def test_returns_results_from_widget_transform(self, mock_fetch_data: Mock): mock_widget.metrics = [slicer.metrics.votes] # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work - result = slicer.query() \ + result = slicer.data \ .dimension(slicer.dimensions.timestamp) \ .widget(mock_widget) \ .render() @@ -1421,7 +1421,7 @@ def test_operations_evaluated(self, mock_fetch_data: Mock): mock_fetch_data.return_value = mock_df # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work - slicer.query() \ + slicer.data \ .dimension(slicer.dimensions.timestamp) \ .widget(mock_widget) \ .render() @@ -1437,7 +1437,7 @@ def test_operations_results_stored_in_data_frame(self, mock_fetch_data: Mock): mock_fetch_data.return_value = mock_df # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work - slicer.query() \ + slicer.data \ .dimension(slicer.dimensions.timestamp) \ .widget(mock_widget) \ .render() diff --git a/fireant/tests/slicer/queries/test_database.py b/fireant/tests/slicer/queries/test_database.py index 15f640ba..70ea3d22 100644 --- a/fireant/tests/slicer/queries/test_database.py +++ b/fireant/tests/slicer/queries/test_database.py @@ -64,9 +64,9 @@ def totals(df): cont_uni_dim_nans_totals_df = cont_uni_dim_nans_df \ - .append(cont_uni_dim_nans_df.groupby(level='timestamp').apply(totals)) \ + .append(cont_uni_dim_nans_df.groupby(level='timestamp').apply(totals))\ .sort_index() \ - .sortlevel(level=[0, 1], ascending=False) # This sorts the DF so that the first instance of NaN is the totals + .sort_index(level=[0, 1], ascending=False) # This sorts the DF so that the first instance of NaN is the totals class FetchDataCleanIndexTests(TestCase): diff --git a/fireant/tests/slicer/queries/test_dimension_options.py b/fireant/tests/slicer/queries/test_dimension_options.py new file mode 100644 index 00000000..1f1c18ea --- /dev/null +++ b/fireant/tests/slicer/queries/test_dimension_options.py @@ -0,0 +1,67 @@ +from unittest import TestCase + +from ..mocks import slicer + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class DimensionsOptionsQueryBuilderTests(TestCase): + maxDiff = None + + def test_query_options_for_cat_dimension(self): + query = slicer.dimensions.political_party \ + .options \ + .query + + self.assertEqual('SELECT DISTINCT ' + '"political_party" "political_party" ' + 'FROM "politics"."politician" ' + 'GROUP BY "political_party" ' + 'ORDER BY "political_party"', query) + + def test_query_options_for_uni_dimension(self): + query = slicer.dimensions.candidate \ + .options \ + .query + + self.assertEqual('SELECT DISTINCT ' + '"candidate_id" "candidate",' + '"candidate_name" "candidate_display" ' + 'FROM "politics"."politician" ' + 'GROUP BY "candidate","candidate_display" ' + 'ORDER BY "candidate_display"', query) + + def test_query_options_for_uni_dimension_with_join(self): + query = slicer.dimensions.district \ + .options \ + .query + + self.assertEqual('SELECT DISTINCT ' + '"politician"."district_id" "district",' + '"district"."district_name" "district_display" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'GROUP BY "district","district_display" ' + 'ORDER BY "district_display"', query) + + def test_no_options_attr_for_datetime_dimension(self): + with self.assertRaises(AttributeError): + slicer.dimensions.timestamp.options + + def test_no_options_attr_for_boolean_dimension(self): + with self.assertRaises(AttributeError): + slicer.dimensions.winner.options + + def test_filter_options(self): + query = slicer.dimensions.candidate \ + .options \ + .filter(slicer.dimensions.political_party.isin(['d', 'r'])) \ + .query + + self.assertEqual('SELECT DISTINCT ' + '"candidate_id" "candidate",' + '"candidate_name" "candidate_display" ' + 'FROM "politics"."politician" ' + 'WHERE "political_party" IN (\'d\',\'r\') ' + 'GROUP BY "candidate","candidate_display" ' + 'ORDER BY "candidate_display"', query) From 4fad225ebcec30f942fff73505b775b54d85960f Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 29 Jan 2018 14:10:48 +0100 Subject: [PATCH 008/123] Added support for pagination through limit and offset parameters --- fireant/database/mysql.py | 2 +- fireant/database/postgresql.py | 4 + fireant/slicer/dimensions.py | 2 +- fireant/slicer/exceptions.py | 3 +- fireant/slicer/intervals.py | 5 - fireant/slicer/queries/builder.py | 9 +- fireant/slicer/slicers.py | 2 +- fireant/slicer/widgets/base.py | 4 +- fireant/slicer/widgets/highcharts.py | 4 + fireant/tests/slicer/matchers.py | 13 ++ fireant/tests/slicer/queries/test_builder.py | 220 ++++++++++++------ .../slicer/queries/test_dimension_options.py | 8 +- requirements.txt | 2 +- 13 files changed, 185 insertions(+), 93 deletions(-) diff --git a/fireant/database/mysql.py b/fireant/database/mysql.py index 74f8bebb..4cf3e938 100644 --- a/fireant/database/mysql.py +++ b/fireant/database/mysql.py @@ -71,5 +71,5 @@ def date_add(self, field, date_part, interval, align_weekday=False): interval_term = terms.Interval(**{'{}s'.format(str(date_part)): interval, 'dialect': Dialects.MYSQL}) return DateAdd(field, interval_term) - def totals(self, query, *terms): + def totals(self, query, terms): raise NotImplementedError diff --git a/fireant/database/postgresql.py b/fireant/database/postgresql.py index a02ab238..daf178a9 100644 --- a/fireant/database/postgresql.py +++ b/fireant/database/postgresql.py @@ -25,6 +25,7 @@ class PostgreSQLDatabase(Database): """ PostgreSQL client that uses the psycopg module. """ + # The pypika query class to use for constructing queries query_cls = PostgreSQLQuery @@ -55,3 +56,6 @@ def trunc_date(self, field, interval): def date_add(self, field, date_part, interval, align_weekday=False): return fn.DateAdd(date_part, interval, field) + + def totals(self, query, terms): + raise NotImplementedError diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 3ca1657b..e2dbd6b6 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -33,7 +33,7 @@ class RollupDimension(Dimension): """ def __init__(self, key, label=None, definition=None): - super(Dimension, self).__init__(key, label, definition) + super(RollupDimension, self).__init__(key, label, definition) self.is_rollup = False @immutable diff --git a/fireant/slicer/exceptions.py b/fireant/slicer/exceptions.py index f93ebefd..554a33d4 100644 --- a/fireant/slicer/exceptions.py +++ b/fireant/slicer/exceptions.py @@ -17,5 +17,6 @@ class CircularJoinsException(SlicerException): class RollupException(SlicerException): pass -class MissingMetricsException(SlicerException): + +class MetricRequiredException(SlicerException): pass diff --git a/fireant/slicer/intervals.py b/fireant/slicer/intervals.py index 3aa0af00..f63b36e6 100644 --- a/fireant/slicer/intervals.py +++ b/fireant/slicer/intervals.py @@ -14,11 +14,6 @@ def __hash__(self): def __repr__(self): return 'NumericInterval(size=%d,offset=%d)' % (self.size, self.offset) - def __eq__(self, other): - return isinstance(other, NumericInterval) \ - and all([self.size == other.size, - self.offset == other.offset]) - class DatetimeInterval(object): def __init__(self, key): diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 4a0746df..dc59d4c7 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -248,7 +248,7 @@ def query(self): else element.definition.as_(element.key) for element in order]) - return str(query) + return query def _join_references(self, query, references): original_query = query.as_('base') @@ -281,12 +281,12 @@ def original_query_field(key): return outer_query - def render(self): + def render(self, limit=None, offset=None): """ :return: """ - query = self.query + query = self.query.limit(limit).offset(offset) data_frame = fetch_data(self.slicer.database, query, @@ -321,4 +321,5 @@ def query(self): if hasattr(element, 'display_definition') else element.definition.as_(element.key) for element in self._dimensions]) - return str(query.distinct()) + + return query.distinct() diff --git a/fireant/slicer/slicers.py b/fireant/slicer/slicers.py index 7c927599..ff8ad4d8 100644 --- a/fireant/slicer/slicers.py +++ b/fireant/slicer/slicers.py @@ -40,7 +40,7 @@ def __eq__(self, other): and all([a is not None and b is not None and a.key == b.key - for a, b in itertools.zip_longest(self._items, other._items)]) + for a, b in itertools.zip_longest(self._items, getattr(other, '_items', ()))]) class Slicer(object): diff --git a/fireant/slicer/widgets/base.py b/fireant/slicer/widgets/base.py index fa0968b7..cfca5426 100644 --- a/fireant/slicer/widgets/base.py +++ b/fireant/slicer/widgets/base.py @@ -1,4 +1,4 @@ -from fireant.slicer.exceptions import MissingMetricsException +from fireant.slicer.exceptions import MetricRequiredException from fireant.utils import immutable @@ -18,7 +18,7 @@ def metric(self, metric): @property def metrics(self): if 0 == len(self._metrics): - raise MissingMetricsException(str(self)) + raise MetricRequiredException(str(self)) return [metric for group in self._metrics diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index e25db233..494d9be6 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -17,6 +17,7 @@ extract_display_values, reference_key, ) +from ..exceptions import MetricRequiredException DEFAULT_COLORS = ( "#DDDF0D", @@ -114,6 +115,9 @@ def metrics(self): :return: A set of metrics used in this chart. This collects all metrics across all axes. """ + if 0 == len(self.axes): + raise MetricRequiredException(str(self)) + seen = set() return [metric for axis in self.axes diff --git a/fireant/tests/slicer/matchers.py b/fireant/tests/slicer/matchers.py index 372f2921..0cc2b45a 100644 --- a/fireant/tests/slicer/matchers.py +++ b/fireant/tests/slicer/matchers.py @@ -2,6 +2,19 @@ Dimension, Metric, ) +from pypika.queries import QueryBuilder + + +class PypikaQueryMatcher: + def __init__(self, query_str): + self.query_str = query_str + + def __eq__(self, other): + return isinstance(other, QueryBuilder) \ + and self.query_str == str(other) + + def __repr__(self): + return self.query_str class _ElementsMatcher: diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 18e49453..1fd33acf 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -8,8 +8,14 @@ from datetime import date import fireant as f -from fireant.slicer.exceptions import RollupException -from ..matchers import DimensionMatcher +from fireant.slicer.exceptions import ( + MetricRequiredException, + RollupException, +) +from ..matchers import ( + DimensionMatcher, + PypikaQueryMatcher, +) from ..mocks import slicer @@ -36,7 +42,7 @@ def test_build_query_with_single_metric(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' - 'FROM "politics"."politician"', query) + 'FROM "politics"."politician"', str(query)) def test_build_query_with_multiple_metrics(self): query = slicer.data \ @@ -46,7 +52,7 @@ def test_build_query_with_multiple_metrics(self): self.assertEqual('SELECT ' 'SUM("votes") "votes",' 'SUM("is_winner") "wins" ' - 'FROM "politics"."politician"', query) + 'FROM "politics"."politician"', str(query)) def test_build_query_with_multiple_visualizations(self): query = slicer.data \ @@ -57,7 +63,7 @@ def test_build_query_with_multiple_visualizations(self): self.assertEqual('SELECT ' 'SUM("votes") "votes",' 'SUM("is_winner") "wins" ' - 'FROM "politics"."politician"', query) + 'FROM "politics"."politician"', str(query)) def test_build_query_for_chart_visualization_with_single_axis(self): query = slicer.data \ @@ -69,7 +75,7 @@ def test_build_query_for_chart_visualization_with_single_axis(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' - 'FROM "politics"."politician"', query) + 'FROM "politics"."politician"', str(query)) def test_build_query_for_chart_visualization_with_multiple_axes(self): query = slicer.data \ @@ -81,7 +87,7 @@ def test_build_query_for_chart_visualization_with_multiple_axes(self): self.assertEqual('SELECT ' 'SUM("votes") "votes",' 'SUM("is_winner") "wins" ' - 'FROM "politics"."politician"', query) + 'FROM "politics"."politician"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -99,7 +105,7 @@ def test_build_query_with_datetime_dimension(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_build_query_with_datetime_dimension_hourly(self): query = slicer.data \ @@ -112,7 +118,7 @@ def test_build_query_with_datetime_dimension_hourly(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_build_query_with_datetime_dimension_daily(self): query = slicer.data \ @@ -125,7 +131,7 @@ def test_build_query_with_datetime_dimension_daily(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_build_query_with_datetime_dimension_weekly(self): query = slicer.data \ @@ -138,7 +144,7 @@ def test_build_query_with_datetime_dimension_weekly(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_build_query_with_datetime_dimension_monthly(self): query = slicer.data \ @@ -151,7 +157,7 @@ def test_build_query_with_datetime_dimension_monthly(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_build_query_with_datetime_dimension_quarterly(self): query = slicer.data \ @@ -164,7 +170,7 @@ def test_build_query_with_datetime_dimension_quarterly(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_build_query_with_datetime_dimension_annually(self): query = slicer.data \ @@ -177,7 +183,7 @@ def test_build_query_with_datetime_dimension_annually(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_build_query_with_boolean_dimension(self): query = slicer.data \ @@ -190,7 +196,7 @@ def test_build_query_with_boolean_dimension(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "winner" ' - 'ORDER BY "winner"', query) + 'ORDER BY "winner"', str(query)) def test_build_query_with_categorical_dimension(self): query = slicer.data \ @@ -203,7 +209,7 @@ def test_build_query_with_categorical_dimension(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "political_party" ' - 'ORDER BY "political_party"', query) + 'ORDER BY "political_party"', str(query)) def test_build_query_with_unique_dimension(self): query = slicer.data \ @@ -217,7 +223,7 @@ def test_build_query_with_unique_dimension(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "election","election_display" ' - 'ORDER BY "election_display"', query) + 'ORDER BY "election_display"', str(query)) def test_build_query_with_multiple_dimensions(self): query = slicer.data \ @@ -233,7 +239,7 @@ def test_build_query_with_multiple_dimensions(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp","candidate","candidate_display" ' - 'ORDER BY "timestamp","candidate_display"', query) + 'ORDER BY "timestamp","candidate_display"', str(query)) def test_build_query_with_multiple_dimensions_and_visualizations(self): query = slicer.data \ @@ -254,7 +260,7 @@ def test_build_query_with_multiple_dimensions_and_visualizations(self): 'SUM("is_winner") "wins" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp","political_party" ' - 'ORDER BY "timestamp","political_party"', query) + 'ORDER BY "timestamp","political_party"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -272,7 +278,7 @@ def test_build_query_with_rollup_cat_dimension(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY ROLLUP("political_party") ' - 'ORDER BY "political_party"', query) + 'ORDER BY "political_party"', str(query)) def test_build_query_with_rollup_uni_dimension(self): query = slicer.data \ @@ -286,7 +292,7 @@ def test_build_query_with_rollup_uni_dimension(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY ROLLUP(("candidate_id","candidate_name")) ' - 'ORDER BY "candidate_display"', query) + 'ORDER BY "candidate_display"', str(query)) def test_rollup_following_non_rolled_up_dimensions(self): query = slicer.data \ @@ -302,7 +308,7 @@ def test_rollup_following_non_rolled_up_dimensions(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp",ROLLUP(("candidate_id","candidate_name")) ' - 'ORDER BY "timestamp","candidate_display"', query) + 'ORDER BY "timestamp","candidate_display"', str(query)) def test_force_all_dimensions_following_rollup_to_be_rolled_up(self): query = slicer.data \ @@ -318,7 +324,7 @@ def test_force_all_dimensions_following_rollup_to_be_rolled_up(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY ROLLUP("political_party",("candidate_id","candidate_name")) ' - 'ORDER BY "political_party","candidate_display"', query) + 'ORDER BY "political_party","candidate_display"', str(query)) def test_force_all_dimensions_following_rollup_to_be_rolled_up_with_split_dimension_calls(self): query = slicer.data \ @@ -334,7 +340,7 @@ def test_force_all_dimensions_following_rollup_to_be_rolled_up_with_split_dimens 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY ROLLUP("political_party",("candidate_id","candidate_name")) ' - 'ORDER BY "political_party","candidate_display"', query) + 'ORDER BY "political_party","candidate_display"', str(query)) def test_raise_exception_when_trying_to_rollup_continuous_dimension(self): with self.assertRaises(RollupException): @@ -344,6 +350,7 @@ def test_raise_exception_when_trying_to_rollup_continuous_dimension(self): slicer.dimensions.timestamp) \ .query + # noinspection SqlDialectInspection,SqlNoDataSourceInspection class QueryBuilderDimensionFilterTests(TestCase): maxDiff = None @@ -357,7 +364,7 @@ def test_build_query_with_filter_isin_categorical_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'WHERE "political_party" IN (\'d\')', query) + 'WHERE "political_party" IN (\'d\')', str(query)) def test_build_query_with_filter_notin_categorical_dim(self): query = slicer.data \ @@ -368,7 +375,7 @@ def test_build_query_with_filter_notin_categorical_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'WHERE "political_party" NOT IN (\'d\')', query) + 'WHERE "political_party" NOT IN (\'d\')', str(query)) def test_build_query_with_filter_isin_unique_dim(self): query = slicer.data \ @@ -379,7 +386,7 @@ def test_build_query_with_filter_isin_unique_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'WHERE "candidate_id" IN (1)', query) + 'WHERE "candidate_id" IN (1)', str(query)) def test_build_query_with_filter_notin_unique_dim(self): query = slicer.data \ @@ -390,7 +397,7 @@ def test_build_query_with_filter_notin_unique_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'WHERE "candidate_id" NOT IN (1)', query) + 'WHERE "candidate_id" NOT IN (1)', str(query)) def test_build_query_with_filter_isin_unique_dim_display(self): query = slicer.data \ @@ -401,7 +408,7 @@ def test_build_query_with_filter_isin_unique_dim_display(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'WHERE "candidate_name" IN (\'Donald Trump\')', query) + 'WHERE "candidate_name" IN (\'Donald Trump\')', str(query)) def test_build_query_with_filter_notin_unique_dim_display(self): query = slicer.data \ @@ -412,7 +419,7 @@ def test_build_query_with_filter_notin_unique_dim_display(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'WHERE "candidate_name" NOT IN (\'Donald Trump\')', query) + 'WHERE "candidate_name" NOT IN (\'Donald Trump\')', str(query)) def test_build_query_with_filter_wildcard_unique_dim(self): query = slicer.data \ @@ -423,7 +430,7 @@ def test_build_query_with_filter_wildcard_unique_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'WHERE "candidate_name" LIKE \'%Trump\'', query) + 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) def test_build_query_with_filter_isin_raise_exception_when_display_definition_undefined(self): with self.assertRaises(f.QueryException): @@ -452,7 +459,7 @@ def test_build_query_with_filter_range_datetime_dimension(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'WHERE "timestamp" BETWEEN \'2009-01-20\' AND \'2017-01-20\'', query) + 'WHERE "timestamp" BETWEEN \'2009-01-20\' AND \'2017-01-20\'', str(query)) def test_build_query_with_filter_boolean_true(self): query = slicer.data \ @@ -463,7 +470,7 @@ def test_build_query_with_filter_boolean_true(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'WHERE "is_winner"', query) + 'WHERE "is_winner"', str(query)) def test_build_query_with_filter_boolean_false(self): query = slicer.data \ @@ -474,7 +481,7 @@ def test_build_query_with_filter_boolean_false(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'WHERE NOT "is_winner"', query) + 'WHERE NOT "is_winner"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -490,7 +497,7 @@ def test_build_query_with_metric_filter_eq(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'HAVING SUM("votes")=5', query) + 'HAVING SUM("votes")=5', str(query)) def test_build_query_with_metric_filter_eq_left(self): query = slicer.data \ @@ -501,7 +508,7 @@ def test_build_query_with_metric_filter_eq_left(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'HAVING SUM("votes")=5', query) + 'HAVING SUM("votes")=5', str(query)) def test_build_query_with_metric_filter_ne(self): query = slicer.data \ @@ -512,7 +519,7 @@ def test_build_query_with_metric_filter_ne(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'HAVING SUM("votes")<>5', query) + 'HAVING SUM("votes")<>5', str(query)) def test_build_query_with_metric_filter_ne_left(self): query = slicer.data \ @@ -523,7 +530,7 @@ def test_build_query_with_metric_filter_ne_left(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'HAVING SUM("votes")<>5', query) + 'HAVING SUM("votes")<>5', str(query)) def test_build_query_with_metric_filter_gt(self): query = slicer.data \ @@ -534,7 +541,7 @@ def test_build_query_with_metric_filter_gt(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'HAVING SUM("votes")>5', query) + 'HAVING SUM("votes")>5', str(query)) def test_build_query_with_metric_filter_gt_left(self): query = slicer.data \ @@ -545,7 +552,7 @@ def test_build_query_with_metric_filter_gt_left(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'HAVING SUM("votes")>5', query) + 'HAVING SUM("votes")>5', str(query)) def test_build_query_with_metric_filter_gte(self): query = slicer.data \ @@ -556,7 +563,7 @@ def test_build_query_with_metric_filter_gte(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'HAVING SUM("votes")>=5', query) + 'HAVING SUM("votes")>=5', str(query)) def test_build_query_with_metric_filter_gte_left(self): query = slicer.data \ @@ -567,7 +574,7 @@ def test_build_query_with_metric_filter_gte_left(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'HAVING SUM("votes")>=5', query) + 'HAVING SUM("votes")>=5', str(query)) def test_build_query_with_metric_filter_lt(self): query = slicer.data \ @@ -578,7 +585,7 @@ def test_build_query_with_metric_filter_lt(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'HAVING SUM("votes")<5', query) + 'HAVING SUM("votes")<5', str(query)) def test_build_query_with_metric_filter_lt_left(self): query = slicer.data \ @@ -589,7 +596,7 @@ def test_build_query_with_metric_filter_lt_left(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'HAVING SUM("votes")<5', query) + 'HAVING SUM("votes")<5', str(query)) def test_build_query_with_metric_filter_lte(self): query = slicer.data \ @@ -600,7 +607,7 @@ def test_build_query_with_metric_filter_lte(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'HAVING SUM("votes")<=5', query) + 'HAVING SUM("votes")<=5', str(query)) def test_build_query_with_metric_filter_lte_left(self): query = slicer.data \ @@ -611,7 +618,7 @@ def test_build_query_with_metric_filter_lte_left(self): self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'HAVING SUM("votes")<=5', query) + 'HAVING SUM("votes")<=5', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -629,7 +636,7 @@ def test_build_query_with_cumsum_operation(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_build_query_with_cummean_operation(self): query = slicer.data \ @@ -642,7 +649,7 @@ def test_build_query_with_cummean_operation(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -681,7 +688,7 @@ def test_dimension_with_single_reference_dod(self): ') "sq1" ' # end-nested 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_dimension_with_single_reference_wow(self): query = slicer.data \ @@ -715,7 +722,7 @@ def test_dimension_with_single_reference_wow(self): ') "sq1" ' # end-nested 'ON "base"."timestamp"=TIMESTAMPADD(\'week\',1,"sq1"."timestamp") ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_dimension_with_single_reference_mom(self): query = slicer.data \ @@ -749,7 +756,7 @@ def test_dimension_with_single_reference_mom(self): ') "sq1" ' # end-nested 'ON "base"."timestamp"=TIMESTAMPADD(\'month\',1,"sq1"."timestamp") ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_dimension_with_single_reference_qoq(self): query = slicer.data \ @@ -783,7 +790,7 @@ def test_dimension_with_single_reference_qoq(self): ') "sq1" ' # end-nested 'ON "base"."timestamp"=TIMESTAMPADD(\'quarter\',1,"sq1"."timestamp") ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_dimension_with_single_reference_yoy(self): query = slicer.data \ @@ -817,7 +824,7 @@ def test_dimension_with_single_reference_yoy(self): ') "sq1" ' # end-nested 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_dimension_with_single_reference_as_a_delta(self): query = slicer.data \ @@ -851,7 +858,7 @@ def test_dimension_with_single_reference_as_a_delta(self): ') "sq1" ' # end-nested 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_dimension_with_single_reference_as_a_delta_percentage(self): query = slicer.data \ @@ -885,7 +892,7 @@ def test_dimension_with_single_reference_as_a_delta_percentage(self): ') "sq1" ' # end-nested 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_dimension_with_multiple_references(self): query = slicer.data \ @@ -931,7 +938,7 @@ def test_dimension_with_multiple_references(self): ') "sq2" ' # end-nested 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq2"."timestamp") ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_reference_joins_nested_query_on_dimensions(self): query = slicer.data \ @@ -970,7 +977,7 @@ def test_reference_joins_nested_query_on_dimensions(self): 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' 'AND "base"."political_party"="sq1"."political_party" ' - 'ORDER BY "timestamp","political_party"', query) + 'ORDER BY "timestamp","political_party"', str(query)) def test_reference_with_unique_dimension_includes_display_definition(self): query = slicer.data \ @@ -1012,7 +1019,7 @@ def test_reference_with_unique_dimension_includes_display_definition(self): 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' 'AND "base"."candidate"="sq1"."candidate" ' - 'ORDER BY "timestamp","candidate_display"', query) + 'ORDER BY "timestamp","candidate_display"', str(query)) def test_adjust_reference_dimension_filters_in_reference_query(self): query = slicer.data \ @@ -1050,7 +1057,7 @@ def test_adjust_reference_dimension_filters_in_reference_query(self): ') "sq1" ' # end-nested 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_filters(self): query = slicer.data \ @@ -1092,7 +1099,7 @@ def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_fil ') "sq1" ' # end-nested 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_reference(self): query = slicer.data \ @@ -1127,7 +1134,7 @@ def test_adapt_dow_for_leap_year_for_yoy_reference(self): 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',-1,' 'TRUNC(TIMESTAMPADD(\'year\',1,"sq1"."timestamp"),\'IW\')) ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): query = slicer.data \ @@ -1162,7 +1169,7 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',-1,' 'TRUNC(TIMESTAMPADD(\'year\',1,"sq1"."timestamp"),\'IW\')) ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): query = slicer.data \ @@ -1197,7 +1204,7 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',-1,' 'TRUNC(TIMESTAMPADD(\'year\',1,"sq1"."timestamp"),\'IW\')) ' - 'ORDER BY "timestamp"', query) + 'ORDER BY "timestamp"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -1220,7 +1227,7 @@ def test_dimension_with_join_includes_join_in_query(self): 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' 'GROUP BY "timestamp","district","district_display" ' - 'ORDER BY "timestamp","district_display"', query) + 'ORDER BY "timestamp","district_display"', str(query)) def test_dimension_with_recursive_join_joins_all_join_tables(self): query = slicer.data \ @@ -1240,7 +1247,7 @@ def test_dimension_with_recursive_join_joins_all_join_tables(self): 'JOIN "locations"."state" ' 'ON "district"."state_id"="state"."id" ' 'GROUP BY "timestamp","state","state_display" ' - 'ORDER BY "timestamp","state_display"', query) + 'ORDER BY "timestamp","state_display"', str(query)) def test_metric_with_join_includes_join_in_query(self): query = slicer.data \ @@ -1258,7 +1265,7 @@ def test_metric_with_join_includes_join_in_query(self): 'JOIN "politics"."voter" ' 'ON "district"."id"="voter"."district_id" ' 'GROUP BY "district","district_display" ' - 'ORDER BY "district_display"', query) + 'ORDER BY "district_display"', str(query)) def test_dimension_filter_with_join_on_display_definition_does_not_include_join_in_query(self): query = slicer.data \ @@ -1269,7 +1276,7 @@ def test_dimension_filter_with_join_on_display_definition_does_not_include_join_ self.assertEqual('SELECT ' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'WHERE "district_id" IN (1)', query) + 'WHERE "district_id" IN (1)', str(query)) def test_dimension_filter_display_field_with_join_includes_join_in_query(self): query = slicer.data \ @@ -1282,7 +1289,7 @@ def test_dimension_filter_display_field_with_join_includes_join_in_query(self): 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' - 'WHERE "district"."district_name" IN (\'District 4\')', query) + 'WHERE "district"."district_name" IN (\'District 4\')', str(query)) def test_dimension_filter_with_recursive_join_includes_join_in_query(self): query = slicer.data \ @@ -1295,7 +1302,7 @@ def test_dimension_filter_with_recursive_join_includes_join_in_query(self): 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' - 'WHERE "district"."state_id" IN (1)', query) + 'WHERE "district"."state_id" IN (1)', str(query)) def test_dimension_filter_with_deep_recursive_join_includes_joins_in_query(self): query = slicer.data \ @@ -1312,15 +1319,81 @@ def test_dimension_filter_with_deep_recursive_join_includes_joins_in_query(self) 'ON "district"."state_id"="state"."id" ' 'JOIN "test"."deep" ' 'ON "deep"."id"="state"."ref_id" ' - 'WHERE "deep"."id" IN (1)', query) + 'WHERE "deep"."id" IN (1)', str(query)) + + +@patch('fireant.slicer.queries.builder.fetch_data') +class QueryBuildPaginationTests(TestCase): + def test_set_limit(self, mock_fetch_data: Mock): + slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .render(limit=20) + + mock_fetch_data.assert_called_once_with(ANY, + PypikaQueryMatcher('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp" LIMIT 20'), + dimensions=ANY) + + def test_set_offset(self, mock_fetch_data: Mock): + slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .render(offset=20) + + mock_fetch_data.assert_called_once_with(ANY, + PypikaQueryMatcher('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp" ' + 'OFFSET 20'), + dimensions=ANY) + + def test_set_limit_and_offset(self, mock_fetch_data: Mock): + slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .render(limit=20, offset=20) + + mock_fetch_data.assert_called_once_with(ANY, + PypikaQueryMatcher('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp" ' + 'LIMIT 20 ' + 'OFFSET 20'), + dimensions=ANY) # noinspection SqlDialectInspection,SqlNoDataSourceInspection class QueryBuilderValidationTests(TestCase): maxDiff = None - def test_query_requires_at_least_one_metric(self): - pass + def test_highcharts_requires_at_least_one_axis(self): + with self.assertRaises(MetricRequiredException): + slicer.data \ + .widget(f.HighCharts([])) \ + .query + + def test_highcharts_axis_requires_at_least_one_metric(self): + with self.assertRaises(MetricRequiredException): + slicer.data \ + .widget(f.HighCharts([f.HighCharts.LineChart([])])) \ + .query + + def test_datatablesjs_requires_at_least_one_metric(self): + with self.assertRaises(MetricRequiredException): + slicer.data \ + .widget(f.DataTablesJS([])) \ + .query # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -1347,7 +1420,8 @@ def test_pass_query_from_builder_as_arg(self, mock_fetch_data: Mock): .render() mock_fetch_data.assert_called_once_with(ANY, - 'SELECT SUM("votes") "votes" FROM "politics"."politician"', + PypikaQueryMatcher('SELECT SUM("votes") "votes" ' + 'FROM "politics"."politician"'), dimensions=ANY) def test_builder_dimensions_as_arg_with_zero_dimensions(self, mock_fetch_data: Mock): diff --git a/fireant/tests/slicer/queries/test_dimension_options.py b/fireant/tests/slicer/queries/test_dimension_options.py index 1f1c18ea..ecb5970f 100644 --- a/fireant/tests/slicer/queries/test_dimension_options.py +++ b/fireant/tests/slicer/queries/test_dimension_options.py @@ -16,7 +16,7 @@ def test_query_options_for_cat_dimension(self): '"political_party" "political_party" ' 'FROM "politics"."politician" ' 'GROUP BY "political_party" ' - 'ORDER BY "political_party"', query) + 'ORDER BY "political_party"', str(query)) def test_query_options_for_uni_dimension(self): query = slicer.dimensions.candidate \ @@ -28,7 +28,7 @@ def test_query_options_for_uni_dimension(self): '"candidate_name" "candidate_display" ' 'FROM "politics"."politician" ' 'GROUP BY "candidate","candidate_display" ' - 'ORDER BY "candidate_display"', query) + 'ORDER BY "candidate_display"', str(query)) def test_query_options_for_uni_dimension_with_join(self): query = slicer.dimensions.district \ @@ -42,7 +42,7 @@ def test_query_options_for_uni_dimension_with_join(self): 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' 'GROUP BY "district","district_display" ' - 'ORDER BY "district_display"', query) + 'ORDER BY "district_display"', str(query)) def test_no_options_attr_for_datetime_dimension(self): with self.assertRaises(AttributeError): @@ -64,4 +64,4 @@ def test_filter_options(self): 'FROM "politics"."politician" ' 'WHERE "political_party" IN (\'d\',\'r\') ' 'GROUP BY "candidate","candidate_display" ' - 'ORDER BY "candidate_display"', query) + 'ORDER BY "candidate_display"', str(query)) diff --git a/requirements.txt b/requirements.txt index 51350847..9336b688 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ six pandas==0.22.0 -pypika==0.10.3 +pypika==0.10.4 pymysql==0.8.0 vertica-python==0.7.3 psycopg2==2.7.3.2 From 2a1d21d8395e2123d4783057ffb46a5151bcd057 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 29 Jan 2018 15:06:23 +0100 Subject: [PATCH 009/123] Bumped pypika version in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1c894b76..e6795a21 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ def find_version(*file_paths): install_requires=[ 'six', 'pandas==0.22.0', - 'pypika==0.10.3', + 'pypika==0.10.4', 'toposort==1.5', 'typing==3.6.2', ], From a7a5530ffc0c43f25486068bae16acc113027d74 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 1 Feb 2018 17:24:08 +0100 Subject: [PATCH 010/123] Polish and fixes for 1.0.0 --- fireant/__init__.py | 9 +- fireant/slicer/__init__.py | 1 - fireant/slicer/base.py | 4 + fireant/slicer/dimensions.py | 29 +- fireant/slicer/exceptions.py | 4 + fireant/slicer/operations.py | 7 + fireant/slicer/queries/builder.py | 147 +++++- fireant/slicer/queries/database.py | 8 +- fireant/slicer/queries/references.py | 13 +- fireant/slicer/references.py | 7 + fireant/slicer/slicers.py | 8 + fireant/slicer/widgets/__init__.py | 1 + fireant/slicer/widgets/base.py | 19 +- fireant/slicer/widgets/datatables.py | 24 +- fireant/slicer/widgets/formats.py | 2 +- fireant/slicer/widgets/helpers.py | 2 +- fireant/slicer/widgets/highcharts.py | 64 +-- fireant/tests/slicer/matchers.py | 2 +- fireant/tests/slicer/mocks.py | 2 + fireant/tests/slicer/queries/test_builder.py | 469 +++++++++++++++--- fireant/tests/slicer/queries/test_database.py | 6 + .../slicer/queries/test_dimension_options.py | 20 +- fireant/tests/slicer/test_dimensions.py | 22 + .../tests/slicer/widgets/test_datatables.py | 122 ++++- .../tests/slicer/widgets/test_highcharts.py | 456 +++++++++-------- 25 files changed, 1023 insertions(+), 425 deletions(-) create mode 100644 fireant/tests/slicer/test_dimensions.py diff --git a/fireant/__init__.py b/fireant/__init__.py index 575178df..e5adb1ca 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -3,11 +3,6 @@ # noinspection PyUnresolvedReferences from .slicer import * # noinspection PyUnresolvedReferences -from .slicer.widgets import ( - DataTablesJS, - HighCharts, - Matplotlib, - Pandas, -) +from .slicer.widgets import * -__version__ = '1.0.0' +__version__ = '1.0.0.dev' diff --git a/fireant/slicer/__init__.py b/fireant/slicer/__init__.py index 722309f8..ed3ef65a 100644 --- a/fireant/slicer/__init__.py +++ b/fireant/slicer/__init__.py @@ -4,7 +4,6 @@ ContinuousDimension, DatetimeDimension, Dimension, - DimensionValue, UniqueDimension, ) from .exceptions import ( diff --git a/fireant/slicer/base.py b/fireant/slicer/base.py index 9dd50a1a..bbaa0ddb 100644 --- a/fireant/slicer/base.py +++ b/fireant/slicer/base.py @@ -20,3 +20,7 @@ def __init__(self, key, label=None, definition=None): def __repr__(self): return self.key + + @property + def has_display_field(self): + return getattr(self, 'display_definition', None) is not None diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index e2dbd6b6..feb6fe8a 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -116,9 +116,13 @@ class UniqueDimension(RollupDimension): def __init__(self, key, label=None, definition=None, display_definition=None): super(UniqueDimension, self).__init__(key=key, label=label, definition=definition) - self.display_key = '{}_display'.format(key) + self.display_definition = display_definition + self.display_key = '{}_display'.format(key) \ + if display_definition is not None \ + else None + def isin(self, values, use_display=False): """ Creates a filter to filter a slicer query. @@ -171,6 +175,19 @@ def wildcard(self, pattern): raise QueryException('No value set for display_definition.') return WildcardFilter(self.display_definition, pattern) + @property + def display(self): + return self + + +class DisplayDimension(Dimension): + """ + WRITEME + """ + + def __init__(self, dimension): + super(DisplayDimension, self).__init__(dimension.display_key, dimension.label, dimension.display_definition) + class ContinuousDimension(Dimension): """ @@ -243,13 +260,3 @@ def between(self, start, stop): start and stop. """ return RangeFilter(self.definition, start, stop) - - -class DimensionValue(object): - """ - An option belongs to a categorical dimension which specifies a fixed set of values - """ - - def __init__(self, key, label=None): - self.key = key - self.label = label or key diff --git a/fireant/slicer/exceptions.py b/fireant/slicer/exceptions.py index 554a33d4..71b28731 100644 --- a/fireant/slicer/exceptions.py +++ b/fireant/slicer/exceptions.py @@ -20,3 +20,7 @@ class RollupException(SlicerException): class MetricRequiredException(SlicerException): pass + + +class ContinuousDimensionRequiredException(SlicerException): + pass diff --git a/fireant/slicer/operations.py b/fireant/slicer/operations.py index 99bb0d12..a7a30fa2 100644 --- a/fireant/slicer/operations.py +++ b/fireant/slicer/operations.py @@ -22,6 +22,13 @@ def apply(self, data_frame): class _Cumulative(Operation): def __init__(self, arg): self.arg = arg + self.key = '{}({})'.format(self.__class__.__name__.lower(), + getattr(arg, 'key', arg)) + self.label = '{}({})'.format(self.__class__.__name__, + getattr(arg, 'label', arg)) + self.prefix = getattr(arg, 'prefix') + self.suffix = getattr(arg, 'suffix') + self.precision = getattr(arg, 'precision') def _group_levels(self, index): """ diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index dc59d4c7..389e0e97 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -1,10 +1,21 @@ from collections import defaultdict +from typing import ( + Dict, + Iterable, +) +import pandas as pd +from pypika import ( + Order, + functions as fn, +) +from pypika.enums import SqlTypes from toposort import ( CircularDependencyError, toposort_flatten, ) +from fireant.slicer.base import SlicerElement from fireant.utils import ( flatten, immutable, @@ -87,7 +98,7 @@ def _tables(self): return ordered_distinct_list([table for element in self._elements # Need extra for-loop to incl. the `display_definition` from `UniqueDimension` - for attr in [element.definition, + for attr in [getattr(element, 'definition', None), getattr(element, 'display_definition', None)] # ... but then filter Nones since most elements do not have `display_definition` if attr is not None @@ -150,7 +161,7 @@ def query(self): dimension_definition = _build_dimension_definition(dimension, self.slicer.database.trunc_date) - if hasattr(dimension, 'display_definition') and dimension.display_definition is not None: + if dimension.has_display_field: # Add display definition field dimension_display_definition = dimension.display_definition.as_(dimension.display_key) fields = [dimension_definition, dimension_display_definition] @@ -197,6 +208,17 @@ def dimension(self, *dimensions): """ self._dimensions += dimensions + @immutable + def orderby(self, element: SlicerElement, orientation=None): + """ + :param element: + The element to order by, either a metric or dimension. + :param orientation: + The directionality to order by, either ascending or descending. + :return: + """ + self._orders += [(element.definition.as_(element.key), orientation)] + @property def metrics(self): """ @@ -213,10 +235,24 @@ def operations(self): :return: an ordered, distinct list of metrics used in all widgets as part of this query. """ - return ordered_distinct_list_by_attr([metric + return ordered_distinct_list_by_attr([item for widget in self._widgets - for metric in widget.metrics - if isinstance(metric, Operation)]) + for item in widget.items + if isinstance(item, Operation)]) + + @property + def orders(self): + if self._orders: + return self._orders + + definitions = [dimension.display_definition.as_(dimension.display_key) + if dimension.has_display_field + else dimension.definition.as_(dimension.key) + for dimension in self._dimensions] + + return [(definition, None) + for definition in definitions] + @property def _elements(self): @@ -225,8 +261,17 @@ def _elements(self): @property def query(self): """ - WRITEME + Build the pypika query for this Slicer query. This collects all of the metrics in each widget, dimensions, and + filters and builds a corresponding pypika query to fetch the data. When references are used, the base query + normally produced is wrapped in an outer query and a query for each reference is joined based on the referenced + dimension shifted. """ + + # Validate + for widget in self._widgets: + if hasattr(widget, 'validate'): + widget.validate(self._dimensions) + query = super(SlicerQueryBuilder, self).query # Add metrics @@ -236,21 +281,37 @@ def query(self): # Add references references = [(reference, dimension) for dimension in self._dimensions - if hasattr(dimension, 'references') - for reference in dimension.references] + for reference in getattr(dimension, 'references', ())] if references: query = self._join_references(query, references) # Add ordering - order = self._orders if self._orders else self._dimensions - query = query.orderby(*[element.display_definition.as_(element.display_key) - if hasattr(element, 'display_definition') - else element.definition.as_(element.key) - for element in order]) + for (definition, orientation) in self.orders: + query = query.orderby(definition, order=orientation) return query def _join_references(self, query, references): + """ + This converts the pypika query built in `self.query()` into a query that includes references. This is achieved + by wrapping the original query with an outer query using the original query as the FROM clause, then joining + copies of the original query for each reference, with the reference dimension shifted by the appropriate + interval. + + The outer query selects everything from the original query and each metric from the reference query using an + alias constructed from the metric key appended with the reference key. For Delta references, the reference + metric is selected as the difference of the metric from the original query and the reference query. For Delta + Percentage references, the reference metric metric is selected as the difference divided by the reference + metric. + + :param query: + The original query built by `self.query` + :param references: + A list of the references that should be included. + :return: + A new pypika query with the dimensions and metrics included from the original query plus each of the + metrics for each of the references. + """ original_query = query.as_('base') def original_query_field(key): @@ -262,7 +323,7 @@ def original_query_field(key): for dimension in self._dimensions: outer_query = outer_query.select(original_query_field(dimension.key)) - if hasattr(dimension, 'display_definition'): + if dimension.has_display_field: outer_query = outer_query.select(original_query_field(dimension.display_key)) # Add metrics @@ -281,15 +342,21 @@ def original_query_field(key): return outer_query - def render(self, limit=None, offset=None): + def fetch(self, limit=None, offset=None) -> Iterable[Dict]: """ + Fetch the data for this query and transform it into the widgets. + :param limit: + A limit on the number of database rows returned. + :param offset: + A offset on the number of database rows returned. :return: + A list of dict (JSON) objects containing the widget configurations. """ query = self.query.limit(limit).offset(offset) data_frame = fetch_data(self.slicer.database, - query, + str(query), dimensions=self._dimensions) # Apply operations @@ -312,14 +379,44 @@ def __init__(self, slicer, dimension): super(DimensionOptionQueryBuilder, self).__init__(slicer, slicer.hint_table or slicer.table) self._dimensions.append(dimension) - @property - def query(self): - query = super(DimensionOptionQueryBuilder, self).query + def fetch(self, limit=None, offset=None, force_include=()) -> pd.Series: + """ + Fetch the data for this query and transform it into the widgets. + + :param limit: + A limit on the number of database rows returned. + :param offset: + A offset on the number of database rows returned. + :param force_include: + A list of dimension values to include in the result set. This can be used to avoid having necessary results + cut off due to the pagination. These results will be returned at the head of the results. + :return: + A list of dict (JSON) objects containing the widget configurations. + """ + query = self.query - # Add ordering - query = query.orderby(*[element.display_definition.as_(element.display_key) - if hasattr(element, 'display_definition') - else element.definition.as_(element.key) - for element in self._dimensions]) + dimension = self._dimensions[0] + definition = dimension.display_definition.as_(dimension.display_key) \ + if dimension.has_display_field \ + else dimension.definition.as_(dimension.key) + + if force_include: + include = fn.Cast(dimension.definition, SqlTypes.VARCHAR) \ + .isin([str(x) for x in force_include]) + + # Ensure that these values are included + query = query.orderby(include, order=Order.desc) + + # Add ordering and pagination + query = query.orderby(definition).limit(limit).offset(offset) + + data = fetch_data(self.slicer.database, + str(query), + dimensions=self._dimensions) + + display_key = getattr(dimension, 'display_key', 'display') + if hasattr(dimension, 'display_values'): + # Include provided display values + data[display_key] = pd.Series(dimension.display_values) - return query.distinct() + return data[display_key] diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index d3041dfc..72946c2e 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -1,7 +1,7 @@ import time +from typing import Iterable import pandas as pd -from typing import Iterable from fireant.database.base import Database from .logger import logger @@ -44,6 +44,9 @@ def clean_and_apply_index(data_frame: pd.DataFrame, dimensions: Iterable[Dimensi :param dimensions: :return: """ + if not dimensions: + return data_frame + dimension_keys = [d.key for d in dimensions] for i, dimension in enumerate(dimensions): @@ -59,7 +62,8 @@ def clean_and_apply_index(data_frame: pd.DataFrame, dimensions: Iterable[Dimensi .groupby(dimension_keys[:i])[level] .fillna('Totals', limit=1) ) if 0 < i else ( - data_frame[level].fillna('Totals', limit=1) + data_frame[level] + .fillna('Totals', limit=1) ) data_frame[level] = data_frame[level] \ diff --git a/fireant/slicer/queries/references.py b/fireant/slicer/queries/references.py index 2158829e..967a00d1 100644 --- a/fireant/slicer/queries/references.py +++ b/fireant/slicer/queries/references.py @@ -1,5 +1,4 @@ from functools import partial - from typing import ( Callable, Iterator, @@ -16,13 +15,17 @@ Criterion, Term, ) + from ..dimensions import ( DatetimeDimension, Dimension, ) from ..intervals import weekly from ..metrics import Metric -from ..references import Reference +from ..references import ( + Reference, + YearOverYear, +) def join_reference(reference: Reference, @@ -37,7 +40,9 @@ def join_reference(reference: Reference, date_add = partial(date_add, date_part=reference.time_unit, interval=reference.interval, - align_weekday=weekly == ref_dimension.interval) + # Only need to adjust this for YoY references with weekly intervals + align_weekday=weekly == ref_dimension.interval + and YearOverYear.time_unit == reference.time_unit) # FIXME this is a bit hacky, need to replace the ref dimension term in all of the filters with the offset if ref_query._wheres: @@ -61,7 +66,7 @@ def join_reference(reference: Reference, ref_query) return outer_query.select(*[ref_metric(metric).as_("{}_{}".format(metric.key, reference.key)) - for metric in metrics]) + for metric in metrics]) def _apply_to_term_in_criterion(target: Term, diff --git a/fireant/slicer/references.py b/fireant/slicer/references.py index f8106cb5..dcbd8764 100644 --- a/fireant/slicer/references.py +++ b/fireant/slicer/references.py @@ -14,6 +14,13 @@ def delta(self, percent=False): label = self.label + ' Δ%' if percent else self.label + ' Δ' return Reference(key, label, self.time_unit, self.interval, delta=True, percent=percent) + def __eq__(self, other): + return isinstance(self, Reference) \ + and self.time_unit == other.time_unit \ + and self.interval == other.interval \ + and self.is_delta == other.is_delta \ + and self.is_percent == other.is_percent + DayOverDay = Reference('dod', 'DoD', 'day', 1) WeekOverWeek = Reference('wow', 'WoW', 'week', 1) diff --git a/fireant/slicer/slicers.py b/fireant/slicer/slicers.py index ff8ad4d8..3f250e59 100644 --- a/fireant/slicer/slicers.py +++ b/fireant/slicer/slicers.py @@ -2,6 +2,7 @@ from .dimensions import ( CategoricalDimension, + DisplayDimension, UniqueDimension, ) from .queries import ( @@ -29,9 +30,16 @@ def __init__(self, items): for item in items: setattr(self, item.key, item) + # Special case to include display definitions for filters + if item.has_display_field: + setattr(self, item.display_key, DisplayDimension(item)) + def __iter__(self): return iter(self._items) + def __getitem__(self, item): + return getattr(self, item) + def __eq__(self, other): """ Checks if the other object is an instance of _Container and has the same number of items with matching keys. diff --git a/fireant/slicer/widgets/__init__.py b/fireant/slicer/widgets/__init__.py index 0104753a..16186bf1 100644 --- a/fireant/slicer/widgets/__init__.py +++ b/fireant/slicer/widgets/__init__.py @@ -1,3 +1,4 @@ +from .base import Widget from .datatables import DataTablesJS from .highcharts import HighCharts from .matplotlib import Matplotlib diff --git a/fireant/slicer/widgets/base.py b/fireant/slicer/widgets/base.py index cfca5426..206a1fdc 100644 --- a/fireant/slicer/widgets/base.py +++ b/fireant/slicer/widgets/base.py @@ -6,27 +6,22 @@ class Widget: def transform(self, data_frame, slicer, dimensions): raise NotImplementedError() - -class MetricsWidget(Widget): - def __init__(self, metrics=()): - self._metrics = list(metrics) + def __init__(self, items=()): + self.items = list(items) @immutable def metric(self, metric): - self._metrics.append(metric) + self.items.append(metric) @property def metrics(self): - if 0 == len(self._metrics): + if 0 == len(self.items): raise MetricRequiredException(str(self)) return [metric - for group in self._metrics + for group in self.items for metric in getattr(group, 'metrics', [group])] - def transform(self, data_frame, slicer, dimensions): - super(MetricsWidget, self).transform(data_frame, slicer, dimensions) - def __repr__(self): - return '{}(metrics={})'.format(self.__class__.__name__, - ','.join(str(m) for m in self._metrics)) + return '{}({})'.format(self.__class__.__name__, + ','.join(str(m) for m in self.items)) diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index b39bbeb3..5ee4938a 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -8,7 +8,7 @@ utils, ) from . import formats -from .base import MetricsWidget +from .base import Widget from .helpers import ( dimensional_metric_label, extract_display_values, @@ -95,9 +95,9 @@ def _format_metric_cell(value, metric): HARD_MAX_COLUMNS = 24 -class DataTablesJS(MetricsWidget): - def __init__(self, metrics=(), pivot=False, max_columns=None): - super(DataTablesJS, self).__init__(metrics) +class DataTablesJS(Widget): + def __init__(self, items=(), pivot=False, max_columns=None): + super(DataTablesJS, self).__init__(items) self.pivot = pivot self.max_columns = min(max_columns, HARD_MAX_COLUMNS) \ if max_columns is not None \ @@ -105,6 +105,7 @@ def __init__(self, metrics=(), pivot=False, max_columns=None): def transform(self, data_frame, slicer, dimensions): """ + WRITEME :param data_frame: :param slicer: @@ -118,7 +119,7 @@ def transform(self, data_frame, slicer, dimensions): for reference in getattr(dimension, 'references', ())] metric_keys = [reference_key(metric, reference) - for metric in self.metrics + for metric in self.items for reference in [None] + references] data_frame = data_frame[metric_keys] @@ -153,6 +154,7 @@ def transform(self, data_frame, slicer, dimensions): @staticmethod def _dimension_columns(dimensions): """ + WRITEME :param dimensions: :return: @@ -163,7 +165,9 @@ def _dimension_columns(dimensions): data=dimension.key, render=dict(_='value')) - if not isinstance(dimension, ContinuousDimension): + is_cont_dim = isinstance(dimension, ContinuousDimension) + is_uni_dim_no_display = (hasattr(dimension, 'display_definition') and dimension.display_definition is None) + if not is_cont_dim and not is_uni_dim_no_display: column['render']['display'] = 'display' columns.append(column) @@ -172,11 +176,12 @@ def _dimension_columns(dimensions): def _metric_columns(self, references): """ + WRITEME :return: """ columns = [] - for metric in self.metrics: + for metric in self.items: for reference in [None] + references: title = reference_label(metric, reference) data = reference_key(metric, reference) @@ -195,7 +200,7 @@ def _metric_columns_pivoted(self, references, df_columns, render_column_label): :return: """ columns = [] - for metric in self.metrics: + for metric in self.items: dimension_value_sets = [list(level) for level in df_columns.levels[1:]] @@ -213,6 +218,7 @@ def _metric_columns_pivoted(self, references, df_columns, render_column_label): def _data_row(self, dimensions, dimension_values, dimension_display_values, references, row_data): """ + WRITEME :param dimensions: :param dimension_values: @@ -225,7 +231,7 @@ def _data_row(self, dimensions, dimension_values, dimension_display_values, refe for dimension, dimension_value in zip(dimensions, utils.wrap_list(dimension_values)): row[dimension.key] = _render_dimension_cell(dimension_value, dimension_display_values.get(dimension.key)) - for metric in self.metrics: + for metric in self.items: for reference in [None] + references: key = reference_key(metric, reference) diff --git a/fireant/slicer/widgets/formats.py b/fireant/slicer/widgets/formats.py index 2169a42f..efb572d8 100644 --- a/fireant/slicer/widgets/formats.py +++ b/fireant/slicer/widgets/formats.py @@ -27,7 +27,7 @@ def dimension_value(value, str_date=True): """ if isinstance(value, date): if not str_date: - return 1000 * value.timestamp() + return int(1000 * value.timestamp()) if not hasattr(value, 'time') or value.time() == NO_TIME: return value.strftime('%Y-%m-%d') diff --git a/fireant/slicer/widgets/helpers.py b/fireant/slicer/widgets/helpers.py index 6186f8b5..f268ce60 100644 --- a/fireant/slicer/widgets/helpers.py +++ b/fireant/slicer/widgets/helpers.py @@ -25,7 +25,7 @@ def extract_display_values(dimensions, data_frame): if hasattr(dimension, 'display_values'): display_values[key] = dimension.display_values - elif hasattr(dimension, 'display_key') and dimension.display_key is not None: + elif getattr(dimension, 'display_key', None): display_values[key] = data_frame[dimension.display_key] \ .groupby(level=key) \ .first() diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 494d9be6..b57ff271 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -4,10 +4,7 @@ from fireant import utils from fireant.utils import immutable -from .base import ( - MetricsWidget, - Widget, -) +from .base import Widget from .formats import ( dimension_value, metric_value, @@ -55,47 +52,54 @@ ) -class ChartWidget(MetricsWidget): +class ChartWidget(Widget): type = None needs_marker = False stacked = False - def __init__(self, metrics=(), name=None, stacked=False): - super(ChartWidget, self).__init__(metrics=metrics) + def __init__(self, items=(), name=None, stacked=False): + super(ChartWidget, self).__init__(items) self.name = name self.stacked = self.stacked or stacked + def transform(self, data_frame, slicer, dimensions): + raise NotImplementedError() -class HighCharts(Widget): - class PieChart(ChartWidget): - type = 'pie' - class LineChart(ChartWidget): +class ContinuousAxisChartWidget(ChartWidget): + pass + + +class HighCharts(Widget): + class LineChart(ContinuousAxisChartWidget): type = 'line' needs_marker = True - class AreaChart(ChartWidget): + class AreaChart(ContinuousAxisChartWidget): type = 'area' needs_marker = True + class AreaPercentageChart(AreaChart): + stacked = True + + class PieChart(ChartWidget): + type = 'pie' + class BarChart(ChartWidget): type = 'bar' - class ColumnChart(ChartWidget): - type = 'column' - class StackedBarChart(BarChart): stacked = True - class StackedColumnChart(ColumnChart): - stacked = True + class ColumnChart(ChartWidget): + type = 'column' - class AreaPercentageChart(AreaChart): + class StackedColumnChart(ColumnChart): stacked = True - def __init__(self, title=None, axes=(), colors=None): + def __init__(self, axes=(), title=None, colors=None): + super(HighCharts, self).__init__(axes) self.title = title - self.axes = list(axes) self.colors = colors or DEFAULT_COLORS @immutable @@ -107,7 +111,7 @@ def axis(self, axis: ChartWidget): :return: """ - self.axes.append(axis) + self.items.append(axis) @property def metrics(self): @@ -115,12 +119,12 @@ def metrics(self): :return: A set of metrics used in this chart. This collects all metrics across all axes. """ - if 0 == len(self.axes): + if 0 == len(self.items): raise MetricRequiredException(str(self)) seen = set() return [metric - for axis in self.axes + for axis in self.items for metric in axis.metrics if not (metric.key in seen or seen.add(metric.key))] @@ -155,10 +159,12 @@ def transform(self, data_frame, slicer, dimensions): for dimension in dimensions for reference in getattr(dimension, 'references', ())] + total_num_items = sum([len(axis.items) for axis in self.items]) + y_axes, series = [], [] - for axis_idx, axis in enumerate(self.axes): + for axis_idx, axis in enumerate(self.items): colors, series_colors = itertools.tee(colors) - axis_color = next(colors) if 1 < len(self.metrics) else None + axis_color = next(colors) if 1 < total_num_items else None # prepend axes, append series, this keeps everything ordered left-to-right y_axes[0:0] = self._render_y_axis(axis_idx, @@ -257,10 +263,10 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, :param references: :return: """ - has_multi_metric = 1 < len(axis.metrics) + has_multi_metric = 1 < len(axis.items) series = [] - for metric in axis.metrics: + for metric in axis.items: visible = True symbols = itertools.cycle(MARKER_SYMBOLS) series_color = next(colors) if has_multi_metric else None @@ -307,9 +313,9 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, def _render_data(self, group_df, metric_key, is_timeseries): if is_timeseries: - return [[dimension_value(utils.wrap_list(dimension_values)[0], + return [(dimension_value(utils.wrap_list(dimension_values)[0], str_date=False), - metric_value(y)] + metric_value(y)) for dimension_values, y in group_df[metric_key].iteritems()] return [metric_value(y) diff --git a/fireant/tests/slicer/matchers.py b/fireant/tests/slicer/matchers.py index 0cc2b45a..1de10758 100644 --- a/fireant/tests/slicer/matchers.py +++ b/fireant/tests/slicer/matchers.py @@ -11,7 +11,7 @@ def __init__(self, query_str): def __eq__(self, other): return isinstance(other, QueryBuilder) \ - and self.query_str == str(other) + and str(self.query_str) == str(other) def __repr__(self): return self.query_str diff --git a/fireant/tests/slicer/mocks.py b/fireant/tests/slicer/mocks.py index 15e4452a..ba4c3d63 100644 --- a/fireant/tests/slicer/mocks.py +++ b/fireant/tests/slicer/mocks.py @@ -258,6 +258,8 @@ def __eq__(self, other): .sum() \ .reset_index('state_display') +cont_dim_operation_df = cont_dim_df.copy() +cont_dim_operation_df['cumsum(votes)'] = cont_dim_df['votes'].cumsum() def ref(data_frame, columns): ref_cols = {column: '%s_eoe' % column diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 1fd33acf..aa2a2d71 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -1,3 +1,4 @@ +from datetime import date from unittest import TestCase from unittest.mock import ( ANY, @@ -5,7 +6,7 @@ patch, ) -from datetime import date +from pypika import Order import fireant as f from fireant.slicer.exceptions import ( @@ -14,7 +15,6 @@ ) from ..matchers import ( DimensionMatcher, - PypikaQueryMatcher, ) from ..mocks import slicer @@ -69,7 +69,7 @@ def test_build_query_for_chart_visualization_with_single_axis(self): query = slicer.data \ .widget(f.HighCharts( axes=[ - f.HighCharts.PieChart(metrics=[slicer.metrics.votes]) + f.HighCharts.PieChart([slicer.metrics.votes]) ])) \ .query @@ -80,8 +80,8 @@ def test_build_query_for_chart_visualization_with_single_axis(self): def test_build_query_for_chart_visualization_with_multiple_axes(self): query = slicer.data \ .widget(f.HighCharts() - .axis(f.HighCharts.PieChart(metrics=[slicer.metrics.votes])) - .axis(f.HighCharts.PieChart(metrics=[slicer.metrics.wins]))) \ + .axis(f.HighCharts.PieChart([slicer.metrics.votes])) + .axis(f.HighCharts.PieChart([slicer.metrics.wins]))) \ .query self.assertEqual('SELECT ' @@ -246,8 +246,8 @@ def test_build_query_with_multiple_dimensions_and_visualizations(self): .widget(f.DataTablesJS([slicer.metrics.votes, slicer.metrics.wins])) \ .widget(f.HighCharts( axes=[ - f.HighCharts.PieChart(metrics=[slicer.metrics.votes]), - f.HighCharts.ColumnChart(metrics=[slicer.metrics.wins]), + f.HighCharts.PieChart([slicer.metrics.votes]), + f.HighCharts.ColumnChart([slicer.metrics.wins]), ])) \ .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.political_party) \ @@ -660,7 +660,7 @@ def test_dimension_with_single_reference_dod(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp .reference(f.DayOverDay)) \ .query @@ -694,7 +694,7 @@ def test_dimension_with_single_reference_wow(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp .reference(f.WeekOverWeek)) \ .query @@ -728,7 +728,7 @@ def test_dimension_with_single_reference_mom(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp .reference(f.MonthOverMonth)) \ .query @@ -762,7 +762,7 @@ def test_dimension_with_single_reference_qoq(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp .reference(f.QuarterOverQuarter)) \ .query @@ -796,7 +796,7 @@ def test_dimension_with_single_reference_yoy(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp .reference(f.YearOverYear)) \ .query @@ -830,7 +830,7 @@ def test_dimension_with_single_reference_as_a_delta(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp .reference(f.DayOverDay.delta())) \ .query @@ -864,7 +864,7 @@ def test_dimension_with_single_reference_as_a_delta_percentage(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp .reference(f.DayOverDay.delta(percent=True))) \ .query @@ -894,11 +894,147 @@ def test_dimension_with_single_reference_as_a_delta_percentage(self): 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' 'ORDER BY "timestamp"', str(query)) + def test_reference_on_dimension_with_weekly_interval(self): + query = slicer.data \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + [slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp(f.weekly) + .reference(f.DayOverDay)) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', str(query)) + + def test_reference_on_dimension_with_monthly_interval(self): + query = slicer.data \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + [slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp(f.monthly) + .reference(f.DayOverDay)) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'MM\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'MM\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', str(query)) + + def test_reference_on_dimension_with_quarterly_interval(self): + query = slicer.data \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + [slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp(f.quarterly) + .reference(f.DayOverDay)) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'Q\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'Q\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', str(query)) + + def test_reference_on_dimension_with_annual_interval(self): + query = slicer.data \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + [slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp(f.annually) + .reference(f.DayOverDay)) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'Y\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'Y\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', str(query)) + def test_dimension_with_multiple_references(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp .reference(f.DayOverDay) .reference(f.YearOverYear.delta(percent=True))) \ @@ -944,7 +1080,7 @@ def test_reference_joins_nested_query_on_dimensions(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp .reference(f.YearOverYear)) \ .dimension(slicer.dimensions.political_party) \ @@ -983,7 +1119,7 @@ def test_reference_with_unique_dimension_includes_display_definition(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp .reference(f.YearOverYear)) \ .dimension(slicer.dimensions.candidate) \ @@ -1025,7 +1161,7 @@ def test_adjust_reference_dimension_filters_in_reference_query(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp .reference(f.DayOverDay)) \ .filter(slicer.dimensions.timestamp @@ -1063,7 +1199,7 @@ def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_fil query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp .reference(f.DayOverDay)) \ .filter(slicer.dimensions.timestamp @@ -1105,7 +1241,7 @@ def test_adapt_dow_for_leap_year_for_yoy_reference(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp(f.weekly) .reference(f.YearOverYear)) \ .query @@ -1132,15 +1268,15 @@ def test_adapt_dow_for_leap_year_for_yoy_reference(self): 'GROUP BY "timestamp"' ') "sq1" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',-1,' - 'TRUNC(TIMESTAMPADD(\'year\',1,"sq1"."timestamp"),\'IW\')) ' + 'ON "base"."timestamp"=' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"sq1"."timestamp"),\'IW\')) ' 'ORDER BY "timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp(f.weekly) .reference(f.YearOverYear.delta())) \ .query @@ -1175,7 +1311,7 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( - metrics=[slicer.metrics.votes])])) \ + [slicer.metrics.votes])])) \ .dimension(slicer.dimensions.timestamp(f.weekly) .reference(f.YearOverYear.delta(True))) \ .query @@ -1322,55 +1458,222 @@ def test_dimension_filter_with_deep_recursive_join_includes_joins_in_query(self) 'WHERE "deep"."id" IN (1)', str(query)) +class QueryBuilderOrderTests(TestCase): + maxDiff = None + + def test_build_query_order_by_dimension(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.dimensions.timestamp) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp"', str(query)) + + def test_build_query_order_by_dimension_display(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.candidate) \ + .orderby(slicer.dimensions.candidate_display) \ + .query + + self.assertEqual('SELECT ' + '"candidate_id" "candidate",' + '"candidate_name" "candidate_display",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "candidate","candidate_display" ' + 'ORDER BY "candidate_display"', str(query)) + + def test_build_query_order_by_dimension_asc(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.dimensions.timestamp, orientation=Order.asc) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp" ASC', str(query)) + + def test_build_query_order_by_dimension_desc(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.dimensions.timestamp, orientation=Order.desc) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp" DESC', str(query)) + + def test_build_query_order_by_metric(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "votes"', str(query)) + + def test_build_query_order_by_metric_asc(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.metrics.votes, orientation=Order.asc) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "votes" ASC', str(query)) + + def test_build_query_order_by_metric_desc(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.metrics.votes, orientation=Order.desc) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "votes" DESC', str(query)) + + def test_build_query_order_by_multiple_dimensions(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp, slicer.dimensions.candidate) \ + .orderby(slicer.dimensions.timestamp) \ + .orderby(slicer.dimensions.candidate) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + '"candidate_id" "candidate",' + '"candidate_name" "candidate_display",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp","candidate","candidate_display" ' + 'ORDER BY "timestamp","candidate"', str(query)) + + def test_build_query_order_by_multiple_dimensions_with_different_orientations(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp, slicer.dimensions.candidate) \ + .orderby(slicer.dimensions.timestamp, orientation=Order.desc) \ + .orderby(slicer.dimensions.candidate, orientation=Order.asc) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + '"candidate_id" "candidate",' + '"candidate_name" "candidate_display",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp","candidate","candidate_display" ' + 'ORDER BY "timestamp" DESC,"candidate" ASC', str(query)) + + def test_build_query_order_by_metrics_and_dimensions(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.dimensions.timestamp) \ + .orderby(slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp","votes"', str(query)) + + def test_build_query_order_by_metrics_and_dimensions_with_different_orientations(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.dimensions.timestamp, orientation=Order.asc) \ + .orderby(slicer.metrics.votes, orientation=Order.desc) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp" ASC,"votes" DESC', str(query)) + + @patch('fireant.slicer.queries.builder.fetch_data') class QueryBuildPaginationTests(TestCase): def test_set_limit(self, mock_fetch_data: Mock): slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp) \ - .render(limit=20) + .fetch(limit=20) mock_fetch_data.assert_called_once_with(ANY, - PypikaQueryMatcher('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp" LIMIT 20'), - dimensions=ANY) + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp" LIMIT 20', + dimensions=DimensionMatcher(slicer.dimensions.timestamp)) def test_set_offset(self, mock_fetch_data: Mock): slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp) \ - .render(offset=20) + .fetch(offset=20) mock_fetch_data.assert_called_once_with(ANY, - PypikaQueryMatcher('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp" ' - 'OFFSET 20'), - dimensions=ANY) + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp" ' + 'OFFSET 20', + dimensions=DimensionMatcher(slicer.dimensions.timestamp)) def test_set_limit_and_offset(self, mock_fetch_data: Mock): slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp) \ - .render(limit=20, offset=20) + .fetch(limit=20, offset=20) mock_fetch_data.assert_called_once_with(ANY, - PypikaQueryMatcher('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp" ' - 'LIMIT 20 ' - 'OFFSET 20'), - dimensions=ANY) + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp" ' + 'ORDER BY "timestamp" ' + 'LIMIT 20 ' + 'OFFSET 20', + dimensions=DimensionMatcher(slicer.dimensions.timestamp)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -1381,12 +1684,14 @@ def test_highcharts_requires_at_least_one_axis(self): with self.assertRaises(MetricRequiredException): slicer.data \ .widget(f.HighCharts([])) \ + .dimension(slicer.dimensions.timestamp) \ .query def test_highcharts_axis_requires_at_least_one_metric(self): with self.assertRaises(MetricRequiredException): slicer.data \ .widget(f.HighCharts([f.HighCharts.LineChart([])])) \ + .dimension(slicer.dimensions.timestamp) \ .query def test_datatablesjs_requires_at_least_one_metric(self): @@ -1400,97 +1705,99 @@ def test_datatablesjs_requires_at_least_one_metric(self): @patch('fireant.slicer.queries.builder.fetch_data') class QueryBuilderRenderTests(TestCase): def test_pass_slicer_database_as_arg(self, mock_fetch_data: Mock): - mock_widget = Mock(name='mock_widget') - mock_widget.metrics = [slicer.metrics.votes] + mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget.transform = Mock() slicer.data \ .widget(mock_widget) \ - .render() + .fetch() mock_fetch_data.assert_called_once_with(slicer.database, ANY, dimensions=ANY) def test_pass_query_from_builder_as_arg(self, mock_fetch_data: Mock): - mock_widget = Mock(name='mock_widget') - mock_widget.metrics = [slicer.metrics.votes] + mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget.transform = Mock() slicer.data \ .widget(mock_widget) \ - .render() + .fetch() mock_fetch_data.assert_called_once_with(ANY, - PypikaQueryMatcher('SELECT SUM("votes") "votes" ' - 'FROM "politics"."politician"'), + 'SELECT SUM("votes") "votes" ' + 'FROM "politics"."politician"', dimensions=ANY) def test_builder_dimensions_as_arg_with_zero_dimensions(self, mock_fetch_data: Mock): - mock_widget = Mock(name='mock_widget') - mock_widget.metrics = [slicer.metrics.votes] + mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget.transform = Mock() slicer.data \ .widget(mock_widget) \ - .render() + .fetch() mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=[]) def test_builder_dimensions_as_arg_with_one_dimension(self, mock_fetch_data: Mock): - mock_widget = Mock(name='mock_widget') - mock_widget.metrics = [slicer.metrics.votes] + mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget.transform = Mock() dimensions = [slicer.dimensions.state] slicer.data \ .widget(mock_widget) \ .dimension(*dimensions) \ - .render() + .fetch() mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=DimensionMatcher(*dimensions)) def test_builder_dimensions_as_arg_with_multiple_dimensions(self, mock_fetch_data: Mock): - mock_widget = Mock(name='mock_widget') - mock_widget.metrics = [slicer.metrics.votes] + mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget.transform = Mock() dimensions = slicer.dimensions.timestamp, slicer.dimensions.state, slicer.dimensions.political_party slicer.data \ .widget(mock_widget) \ .dimension(*dimensions) \ - .render() + .fetch() mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=DimensionMatcher(*dimensions)) def test_call_transform_on_widget(self, mock_fetch_data: Mock): - mock_widget = Mock(name='mock_widget') - mock_widget.metrics = [slicer.metrics.votes] + mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget.transform = Mock() # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work slicer.data \ .dimension(slicer.dimensions.timestamp) \ .widget(mock_widget) \ - .render() + .fetch() mock_widget.transform.assert_called_once_with(mock_fetch_data.return_value, slicer, DimensionMatcher(slicer.dimensions.timestamp)) def test_returns_results_from_widget_transform(self, mock_fetch_data: Mock): - mock_widget = Mock(name='mock_widget') - mock_widget.metrics = [slicer.metrics.votes] + mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget.transform = Mock() # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work result = slicer.data \ .dimension(slicer.dimensions.timestamp) \ .widget(mock_widget) \ - .render() + .fetch() self.assertListEqual(result, [mock_widget.transform.return_value]) def test_operations_evaluated(self, mock_fetch_data: Mock): - mock_widget = Mock(name='mock_widget') mock_operation = Mock(name='mock_operation ', spec=f.Operation) mock_operation.key, mock_operation.definition = 'mock_operation', slicer.table.abc - mock_widget.metrics = [mock_operation] + + mock_widget = f.Widget([mock_operation]) + mock_widget.transform = Mock() + mock_df = {} mock_fetch_data.return_value = mock_df @@ -1498,15 +1805,17 @@ def test_operations_evaluated(self, mock_fetch_data: Mock): slicer.data \ .dimension(slicer.dimensions.timestamp) \ .widget(mock_widget) \ - .render() + .fetch() mock_operation.apply.assert_called_once_with(mock_df) def test_operations_results_stored_in_data_frame(self, mock_fetch_data: Mock): - mock_widget = Mock(name='mock_widget') mock_operation = Mock(name='mock_operation ', spec=f.Operation) mock_operation.key, mock_operation.definition = 'mock_operation', slicer.table.abc - mock_widget.metrics = [mock_operation] + + mock_widget = f.Widget([mock_operation]) + mock_widget.transform = Mock() + mock_df = {} mock_fetch_data.return_value = mock_df @@ -1514,7 +1823,7 @@ def test_operations_results_stored_in_data_frame(self, mock_fetch_data: Mock): slicer.data \ .dimension(slicer.dimensions.timestamp) \ .widget(mock_widget) \ - .render() + .fetch() self.assertIn(mock_operation.key, mock_df) self.assertEqual(mock_df[mock_operation.key], mock_operation.apply.return_value) diff --git a/fireant/tests/slicer/queries/test_database.py b/fireant/tests/slicer/queries/test_database.py index 70ea3d22..12f27277 100644 --- a/fireant/tests/slicer/queries/test_database.py +++ b/fireant/tests/slicer/queries/test_database.py @@ -12,6 +12,7 @@ cat_dim_df, cont_dim_df, cont_uni_dim_df, + single_metric_df, slicer, uni_dim_df, ) @@ -70,6 +71,11 @@ def totals(df): class FetchDataCleanIndexTests(TestCase): + def test_do_nothing_when_no_dimensions(self): + result = clean_and_apply_index(single_metric_df, []) + + pd.testing.assert_frame_equal(result, single_metric_df) + def test_set_time_series_index_level(self): result = clean_and_apply_index(cont_dim_df.reset_index(), [slicer.dimensions.timestamp]) diff --git a/fireant/tests/slicer/queries/test_dimension_options.py b/fireant/tests/slicer/queries/test_dimension_options.py index ecb5970f..c1729f6f 100644 --- a/fireant/tests/slicer/queries/test_dimension_options.py +++ b/fireant/tests/slicer/queries/test_dimension_options.py @@ -12,37 +12,34 @@ def test_query_options_for_cat_dimension(self): .options \ .query - self.assertEqual('SELECT DISTINCT ' + self.assertEqual('SELECT ' '"political_party" "political_party" ' 'FROM "politics"."politician" ' - 'GROUP BY "political_party" ' - 'ORDER BY "political_party"', str(query)) + 'GROUP BY "political_party"', str(query)) def test_query_options_for_uni_dimension(self): query = slicer.dimensions.candidate \ .options \ .query - self.assertEqual('SELECT DISTINCT ' + self.assertEqual('SELECT ' '"candidate_id" "candidate",' '"candidate_name" "candidate_display" ' 'FROM "politics"."politician" ' - 'GROUP BY "candidate","candidate_display" ' - 'ORDER BY "candidate_display"', str(query)) + 'GROUP BY "candidate","candidate_display"', str(query)) def test_query_options_for_uni_dimension_with_join(self): query = slicer.dimensions.district \ .options \ .query - self.assertEqual('SELECT DISTINCT ' + self.assertEqual('SELECT ' '"politician"."district_id" "district",' '"district"."district_name" "district_display" ' 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' - 'GROUP BY "district","district_display" ' - 'ORDER BY "district_display"', str(query)) + 'GROUP BY "district","district_display"', str(query)) def test_no_options_attr_for_datetime_dimension(self): with self.assertRaises(AttributeError): @@ -58,10 +55,9 @@ def test_filter_options(self): .filter(slicer.dimensions.political_party.isin(['d', 'r'])) \ .query - self.assertEqual('SELECT DISTINCT ' + self.assertEqual('SELECT ' '"candidate_id" "candidate",' '"candidate_name" "candidate_display" ' 'FROM "politics"."politician" ' 'WHERE "political_party" IN (\'d\',\'r\') ' - 'GROUP BY "candidate","candidate_display" ' - 'ORDER BY "candidate_display"', str(query)) + 'GROUP BY "candidate","candidate_display"', str(query)) diff --git a/fireant/tests/slicer/test_dimensions.py b/fireant/tests/slicer/test_dimensions.py new file mode 100644 index 00000000..5d06de82 --- /dev/null +++ b/fireant/tests/slicer/test_dimensions.py @@ -0,0 +1,22 @@ +from unittest import TestCase + +from pypika import Table + +from fireant import UniqueDimension + +test = Table('test') + + +class UniqueDimensionTests(TestCase): + def test_display_key_is_set_when_display_definition_is_used(self): + dim = UniqueDimension('test', + definition=test.definition, + display_definition=test.display_definition) + + self.assertEqual(dim.display_key, "test_display") + + def test_display_key_is_none_when_display_definition_is_none(self): + dim = UniqueDimension('test', + definition=test.definition) + + self.assertIsNone(dim.display_key) diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py index afadc4e5..1575694d 100644 --- a/fireant/tests/slicer/widgets/test_datatables.py +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -1,17 +1,20 @@ +from datetime import date from unittest import TestCase from unittest.mock import Mock import pandas as pd -from datetime import date + from fireant.slicer.widgets.datatables import ( DataTablesJS, _format_metric_cell, ) from fireant.tests.slicer.mocks import ( + CumSum, ElectionOverElection, cat_dim_df, cont_cat_dim_df, cont_dim_df, + cont_dim_operation_df, cont_uni_dim_df, cont_uni_dim_ref_df, multi_metric_df, @@ -25,7 +28,7 @@ class DataTablesTransformerTests(TestCase): maxDiff = None def test_single_metric(self): - result = DataTablesJS(metrics=[slicer.metrics.votes]) \ + result = DataTablesJS(items=[slicer.metrics.votes]) \ .transform(single_metric_df, slicer, []) self.assertEqual({ @@ -40,7 +43,7 @@ def test_single_metric(self): }, result) def test_single_metric_with_dataframe_containing_more(self): - result = DataTablesJS(metrics=[slicer.metrics.votes]) \ + result = DataTablesJS(items=[slicer.metrics.votes]) \ .transform(multi_metric_df, slicer, []) self.assertEqual({ @@ -55,7 +58,7 @@ def test_single_metric_with_dataframe_containing_more(self): }, result) def test_multiple_metrics(self): - result = DataTablesJS(metrics=[slicer.metrics.votes, slicer.metrics.wins]) \ + result = DataTablesJS(items=[slicer.metrics.votes, slicer.metrics.wins]) \ .transform(multi_metric_df, slicer, []) self.assertEqual({ @@ -75,7 +78,7 @@ def test_multiple_metrics(self): }, result) def test_multiple_metrics_reversed(self): - result = DataTablesJS(metrics=[slicer.metrics.wins, slicer.metrics.votes]) \ + result = DataTablesJS(items=[slicer.metrics.wins, slicer.metrics.votes]) \ .transform(multi_metric_df, slicer, []) self.assertEqual({ @@ -95,7 +98,7 @@ def test_multiple_metrics_reversed(self): }, result) def test_time_series_dim(self): - result = DataTablesJS(metrics=[slicer.metrics.wins]) \ + result = DataTablesJS(items=[slicer.metrics.wins]) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) self.assertEqual({ @@ -129,8 +132,43 @@ def test_time_series_dim(self): }], }, result) + def test_time_series_dim_with_operation(self): + result = DataTablesJS(items=[CumSum(slicer.metrics.votes)]) \ + .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp]) + + self.assertEqual({ + 'columns': [{ + 'data': 'timestamp', + 'title': 'Timestamp', + 'render': {'_': 'value'}, + }, { + 'data': 'cumsum(votes)', + 'title': 'CumSum(Votes)', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'cumsum(votes)': {'display': '15220449', 'value': 15220449}, + 'timestamp': {'value': '1996-01-01'} + }, { + 'cumsum(votes)': {'display': '31882466', 'value': 31882466}, + 'timestamp': {'value': '2000-01-01'} + }, { + 'cumsum(votes)': {'display': '51497398', 'value': 51497398}, + 'timestamp': {'value': '2004-01-01'} + }, { + 'cumsum(votes)': {'display': '72791613', 'value': 72791613}, + 'timestamp': {'value': '2008-01-01'} + }, { + 'cumsum(votes)': {'display': '93363823', 'value': 93363823}, + 'timestamp': {'value': '2012-01-01'} + }, { + 'cumsum(votes)': {'display': '111674336', 'value': 111674336}, + 'timestamp': {'value': '2016-01-01'} + }], + }, result) + def test_cat_dim(self): - result = DataTablesJS(metrics=[slicer.metrics.wins]) \ + result = DataTablesJS(items=[slicer.metrics.wins]) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) self.assertEqual({ @@ -156,7 +194,7 @@ def test_cat_dim(self): }, result) def test_uni_dim(self): - result = DataTablesJS(metrics=[slicer.metrics.wins]) \ + result = DataTablesJS(items=[slicer.metrics.wins]) \ .transform(uni_dim_df, slicer, [slicer.dimensions.candidate]) self.assertEqual({ @@ -205,8 +243,66 @@ def test_uni_dim(self): }], }, result) + def test_uni_dim_no_display_definition(self): + import copy + candidate = copy.copy(slicer.dimensions.candidate) + candidate.display_key = None + candidate.display_definition = None + + uni_dim_df_copy = uni_dim_df.copy() + del uni_dim_df_copy[slicer.dimensions.candidate.display_key] + + result = DataTablesJS(items=[slicer.metrics.wins]) \ + .transform(uni_dim_df_copy, slicer, [candidate]) + + self.assertEqual({ + 'columns': [{ + 'data': 'candidate', + 'render': {'_': 'value'}, + 'title': 'Candidate' + }, { + 'data': 'wins', + 'render': {'_': 'value', 'display': 'display'}, + 'title': 'Wins' + }], + 'data': [{ + 'candidate': {'value': 1}, + 'wins': {'display': '2', 'value': 2} + }, { + 'candidate': {'value': 2}, + 'wins': {'display': '0', 'value': 0} + }, { + 'candidate': {'value': 3}, + 'wins': {'display': '0', 'value': 0} + }, { + 'candidate': {'value': 4}, + 'wins': {'display': '4', 'value': 4} + }, { + 'candidate': {'value': 5}, + 'wins': {'display': '0', 'value': 0} + }, { + 'candidate': {'value': 6}, + 'wins': {'display': '0', 'value': 0} + }, { + 'candidate': {'value': 7}, + 'wins': {'display': '4', 'value': 4} + }, { + 'candidate': {'value': 8}, + 'wins': {'display': '0', 'value': 0} + }, { + 'candidate': {'value': 9}, + 'wins': {'display': '0', 'value': 0} + }, { + 'candidate': {'value': 10}, + 'wins': {'display': '2', 'value': 2} + }, { + 'candidate': {'value': 11}, + 'wins': {'display': '0', 'value': 0} + }], + }, result) + def test_multi_dims_time_series_and_uni(self): - result = DataTablesJS(metrics=[slicer.metrics.wins]) \ + result = DataTablesJS(items=[slicer.metrics.wins]) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) self.assertEqual({ @@ -275,7 +371,7 @@ def test_multi_dims_time_series_and_uni(self): }, result) def test_pivoted_single_dimension_no_effect(self): - result = DataTablesJS(metrics=[slicer.metrics.wins], pivot=True) \ + result = DataTablesJS(items=[slicer.metrics.wins], pivot=True) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) self.assertEqual({ @@ -301,7 +397,7 @@ def test_pivoted_single_dimension_no_effect(self): }, result) def test_pivoted_multi_dims_time_series_and_cat(self): - result = DataTablesJS(metrics=[slicer.metrics.wins], pivot=True) \ + result = DataTablesJS(items=[slicer.metrics.wins], pivot=True) \ .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party]) self.assertEqual({ @@ -368,7 +464,7 @@ def test_pivoted_multi_dims_time_series_and_cat(self): }, result) def test_pivoted_multi_dims_time_series_and_uni(self): - result = DataTablesJS(metrics=[slicer.metrics.votes], pivot=True) \ + result = DataTablesJS(items=[slicer.metrics.votes], pivot=True) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) self.assertEqual({ @@ -425,7 +521,7 @@ def test_pivoted_multi_dims_time_series_and_uni(self): }, result) def test_time_series_ref(self): - result = DataTablesJS(metrics=[slicer.metrics.votes]) \ + result = DataTablesJS(items=[slicer.metrics.votes]) \ .transform(cont_uni_dim_ref_df, slicer, [slicer.dimensions.timestamp.reference(ElectionOverElection), slicer.dimensions.state]) diff --git a/fireant/tests/slicer/widgets/test_highcharts.py b/fireant/tests/slicer/widgets/test_highcharts.py index 22724ff2..c1a3a569 100644 --- a/fireant/tests/slicer/widgets/test_highcharts.py +++ b/fireant/tests/slicer/widgets/test_highcharts.py @@ -1,10 +1,12 @@ from unittest import TestCase +from fireant import CumSum from fireant.slicer.widgets.highcharts import HighCharts from fireant.tests.slicer.mocks import ( ElectionOverElection, cat_dim_df, cont_dim_df, + cont_dim_operation_df, cont_uni_dim_df, cont_uni_dim_ref_delta_df, cont_uni_dim_ref_df, @@ -22,9 +24,8 @@ class HighchartsLineChartTransformerTests(TestCase): stacking = None def test_single_metric_line_chart(self): - result = HighCharts("Time Series, Single Metric", axes=[ - self.chart_class(metrics=[slicer.metrics.votes]) - ]) \ + result = HighCharts(title="Time Series, Single Metric", + axes=[self.chart_class([slicer.metrics.votes])]) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) self.assertEqual({ @@ -41,12 +42,45 @@ def test_single_metric_line_chart(self): "type": self.chart_type, "name": "Votes", "yAxis": "0", - "data": [[820454400000.0, 15220449], - [946684800000.0, 16662017], - [1072915200000.0, 19614932], - [1199145600000.0, 21294215], - [1325376000000.0, 20572210], - [1451606400000.0, 18310513]], + "data": [(820454400000, 15220449), + (946684800000, 16662017), + (1072915200000, 19614932), + (1199145600000, 21294215), + (1325376000000, 20572210), + (1451606400000, 18310513)], + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + "visible": True, + }] + }, result) + + def test_single_operation_line_chart(self): + result = HighCharts(title="Time Series, Single Metric", + axes=[self.chart_class([CumSum(slicer.metrics.votes)])]) \ + .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp]) + + self.assertEqual({ + "title": {"text": "Time Series, Single Metric"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}} + }], + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "CumSum(Votes)", + "yAxis": "0", + "data": [(820454400000, 15220449), + (946684800000, 31882466), + (1072915200000, 51497398), + (1199145600000, 72791613), + (1325376000000, 93363823), + (1451606400000, 111674336)], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -56,9 +90,8 @@ def test_single_metric_line_chart(self): }, result) def test_single_metric_with_uni_dim_line_chart(self): - result = HighCharts("Time Series with Unique Dimension and Single Metric", axes=[ - self.chart_class(metrics=[slicer.metrics.votes]) - ]) \ + result = HighCharts(title="Time Series with Unique Dimension and Single Metric", + axes=[self.chart_class([slicer.metrics.votes])]) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) @@ -76,12 +109,12 @@ def test_single_metric_with_uni_dim_line_chart(self): "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [[820454400000.0, 5574387], - [946684800000.0, 6233385], - [1072915200000.0, 7359621], - [1199145600000.0, 8007961], - [1325376000000.0, 7877967], - [1451606400000.0, 5072915]], + "data": [(820454400000, 5574387), + (946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -91,12 +124,12 @@ def test_single_metric_with_uni_dim_line_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [[820454400000.0, 9646062], - [946684800000.0, 10428632], - [1072915200000.0, 12255311], - [1199145600000.0, 13286254], - [1325376000000.0, 12694243], - [1451606400000.0, 13237598]], + "data": [(820454400000, 9646062), + (946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -106,10 +139,9 @@ def test_single_metric_with_uni_dim_line_chart(self): }, result) def test_multi_metrics_single_axis_line_chart(self): - result = HighCharts("Time Series with Unique Dimension and Multiple Metrics", axes=[ - self.chart_class(metrics=[slicer.metrics.votes, - slicer.metrics.wins]) - ]) \ + result = HighCharts(title="Time Series with Unique Dimension and Multiple Metrics", + axes=[self.chart_class([slicer.metrics.votes, + slicer.metrics.wins])]) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) @@ -127,12 +159,12 @@ def test_multi_metrics_single_axis_line_chart(self): "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [[820454400000.0, 5574387], - [946684800000.0, 6233385], - [1072915200000.0, 7359621], - [1199145600000.0, 8007961], - [1325376000000.0, 7877967], - [1451606400000.0, 5072915]], + "data": [(820454400000, 5574387), + (946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -142,12 +174,12 @@ def test_multi_metrics_single_axis_line_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [[820454400000.0, 9646062], - [946684800000.0, 10428632], - [1072915200000.0, 12255311], - [1199145600000.0, 13286254], - [1325376000000.0, 12694243], - [1451606400000.0, 13237598]], + "data": [(820454400000, 9646062), + (946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], "color": "#DDDF0D", "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -157,12 +189,12 @@ def test_multi_metrics_single_axis_line_chart(self): "type": self.chart_type, "name": "Wins (Texas)", "yAxis": "0", - "data": [[820454400000.0, 1], - [946684800000.0, 1], - [1072915200000.0, 1], - [1199145600000.0, 1], - [1325376000000.0, 1], - [1451606400000.0, 1]], + "data": [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], "color": "#55BF3B", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -172,12 +204,12 @@ def test_multi_metrics_single_axis_line_chart(self): "type": self.chart_type, "name": "Wins (California)", "yAxis": "0", - "data": [[820454400000.0, 1], - [946684800000.0, 1], - [1072915200000.0, 1], - [1199145600000.0, 1], - [1325376000000.0, 1], - [1451606400000.0, 1]], + "data": [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -187,10 +219,9 @@ def test_multi_metrics_single_axis_line_chart(self): }, result) def test_multi_metrics_multi_axis_line_chart(self): - result = HighCharts("Time Series with Unique Dimension and Multiple Metrics, Multi-Axis", axes=[ - self.chart_class(metrics=[slicer.metrics.votes]), - self.chart_class(metrics=[slicer.metrics.wins]), - ]) \ + result = HighCharts(title="Time Series with Unique Dimension and Multiple Metrics, Multi-Axis", + axes=[self.chart_class([slicer.metrics.votes]), + self.chart_class([slicer.metrics.wins]), ]) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) @@ -212,12 +243,12 @@ def test_multi_metrics_multi_axis_line_chart(self): "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [[820454400000.0, 5574387], - [946684800000.0, 6233385], - [1072915200000.0, 7359621], - [1199145600000.0, 8007961], - [1325376000000.0, 7877967], - [1451606400000.0, 5072915]], + "data": [(820454400000, 5574387), + (946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -227,12 +258,12 @@ def test_multi_metrics_multi_axis_line_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [[820454400000.0, 9646062], - [946684800000.0, 10428632], - [1072915200000.0, 12255311], - [1199145600000.0, 13286254], - [1325376000000.0, 12694243], - [1451606400000.0, 13237598]], + "data": [(820454400000, 9646062), + (946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -242,12 +273,12 @@ def test_multi_metrics_multi_axis_line_chart(self): "type": self.chart_type, "name": "Wins (Texas)", "yAxis": "1", - "data": [[820454400000.0, 1], - [946684800000.0, 1], - [1072915200000.0, 1], - [1199145600000.0, 1], - [1325376000000.0, 1], - [1451606400000.0, 1]], + "data": [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], "color": "#55BF3B", "marker": {"symbol": "circle", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -257,12 +288,12 @@ def test_multi_metrics_multi_axis_line_chart(self): "type": self.chart_type, "name": "Wins (California)", "yAxis": "1", - "data": [[820454400000.0, 1], - [946684800000.0, 1], - [1072915200000.0, 1], - [1199145600000.0, 1], - [1325376000000.0, 1], - [1451606400000.0, 1]], + "data": [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], "color": "#DF5353", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -272,9 +303,8 @@ def test_multi_metrics_multi_axis_line_chart(self): }, result) def test_uni_dim_with_ref_line_chart(self): - result = HighCharts("Time Series with Unique Dimension and Reference", axes=[ - self.chart_class(metrics=[slicer.metrics.votes]) - ]) \ + result = HighCharts(title="Time Series with Unique Dimension and Reference", + axes=[self.chart_class([slicer.metrics.votes])]) \ .transform(cont_uni_dim_ref_df, slicer, [slicer.dimensions.timestamp.reference(ElectionOverElection), slicer.dimensions.state]) @@ -292,11 +322,11 @@ def test_uni_dim_with_ref_line_chart(self): "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [[946684800000.0, 6233385], - [1072915200000.0, 7359621], - [1199145600000.0, 8007961], - [1325376000000.0, 7877967], - [1451606400000.0, 5072915]], + "data": [(946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -306,11 +336,11 @@ def test_uni_dim_with_ref_line_chart(self): "type": self.chart_type, "name": "Votes (EoE) (Texas)", "yAxis": "0", - "data": [[946684800000.0, 5574387.0], - [1072915200000.0, 6233385.0], - [1199145600000.0, 7359621.0], - [1325376000000.0, 8007961.0], - [1451606400000.0, 7877967.0]], + "data": [(946684800000, 5574387), + (1072915200000, 6233385), + (1199145600000, 7359621), + (1325376000000, 8007961), + (1451606400000, 7877967)], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "ShortDash", @@ -320,11 +350,11 @@ def test_uni_dim_with_ref_line_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [[946684800000.0, 10428632], - [1072915200000.0, 12255311], - [1199145600000.0, 13286254], - [1325376000000.0, 12694243], - [1451606400000.0, 13237598]], + "data": [(946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -334,11 +364,11 @@ def test_uni_dim_with_ref_line_chart(self): "type": self.chart_type, "name": "Votes (EoE) (California)", "yAxis": "0", - "data": [[946684800000.0, 9646062.0], - [1072915200000.0, 10428632.0], - [1199145600000.0, 12255311.0], - [1325376000000.0, 13286254.0], - [1451606400000.0, 12694243.0]], + "data": [(946684800000, 9646062), + (1072915200000, 10428632), + (1199145600000, 12255311), + (1325376000000, 13286254), + (1451606400000, 12694243)], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "ShortDash", @@ -348,9 +378,8 @@ def test_uni_dim_with_ref_line_chart(self): }, result) def test_uni_dim_with_ref_delta_line_chart(self): - result = HighCharts("Time Series with Unique Dimension and Delta Reference", axes=[ - self.chart_class(metrics=[slicer.metrics.votes]) - ]) \ + result = HighCharts(title="Time Series with Unique Dimension and Delta Reference", + axes=[self.chart_class([slicer.metrics.votes])]) \ .transform(cont_uni_dim_ref_delta_df, slicer, [slicer.dimensions.timestamp.reference(ElectionOverElection.delta()), @@ -375,11 +404,11 @@ def test_uni_dim_with_ref_delta_line_chart(self): "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [[946684800000.0, 6233385], - [1072915200000.0, 7359621], - [1199145600000.0, 8007961], - [1325376000000.0, 7877967], - [1451606400000.0, 5072915]], + "data": [(946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -389,11 +418,11 @@ def test_uni_dim_with_ref_delta_line_chart(self): "type": self.chart_type, "name": "Votes (EoE Δ) (Texas)", "yAxis": "0_eoe_delta", - "data": [[946684800000.0, -658998.0], - [1072915200000.0, -1126236.0], - [1199145600000.0, -648340.0], - [1325376000000.0, 129994.0], - [1451606400000.0, 2805052.0]], + "data": [(946684800000, -658998), + (1072915200000, -1126236), + (1199145600000, -648340), + (1325376000000, 129994), + (1451606400000, 2805052)], "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "ShortDash", @@ -403,11 +432,11 @@ def test_uni_dim_with_ref_delta_line_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [[946684800000.0, 10428632], - [1072915200000.0, 12255311], - [1199145600000.0, 13286254], - [1325376000000.0, 12694243], - [1451606400000.0, 13237598]], + "data": [(946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -417,11 +446,11 @@ def test_uni_dim_with_ref_delta_line_chart(self): "type": self.chart_type, "name": "Votes (EoE Δ) (California)", "yAxis": "0_eoe_delta", - "data": [[946684800000.0, -782570.0], - [1072915200000.0, -1826679.0], - [1199145600000.0, -1030943.0], - [1325376000000.0, 592011.0], - [1451606400000.0, -543355.0]], + "data": [(946684800000, -782570), + (1072915200000, -1826679), + (1199145600000, -1030943), + (1325376000000, 592011), + (1451606400000, -543355)], "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "ShortDash", @@ -439,9 +468,8 @@ class HighchartsBarChartTransformerTests(TestCase): stacking = None def test_single_metric_bar_chart(self): - result = HighCharts("All Votes", axes=[ - self.chart_class(metrics=[slicer.metrics.votes]) - ]) \ + result = HighCharts(title="All Votes", + axes=[self.chart_class([slicer.metrics.votes])]) \ .transform(single_metric_df, slicer, []) self.assertEqual({ @@ -461,7 +489,7 @@ def test_single_metric_bar_chart(self): "type": self.chart_type, "name": "Votes", "yAxis": "0", - "data": [111674336.0], + "data": [111674336], "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -471,10 +499,9 @@ def test_single_metric_bar_chart(self): }, result) def test_multi_metric_bar_chart(self): - result = HighCharts("Votes and Wins", axes=[ - self.chart_class(metrics=[slicer.metrics.votes, - slicer.metrics.wins]) - ]) \ + result = HighCharts(title="Votes and Wins", + axes=[self.chart_class([slicer.metrics.votes, + slicer.metrics.wins])]) \ .transform(multi_metric_df, slicer, []) self.assertEqual({ @@ -495,7 +522,7 @@ def test_multi_metric_bar_chart(self): "type": self.chart_type, "name": "Votes", "yAxis": "0", - "data": [111674336.0], + "data": [111674336], "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -505,7 +532,7 @@ def test_multi_metric_bar_chart(self): "type": self.chart_type, "name": "Wins", "yAxis": "0", - "data": [12.0], + "data": [12], "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -515,9 +542,8 @@ def test_multi_metric_bar_chart(self): }, result) def test_cat_dim_single_metric_bar_chart(self): - result = HighCharts("Votes and Wins", axes=[ - self.chart_class(metrics=[slicer.metrics.votes]) - ]) \ + result = HighCharts(title="Votes and Wins", + axes=[self.chart_class([slicer.metrics.votes])]) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) self.assertEqual({ @@ -538,7 +564,7 @@ def test_cat_dim_single_metric_bar_chart(self): "type": self.chart_type, "name": "Votes", "yAxis": "0", - "data": [54551568.0, 1076384.0, 56046384.0], + "data": [54551568, 1076384, 56046384], "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -548,10 +574,9 @@ def test_cat_dim_single_metric_bar_chart(self): }, result) def test_cat_dim_multi_metric_bar_chart(self): - result = HighCharts("Votes and Wins", axes=[ - self.chart_class(metrics=[slicer.metrics.votes, - slicer.metrics.wins]) - ]) \ + result = HighCharts(title="Votes and Wins", + axes=[self.chart_class([slicer.metrics.votes, + slicer.metrics.wins])]) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) self.assertEqual({ @@ -572,7 +597,7 @@ def test_cat_dim_multi_metric_bar_chart(self): "type": self.chart_type, "name": "Votes", "yAxis": "0", - "data": [54551568.0, 1076384.0, 56046384.0], + "data": [54551568, 1076384, 56046384], "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -582,7 +607,7 @@ def test_cat_dim_multi_metric_bar_chart(self): "type": self.chart_type, "name": "Wins", "yAxis": "0", - "data": [6.0, 0.0, 6.0], + "data": [6, 0, 6], "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -592,9 +617,8 @@ def test_cat_dim_multi_metric_bar_chart(self): }, result) def test_cont_uni_dims_single_metric_bar_chart(self): - result = HighCharts("Election Votes by State", axes=[ - self.chart_class(metrics=[slicer.metrics.votes]) - ]) \ + result = HighCharts(title="Election Votes by State", + axes=[self.chart_class([slicer.metrics.votes])]) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) self.assertEqual({ @@ -612,12 +636,12 @@ def test_cont_uni_dims_single_metric_bar_chart(self): "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [[820454400000.0, 5574387], - [946684800000.0, 6233385], - [1072915200000.0, 7359621], - [1199145600000.0, 8007961], - [1325376000000.0, 7877967], - [1451606400000.0, 5072915]], + "data": [(820454400000, 5574387), + (946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -627,12 +651,12 @@ def test_cont_uni_dims_single_metric_bar_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [[820454400000.0, 9646062], - [946684800000.0, 10428632], - [1072915200000.0, 12255311], - [1199145600000.0, 13286254], - [1325376000000.0, 12694243], - [1451606400000.0, 13237598]], + "data": [(820454400000, 9646062), + (946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -642,10 +666,9 @@ def test_cont_uni_dims_single_metric_bar_chart(self): }, result) def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): - result = HighCharts("Election Votes by State", axes=[ - self.chart_class(metrics=[slicer.metrics.votes, - slicer.metrics.wins]), - ]) \ + result = HighCharts(title="Election Votes by State", + axes=[self.chart_class([slicer.metrics.votes, + slicer.metrics.wins]), ]) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) self.assertEqual({ @@ -663,12 +686,12 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [[820454400000.0, 5574387], - [946684800000.0, 6233385], - [1072915200000.0, 7359621], - [1199145600000.0, 8007961], - [1325376000000.0, 7877967], - [1451606400000.0, 5072915]], + "data": [(820454400000, 5574387), + (946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -678,12 +701,12 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [[820454400000.0, 9646062], - [946684800000.0, 10428632], - [1072915200000.0, 12255311], - [1199145600000.0, 13286254], - [1325376000000.0, 12694243], - [1451606400000.0, 13237598]], + "data": [(820454400000, 9646062), + (946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -693,12 +716,12 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): "type": self.chart_type, "name": "Wins (Texas)", "yAxis": "0", - "data": [[820454400000.0, 1], - [946684800000.0, 1], - [1072915200000.0, 1], - [1199145600000.0, 1], - [1325376000000.0, 1], - [1451606400000.0, 1]], + "data": [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -708,12 +731,12 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): "type": self.chart_type, "name": "Wins (California)", "yAxis": "0", - "data": [[820454400000.0, 1], - [946684800000.0, 1], - [1072915200000.0, 1], - [1199145600000.0, 1], - [1325376000000.0, 1], - [1451606400000.0, 1]], + "data": [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -723,10 +746,9 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): }, result) def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): - result = HighCharts("Election Votes by State", axes=[ - self.chart_class(metrics=[slicer.metrics.votes]), - self.chart_class(metrics=[slicer.metrics.wins]), - ]) \ + result = HighCharts(title="Election Votes by State", + axes=[self.chart_class([slicer.metrics.votes]), + self.chart_class([slicer.metrics.wins]), ]) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) self.assertEqual({ @@ -749,12 +771,12 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "type": self.chart_type, "name": "Votes (Texas)", "yAxis": "0", - "data": [[820454400000.0, 5574387], - [946684800000.0, 6233385], - [1072915200000.0, 7359621], - [1199145600000.0, 8007961], - [1325376000000.0, 7877967], - [1451606400000.0, 5072915]], + "data": [(820454400000, 5574387), + (946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -764,12 +786,12 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "type": self.chart_type, "name": "Votes (California)", "yAxis": "0", - "data": [[820454400000.0, 9646062], - [946684800000.0, 10428632], - [1072915200000.0, 12255311], - [1199145600000.0, 13286254], - [1325376000000.0, 12694243], - [1451606400000.0, 13237598]], + "data": [(820454400000, 9646062), + (946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -779,12 +801,12 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "type": self.chart_type, "name": "Wins (Texas)", "yAxis": "1", - "data": [[820454400000.0, 1], - [946684800000.0, 1], - [1072915200000.0, 1], - [1199145600000.0, 1], - [1325376000000.0, 1], - [1451606400000.0, 1]], + "data": [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -794,12 +816,12 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "type": self.chart_type, "name": "Wins (California)", "yAxis": "1", - "data": [[820454400000.0, 1], - [946684800000.0, 1], - [1072915200000.0, 1], - [1199145600000.0, 1], - [1325376000000.0, 1], - [1451606400000.0, 1]], + "data": [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], "color": "#DF5353", "dashStyle": "Solid", "marker": {}, From c7d8b0b2c7549515bfd7c71a47e2c3e14be67d06 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 1 Feb 2018 18:34:12 +0100 Subject: [PATCH 011/123] Added pandas transformer --- fireant/slicer/widgets/pandas.py | 77 +++++++- fireant/tests/slicer/test_slicer.py | 49 +++++ .../tests/slicer/widgets/test_datatables.py | 15 -- fireant/tests/slicer/widgets/test_pandas.py | 171 ++++++++++++++++++ 4 files changed, 296 insertions(+), 16 deletions(-) create mode 100644 fireant/tests/slicer/test_slicer.py create mode 100644 fireant/tests/slicer/widgets/test_pandas.py diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index 11f8e14d..93ac3f13 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -1,6 +1,81 @@ +import pandas as pd + from .base import Widget +from .helpers import ( + reference_key, + reference_label, +) + +HARD_MAX_COLUMNS = 24 class Pandas(Widget): + def __init__(self, items=(), pivot=False, max_columns=None): + super(Pandas, self).__init__(items) + self.pivot = pivot + self.max_columns = min(max_columns, HARD_MAX_COLUMNS) \ + if max_columns is not None \ + else HARD_MAX_COLUMNS + def transform(self, data_frame, slicer, dimensions): - raise NotImplementedError() + """ + WRITEME + + :param data_frame: + :param slicer: + :param dimensions: + :return: + """ + result = data_frame.copy() + references = [] + + for dimension in dimensions: + references += getattr(dimension, 'references', []) + + if dimension.has_display_field: + result = result.set_index(dimension.display_key, append=True) + result = result.reset_index(dimension.key, drop=True) + + if hasattr(dimension, 'display_values'): + self._replace_display_values_in_index(dimension, result) + + if isinstance(data_frame.index, pd.MultiIndex): + index_levels = [dimension.display_key + if dimension.has_display_field + else dimension.key + for dimension in dimensions] + + result = result.reorder_levels(index_levels) + + result = result[[reference_key(item, reference) + for reference in [None] + references + for item in self.items]] + + if dimensions: + result.index.names = [dimension.label or dimension.key + for dimension in dimensions] + + result.columns = [reference_label(item, reference) + for reference in [None] + references + for item in self.items] + + if not self.pivot: + return result + + pivot_levels = result.index.names[1:] + return result.unstack(level=pivot_levels) + + def _replace_display_values_in_index(self, dimension, result): + """ + Replaces the raw values of a (categorical) dimension in the index with their corresponding display values. + """ + if isinstance(result.index, pd.MultiIndex): + values = [dimension.display_values.get(x, x) + for x in result.index.get_level_values(dimension.key)] + result.index.set_levels(level=dimension.key, levels=values) + return result + + values = [dimension.display_values.get(x, x) + for x in result.index] + result.index = pd.Index(values, name=result.index.name) + return result diff --git a/fireant/tests/slicer/test_slicer.py b/fireant/tests/slicer/test_slicer.py new file mode 100644 index 00000000..e93adc04 --- /dev/null +++ b/fireant/tests/slicer/test_slicer.py @@ -0,0 +1,49 @@ +from unittest import TestCase + +from fireant import ( + Dimension, + Metric, +) +from fireant.slicer.dimensions import DisplayDimension +from .mocks import slicer + + +class SlicerContainerTests(TestCase): + def test_access_metric_via_attr(self): + votes = slicer.metrics.votes + self.assertIsInstance(votes, Metric) + self.assertEqual(votes.key, 'votes') + + def test_access_metric_via_item(self): + votes = slicer.metrics['votes'] + self.assertIsInstance(votes, Metric) + self.assertEqual(votes.key, 'votes') + + def test_access_dimension_via_attr(self): + timestamp = slicer.dimensions.timestamp + self.assertIsInstance(timestamp, Dimension) + self.assertEqual(timestamp.key, 'timestamp') + + def test_access_dimension_via_item(self): + timestamp = slicer.dimensions['timestamp'] + self.assertIsInstance(timestamp, Dimension) + self.assertEqual(timestamp.key, 'timestamp') + + def test_access_dimension_display_via_attr(self): + candidate_display = slicer.dimensions.candidate_display + self.assertIsInstance(candidate_display, DisplayDimension) + self.assertEqual(candidate_display.key, 'candidate_display') + + def test_access_dimension_display_via_item(self): + candidate_display = slicer.dimensions['candidate_display'] + self.assertIsInstance(candidate_display, DisplayDimension) + self.assertEqual(candidate_display.key, 'candidate_display') + + def test_iter_metrics(self): + metric_keys = [metric.key for metric in slicer.metrics] + self.assertListEqual(metric_keys, ['votes', 'wins', 'voters', 'turnout']) + + def test_iter_dimensions(self): + dimension_keys = [dimension.key for dimension in slicer.dimensions] + self.assertListEqual(dimension_keys, ['timestamp', 'political_party', 'candidate', 'election', 'district', + 'state', 'winner', 'deepjoin']) diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py index 1575694d..e930cc90 100644 --- a/fireant/tests/slicer/widgets/test_datatables.py +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -42,21 +42,6 @@ def test_single_metric(self): }], }, result) - def test_single_metric_with_dataframe_containing_more(self): - result = DataTablesJS(items=[slicer.metrics.votes]) \ - .transform(multi_metric_df, slicer, []) - - self.assertEqual({ - 'columns': [{ - 'data': 'votes', - 'title': 'Votes', - 'render': {'_': 'value', 'display': 'display'}, - }], - 'data': [{ - 'votes': {'value': 111674336, 'display': '111674336'} - }], - }, result) - def test_multiple_metrics(self): result = DataTablesJS(items=[slicer.metrics.votes, slicer.metrics.wins]) \ .transform(multi_metric_df, slicer, []) diff --git a/fireant/tests/slicer/widgets/test_pandas.py b/fireant/tests/slicer/widgets/test_pandas.py new file mode 100644 index 00000000..00e1c04a --- /dev/null +++ b/fireant/tests/slicer/widgets/test_pandas.py @@ -0,0 +1,171 @@ +from unittest import TestCase + +import pandas as pd +import pandas.testing + +from fireant.slicer.widgets.pandas import Pandas +from fireant.tests.slicer.mocks import ( + CumSum, + ElectionOverElection, + cat_dim_df, + cont_cat_dim_df, + cont_dim_df, + cont_dim_operation_df, + cont_uni_dim_df, + cont_uni_dim_ref_df, + multi_metric_df, + single_metric_df, + slicer, + uni_dim_df, +) + + +class DataTablesTransformerTests(TestCase): + maxDiff = None + + def test_single_metric(self): + result = Pandas(items=[slicer.metrics.votes]) \ + .transform(single_metric_df, slicer, []) + + expected = single_metric_df.copy()[['votes']] + expected.columns = ['Votes'] + + pandas.testing.assert_frame_equal(result, expected) + + def test_multiple_metrics(self): + result = Pandas(items=[slicer.metrics.votes, slicer.metrics.wins]) \ + .transform(multi_metric_df, slicer, []) + + expected = multi_metric_df.copy()[['votes', 'wins']] + expected.columns = ['Votes', 'Wins'] + + pandas.testing.assert_frame_equal(result, expected) + + def test_multiple_metrics_reversed(self): + result = Pandas(items=[slicer.metrics.wins, slicer.metrics.votes]) \ + .transform(multi_metric_df, slicer, []) + + expected = multi_metric_df.copy()[['wins', 'votes']] + expected.columns = ['Wins', 'Votes'] + + pandas.testing.assert_frame_equal(result, expected) + + def test_time_series_dim(self): + result = Pandas(items=[slicer.metrics.wins]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) + + expected = cont_dim_df.copy()[['wins']] + expected.index.names = ['Timestamp'] + expected.columns = ['Wins'] + + pandas.testing.assert_frame_equal(result, expected) + + def test_time_series_dim_with_operation(self): + result = Pandas(items=[CumSum(slicer.metrics.votes)]) \ + .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp]) + + expected = cont_dim_operation_df.copy()[['cumsum(votes)']] + expected.index.names = ['Timestamp'] + expected.columns = ['CumSum(Votes)'] + + pandas.testing.assert_frame_equal(result, expected) + + def test_cat_dim(self): + result = Pandas(items=[slicer.metrics.wins]) \ + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + + expected = cat_dim_df.copy()[['wins']] + expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') + expected.columns = ['Wins'] + + pandas.testing.assert_frame_equal(result, expected) + + def test_uni_dim(self): + result = Pandas(items=[slicer.metrics.wins]) \ + .transform(uni_dim_df, slicer, [slicer.dimensions.candidate]) + + expected = uni_dim_df.copy() \ + .set_index('candidate_display', append=True) \ + .reset_index('candidate', drop=True) \ + [['wins']] + expected.index.names = ['Candidate'] + expected.columns = ['Wins'] + + pandas.testing.assert_frame_equal(result, expected) + + def test_uni_dim_no_display_definition(self): + import copy + candidate = copy.copy(slicer.dimensions.candidate) + candidate.display_key = None + candidate.display_definition = None + + uni_dim_df_copy = uni_dim_df.copy() + del uni_dim_df_copy[slicer.dimensions.candidate.display_key] + + result = Pandas(items=[slicer.metrics.wins]) \ + .transform(uni_dim_df_copy, slicer, [candidate]) + + expected = uni_dim_df_copy.copy()[['wins']] + expected.index.names = ['Candidate'] + expected.columns = ['Wins'] + + pandas.testing.assert_frame_equal(result, expected) + + def test_multi_dims_time_series_and_uni(self): + result = Pandas(items=[slicer.metrics.wins]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + + expected = cont_uni_dim_df.copy() \ + .set_index('state_display', append=True) \ + .reset_index('state', drop=False)[['wins']] + expected.index.names = ['Timestamp', 'State'] + expected.columns = ['Wins'] + + pandas.testing.assert_frame_equal(result, expected) + + def test_pivoted_single_dimension_no_effect(self): + result = Pandas(items=[slicer.metrics.wins], pivot=True) \ + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + + expected = cat_dim_df.copy()[['wins']] + expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') + expected.columns = ['Wins'] + + pandas.testing.assert_frame_equal(result, expected) + + def test_pivoted_multi_dims_time_series_and_cat(self): + result = Pandas(items=[slicer.metrics.wins], pivot=True) \ + .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party]) + + expected = cont_cat_dim_df.copy()[['wins']] + expected.index.names = ['Timestamp', 'Party'] + expected.columns = ['Wins'] + expected = expected.unstack(level=[1]) + + pandas.testing.assert_frame_equal(result, expected) + + def test_pivoted_multi_dims_time_series_and_uni(self): + result = Pandas(items=[slicer.metrics.votes], pivot=True) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + + expected = cont_uni_dim_df.copy() \ + .set_index('state_display', append=True) \ + .reset_index('state', drop=True)[['votes']] + expected.index.names = ['Timestamp', 'State'] + expected.columns = ['Votes'] + expected = expected.unstack(level=[1]) + + pandas.testing.assert_frame_equal(result, expected) + + def test_time_series_ref(self): + result = Pandas(items=[slicer.metrics.votes]) \ + .transform(cont_uni_dim_ref_df, slicer, [slicer.dimensions.timestamp.reference(ElectionOverElection), + slicer.dimensions.state]) + + expected = cont_uni_dim_ref_df.copy() \ + .set_index('state_display', append=True) \ + .reset_index('state', drop=True)[['votes', 'votes_eoe']] + expected.index.names = ['Timestamp', 'State'] + expected.columns = ['Votes', 'Votes (EoE)'] + + pandas.testing.assert_frame_equal(result, expected) From af32d034d8f6ece6ed2568a3316e256c4b643067 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 1 Feb 2018 18:43:48 +0100 Subject: [PATCH 012/123] Added CSV transformer --- fireant/slicer/widgets/__init__.py | 1 + fireant/slicer/widgets/csv.py | 8 ++ fireant/tests/slicer/widgets/test_csv.py | 170 +++++++++++++++++++++++ 3 files changed, 179 insertions(+) create mode 100644 fireant/slicer/widgets/csv.py create mode 100644 fireant/tests/slicer/widgets/test_csv.py diff --git a/fireant/slicer/widgets/__init__.py b/fireant/slicer/widgets/__init__.py index 16186bf1..551cd585 100644 --- a/fireant/slicer/widgets/__init__.py +++ b/fireant/slicer/widgets/__init__.py @@ -3,3 +3,4 @@ from .highcharts import HighCharts from .matplotlib import Matplotlib from .pandas import Pandas +from .csv import CSV \ No newline at end of file diff --git a/fireant/slicer/widgets/csv.py b/fireant/slicer/widgets/csv.py new file mode 100644 index 00000000..ade1dc3f --- /dev/null +++ b/fireant/slicer/widgets/csv.py @@ -0,0 +1,8 @@ +from .pandas import Pandas + + +class CSV(Pandas): + + def transform(self, data_frame, slicer, dimensions): + result_df = super(CSV, self).transform(data_frame, slicer, dimensions) + return result_df.to_csv() diff --git a/fireant/tests/slicer/widgets/test_csv.py b/fireant/tests/slicer/widgets/test_csv.py new file mode 100644 index 00000000..963d68f2 --- /dev/null +++ b/fireant/tests/slicer/widgets/test_csv.py @@ -0,0 +1,170 @@ +from unittest import TestCase + +import pandas as pd + +from fireant.slicer.widgets.csv import CSV +from fireant.tests.slicer.mocks import ( + CumSum, + ElectionOverElection, + cat_dim_df, + cont_cat_dim_df, + cont_dim_df, + cont_dim_operation_df, + cont_uni_dim_df, + cont_uni_dim_ref_df, + multi_metric_df, + single_metric_df, + slicer, + uni_dim_df, +) + + +class DataTablesTransformerTests(TestCase): + maxDiff = None + + def test_single_metric(self): + result = CSV(items=[slicer.metrics.votes]) \ + .transform(single_metric_df, slicer, []) + + expected = single_metric_df.copy()[['votes']] + expected.columns = ['Votes'] + + self.assertEqual(result, expected.to_csv()) + + def test_multiple_metrics(self): + result = CSV(items=[slicer.metrics.votes, slicer.metrics.wins]) \ + .transform(multi_metric_df, slicer, []) + + expected = multi_metric_df.copy()[['votes', 'wins']] + expected.columns = ['Votes', 'Wins'] + + self.assertEqual(result, expected.to_csv()) + + def test_multiple_metrics_reversed(self): + result = CSV(items=[slicer.metrics.wins, slicer.metrics.votes]) \ + .transform(multi_metric_df, slicer, []) + + expected = multi_metric_df.copy()[['wins', 'votes']] + expected.columns = ['Wins', 'Votes'] + + self.assertEqual(result, expected.to_csv()) + + def test_time_series_dim(self): + result = CSV(items=[slicer.metrics.wins]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) + + expected = cont_dim_df.copy()[['wins']] + expected.index.names = ['Timestamp'] + expected.columns = ['Wins'] + + self.assertEqual(result, expected.to_csv()) + + def test_time_series_dim_with_operation(self): + result = CSV(items=[CumSum(slicer.metrics.votes)]) \ + .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp]) + + expected = cont_dim_operation_df.copy()[['cumsum(votes)']] + expected.index.names = ['Timestamp'] + expected.columns = ['CumSum(Votes)'] + + self.assertEqual(result, expected.to_csv()) + + def test_cat_dim(self): + result = CSV(items=[slicer.metrics.wins]) \ + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + + expected = cat_dim_df.copy()[['wins']] + expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') + expected.columns = ['Wins'] + + self.assertEqual(result, expected.to_csv()) + + def test_uni_dim(self): + result = CSV(items=[slicer.metrics.wins]) \ + .transform(uni_dim_df, slicer, [slicer.dimensions.candidate]) + + expected = uni_dim_df.copy() \ + .set_index('candidate_display', append=True) \ + .reset_index('candidate', drop=True) \ + [['wins']] + expected.index.names = ['Candidate'] + expected.columns = ['Wins'] + + self.assertEqual(result, expected.to_csv()) + + def test_uni_dim_no_display_definition(self): + import copy + candidate = copy.copy(slicer.dimensions.candidate) + candidate.display_key = None + candidate.display_definition = None + + uni_dim_df_copy = uni_dim_df.copy() + del uni_dim_df_copy[slicer.dimensions.candidate.display_key] + + result = CSV(items=[slicer.metrics.wins]) \ + .transform(uni_dim_df_copy, slicer, [candidate]) + + expected = uni_dim_df_copy.copy()[['wins']] + expected.index.names = ['Candidate'] + expected.columns = ['Wins'] + + self.assertEqual(result, expected.to_csv()) + + def test_multi_dims_time_series_and_uni(self): + result = CSV(items=[slicer.metrics.wins]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + + expected = cont_uni_dim_df.copy() \ + .set_index('state_display', append=True) \ + .reset_index('state', drop=False)[['wins']] + expected.index.names = ['Timestamp', 'State'] + expected.columns = ['Wins'] + + self.assertEqual(result, expected.to_csv()) + + def test_pivoted_single_dimension_no_effect(self): + result = CSV(items=[slicer.metrics.wins], pivot=True) \ + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + + expected = cat_dim_df.copy()[['wins']] + expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') + expected.columns = ['Wins'] + + self.assertEqual(result, expected.to_csv()) + + def test_pivoted_multi_dims_time_series_and_cat(self): + result = CSV(items=[slicer.metrics.wins], pivot=True) \ + .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party]) + + expected = cont_cat_dim_df.copy()[['wins']] + expected.index.names = ['Timestamp', 'Party'] + expected.columns = ['Wins'] + expected = expected.unstack(level=[1]) + + self.assertEqual(result, expected.to_csv()) + + def test_pivoted_multi_dims_time_series_and_uni(self): + result = CSV(items=[slicer.metrics.votes], pivot=True) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + + expected = cont_uni_dim_df.copy() \ + .set_index('state_display', append=True) \ + .reset_index('state', drop=True)[['votes']] + expected.index.names = ['Timestamp', 'State'] + expected.columns = ['Votes'] + expected = expected.unstack(level=[1]) + + self.assertEqual(result, expected.to_csv()) + + def test_time_series_ref(self): + result = CSV(items=[slicer.metrics.votes]) \ + .transform(cont_uni_dim_ref_df, slicer, [slicer.dimensions.timestamp.reference(ElectionOverElection), + slicer.dimensions.state]) + + expected = cont_uni_dim_ref_df.copy() \ + .set_index('state_display', append=True) \ + .reset_index('state', drop=True)[['votes', 'votes_eoe']] + expected.index.names = ['Timestamp', 'State'] + expected.columns = ['Votes', 'Votes (EoE)'] + + self.assertEqual(result, expected.to_csv()) From 81c9abd91519e31b18c0488717f11fa5fabdf0c7 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 2 Feb 2018 10:30:45 +0100 Subject: [PATCH 013/123] Fixed some issues reported by codacy --- fireant/slicer/queries/references.py | 10 ++++++---- fireant/slicer/widgets/__init__.py | 2 +- fireant/slicer/widgets/base.py | 8 +++++--- fireant/slicer/widgets/datatables.py | 6 ++++-- fireant/slicer/widgets/highcharts.py | 10 +++++----- fireant/slicer/widgets/matplotlib.py | 7 +++++-- fireant/slicer/widgets/pandas.py | 6 ++++-- 7 files changed, 30 insertions(+), 19 deletions(-) diff --git a/fireant/slicer/queries/references.py b/fireant/slicer/queries/references.py index 967a00d1..e971a087 100644 --- a/fireant/slicer/queries/references.py +++ b/fireant/slicer/queries/references.py @@ -45,10 +45,12 @@ def join_reference(reference: Reference, and YearOverYear.time_unit == reference.time_unit) # FIXME this is a bit hacky, need to replace the ref dimension term in all of the filters with the offset - if ref_query._wheres: - ref_query._wheres = _apply_to_term_in_criterion(ref_dimension.definition, - date_add(ref_dimension.definition), - ref_query._wheres) + query_wheres = getattr(ref_query, '_wheres') + if query_wheres: + wheres = _apply_to_term_in_criterion(ref_dimension.definition, + date_add(ref_dimension.definition), + query_wheres) + setattr(ref_query, '_wheres', wheres) # Join inner query join_criterion = _build_reference_join_criterion(ref_dimension, diff --git a/fireant/slicer/widgets/__init__.py b/fireant/slicer/widgets/__init__.py index 551cd585..87a36f36 100644 --- a/fireant/slicer/widgets/__init__.py +++ b/fireant/slicer/widgets/__init__.py @@ -1,6 +1,6 @@ from .base import Widget +from .csv import CSV from .datatables import DataTablesJS from .highcharts import HighCharts from .matplotlib import Matplotlib from .pandas import Pandas -from .csv import CSV \ No newline at end of file diff --git a/fireant/slicer/widgets/base.py b/fireant/slicer/widgets/base.py index 206a1fdc..6e2e4529 100644 --- a/fireant/slicer/widgets/base.py +++ b/fireant/slicer/widgets/base.py @@ -3,9 +3,6 @@ class Widget: - def transform(self, data_frame, slicer, dimensions): - raise NotImplementedError() - def __init__(self, items=()): self.items = list(items) @@ -25,3 +22,8 @@ def metrics(self): def __repr__(self): return '{}({})'.format(self.__class__.__name__, ','.join(str(m) for m in self.items)) + + +class TransformableWidget(Widget): + def transform(self, data_frame, slicer, dimensions): + raise NotImplementedError() diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index 5ee4938a..18b239e0 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -8,7 +8,9 @@ utils, ) from . import formats -from .base import Widget +from .base import ( + TransformableWidget, +) from .helpers import ( dimensional_metric_label, extract_display_values, @@ -95,7 +97,7 @@ def _format_metric_cell(value, metric): HARD_MAX_COLUMNS = 24 -class DataTablesJS(Widget): +class DataTablesJS(TransformableWidget): def __init__(self, items=(), pivot=False, max_columns=None): super(DataTablesJS, self).__init__(items) self.pivot = pivot diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index b57ff271..83e41049 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -4,7 +4,10 @@ from fireant import utils from fireant.utils import immutable -from .base import Widget +from .base import ( + TransformableWidget, + Widget, +) from .formats import ( dimension_value, metric_value, @@ -62,15 +65,12 @@ def __init__(self, items=(), name=None, stacked=False): self.name = name self.stacked = self.stacked or stacked - def transform(self, data_frame, slicer, dimensions): - raise NotImplementedError() - class ContinuousAxisChartWidget(ChartWidget): pass -class HighCharts(Widget): +class HighCharts(TransformableWidget): class LineChart(ContinuousAxisChartWidget): type = 'line' needs_marker = True diff --git a/fireant/slicer/widgets/matplotlib.py b/fireant/slicer/widgets/matplotlib.py index c3c61adb..9f0d2e70 100644 --- a/fireant/slicer/widgets/matplotlib.py +++ b/fireant/slicer/widgets/matplotlib.py @@ -1,6 +1,9 @@ -from .base import Widget +from .base import ( + Widget, + TransformableWidget, +) -class Matplotlib(Widget): +class Matplotlib(TransformableWidget): def transform(self, data_frame, slicer, dimensions): raise NotImplementedError() diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index 93ac3f13..2709923a 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -1,6 +1,8 @@ import pandas as pd -from .base import Widget +from .base import ( + TransformableWidget, +) from .helpers import ( reference_key, reference_label, @@ -9,7 +11,7 @@ HARD_MAX_COLUMNS = 24 -class Pandas(Widget): +class Pandas(TransformableWidget): def __init__(self, items=(), pivot=False, max_columns=None): super(Pandas, self).__init__(items) self.pivot = pivot From 98ad44d74e22abe5d03a9d708004a36b8688343c Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 5 Feb 2018 15:23:52 +0100 Subject: [PATCH 014/123] Added more tests --- fireant/slicer/widgets/base.py | 4 +-- fireant/slicer/widgets/matplotlib.py | 5 +--- fireant/tests/slicer/queries/test_builder.py | 29 ++++++++++++++----- fireant/tests/slicer/queries/test_database.py | 4 +++ fireant/tests/slicer/widgets/test_widgets.py | 25 ++++++++++++++++ 5 files changed, 53 insertions(+), 14 deletions(-) create mode 100644 fireant/tests/slicer/widgets/test_widgets.py diff --git a/fireant/slicer/widgets/base.py b/fireant/slicer/widgets/base.py index 6e2e4529..dc588057 100644 --- a/fireant/slicer/widgets/base.py +++ b/fireant/slicer/widgets/base.py @@ -7,8 +7,8 @@ def __init__(self, items=()): self.items = list(items) @immutable - def metric(self, metric): - self.items.append(metric) + def item(self, item): + self.items.append(item) @property def metrics(self): diff --git a/fireant/slicer/widgets/matplotlib.py b/fireant/slicer/widgets/matplotlib.py index 9f0d2e70..1d695047 100644 --- a/fireant/slicer/widgets/matplotlib.py +++ b/fireant/slicer/widgets/matplotlib.py @@ -1,7 +1,4 @@ -from .base import ( - Widget, - TransformableWidget, -) +from .base import TransformableWidget class Matplotlib(TransformableWidget): diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index aa2a2d71..f541837e 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -19,17 +19,30 @@ from ..mocks import slicer -class SlicerShortcutTests(TestCase): - maxDiff = None +class QueryBuilderTests(TestCase): + def test_widget_is_immutable(self): + query1 = slicer.data + query2 = query1.widget(f.DataTablesJS([slicer.metrics.votes])) + + self.assertIsNot(query1, query2) + + def test_dimension_is_immutable(self): + query1 = slicer.data + query2 = query1.dimension(slicer.dimensions.timestamp) + + self.assertIsNot(query1, query2) + + def test_filter_is_immutable(self): + query1 = slicer.data + query2 = query1.filter(slicer.dimensions.timestamp == 'ok') - def test_get_attr_from_slicer_dimensions_returns_dimension(self): - timestamp_dimension = slicer.dimensions.timestamp - self.assertTrue(hasattr(timestamp_dimension, 'definition')) + self.assertIsNot(query1, query2) - def test_get_attr_from_slicer_metrics_returns_metric(self): - votes_metric = slicer.metrics.votes - self.assertTrue(hasattr(votes_metric, 'definition')) + def test_orderby_is_immutable(self): + query1 = slicer.data + query2 = query1.orderby(slicer.dimensions.timestamp) + self.assertIsNot(query1, query2) # noinspection SqlDialectInspection,SqlNoDataSourceInspection class QueryBuilderMetricTests(TestCase): diff --git a/fireant/tests/slicer/queries/test_database.py b/fireant/tests/slicer/queries/test_database.py index 12f27277..bfa71ac8 100644 --- a/fireant/tests/slicer/queries/test_database.py +++ b/fireant/tests/slicer/queries/test_database.py @@ -127,3 +127,7 @@ def test_set_index_for_multiindex_with_nans_and_totals(self): [slicer.dimensions.timestamp, slicer.dimensions.state.rollup()]) self.assertListEqual(list(result.index.levels[1]), ['', '1', '2', 'Totals']) + + +class FetchDimensionOptionsTests(TestCase): + pass # TODO diff --git a/fireant/tests/slicer/widgets/test_widgets.py b/fireant/tests/slicer/widgets/test_widgets.py new file mode 100644 index 00000000..40d0fb7b --- /dev/null +++ b/fireant/tests/slicer/widgets/test_widgets.py @@ -0,0 +1,25 @@ +from unittest import TestCase + +from fireant.slicer.widgets.base import ( + TransformableWidget, + Widget, +) + + +class BaseWidgetTests(TestCase): + def test_create_widget_with_items(self): + widget = Widget(items=[0, 1, 2]) + self.assertListEqual(widget.items, [0, 1, 2]) + + def test_add_widget_to_items(self): + widget = Widget(items=[0, 1, 2]).item(3) + self.assertListEqual(widget.items, [0, 1, 2, 3]) + + def test_item_func_immuatable(self): + widget1 = Widget(items=[0, 1, 2]) + widget2 = widget1.item(3) + self.assertIsNot(widget1, widget2) + + def test_transformable_widget_has_transform_function(self): + self.assertTrue(hasattr(TransformableWidget, 'transform')) + self.assertTrue(callable(TransformableWidget.transform)) From 11a7fab41a9f52bcbfa53aa74ddd4b9b7d83d543 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 8 Feb 2018 14:56:26 +0100 Subject: [PATCH 015/123] fixes for totals, colors, etc issues found during manual testing --- fireant/__init__.py | 2 +- fireant/database/mysql.py | 2 +- fireant/database/postgresql.py | 8 +- fireant/database/vertica.py | 31 +-- fireant/slicer/dimensions.py | 17 +- fireant/slicer/intervals.py | 15 +- fireant/slicer/operations.py | 3 + fireant/slicer/queries/builder.py | 25 +- fireant/slicer/queries/database.py | 72 +++-- fireant/slicer/widgets/base.py | 7 + fireant/slicer/widgets/datatables.py | 6 +- fireant/slicer/widgets/formats.py | 16 +- fireant/slicer/widgets/helpers.py | 6 +- fireant/slicer/widgets/highcharts.py | 90 ++++--- fireant/tests/database/test_databases.py | 2 +- fireant/tests/database/test_mysql.py | 24 +- fireant/tests/database/test_postgresql.py | 28 +- fireant/tests/slicer/mocks.py | 42 ++- fireant/tests/slicer/queries/test_builder.py | 8 - fireant/tests/slicer/queries/test_database.py | 28 +- .../tests/slicer/widgets/test_datatables.py | 193 +++++++++++++ .../tests/slicer/widgets/test_highcharts.py | 255 +++++++++++++++--- fireant/utils.py | 7 + 23 files changed, 683 insertions(+), 204 deletions(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index e5adb1ca..3c87c175 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev' +__version__ = '1.0.0.dev2' diff --git a/fireant/database/mysql.py b/fireant/database/mysql.py index 4cf3e938..5fb3f8fe 100644 --- a/fireant/database/mysql.py +++ b/fireant/database/mysql.py @@ -64,7 +64,7 @@ def fetch_data(self, query): return pd.read_sql(query, self.connect()) def trunc_date(self, field, interval): - return Trunc(field, interval) + return Trunc(field, str(interval)) def date_add(self, field, date_part, interval, align_weekday=False): # adding an extra 's' as MySQL's interval doesn't work with 'year', 'week' etc, it expects a plural diff --git a/fireant/database/postgresql.py b/fireant/database/postgresql.py index daf178a9..34047ba4 100644 --- a/fireant/database/postgresql.py +++ b/fireant/database/postgresql.py @@ -8,13 +8,13 @@ from .base import Database -class Trunc(terms.Function): +class DateTrunc(terms.Function): """ Wrapper for the PostgreSQL date_trunc function """ def __init__(self, field, date_format, alias=None): - super(Trunc, self).__init__('date_trunc', date_format, field, alias=alias) + super(DateTrunc, self).__init__('DATE_TRUNC', date_format, field, alias=alias) # Setting the fields here means we can access the TRUNC args by name. self.field = field self.date_format = date_format @@ -52,10 +52,10 @@ def fetch_data(self, query): return pd.read_sql(query, self.connect()) def trunc_date(self, field, interval): - return Trunc(field, interval) + return DateTrunc(field, str(interval)) def date_add(self, field, date_part, interval, align_weekday=False): - return fn.DateAdd(date_part, interval, field) + return fn.DateAdd(str(date_part), interval, field) def totals(self, query, terms): raise NotImplementedError diff --git a/fireant/database/vertica.py b/fireant/database/vertica.py index 9238d252..5cb48e0e 100644 --- a/fireant/database/vertica.py +++ b/fireant/database/vertica.py @@ -1,16 +1,12 @@ -from fireant.slicer import ( - annually, - daily, - hourly, - monthly, - quarterly, - weekly, -) from pypika import ( VerticaQuery, functions as fn, terms, ) + +from fireant.slicer import ( + weekly, +) from .base import Database @@ -31,16 +27,17 @@ class VerticaDatabase(Database): """ Vertica client that uses the vertica_python driver. """ + # The pypika query class to use for constructing queries query_cls = VerticaQuery DATETIME_INTERVALS = { - hourly: 'HH', - daily: 'DD', - weekly: 'IW', - monthly: 'MM', - quarterly: 'Q', - annually: 'Y' + 'hour': 'HH', + 'day': 'DD', + 'week': 'IW', + 'month': 'MM', + 'quarter': 'Q', + 'year': 'Y' } def __init__(self, host='localhost', port=5433, database='vertica', user='vertica', password=None, @@ -60,15 +57,15 @@ def connect(self): read_timeout=self.read_timeout) def trunc_date(self, field, interval): - trunc_date_interval = self.DATETIME_INTERVALS.get(interval, 'DD') + trunc_date_interval = self.DATETIME_INTERVALS.get(str(interval), 'DD') return Trunc(field, trunc_date_interval) def date_add(self, field, date_part, interval, align_weekday=False): - shifted_date = fn.TimestampAdd(date_part, interval, field) + shifted_date = fn.TimestampAdd(str(date_part), interval, field) if align_weekday: truncated = self.trunc_date(shifted_date, weekly) - return fn.TimestampAdd(date_part, -interval, truncated) + return fn.TimestampAdd(str(date_part), -interval, truncated) return shifted_date diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index feb6fe8a..1cf63445 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -23,17 +23,6 @@ class Dimension(SlicerElement): def __init__(self, key, label=None, definition=None): super(Dimension, self).__init__(key, label, definition) - - -class RollupDimension(Dimension): - """ - This represents a dimension which can be rolled up to display the totals across the dimension in addition to the - break-down by dimension. Rollup returns `NULL` for the dimension value in most databases and therefore it is not - safe to use roll up in combination with dimension definitions that can return `NULL`. - """ - - def __init__(self, key, label=None, definition=None): - super(RollupDimension, self).__init__(key, label, definition) self.is_rollup = False @immutable @@ -46,7 +35,7 @@ def rollup(self): self.is_rollup = True -class BooleanDimension(RollupDimension): +class BooleanDimension(Dimension): """ This is a dimension that represents a boolean true/false value. The expression should always result in a boolean value. @@ -69,7 +58,7 @@ def is_(self, value: bool): return BooleanFilter(self.definition, value) -class CategoricalDimension(RollupDimension): +class CategoricalDimension(Dimension): """ This is a dimension that represents an enum-like database field, with a finite list of options to chose from. It provides support for configuring a display value for each of the possible values. @@ -108,7 +97,7 @@ def notin(self, values): return ExcludesFilter(self.definition, values) -class UniqueDimension(RollupDimension): +class UniqueDimension(Dimension): """ This is a dimension that represents a field in a database which is a unique identifier, such as a primary/foreign key. It provides support for a display value field which is selected and used in the results. diff --git a/fireant/slicer/intervals.py b/fireant/slicer/intervals.py index f63b36e6..e7020fff 100644 --- a/fireant/slicer/intervals.py +++ b/fireant/slicer/intervals.py @@ -26,10 +26,13 @@ def __eq__(self, other): return isinstance(other, DatetimeInterval) \ and self.key == other.key + def __str__(self): + return self.key -hourly = DatetimeInterval('hourly') -daily = DatetimeInterval('daily') -weekly = DatetimeInterval('weekly') -monthly = DatetimeInterval('monthly') -quarterly = DatetimeInterval('quarterly') -annually = DatetimeInterval('annually') + +hourly = DatetimeInterval('hour') +daily = DatetimeInterval('day') +weekly = DatetimeInterval('week') +monthly = DatetimeInterval('month') +quarterly = DatetimeInterval('quarter') +annually = DatetimeInterval('year') diff --git a/fireant/slicer/operations.py b/fireant/slicer/operations.py index a7a30fa2..419009a0 100644 --- a/fireant/slicer/operations.py +++ b/fireant/slicer/operations.py @@ -49,6 +49,9 @@ def metrics(self): def apply(self, data_frame): raise NotImplementedError() + def __repr__(self): + return self.key + class CumSum(_Cumulative): def apply(self, data_frame): diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 389e0e97..8d0f6521 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -24,21 +24,18 @@ ) from .database import fetch_data from .references import join_reference -from ..dimensions import RollupDimension from ..exceptions import ( CircularJoinsException, MissingTableJoinException, - RollupException, ) from ..filters import DimensionFilter -from ..intervals import DatetimeInterval from ..operations import Operation def _build_dimension_definition(dimension, interval_func): - if hasattr(dimension, 'interval') and isinstance(dimension.interval, DatetimeInterval): - return interval_func(dimension.definition, - dimension.interval).as_(dimension.key) + if hasattr(dimension, 'interval'): + return interval_func(dimension.definition, dimension.interval) \ + .as_(dimension.key) return dimension.definition.as_(dimension.key) @@ -58,11 +55,7 @@ def _select_groups(terms, query, rollup, database): def is_rolling_up(dimension, rolling_up): - if rolling_up: - if not isinstance(dimension, RollupDimension): - raise RollupException('Cannot roll up dimension {}'.format(dimension)) - return True - return getattr(dimension, "is_rollup", False) + return rolling_up or getattr(dimension, "is_rollup", False) class QueryBuilder(object): @@ -94,7 +87,6 @@ def _tables(self): :return: A collection of tables required to execute a query, """ - return ordered_distinct_list([table for element in self._elements # Need extra for-loop to incl. the `display_definition` from `UniqueDimension` @@ -206,7 +198,9 @@ def dimension(self, *dimensions): :param dimensions: :return: """ - self._dimensions += dimensions + self._dimensions += [dimension + for dimension in dimensions + if dimension not in self._dimensions] @immutable def orderby(self, element: SlicerElement, orientation=None): @@ -235,10 +229,9 @@ def operations(self): :return: an ordered, distinct list of metrics used in all widgets as part of this query. """ - return ordered_distinct_list_by_attr([item + return ordered_distinct_list_by_attr([operation for widget in self._widgets - for item in widget.items - if isinstance(item, Operation)]) + for operation in widget.operations]) @property def orders(self): diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index 72946c2e..84deebb2 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -1,16 +1,16 @@ import time -from typing import Iterable import pandas as pd +from typing import Iterable from fireant.database.base import Database from .logger import logger from ..dimensions import ( + ContinuousDimension, Dimension, - RollupDimension, ) -CONTINUOUS_DIMS = (pd.DatetimeIndex, pd.RangeIndex) +NULL_VALUE = 'null' def fetch_data(database: Database, query: str, dimensions: Iterable[Dimension]): @@ -39,6 +39,8 @@ def fetch_data(database: Database, query: str, dimensions: Iterable[Dimension]): def clean_and_apply_index(data_frame: pd.DataFrame, dimensions: Iterable[Dimension]): """ + Sets the index on a data frame. This will also replace any nulls from the database with an empty string for + non-continuous dimensions. Totals will be indexed with Nones. :param data_frame: :param dimensions: @@ -50,25 +52,57 @@ def clean_and_apply_index(data_frame: pd.DataFrame, dimensions: Iterable[Dimensi dimension_keys = [d.key for d in dimensions] for i, dimension in enumerate(dimensions): - if not isinstance(dimension, RollupDimension): + if isinstance(dimension, ContinuousDimension): + # Continuous dimensions are can never contain null values since they are selected as windows of values + # With that in mind, we leave the NaNs in them to represent Totals. continue level = dimension.key - if dimension.is_rollup: - # Rename the first instance of NaN to totals for each dimension value - # If there are multiple dimensions, we need to group by the preceding dimensions for each dimension - data_frame[level] = ( - data_frame - .groupby(dimension_keys[:i])[level] - .fillna('Totals', limit=1) - ) if 0 < i else ( - data_frame[level] - .fillna('Totals', limit=1) - ) - - data_frame[level] = data_frame[level] \ - .fillna('') \ - .astype('str') + data_frame[level] = fill_nans_in_level(data_frame, dimension, dimension_keys[:i]) # Set index on dimension columns return data_frame.set_index(dimension_keys) + + +def fill_nans_in_level(data_frame, dimension, preceding_dimension_keys): + """ + In case there are NaN values representing both totals (from ROLLUP) and database nulls, we need to replace the real + nulls with an empty string in order to distinguish between them. We choose to replace the actual database nulls + with an empty string in order to let NaNs represent Totals because mixing string values in the pandas index types + used by continuous dimensions does work. + + :param data_frame: + The data_frame we are replacing values in. + :param level: + The level of the data frame to replace nulls in. This function should be called once per non-conitnuous + dimension, in the order of the dimensions. + :param is_rollup: + + :param preceding_dimension_keys: + :return: + The level in the data_frame with the nulls replaced with empty string + """ + level = dimension.key + + if dimension.is_rollup: + if preceding_dimension_keys: + return (data_frame + .groupby(preceding_dimension_keys)[level] + .apply(_fill_nan_for_nulls)) + + return _fill_nan_for_nulls(data_frame[level]) + + return data_frame[level].fillna(NULL_VALUE) + + +def _fill_nan_for_nulls(df): + """ + Fills the first NaN with a literal string "null" if there are two NaN values, otherwise nothing is filled. + + + :param df: + :return: + """ + if 1 < pd.isnull(df).sum(): + return df.fillna(NULL_VALUE, limit=1) + return df diff --git a/fireant/slicer/widgets/base.py b/fireant/slicer/widgets/base.py index dc588057..a77a45c6 100644 --- a/fireant/slicer/widgets/base.py +++ b/fireant/slicer/widgets/base.py @@ -1,5 +1,6 @@ from fireant.slicer.exceptions import MetricRequiredException from fireant.utils import immutable +from ..operations import Operation class Widget: @@ -19,6 +20,12 @@ def metrics(self): for group in self.items for metric in getattr(group, 'metrics', [group])] + @property + def operations(self): + return [item + for item in self.items + if isinstance(item, Operation)] + def __repr__(self): return '{}({})'.format(self.__class__.__name__, ','.join(str(m) for m in self.items)) diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index 18b239e0..bd05d9d6 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -35,7 +35,11 @@ def _render_dimension_cell(dimension_value: str, display_values: dict): dimension_cell = {'value': formats.dimension_value(dimension_value)} if display_values is not None: - dimension_cell['display'] = display_values.get(dimension_value, dimension_value) + if pd.isnull(dimension_value): + dimension_cell['display'] = 'Totals' + else: + display_value = display_values.get(dimension_value, dimension_value) + dimension_cell['display'] = formats.dimension_value(display_value) return dimension_cell diff --git a/fireant/slicer/widgets/formats.py b/fireant/slicer/widgets/formats.py index efb572d8..c17f3e08 100644 --- a/fireant/slicer/widgets/formats.py +++ b/fireant/slicer/widgets/formats.py @@ -14,7 +14,13 @@ milliseconds = np.timedelta64(1, 'ms') -def dimension_value(value, str_date=True): +def date_as_millis(value): + if not isinstance(value, date): + value = datetime.strptime(value, '%Y-%m-%d %H:%M:%S') + return int(1000 * value.timestamp()) + + +def dimension_value(value): """ Format a dimension value. This will coerce the raw string or date values into a proper primitive value like a string, float, or int. @@ -25,14 +31,14 @@ def dimension_value(value, str_date=True): When True, dates and datetimes will be converted to ISO strings. The time is omitted for dates. When False, the datetime will be converted to a POSIX timestamp (millis-since-epoch). """ - if isinstance(value, date): - if not str_date: - return int(1000 * value.timestamp()) + if pd.isnull(value): + return 'Totals' + if isinstance(value, date): if not hasattr(value, 'time') or value.time() == NO_TIME: return value.strftime('%Y-%m-%d') else: - return value.strftime('%Y-%m-%dT%H:%M:%S') + return value.strftime('%Y-%m-%d %H:%M:%S') for type_cast in (int, float): try: diff --git a/fireant/slicer/widgets/helpers.py b/fireant/slicer/widgets/helpers.py index f268ce60..15e391be 100644 --- a/fireant/slicer/widgets/helpers.py +++ b/fireant/slicer/widgets/helpers.py @@ -1,3 +1,5 @@ +import pandas as pd + from fireant import utils @@ -91,12 +93,14 @@ def render_series_label(metric, reference, dimension_values): dimension_labels = [utils.deep_get(dimension_display_values, [dimension.key, dimension_value], dimension_value) + if not pd.isnull(dimension_value) + else 'Totals' for dimension, dimension_value in zip(dimensions[1:], dimension_values)] if dimension_labels: return '{} ({})'.format(reference_label(metric, reference), ', '.join(dimension_labels)) - return metric.label + return reference_label(metric, reference) return render_series_label diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 83e41049..c2cb5643 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -1,15 +1,20 @@ import itertools import pandas as pd +from datetime import ( + datetime, +) -from fireant import utils -from fireant.utils import immutable +from fireant import ( + DatetimeDimension, + utils, +) from .base import ( TransformableWidget, Widget, ) from .formats import ( - dimension_value, + date_as_millis, metric_value, ) from .helpers import ( @@ -102,7 +107,7 @@ def __init__(self, axes=(), title=None, colors=None): self.title = title self.colors = colors or DEFAULT_COLORS - @immutable + @utils.immutable def axis(self, axis: ChartWidget): """ (Immutable) Adds an axis to the Chart. @@ -128,6 +133,12 @@ def metrics(self): for metric in axis.metrics if not (metric.key in seen or seen.add(metric.key))] + @property + def operations(self): + return utils.ordered_distinct_list_by_attr([operation + for item in self.items + for operation in item.operations]) + def transform(self, data_frame, slicer, dimensions): """ - Main entry point - @@ -147,9 +158,13 @@ def transform(self, data_frame, slicer, dimensions): """ colors = itertools.cycle(self.colors) - levels = data_frame.index.names[1:] - groups = list(data_frame.groupby(level=levels)) \ - if levels \ + def group_series(keys): + if isinstance(keys[0], datetime) and pd.isnull(keys[0]): + return tuple('Totals' for _ in keys[1:]) + return tuple(str(key) if not pd.isnull(key) else 'Totals' for key in keys[1:]) + + groups = list(data_frame.groupby(group_series)) \ + if isinstance(data_frame.index, pd.MultiIndex) \ else [([], data_frame)] dimension_display_values = extract_display_values(dimensions, data_frame) @@ -170,15 +185,17 @@ def transform(self, data_frame, slicer, dimensions): y_axes[0:0] = self._render_y_axis(axis_idx, axis_color, references) + is_timeseries = dimensions and isinstance(dimensions[0], DatetimeDimension) series += self._render_series(axis, axis_idx, axis_color, series_colors, groups, render_series_label, - references) + references, + is_timeseries) - x_axis = self._render_x_axis(data_frame, dimension_display_values) + x_axis = self._render_x_axis(data_frame, dimensions, dimension_display_values) return { "title": {"text": self.title}, @@ -190,7 +207,7 @@ def transform(self, data_frame, slicer, dimensions): } @staticmethod - def _render_x_axis(data_frame, dimension_display_values): + def _render_x_axis(data_frame, dimensions, dimension_display_values): """ Renders the xAxis configuraiton. @@ -204,7 +221,7 @@ def _render_x_axis(data_frame, dimension_display_values): if isinstance(data_frame.index, pd.MultiIndex) \ else data_frame.index - if isinstance(first_level, pd.DatetimeIndex): + if dimensions and isinstance(dimensions[0], DatetimeDimension): return {"type": "datetime"} categories = ["All"] \ @@ -238,17 +255,18 @@ def _render_y_axis(axis_idx, color, references): }] y_axes += [{ - "id": "{}_{}".format(axis_idx, reference.key), - "title": {"text": reference.label}, - "opposite": True, - "labels": {"style": {"color": color}} - } - for reference in references - if reference.is_delta] + "id": "{}_{}".format(axis_idx, reference.key), + "title": {"text": reference.label}, + "opposite": True, + "labels": {"style": {"color": color}} + } + for reference in references + if reference.is_delta] return y_axes - def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, render_series_label, references): + def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, render_series_label, + references, is_timeseries=False): """ Renders the series configuration. @@ -258,26 +276,23 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, :param axis_idx: :param axis_color: :param colors: - :param data_frame_groups: + :param index_rows: + :param data_frame: :param render_series_label: :param references: + :param is_timeseries: :return: """ has_multi_metric = 1 < len(axis.items) series = [] for metric in axis.items: - visible = True symbols = itertools.cycle(MARKER_SYMBOLS) series_color = next(colors) if has_multi_metric else None for (dimension_values, group_df), symbol in zip(data_frame_groups, symbols): dimension_values = utils.wrap_list(dimension_values) - is_timeseries = isinstance(group_df.index.levels[0] - if isinstance(group_df.index, pd.MultiIndex) - else group_df.index, pd.DatetimeIndex) - if not has_multi_metric: series_color = next(colors) @@ -288,7 +303,6 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, "type": axis.type, "color": series_color, "dashStyle": dash_style, - "visible": visible, "name": render_series_label(metric, reference, dimension_values), @@ -307,16 +321,20 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, else None), }) - visible = False # Only display the first in each group - return series def _render_data(self, group_df, metric_key, is_timeseries): - if is_timeseries: - return [(dimension_value(utils.wrap_list(dimension_values)[0], - str_date=False), - metric_value(y)) - for dimension_values, y in group_df[metric_key].iteritems()] - - return [metric_value(y) - for y in group_df[metric_key].values] + if not is_timeseries: + return [metric_value(y) for y in group_df[metric_key].values] + + series = [] + for dimension_values, y in group_df[metric_key].iteritems(): + first_dimension_value = utils.wrap_list(dimension_values)[0] + + if pd.isnull(first_dimension_value): + # Ignore totals on the x-axis. + continue + + series.append((date_as_millis(first_dimension_value), metric_value(y))) + + return series diff --git a/fireant/tests/database/test_databases.py b/fireant/tests/database/test_databases.py index a21898c5..c76e6372 100644 --- a/fireant/tests/database/test_databases.py +++ b/fireant/tests/database/test_databases.py @@ -39,4 +39,4 @@ def test_database_api(self): db.connect() with self.assertRaises(NotImplementedError): - db.trunc_date(Field('abc'), 'DAY') + db.trunc_date(Field('abc'), 'day') diff --git a/fireant/tests/database/test_mysql.py b/fireant/tests/database/test_mysql.py index fb0cf679..1f24fe8a 100644 --- a/fireant/tests/database/test_mysql.py +++ b/fireant/tests/database/test_mysql.py @@ -1,14 +1,22 @@ from unittest import TestCase - from unittest.mock import ( ANY, Mock, patch, ) -from fireant.database import MySQLDatabase from pypika import Field +from fireant import ( + annually, + daily, + hourly, + monthly, + quarterly, + weekly, +) +from fireant.database import MySQLDatabase + class TestMySQLDatabase(TestCase): @classmethod @@ -38,32 +46,32 @@ def test_connect(self): ) def test_trunc_hour(self): - result = self.mysql.trunc_date(Field('date'), 'hour') + result = self.mysql.trunc_date(Field('date'), hourly) self.assertEqual('dashmore.TRUNC("date",\'hour\')', str(result)) def test_trunc_day(self): - result = self.mysql.trunc_date(Field('date'), 'day') + result = self.mysql.trunc_date(Field('date'), daily) self.assertEqual('dashmore.TRUNC("date",\'day\')', str(result)) def test_trunc_week(self): - result = self.mysql.trunc_date(Field('date'), 'week') + result = self.mysql.trunc_date(Field('date'), weekly) self.assertEqual('dashmore.TRUNC("date",\'week\')', str(result)) def test_trunc_month(self): - result = self.mysql.trunc_date(Field('date'), 'month') + result = self.mysql.trunc_date(Field('date'), monthly) self.assertEqual('dashmore.TRUNC("date",\'month\')', str(result)) def test_trunc_quarter(self): - result = self.mysql.trunc_date(Field('date'), 'quarter') + result = self.mysql.trunc_date(Field('date'), quarterly) self.assertEqual('dashmore.TRUNC("date",\'quarter\')', str(result)) def test_trunc_year(self): - result = self.mysql.trunc_date(Field('date'), 'year') + result = self.mysql.trunc_date(Field('date'), annually) self.assertEqual('dashmore.TRUNC("date",\'year\')', str(result)) diff --git a/fireant/tests/database/test_postgresql.py b/fireant/tests/database/test_postgresql.py index 0f498c4e..743ab2e9 100644 --- a/fireant/tests/database/test_postgresql.py +++ b/fireant/tests/database/test_postgresql.py @@ -6,6 +6,14 @@ ) from pypika import Field +from fireant import ( + DatetimeInterval, + hourly, + daily, + weekly, + quarterly, + annually, +) from fireant.database import PostgreSQLDatabase @@ -36,29 +44,29 @@ def test_connect(self): ) def test_trunc_hour(self): - result = self.database.trunc_date(Field('date'), 'hour') + result = self.database.trunc_date(Field('date'), hourly) - self.assertEqual('date_trunc(\'hour\',"date")', str(result)) + self.assertEqual('DATE_TRUNC(\'hour\',"date")', str(result)) def test_trunc_day(self): - result = self.database.trunc_date(Field('date'), 'day') + result = self.database.trunc_date(Field('date'), daily) - self.assertEqual('date_trunc(\'day\',"date")', str(result)) + self.assertEqual('DATE_TRUNC(\'day\',"date")', str(result)) def test_trunc_week(self): - result = self.database.trunc_date(Field('date'), 'week') + result = self.database.trunc_date(Field('date'), weekly) - self.assertEqual('date_trunc(\'week\',"date")', str(result)) + self.assertEqual('DATE_TRUNC(\'week\',"date")', str(result)) def test_trunc_quarter(self): - result = self.database.trunc_date(Field('date'), 'quarter') + result = self.database.trunc_date(Field('date'), quarterly) - self.assertEqual('date_trunc(\'quarter\',"date")', str(result)) + self.assertEqual('DATE_TRUNC(\'quarter\',"date")', str(result)) def test_trunc_year(self): - result = self.database.trunc_date(Field('date'), 'year') + result = self.database.trunc_date(Field('date'), annually) - self.assertEqual('date_trunc(\'year\',"date")', str(result)) + self.assertEqual('DATE_TRUNC(\'year\',"date")', str(result)) def test_date_add_hour(self): result = self.database.date_add(Field('date'), 'hour', 1) diff --git a/fireant/tests/slicer/mocks.py b/fireant/tests/slicer/mocks.py index ba4c3d63..4d7bfc3b 100644 --- a/fireant/tests/slicer/mocks.py +++ b/fireant/tests/slicer/mocks.py @@ -2,21 +2,21 @@ OrderedDict, namedtuple, ) -from unittest.mock import Mock - -import pandas as pd from datetime import ( datetime, ) +from unittest.mock import Mock -from fireant import * -from fireant import VerticaDatabase +import pandas as pd from pypika import ( JoinType, Table, functions as fn, ) +from fireant import * +from fireant import VerticaDatabase + class TestDatabase(VerticaDatabase): # Vertica client that uses the vertica_python driver. @@ -295,11 +295,34 @@ def totals(data_frame, dimensions, columns): """ Computes the totals across a dimension and adds the total as an extra row. """ - dfx = data_frame.unstack(level=dimensions) - for c in columns: - dfx[(c, 'Total')] = dfx[c].sum(axis=1) - return dfx.stack(level=dimensions) + def _totals(df): + if isinstance(df, pd.Series): + return df.sum() + + return pd.DataFrame( + [df.sum()], + columns=columns, + index=pd.Index([None], + name=df.index.names[-1])) + + totals_df = None + for i in range(-1, -1 - len(dimensions), -1): + groupby_levels = data_frame.index.names[:i] + + if groupby_levels: + level_totals_df = data_frame[columns].groupby(level=groupby_levels).apply(_totals) + else: + level_totals_df = pd.DataFrame([data_frame[columns].apply(_totals)], + columns=columns, + index=pd.MultiIndex.from_tuples([[None] * len(data_frame.index.levels)], + names=data_frame.index.names)) + + totals_df = totals_df.append(level_totals_df) \ + if totals_df is not None \ + else level_totals_df + + return data_frame.append(totals_df).sort_index() # Convert all index values to string @@ -318,5 +341,6 @@ def totals(data_frame, dimensions, columns): cont_cat_dim_totals_df = totals(cont_cat_dim_df, ['political_party'], _columns) cont_uni_dim_totals_df = totals(cont_uni_dim_df, ['state'], _columns) +cont_uni_dim_all_totals_df = totals(cont_uni_dim_df, ['timestamp', 'state'], _columns) ElectionOverElection = Reference('eoe', 'EoE', 'year', 4) \ No newline at end of file diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index f541837e..3ac99fd7 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -355,14 +355,6 @@ def test_force_all_dimensions_following_rollup_to_be_rolled_up_with_split_dimens 'GROUP BY ROLLUP("political_party",("candidate_id","candidate_name")) ' 'ORDER BY "political_party","candidate_display"', str(query)) - def test_raise_exception_when_trying_to_rollup_continuous_dimension(self): - with self.assertRaises(RollupException): - slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ - .dimension(slicer.dimensions.political_party.rollup(), - slicer.dimensions.timestamp) \ - .query - # noinspection SqlDialectInspection,SqlNoDataSourceInspection class QueryBuilderDimensionFilterTests(TestCase): diff --git a/fireant/tests/slicer/queries/test_database.py b/fireant/tests/slicer/queries/test_database.py index bfa71ac8..7dfecd0d 100644 --- a/fireant/tests/slicer/queries/test_database.py +++ b/fireant/tests/slicer/queries/test_database.py @@ -1,5 +1,9 @@ from unittest import TestCase -from unittest.mock import Mock +from unittest.mock import ( + MagicMock, + Mock, + patch, +) import numpy as np import pandas as pd @@ -22,9 +26,11 @@ class FetchDataTests(TestCase): @classmethod def setUpClass(cls): cls.mock_database = Mock(name='database') - cls.mock_data_frame = cls.mock_database.fetch_data.return_value = Mock(name='data_frame') + cls.mock_data_frame = cls.mock_database.fetch_data.return_value = MagicMock(name='data_frame') cls.mock_query = 'SELECT *' cls.mock_dimensions = [Mock(), Mock()] + cls.mock_dimensions[0].is_rollup = False + cls.mock_dimensions[1].is_rollup = True cls.result = fetch_data(cls.mock_database, cls.mock_query, cls.mock_dimensions) @@ -71,6 +77,8 @@ def totals(df): class FetchDataCleanIndexTests(TestCase): + maxDiff = None + def test_do_nothing_when_no_dimensions(self): result = clean_and_apply_index(single_metric_df, []) @@ -86,19 +94,19 @@ def test_set_cat_dim_index(self): result = clean_and_apply_index(cat_dim_df.reset_index(), [slicer.dimensions.political_party]) - self.assertListEqual(list(result.index), ['d', 'i', 'r']) + self.assertListEqual(result.index.tolist(), ['d', 'i', 'r']) def test_set_cat_dim_index_with_nan_converted_to_empty_str(self): result = clean_and_apply_index(cat_dim_nans_df.reset_index(), [slicer.dimensions.political_party]) - self.assertListEqual(list(result.index), ['d', 'i', 'r', '']) + self.assertListEqual(result.index.tolist(), ['d', 'i', 'r', 'null']) - def test_convert_cat_totals(self): + def test_convert_cat_totals_converted_to_none(self): result = clean_and_apply_index(cat_dim_nans_df.reset_index(), [slicer.dimensions.political_party.rollup()]) - self.assertListEqual(list(result.index), ['d', 'i', 'r', 'Totals']) + self.assertListEqual(result.index.tolist(), ['d', 'i', 'r', None]) def test_convert_numeric_values_to_string(self): result = clean_and_apply_index(uni_dim_df.reset_index(), [slicer.dimensions.candidate]) @@ -108,25 +116,25 @@ def test_set_uni_dim_index(self): result = clean_and_apply_index(uni_dim_df.reset_index(), [slicer.dimensions.candidate]) - self.assertListEqual(list(result.index), [str(x + 1) for x in range(11)]) + self.assertListEqual(result.index.tolist(), [str(x + 1) for x in range(11)]) def test_set_uni_dim_index_with_nan_converted_to_empty_str(self): result = clean_and_apply_index(uni_dim_nans_df.reset_index(), [slicer.dimensions.candidate]) - self.assertListEqual(list(result.index), [str(x + 1) for x in range(11)] + ['']) + self.assertListEqual(result.index.tolist(), [str(x + 1) for x in range(11)] + ['null']) def test_convert_uni_totals(self): result = clean_and_apply_index(uni_dim_nans_df.reset_index(), [slicer.dimensions.candidate.rollup()]) - self.assertListEqual(list(result.index), [str(x + 1) for x in range(11)] + ['Totals']) + self.assertListEqual(result.index.tolist(), [str(x + 1) for x in range(11)] + [None]) def test_set_index_for_multiindex_with_nans_and_totals(self): result = clean_and_apply_index(cont_uni_dim_nans_totals_df.reset_index(), [slicer.dimensions.timestamp, slicer.dimensions.state.rollup()]) - self.assertListEqual(list(result.index.levels[1]), ['', '1', '2', 'Totals']) + self.assertListEqual(result.index.get_level_values(1).unique().tolist(), ['2', '1', 'null', np.nan]) class FetchDimensionOptionsTests(TestCase): diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py index e930cc90..55783755 100644 --- a/fireant/tests/slicer/widgets/test_datatables.py +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -13,10 +13,12 @@ ElectionOverElection, cat_dim_df, cont_cat_dim_df, + cont_uni_dim_all_totals_df, cont_dim_df, cont_dim_operation_df, cont_uni_dim_df, cont_uni_dim_ref_df, + cont_uni_dim_totals_df, multi_metric_df, single_metric_df, slicer, @@ -355,6 +357,197 @@ def test_multi_dims_time_series_and_uni(self): }], }, result) + def test_multi_dims_with_one_level_totals(self): + result = DataTablesJS(items=[slicer.metrics.wins]) \ + .transform(cont_uni_dim_totals_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state.rollup()]) + + self.assertEqual({ + 'columns': [{ + 'data': 'timestamp', + 'title': 'Timestamp', + 'render': {'_': 'value'}, + }, { + 'data': 'state', + 'render': {'_': 'value', 'display': 'display'}, + 'title': 'State' + }, { + 'data': 'wins', + 'title': 'Wins', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'state': {'display': 'Texas', 'value': 1}, + 'timestamp': {'value': '1996-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'California', 'value': 2}, + 'timestamp': {'value': '1996-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'Totals', 'value': 'Totals'}, + 'timestamp': {'value': '1996-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'state': {'display': 'Texas', 'value': 1}, + 'timestamp': {'value': '2000-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'California', 'value': 2}, + 'timestamp': {'value': '2000-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'Totals', 'value': 'Totals'}, + 'timestamp': {'value': '2000-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'state': {'display': 'Texas', 'value': 1}, + 'timestamp': {'value': '2004-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'California', 'value': 2}, + 'timestamp': {'value': '2004-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'Totals', 'value': 'Totals'}, + 'timestamp': {'value': '2004-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'state': {'display': 'Texas', 'value': 1}, + 'timestamp': {'value': '2008-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'California', 'value': 2}, + 'timestamp': {'value': '2008-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'Totals', 'value': 'Totals'}, + 'timestamp': {'value': '2008-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'state': {'display': 'Texas', 'value': 1}, + 'timestamp': {'value': '2012-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'California', 'value': 2}, + 'timestamp': {'value': '2012-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'Totals', 'value': 'Totals'}, + 'timestamp': {'value': '2012-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'state': {'display': 'Texas', 'value': 1}, + 'timestamp': {'value': '2016-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'California', 'value': 2}, + 'timestamp': {'value': '2016-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'Totals', 'value': 'Totals'}, + 'timestamp': {'value': '2016-01-01'}, + 'wins': {'display': '2', 'value': 2} + }], + }, result) + + def test_multi_dims_with_all_levels_totals(self): + result = DataTablesJS(items=[slicer.metrics.wins]) \ + .transform(cont_uni_dim_all_totals_df, slicer, [slicer.dimensions.timestamp.rollup(), + slicer.dimensions.state.rollup()]) + + self.assertEqual({ + 'columns': [{ + 'data': 'timestamp', + 'title': 'Timestamp', + 'render': {'_': 'value'}, + }, { + 'data': 'state', + 'render': {'_': 'value', 'display': 'display'}, + 'title': 'State' + }, { + 'data': 'wins', + 'title': 'Wins', + 'render': {'_': 'value', 'display': 'display'}, + }], + 'data': [{ + 'state': {'display': 'Texas', 'value': 1}, + 'timestamp': {'value': '1996-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'California', 'value': 2}, + 'timestamp': {'value': '1996-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'Totals', 'value': 'Totals'}, + 'timestamp': {'value': '1996-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'state': {'display': 'Texas', 'value': 1}, + 'timestamp': {'value': '2000-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'California', 'value': 2}, + 'timestamp': {'value': '2000-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'Totals', 'value': 'Totals'}, + 'timestamp': {'value': '2000-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'state': {'display': 'Texas', 'value': 1}, + 'timestamp': {'value': '2004-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'California', 'value': 2}, + 'timestamp': {'value': '2004-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'Totals', 'value': 'Totals'}, + 'timestamp': {'value': '2004-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'state': {'display': 'Texas', 'value': 1}, + 'timestamp': {'value': '2008-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'California', 'value': 2}, + 'timestamp': {'value': '2008-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'Totals', 'value': 'Totals'}, + 'timestamp': {'value': '2008-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'state': {'display': 'Texas', 'value': 1}, + 'timestamp': {'value': '2012-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'California', 'value': 2}, + 'timestamp': {'value': '2012-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'Totals', 'value': 'Totals'}, + 'timestamp': {'value': '2012-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'state': {'display': 'Texas', 'value': 1}, + 'timestamp': {'value': '2016-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'California', 'value': 2}, + 'timestamp': {'value': '2016-01-01'}, + 'wins': {'display': '1', 'value': 1} + }, { + 'state': {'display': 'Totals', 'value': 'Totals'}, + 'timestamp': {'value': '2016-01-01'}, + 'wins': {'display': '2', 'value': 2} + }, { + 'state': {'display': 'Totals', 'value': 'Totals'}, + 'timestamp': {'value': 'Totals'}, + 'wins': {'display': '12', 'value': 12} + }], + }, result) + def test_pivoted_single_dimension_no_effect(self): result = DataTablesJS(items=[slicer.metrics.wins], pivot=True) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) diff --git a/fireant/tests/slicer/widgets/test_highcharts.py b/fireant/tests/slicer/widgets/test_highcharts.py index c1a3a569..12cefb63 100644 --- a/fireant/tests/slicer/widgets/test_highcharts.py +++ b/fireant/tests/slicer/widgets/test_highcharts.py @@ -7,9 +7,11 @@ cat_dim_df, cont_dim_df, cont_dim_operation_df, + cont_uni_dim_all_totals_df, cont_uni_dim_df, cont_uni_dim_ref_delta_df, cont_uni_dim_ref_df, + cont_uni_dim_totals_df, multi_metric_df, single_metric_df, slicer, @@ -52,7 +54,6 @@ def test_single_metric_line_chart(self): "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": True, }] }, result) @@ -85,7 +86,6 @@ def test_single_operation_line_chart(self): "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": True, }] }, result) @@ -119,7 +119,6 @@ def test_single_metric_with_uni_dim_line_chart(self): "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Votes (California)", @@ -134,7 +133,6 @@ def test_single_metric_with_uni_dim_line_chart(self): "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": False, }] }, result) @@ -169,7 +167,6 @@ def test_multi_metrics_single_axis_line_chart(self): "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Votes (California)", @@ -184,7 +181,6 @@ def test_multi_metrics_single_axis_line_chart(self): "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": False, }, { "type": self.chart_type, "name": "Wins (Texas)", @@ -199,7 +195,6 @@ def test_multi_metrics_single_axis_line_chart(self): "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Wins (California)", @@ -214,7 +209,6 @@ def test_multi_metrics_single_axis_line_chart(self): "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": False, }] }, result) @@ -253,7 +247,6 @@ def test_multi_metrics_multi_axis_line_chart(self): "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Votes (California)", @@ -268,7 +261,6 @@ def test_multi_metrics_multi_axis_line_chart(self): "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": False, }, { "type": self.chart_type, "name": "Wins (Texas)", @@ -283,7 +275,6 @@ def test_multi_metrics_multi_axis_line_chart(self): "marker": {"symbol": "circle", "fillColor": "#55BF3B"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Wins (California)", @@ -298,10 +289,225 @@ def test_multi_metrics_multi_axis_line_chart(self): "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": False, }] }, result) + def test_multi_dim_with_totals_line_chart(self): + result = HighCharts(title="Time Series with Unique Dimension and Multiple Metrics, Multi-Axis", + axes=[self.chart_class([slicer.metrics.votes]), + self.chart_class([slicer.metrics.wins]), ]) \ + .transform(cont_uni_dim_totals_df, slicer, [slicer.dimensions.timestamp.rollup(), + slicer.dimensions.state.rollup()]) + + self.assertEqual({ + "title": {"text": "Time Series with Unique Dimension and Multiple Metrics, Multi-Axis"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "1", + "title": {"text": None}, + "labels": {"style": {"color": "#55BF3B"}} + }, { + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": "#DDDF0D"}} + }], + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + 'color': '#DDDF0D', + 'dashStyle': 'Solid', + 'data': [(820454400000, 5574387), + (946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], + 'marker': {'fillColor': '#DDDF0D', 'symbol': 'circle'}, + 'name': 'Votes (Texas)', + 'stacking': self.stacking, + 'type': self.chart_type, + 'yAxis': '0' + }, { + 'color': '#55BF3B', + 'dashStyle': 'Solid', + 'data': [(820454400000, 9646062), + (946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], + 'marker': {'fillColor': '#DDDF0D', 'symbol': 'square'}, + 'name': 'Votes (California)', + 'stacking': self.stacking, + 'type': self.chart_type, + 'yAxis': '0' + }, { + 'color': '#DF5353', + 'dashStyle': 'Solid', + 'data': [(820454400000, 15220449), + (946684800000, 16662017), + (1072915200000, 19614932), + (1199145600000, 21294215), + (1325376000000, 20572210), + (1451606400000, 18310513)], + 'marker': {'fillColor': '#DDDF0D', 'symbol': 'diamond'}, + 'name': 'Votes (Totals)', + 'stacking': self.stacking, + 'type': self.chart_type, + 'yAxis': '0' + }, { + 'color': '#55BF3B', + 'dashStyle': 'Solid', + 'data': [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], + 'marker': {'fillColor': '#55BF3B', 'symbol': 'circle'}, + 'name': 'Wins (Texas)', + 'stacking': self.stacking, + 'type': self.chart_type, + 'yAxis': '1' + }, { + 'color': '#DF5353', + 'dashStyle': 'Solid', + 'data': [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], + 'marker': {'fillColor': '#55BF3B', 'symbol': 'square'}, + 'name': 'Wins (California)', + 'stacking': self.stacking, + 'type': self.chart_type, + 'yAxis': '1' + }, { + 'color': '#7798BF', + 'dashStyle': 'Solid', + 'data': [(820454400000, 2), + (946684800000, 2), + (1072915200000, 2), + (1199145600000, 2), + (1325376000000, 2), + (1451606400000, 2)], + 'marker': {'fillColor': '#55BF3B', 'symbol': 'diamond'}, + 'name': 'Wins (Totals)', + 'stacking': self.stacking, + 'type': self.chart_type, + 'yAxis': '1' + }], + }, result) + + def test_multi_dim_with_totals_on_first_dim_line_chart(self): + result = HighCharts(title="Time Series with Unique Dimension and Multiple Metrics, Multi-Axis", + axes=[self.chart_class([slicer.metrics.votes]), + self.chart_class([slicer.metrics.wins]), ]) \ + .transform(cont_uni_dim_all_totals_df, slicer, [slicer.dimensions.timestamp.rollup(), + slicer.dimensions.state.rollup()]) + + self.assertEqual({ + "title": {"text": "Time Series with Unique Dimension and Multiple Metrics, Multi-Axis"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "1", + "title": {"text": None}, + "labels": {"style": {"color": "#55BF3B"}} + }, { + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": "#DDDF0D"}} + }], + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + 'color': '#DDDF0D', + 'dashStyle': 'Solid', + 'data': [(820454400000, 5574387), + (946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], + 'marker': {'fillColor': '#DDDF0D', 'symbol': 'circle'}, + 'name': 'Votes (Texas)', + 'stacking': self.stacking, + 'type': self.chart_type, + 'yAxis': '0' + }, { + 'color': '#55BF3B', + 'dashStyle': 'Solid', + 'data': [(820454400000, 9646062), + (946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], + 'marker': {'fillColor': '#DDDF0D', 'symbol': 'square'}, + 'name': 'Votes (California)', + 'stacking': self.stacking, + 'type': self.chart_type, + 'yAxis': '0' + }, { + 'color': '#DF5353', + 'dashStyle': 'Solid', + 'data': [(820454400000, 15220449), + (946684800000, 16662017), + (1072915200000, 19614932), + (1199145600000, 21294215), + (1325376000000, 20572210), + (1451606400000, 18310513)], + 'marker': {'fillColor': '#DDDF0D', 'symbol': 'diamond'}, + 'name': 'Votes (Totals)', + 'stacking': self.stacking, + 'type': self.chart_type, + 'yAxis': '0' + }, { + 'color': '#55BF3B', + 'dashStyle': 'Solid', + 'data': [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], + 'marker': {'fillColor': '#55BF3B', 'symbol': 'circle'}, + 'name': 'Wins (Texas)', + 'stacking': self.stacking, + 'type': self.chart_type, + 'yAxis': '1' + }, { + 'color': '#DF5353', + 'dashStyle': 'Solid', + 'data': [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], + 'marker': {'fillColor': '#55BF3B', 'symbol': 'square'}, + 'name': 'Wins (California)', + 'stacking': self.stacking, + 'type': self.chart_type, + 'yAxis': '1' + }, { + 'color': '#7798BF', + 'dashStyle': 'Solid', + 'data': [(820454400000, 2), + (946684800000, 2), + (1072915200000, 2), + (1199145600000, 2), + (1325376000000, 2), + (1451606400000, 2)], + 'marker': {'fillColor': '#55BF3B', 'symbol': 'diamond'}, + 'name': 'Wins (Totals)', + 'stacking': self.stacking, + 'type': self.chart_type, + 'yAxis': '1' + }], + }, result) + def test_uni_dim_with_ref_line_chart(self): result = HighCharts(title="Time Series with Unique Dimension and Reference", axes=[self.chart_class([slicer.metrics.votes])]) \ @@ -331,7 +537,6 @@ def test_uni_dim_with_ref_line_chart(self): "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Votes (EoE) (Texas)", @@ -345,7 +550,6 @@ def test_uni_dim_with_ref_line_chart(self): "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "ShortDash", "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Votes (California)", @@ -359,7 +563,6 @@ def test_uni_dim_with_ref_line_chart(self): "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": False, }, { "type": self.chart_type, "name": "Votes (EoE) (California)", @@ -373,7 +576,6 @@ def test_uni_dim_with_ref_line_chart(self): "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "ShortDash", "stacking": self.stacking, - "visible": False, }] }, result) @@ -413,7 +615,6 @@ def test_uni_dim_with_ref_delta_line_chart(self): "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Votes (EoE Δ) (Texas)", @@ -427,7 +628,6 @@ def test_uni_dim_with_ref_delta_line_chart(self): "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "ShortDash", "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Votes (California)", @@ -441,7 +641,6 @@ def test_uni_dim_with_ref_delta_line_chart(self): "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", "stacking": self.stacking, - "visible": False, }, { "type": self.chart_type, "name": "Votes (EoE Δ) (California)", @@ -455,7 +654,6 @@ def test_uni_dim_with_ref_delta_line_chart(self): "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "ShortDash", "stacking": self.stacking, - "visible": False, }] }, result) @@ -494,7 +692,6 @@ def test_single_metric_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": True, }] }, result) @@ -527,7 +724,6 @@ def test_multi_metric_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Wins", @@ -537,7 +733,6 @@ def test_multi_metric_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": True, }] }, result) @@ -569,7 +764,6 @@ def test_cat_dim_single_metric_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": True, }] }, result) @@ -602,7 +796,6 @@ def test_cat_dim_multi_metric_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Wins", @@ -612,7 +805,6 @@ def test_cat_dim_multi_metric_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": True, }] }, result) @@ -646,7 +838,6 @@ def test_cont_uni_dims_single_metric_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Votes (California)", @@ -661,7 +852,6 @@ def test_cont_uni_dims_single_metric_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": False, }] }, result) @@ -696,7 +886,6 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Votes (California)", @@ -711,7 +900,6 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": False, }, { "type": self.chart_type, "name": "Wins (Texas)", @@ -726,7 +914,6 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Wins (California)", @@ -741,7 +928,6 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": False, }] }, result) @@ -758,7 +944,6 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "id": "1", "title": {"text": None}, "labels": {"style": {"color": "#55BF3B"}} - }, { "id": "0", "title": {"text": None}, @@ -781,7 +966,6 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Votes (California)", @@ -796,7 +980,6 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": False, }, { "type": self.chart_type, "name": "Wins (Texas)", @@ -811,7 +994,6 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": True, }, { "type": self.chart_type, "name": "Wins (California)", @@ -826,7 +1008,6 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "dashStyle": "Solid", "marker": {}, "stacking": self.stacking, - "visible": False, }] }, result) diff --git a/fireant/utils.py b/fireant/utils.py index eaecb35c..6f8bcf3c 100644 --- a/fireant/utils.py +++ b/fireant/utils.py @@ -86,3 +86,10 @@ def ordered_distinct_list_by_attr(l, attr='key'): for x in l if not getattr(x, attr) in seen and not seen.add(getattr(x, attr))] + + +def groupby_first_level(index): + seen = set() + return [x[1:] + for x in list(index) + if x[1:] not in seen and not seen.add(x[1:])] From 14eff1df555911f6609699ce331c15fe45de7135 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 8 Feb 2018 16:59:01 +0100 Subject: [PATCH 016/123] Added formatting to highcharts --- fireant/slicer/widgets/highcharts.py | 12 +- .../tests/slicer/widgets/test_highcharts.py | 361 ++++++++++++++++++ 2 files changed, 371 insertions(+), 2 deletions(-) diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index c2cb5643..6adc3974 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -276,8 +276,7 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, :param axis_idx: :param axis_color: :param colors: - :param index_rows: - :param data_frame: + :param data_frame_groups: :param render_series_label: :param references: :param is_timeseries: @@ -308,6 +307,8 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, "data": self._render_data(group_df, metric_key, is_timeseries), + "tooltip": self._render_tooltip(metric), + "yAxis": ("{}_{}".format(axis_idx, reference.key) if reference is not None and reference.is_delta else str(axis_idx)), @@ -338,3 +339,10 @@ def _render_data(self, group_df, metric_key, is_timeseries): series.append((date_as_millis(first_dimension_value), metric_value(y))) return series + + def _render_tooltip(self, metric): + return { + "valuePrefix": metric.prefix, + "valueSuffix": metric.suffix, + "valueDecimals": metric.precision, + } diff --git a/fireant/tests/slicer/widgets/test_highcharts.py b/fireant/tests/slicer/widgets/test_highcharts.py index 12cefb63..31ea6e9c 100644 --- a/fireant/tests/slicer/widgets/test_highcharts.py +++ b/fireant/tests/slicer/widgets/test_highcharts.py @@ -1,3 +1,5 @@ +import copy +import json from unittest import TestCase from fireant import CumSum @@ -50,6 +52,130 @@ def test_single_metric_line_chart(self): (1199145600000, 21294215), (1325376000000, 20572210), (1451606400000, 18310513)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + }] + }, result) + + def test_metric_prefix_line_chart(self): + votes = copy.copy(slicer.metrics.votes) + votes.prefix = '$' + result = HighCharts(title="Time Series, Single Metric", + axes=[self.chart_class([votes])]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) + + print(json.dumps(result)) + + self.assertEqual({ + "title": {"text": "Time Series, Single Metric"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}} + }], + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes", + "yAxis": "0", + "data": [(820454400000, 15220449), + (946684800000, 16662017), + (1072915200000, 19614932), + (1199145600000, 21294215), + (1325376000000, 20572210), + (1451606400000, 18310513)], + 'tooltip': { + 'valuePrefix': '$', + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + }] + }, result) + + def test_metric_suffix_line_chart(self): + votes = copy.copy(slicer.metrics.votes) + votes.suffix = '%' + result = HighCharts(title="Time Series, Single Metric", + axes=[self.chart_class([votes])]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) + + self.assertEqual({ + "title": {"text": "Time Series, Single Metric"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}} + }], + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes", + "yAxis": "0", + "data": [(820454400000, 15220449), + (946684800000, 16662017), + (1072915200000, 19614932), + (1199145600000, 21294215), + (1325376000000, 20572210), + (1451606400000, 18310513)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': '%', + 'valueDecimals': None, + }, + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + }] + }, result) + + def test_metric_precision_line_chart(self): + votes = copy.copy(slicer.metrics.votes) + votes.precision = 2 + result = HighCharts(title="Time Series, Single Metric", + axes=[self.chart_class([votes])]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) + + self.assertEqual({ + "title": {"text": "Time Series, Single Metric"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}} + }], + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes", + "yAxis": "0", + "data": [(820454400000, 15220449), + (946684800000, 16662017), + (1072915200000, 19614932), + (1199145600000, 21294215), + (1325376000000, 20572210), + (1451606400000, 18310513)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': 2, + }, "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -82,6 +208,11 @@ def test_single_operation_line_chart(self): (1199145600000, 72791613), (1325376000000, 93363823), (1451606400000, 111674336)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -115,6 +246,11 @@ def test_single_metric_with_uni_dim_line_chart(self): (1199145600000, 8007961), (1325376000000, 7877967), (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -129,6 +265,11 @@ def test_single_metric_with_uni_dim_line_chart(self): (1199145600000, 13286254), (1325376000000, 12694243), (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -163,6 +304,11 @@ def test_multi_metrics_single_axis_line_chart(self): (1199145600000, 8007961), (1325376000000, 7877967), (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -177,6 +323,11 @@ def test_multi_metrics_single_axis_line_chart(self): (1199145600000, 13286254), (1325376000000, 12694243), (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -191,6 +342,11 @@ def test_multi_metrics_single_axis_line_chart(self): (1199145600000, 1), (1325376000000, 1), (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -205,6 +361,11 @@ def test_multi_metrics_single_axis_line_chart(self): (1199145600000, 1), (1325376000000, 1), (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -243,6 +404,11 @@ def test_multi_metrics_multi_axis_line_chart(self): (1199145600000, 8007961), (1325376000000, 7877967), (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -257,6 +423,11 @@ def test_multi_metrics_multi_axis_line_chart(self): (1199145600000, 13286254), (1325376000000, 12694243), (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -271,6 +442,11 @@ def test_multi_metrics_multi_axis_line_chart(self): (1199145600000, 1), (1325376000000, 1), (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "marker": {"symbol": "circle", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -285,6 +461,11 @@ def test_multi_metrics_multi_axis_line_chart(self): (1199145600000, 1), (1325376000000, 1), (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DF5353", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -322,6 +503,11 @@ def test_multi_dim_with_totals_line_chart(self): (1199145600000, 8007961), (1325376000000, 7877967), (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, 'marker': {'fillColor': '#DDDF0D', 'symbol': 'circle'}, 'name': 'Votes (Texas)', 'stacking': self.stacking, @@ -336,6 +522,11 @@ def test_multi_dim_with_totals_line_chart(self): (1199145600000, 13286254), (1325376000000, 12694243), (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, 'marker': {'fillColor': '#DDDF0D', 'symbol': 'square'}, 'name': 'Votes (California)', 'stacking': self.stacking, @@ -350,6 +541,11 @@ def test_multi_dim_with_totals_line_chart(self): (1199145600000, 21294215), (1325376000000, 20572210), (1451606400000, 18310513)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, 'marker': {'fillColor': '#DDDF0D', 'symbol': 'diamond'}, 'name': 'Votes (Totals)', 'stacking': self.stacking, @@ -364,6 +560,11 @@ def test_multi_dim_with_totals_line_chart(self): (1199145600000, 1), (1325376000000, 1), (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, 'marker': {'fillColor': '#55BF3B', 'symbol': 'circle'}, 'name': 'Wins (Texas)', 'stacking': self.stacking, @@ -378,6 +579,11 @@ def test_multi_dim_with_totals_line_chart(self): (1199145600000, 1), (1325376000000, 1), (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, 'marker': {'fillColor': '#55BF3B', 'symbol': 'square'}, 'name': 'Wins (California)', 'stacking': self.stacking, @@ -392,6 +598,11 @@ def test_multi_dim_with_totals_line_chart(self): (1199145600000, 2), (1325376000000, 2), (1451606400000, 2)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, 'marker': {'fillColor': '#55BF3B', 'symbol': 'diamond'}, 'name': 'Wins (Totals)', 'stacking': self.stacking, @@ -430,6 +641,11 @@ def test_multi_dim_with_totals_on_first_dim_line_chart(self): (1199145600000, 8007961), (1325376000000, 7877967), (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, 'marker': {'fillColor': '#DDDF0D', 'symbol': 'circle'}, 'name': 'Votes (Texas)', 'stacking': self.stacking, @@ -444,6 +660,11 @@ def test_multi_dim_with_totals_on_first_dim_line_chart(self): (1199145600000, 13286254), (1325376000000, 12694243), (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, 'marker': {'fillColor': '#DDDF0D', 'symbol': 'square'}, 'name': 'Votes (California)', 'stacking': self.stacking, @@ -458,6 +679,11 @@ def test_multi_dim_with_totals_on_first_dim_line_chart(self): (1199145600000, 21294215), (1325376000000, 20572210), (1451606400000, 18310513)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, 'marker': {'fillColor': '#DDDF0D', 'symbol': 'diamond'}, 'name': 'Votes (Totals)', 'stacking': self.stacking, @@ -472,6 +698,11 @@ def test_multi_dim_with_totals_on_first_dim_line_chart(self): (1199145600000, 1), (1325376000000, 1), (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, 'marker': {'fillColor': '#55BF3B', 'symbol': 'circle'}, 'name': 'Wins (Texas)', 'stacking': self.stacking, @@ -486,6 +717,11 @@ def test_multi_dim_with_totals_on_first_dim_line_chart(self): (1199145600000, 1), (1325376000000, 1), (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, 'marker': {'fillColor': '#55BF3B', 'symbol': 'square'}, 'name': 'Wins (California)', 'stacking': self.stacking, @@ -500,6 +736,11 @@ def test_multi_dim_with_totals_on_first_dim_line_chart(self): (1199145600000, 2), (1325376000000, 2), (1451606400000, 2)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, 'marker': {'fillColor': '#55BF3B', 'symbol': 'diamond'}, 'name': 'Wins (Totals)', 'stacking': self.stacking, @@ -533,6 +774,11 @@ def test_uni_dim_with_ref_line_chart(self): (1199145600000, 8007961), (1325376000000, 7877967), (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -546,6 +792,11 @@ def test_uni_dim_with_ref_line_chart(self): (1199145600000, 7359621), (1325376000000, 8007961), (1451606400000, 7877967)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "ShortDash", @@ -559,6 +810,11 @@ def test_uni_dim_with_ref_line_chart(self): (1199145600000, 13286254), (1325376000000, 12694243), (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -572,6 +828,11 @@ def test_uni_dim_with_ref_line_chart(self): (1199145600000, 12255311), (1325376000000, 13286254), (1451606400000, 12694243)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "ShortDash", @@ -611,6 +872,11 @@ def test_uni_dim_with_ref_delta_line_chart(self): (1199145600000, 8007961), (1325376000000, 7877967), (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "Solid", @@ -624,6 +890,11 @@ def test_uni_dim_with_ref_delta_line_chart(self): (1199145600000, -648340), (1325376000000, 129994), (1451606400000, 2805052)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, "dashStyle": "ShortDash", @@ -637,6 +908,11 @@ def test_uni_dim_with_ref_delta_line_chart(self): (1199145600000, 13286254), (1325376000000, 12694243), (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "Solid", @@ -650,6 +926,11 @@ def test_uni_dim_with_ref_delta_line_chart(self): (1199145600000, -1030943), (1325376000000, 592011), (1451606400000, -543355)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, "dashStyle": "ShortDash", @@ -688,6 +969,11 @@ def test_single_metric_bar_chart(self): "name": "Votes", "yAxis": "0", "data": [111674336], + 'tooltip': { + 'valueDecimals': None, + 'valuePrefix': None, + 'valueSuffix': None + }, "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -720,6 +1006,11 @@ def test_multi_metric_bar_chart(self): "name": "Votes", "yAxis": "0", "data": [111674336], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -729,6 +1020,11 @@ def test_multi_metric_bar_chart(self): "name": "Wins", "yAxis": "0", "data": [12], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -760,6 +1056,11 @@ def test_cat_dim_single_metric_bar_chart(self): "name": "Votes", "yAxis": "0", "data": [54551568, 1076384, 56046384], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -792,6 +1093,11 @@ def test_cat_dim_multi_metric_bar_chart(self): "name": "Votes", "yAxis": "0", "data": [54551568, 1076384, 56046384], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -801,6 +1107,11 @@ def test_cat_dim_multi_metric_bar_chart(self): "name": "Wins", "yAxis": "0", "data": [6, 0, 6], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -834,6 +1145,11 @@ def test_cont_uni_dims_single_metric_bar_chart(self): (1199145600000, 8007961), (1325376000000, 7877967), (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -848,6 +1164,11 @@ def test_cont_uni_dims_single_metric_bar_chart(self): (1199145600000, 13286254), (1325376000000, 12694243), (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -882,6 +1203,11 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): (1199145600000, 8007961), (1325376000000, 7877967), (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -896,6 +1222,11 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): (1199145600000, 13286254), (1325376000000, 12694243), (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -910,6 +1241,11 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): (1199145600000, 1), (1325376000000, 1), (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -924,6 +1260,11 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): (1199145600000, 1), (1325376000000, 1), (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -962,6 +1303,11 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): (1199145600000, 8007961), (1325376000000, 7877967), (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DDDF0D", "dashStyle": "Solid", "marker": {}, @@ -976,6 +1322,11 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): (1199145600000, 13286254), (1325376000000, 12694243), (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -990,6 +1341,11 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): (1199145600000, 1), (1325376000000, 1), (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#55BF3B", "dashStyle": "Solid", "marker": {}, @@ -1004,6 +1360,11 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): (1199145600000, 1), (1325376000000, 1), (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, "color": "#DF5353", "dashStyle": "Solid", "marker": {}, From 2c3aeda5a355c46846b9cf7fc09caf2456bbfbbe Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 8 Feb 2018 17:08:20 +0100 Subject: [PATCH 017/123] bumped dev version --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 3c87c175..e2784380 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev2' +__version__ = '1.0.0.dev3' From d74ffe3a83314f5568583946d04be5d9d244eb77 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 9 Feb 2018 12:17:00 +0100 Subject: [PATCH 018/123] - Fixed references to share a query if they're for the same dimension and time interval, for example a YoY and YoY delta for the same dimension can share a join query - Improved the order of reference line styles in highcharts - Fixed the query for YoY references with weekly date intervals --- fireant/database/base.py | 2 +- fireant/database/mysql.py | 2 +- fireant/database/postgresql.py | 2 +- fireant/database/vertica.py | 10 +- fireant/slicer/base.py | 3 + fireant/slicer/queries/builder.py | 91 ++-- fireant/slicer/queries/references.py | 151 ++++-- fireant/slicer/references.py | 33 ++ fireant/slicer/widgets/datatables.py | 2 + fireant/slicer/widgets/helpers.py | 31 +- fireant/slicer/widgets/highcharts.py | 14 +- fireant/slicer/widgets/pandas.py | 2 +- fireant/tests/slicer/queries/test_builder.py | 177 ++++++- .../tests/slicer/widgets/test_highcharts.py | 461 +++++++++++++++++- fireant/utils.py | 22 + 15 files changed, 842 insertions(+), 161 deletions(-) diff --git a/fireant/database/base.py b/fireant/database/base.py index d9edd45b..c3df135d 100644 --- a/fireant/database/base.py +++ b/fireant/database/base.py @@ -19,7 +19,7 @@ def connect(self): def trunc_date(self, field, interval): raise NotImplementedError - def date_add(self, field: terms.Term, date_part: str, interval: int, align_weekday=False): + def date_add(self, field: terms.Term, date_part: str, interval: int): """ Database specific function for adding or subtracting dates """ raise NotImplementedError diff --git a/fireant/database/mysql.py b/fireant/database/mysql.py index 5fb3f8fe..7b25284b 100644 --- a/fireant/database/mysql.py +++ b/fireant/database/mysql.py @@ -66,7 +66,7 @@ def fetch_data(self, query): def trunc_date(self, field, interval): return Trunc(field, str(interval)) - def date_add(self, field, date_part, interval, align_weekday=False): + def date_add(self, field, date_part, interval): # adding an extra 's' as MySQL's interval doesn't work with 'year', 'week' etc, it expects a plural interval_term = terms.Interval(**{'{}s'.format(str(date_part)): interval, 'dialect': Dialects.MYSQL}) return DateAdd(field, interval_term) diff --git a/fireant/database/postgresql.py b/fireant/database/postgresql.py index 34047ba4..48a9ff49 100644 --- a/fireant/database/postgresql.py +++ b/fireant/database/postgresql.py @@ -54,7 +54,7 @@ def fetch_data(self, query): def trunc_date(self, field, interval): return DateTrunc(field, str(interval)) - def date_add(self, field, date_part, interval, align_weekday=False): + def date_add(self, field, date_part, interval): return fn.DateAdd(str(date_part), interval, field) def totals(self, query, terms): diff --git a/fireant/database/vertica.py b/fireant/database/vertica.py index 5cb48e0e..9a45fd2c 100644 --- a/fireant/database/vertica.py +++ b/fireant/database/vertica.py @@ -60,14 +60,8 @@ def trunc_date(self, field, interval): trunc_date_interval = self.DATETIME_INTERVALS.get(str(interval), 'DD') return Trunc(field, trunc_date_interval) - def date_add(self, field, date_part, interval, align_weekday=False): - shifted_date = fn.TimestampAdd(str(date_part), interval, field) - - if align_weekday: - truncated = self.trunc_date(shifted_date, weekly) - return fn.TimestampAdd(str(date_part), -interval, truncated) - - return shifted_date + def date_add(self, field, date_part, interval): + return fn.TimestampAdd(str(date_part), interval, field) def totals(self, query, terms): return query.rollup(*terms) diff --git a/fireant/slicer/base.py b/fireant/slicer/base.py index bbaa0ddb..ef693781 100644 --- a/fireant/slicer/base.py +++ b/fireant/slicer/base.py @@ -24,3 +24,6 @@ def __repr__(self): @property def has_display_field(self): return getattr(self, 'display_definition', None) is not None + + def __hash__(self): + return hash(self.__class__.__name__ + self.key) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 8d0f6521..c338c425 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -1,7 +1,6 @@ -from collections import defaultdict -from typing import ( - Dict, - Iterable, +from collections import ( + OrderedDict, + defaultdict, ) import pandas as pd @@ -14,22 +13,30 @@ CircularDependencyError, toposort_flatten, ) +from typing import ( + Dict, + Iterable, +) from fireant.slicer.base import SlicerElement from fireant.utils import ( flatten, + groupby, immutable, ordered_distinct_list, ordered_distinct_list_by_attr, ) from .database import fetch_data -from .references import join_reference +from .references import ( + create_container_query, + create_joined_reference_query, + select_reference_metrics, +) from ..exceptions import ( CircularJoinsException, MissingTableJoinException, ) from ..filters import DimensionFilter -from ..operations import Operation def _build_dimension_definition(dimension, interval_func): @@ -272,11 +279,11 @@ def query(self): for metric in self.metrics]) # Add references - references = [(reference, dimension) - for dimension in self._dimensions - for reference in getattr(dimension, 'references', ())] - if references: - query = self._join_references(query, references) + references_for_dimensions = OrderedDict([(dimension, dimension.references) + for dimension in self._dimensions + if hasattr(dimension, 'references') and dimension.references]) + if references_for_dimensions: + query = self._join_references(query, references_for_dimensions) # Add ordering for (definition, orientation) in self.orders: @@ -284,7 +291,7 @@ def query(self): return query - def _join_references(self, query, references): + def _join_references(self, query, references_for_dimensions): """ This converts the pypika query built in `self.query()` into a query that includes references. This is achieved by wrapping the original query with an outer query using the original query as the FROM clause, then joining @@ -299,41 +306,41 @@ def _join_references(self, query, references): :param query: The original query built by `self.query` - :param references: - A list of the references that should be included. + :param references_for_dimensions: + An ordered dict with the dimension as the key and a list of references to add for that dimension. :return: A new pypika query with the dimensions and metrics included from the original query plus each of the metrics for each of the references. """ - original_query = query.as_('base') + metrics = self.metrics - def original_query_field(key): - return original_query.field(key).as_(key) - - outer_query = self.slicer.database.query_cls.from_(original_query) - - # Add dimensions - for dimension in self._dimensions: - outer_query = outer_query.select(original_query_field(dimension.key)) - - if dimension.has_display_field: - outer_query = outer_query.select(original_query_field(dimension.display_key)) - - # Add metrics - outer_query = outer_query.select(*[original_query_field(metric.key) - for metric in self.metrics]) - - # Build nested reference queries - for reference, dimension in references: - outer_query = join_reference(reference, - self.metrics, - self._dimensions, - dimension, - self.slicer.database.date_add, - original_query, - outer_query) - - return outer_query + original_query = query.as_('base') + container_query = create_container_query(original_query, + self.slicer.database.query_cls, + self._dimensions, + metrics) + + # join reference queries + for dimension, references in references_for_dimensions.items(): + get_unit_and_interval = lambda reference: (reference.time_unit, reference.interval) + grouped_references = groupby(references, get_unit_and_interval) + + for (time_unit, interval), references_by_interval in grouped_references.items(): + container_query, ref_query = create_joined_reference_query(time_unit, + interval, + self._dimensions, + dimension, + original_query, + container_query, + self.slicer.database) + + container_query = select_reference_metrics(references_by_interval, + container_query, + original_query, + ref_query, + metrics) + + return container_query def fetch(self, limit=None, offset=None) -> Iterable[Dict]: """ diff --git a/fireant/slicer/queries/references.py b/fireant/slicer/queries/references.py index e971a087..89c09963 100644 --- a/fireant/slicer/queries/references.py +++ b/fireant/slicer/queries/references.py @@ -1,8 +1,4 @@ from functools import partial -from typing import ( - Callable, - Iterator, -) from pypika import ( JoinType, @@ -15,60 +11,129 @@ Criterion, Term, ) +from typing import ( + Callable, + Iterable, +) from ..dimensions import ( DatetimeDimension, Dimension, ) from ..intervals import weekly -from ..metrics import Metric from ..references import ( Reference, YearOverYear, + reference_key, ) -def join_reference(reference: Reference, - metrics: Iterator[Metric], - dimensions: Iterator[Dimension], - ref_dimension: DatetimeDimension, - date_add: Callable, - original_query, - outer_query: QueryBuilder): - ref_query = original_query.as_(reference.key) +def create_container_query(original_query, query_cls, dimensions, metrics): + """ + Creates a container query with the original query used as the FROM clause and selects all of the metrics. The + container query is used as a base for joining reference queries. + + :param original_query: + :param query_cls: + :param dimensions: + :param metrics: + :return: + """ + + def original_query_field(key): + return original_query.field(key).as_(key) + + outer_query = query_cls.from_(original_query) + + # Add dimensions + for dimension in dimensions: + outer_query = outer_query.select(original_query_field(dimension.key)) + + if dimension.has_display_field: + outer_query = outer_query.select(original_query_field(dimension.display_key)) + + # Add base metrics + return outer_query.select(*[original_query_field(metric.key) + for metric in metrics]) + + +def create_joined_reference_query(time_unit: str, + interval: int, + dimensions: Iterable[Dimension], + ref_dimension: DatetimeDimension, + original_query, + container_query, + database): + ref_query = original_query.as_('{}_ref'.format(ref_dimension.key)) + + offset_func = partial(database.date_add, + date_part=time_unit, + interval=interval) - date_add = partial(date_add, - date_part=reference.time_unit, - interval=reference.interval, - # Only need to adjust this for YoY references with weekly intervals - align_weekday=weekly == ref_dimension.interval - and YearOverYear.time_unit == reference.time_unit) + _hack_fixes_into_the_query(database, interval, offset_func, ref_dimension, ref_query, time_unit) - # FIXME this is a bit hacky, need to replace the ref dimension term in all of the filters with the offset + # Join inner query + join_criterion = _create_reference_join_criterion(ref_dimension, + dimensions, + original_query, + ref_query, + offset_func) + container_query = container_query \ + .join(ref_query, JoinType.left) \ + .on(join_criterion) + + return container_query, ref_query + + +def _hack_fixes_into_the_query(database, interval, offset_func, ref_dimension, ref_query, time_unit): + # for weekly interval dimensions with YoY references, correct the day of week by adding a year, truncating to a + # week, then subtracting a year + offset_for_weekday = weekly == ref_dimension.interval and YearOverYear.time_unit == time_unit + if offset_for_weekday: + shift_forward = database.date_add(ref_dimension.definition, time_unit, interval) + offset = database.trunc_date(shift_forward, 'week') + shift_back = database.date_add(offset, time_unit, -interval) + + getattr(ref_query, '_selects')[0] = shift_back.as_(ref_dimension.key) + + # need to replace the ref dimension term in all of the filters with the offset query_wheres = getattr(ref_query, '_wheres') if query_wheres: wheres = _apply_to_term_in_criterion(ref_dimension.definition, - date_add(ref_dimension.definition), + offset_func(ref_dimension.definition), query_wheres) setattr(ref_query, '_wheres', wheres) - # Join inner query - join_criterion = _build_reference_join_criterion(ref_dimension, - dimensions, - original_query, - ref_query, - date_add) - outer_query = outer_query \ - .join(ref_query, JoinType.left) \ - .on(join_criterion) - # Add metrics - ref_metric = _reference_metric(reference, - original_query, - ref_query) +def select_reference_metrics(references, container_query, original_query, ref_query, metrics): + """ + Select the metrics for a list of references with a reference query. The list of references is expected to share + the same time unit and interval. - return outer_query.select(*[ref_metric(metric).as_("{}_{}".format(metric.key, reference.key)) - for metric in metrics]) + :param references: + :param container_query: + :param original_query: + :param ref_query: + :param metrics: + :return: + """ + seen = set() + for reference in references: + # Don't select duplicate references twice + if reference.key in seen: + continue + else: + seen.add(reference.key) + + # Get function to select reference metrics + ref_metric = _reference_metric(reference, + original_query, + ref_query) + + # Select metrics + container_query = container_query.select(*[ref_metric(metric).as_(reference_key(metric, reference)) + for metric in metrics]) + return container_query def _apply_to_term_in_criterion(target: Term, @@ -101,13 +166,13 @@ def _apply_to_term_in_criterion(target: Term, return criterion -def _build_reference_join_criterion(dimension: Dimension, - all_dimensions: Iterator[Dimension], - original_query: QueryBuilder, - ref_query: QueryBuilder, - offset_func: Callable): +def _create_reference_join_criterion(dimension: Dimension, + all_dimensions: Iterable[Dimension], + original_query: QueryBuilder, + ref_query: QueryBuilder, + offset_func: Callable): """ - This builds the criterion for joining a reference query to the base query. It matches the referenced dimension + This creates the criterion for joining a reference query to the base query. It matches the referenced dimension in the base query to the offset referenced dimension in the reference query and all other dimensions. :param dimension: @@ -121,7 +186,7 @@ def _build_reference_join_criterion(dimension: Dimension, :return: pypika.Criterion """ - join_criterion = original_query.field(dimension.key) == offset_func(field=ref_query.field(dimension.key)) + join_criterion = original_query.field(dimension.key) == offset_func(ref_query.field(dimension.key)) for dimension_ in all_dimensions: if dimension == dimension_: diff --git a/fireant/slicer/references.py b/fireant/slicer/references.py index dcbd8764..9c27ef80 100644 --- a/fireant/slicer/references.py +++ b/fireant/slicer/references.py @@ -21,9 +21,42 @@ def __eq__(self, other): and self.is_delta == other.is_delta \ and self.is_percent == other.is_percent + def __hash__(self): + return hash('reference' + self.key) + DayOverDay = Reference('dod', 'DoD', 'day', 1) WeekOverWeek = Reference('wow', 'WoW', 'week', 1) MonthOverMonth = Reference('mom', 'MoM', 'month', 1) QuarterOverQuarter = Reference('qoq', 'QoQ', 'quarter', 1) YearOverYear = Reference('yoy', 'YoY', 'year', 1) + + +def reference_key(metric, reference): + """ + Format a metric key for a reference. + + :return: + A string that is used as the key for a reference metric. + """ + key = metric.key + + if reference is not None: + return '{}_{}'.format(key, reference.key) + + return key + + +def reference_label(metric, reference): + """ + Format a metric label for a reference. + + :return: + A string that is used as the display value for a reference metric. + """ + label = metric.label or metric.key + + if reference is not None: + return '{} ({})'.format(label, reference.label) + + return label diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index bd05d9d6..cc811c4e 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -14,6 +14,8 @@ from .helpers import ( dimensional_metric_label, extract_display_values, +) +from ..references import ( reference_key, reference_label, ) diff --git a/fireant/slicer/widgets/helpers.py b/fireant/slicer/widgets/helpers.py index 15e391be..c7cf2d0b 100644 --- a/fireant/slicer/widgets/helpers.py +++ b/fireant/slicer/widgets/helpers.py @@ -1,6 +1,7 @@ import pandas as pd from fireant import utils +from ..references import reference_label def extract_display_values(dimensions, data_frame): @@ -35,36 +36,6 @@ def extract_display_values(dimensions, data_frame): return display_values -def reference_key(metric, reference): - """ - Format a metric key for a reference. - - :return: - A string that is used as the key for a reference metric. - """ - key = metric.key - - if reference is not None: - return '{}_{}'.format(key, reference.key) - - return key - - -def reference_label(metric, reference): - """ - Format a metric label for a reference. - - :return: - A string that is used as the display value for a reference metric. - """ - label = metric.label or metric.key - - if reference is not None: - return '{} ({})'.format(label, reference.label) - - return label - - def dimensional_metric_label(dimensions, dimension_display_values): """ Creates a callback function for rendering series labels. diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 6adc3974..a67b53f4 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -20,9 +20,9 @@ from .helpers import ( dimensional_metric_label, extract_display_values, - reference_key, ) from ..exceptions import MetricRequiredException +from ..references import reference_key DEFAULT_COLORS = ( "#DDDF0D", @@ -39,16 +39,16 @@ DASH_STYLES = ( 'Solid', - 'ShortDash', - 'ShortDot', - 'ShortDashDot', - 'ShortDashDotDot', - 'Dot', 'Dash', - 'LongDash', + 'Dot', 'DashDot', + 'LongDash', 'LongDashDot', + 'ShortDash', + 'ShortDashDot', 'LongDashDotDot', + 'ShortDashDotDot', + 'ShortDot', ) MARKER_SYMBOLS = ( diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index 2709923a..cdcd088e 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -3,7 +3,7 @@ from .base import ( TransformableWidget, ) -from .helpers import ( +from ..references import ( reference_key, reference_label, ) diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 3ac99fd7..88fb14a5 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -1,4 +1,3 @@ -from datetime import date from unittest import TestCase from unittest.mock import ( ANY, @@ -6,12 +5,12 @@ patch, ) +from datetime import date from pypika import Order import fireant as f from fireant.slicer.exceptions import ( MetricRequiredException, - RollupException, ) from ..matchers import ( DimensionMatcher, @@ -1267,14 +1266,13 @@ def test_adapt_dow_for_leap_year_for_yoy_reference(self): 'LEFT JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "timestamp",' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' ') "sq1" ' # end-nested - 'ON "base"."timestamp"=' - 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"sq1"."timestamp"),\'IW\')) ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): @@ -1302,16 +1300,16 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): 'LEFT JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "timestamp",' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' ') "sq1" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',-1,' - 'TRUNC(TIMESTAMPADD(\'year\',1,"sq1"."timestamp"),\'IW\')) ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' 'ORDER BY "timestamp"', str(query)) + def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): query = slicer.data \ .widget(f.HighCharts( @@ -1337,14 +1335,173 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): 'LEFT JOIN (' # nested 'SELECT ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', str(query)) + + def test_adapt_dow_for_leap_year_for_yoy_reference_with_date_filter(self): + query = slicer.data \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + [slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp(f.weekly) + .reference(f.YearOverYear)) \ + .filter(slicer.dimensions.timestamp.between(date(2018, 1, 1), date(2018, 1, 31))) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_yoy" ' + 'FROM ' + + '(' # nested + 'SELECT ' 'TRUNC("timestamp",\'IW\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE TIMESTAMPADD(\'year\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' 'GROUP BY "timestamp"' ') "sq1" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',-1,' - 'TRUNC(TIMESTAMPADD(\'year\',1,"sq1"."timestamp"),\'IW\')) ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', str(query)) + + def test_adding_duplicate_reference_does_not_join_more_queries(self): + query = slicer.data \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + [slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.DayOverDay) + .reference(f.DayOverDay)) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', str(query)) + + def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension(self): + query = slicer.data \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + [slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.DayOverDay) + .reference(f.DayOverDay.delta()) + .reference(f.DayOverDay.delta(percent=True))) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_dod",' + '"base"."votes"-"sq1"."votes" "votes_dod_delta",' + '("base"."votes"-"sq1"."votes")*100/NULLIF("sq1"."votes",0) "votes_dod_delta_percent" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ORDER BY "timestamp"', str(query)) + + def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension_with_different_periods(self): + query = slicer.data \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + [slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.timestamp + .reference(f.DayOverDay) + .reference(f.DayOverDay.delta()) + .reference(f.YearOverYear) + .reference(f.YearOverYear.delta())) \ + .query + + self.assertEqual('SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."votes" "votes",' + '"sq1"."votes" "votes_dod",' + '"base"."votes"-"sq1"."votes" "votes_dod_delta",' + '"sq2"."votes" "votes_yoy",' + '"base"."votes"-"sq2"."votes" "votes_yoy_delta" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq1" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + + 'LEFT JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "sq2" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq2"."timestamp") ' 'ORDER BY "timestamp"', str(query)) diff --git a/fireant/tests/slicer/widgets/test_highcharts.py b/fireant/tests/slicer/widgets/test_highcharts.py index 31ea6e9c..bd529f64 100644 --- a/fireant/tests/slicer/widgets/test_highcharts.py +++ b/fireant/tests/slicer/widgets/test_highcharts.py @@ -1,6 +1,8 @@ import copy -import json -from unittest import TestCase +from unittest import ( + TestCase, + skip, +) from fireant import CumSum from fireant.slicer.widgets.highcharts import HighCharts @@ -20,7 +22,7 @@ ) -class HighchartsLineChartTransformerTests(TestCase): +class HighChartsLineChartTransformerTests(TestCase): maxDiff = None chart_class = HighCharts.LineChart @@ -71,8 +73,6 @@ def test_metric_prefix_line_chart(self): axes=[self.chart_class([votes])]) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) - print(json.dumps(result)) - self.assertEqual({ "title": {"text": "Time Series, Single Metric"}, "xAxis": {"type": "datetime"}, @@ -799,7 +799,7 @@ def test_uni_dim_with_ref_line_chart(self): }, "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, - "dashStyle": "ShortDash", + "dashStyle": "Dash", "stacking": self.stacking, }, { "type": self.chart_type, @@ -835,7 +835,7 @@ def test_uni_dim_with_ref_line_chart(self): }, "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, - "dashStyle": "ShortDash", + "dashStyle": "Dash", "stacking": self.stacking, }] }, result) @@ -897,7 +897,7 @@ def test_uni_dim_with_ref_delta_line_chart(self): }, "color": "#DDDF0D", "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, - "dashStyle": "ShortDash", + "dashStyle": "Dash", "stacking": self.stacking, }, { "type": self.chart_type, @@ -933,13 +933,13 @@ def test_uni_dim_with_ref_delta_line_chart(self): }, "color": "#55BF3B", "marker": {"symbol": "square", "fillColor": "#55BF3B"}, - "dashStyle": "ShortDash", + "dashStyle": "Dash", "stacking": self.stacking, }] }, result) -class HighchartsBarChartTransformerTests(TestCase): +class HighChartsBarChartTransformerTests(TestCase): maxDiff = None chart_class = HighCharts.BarChart @@ -1373,12 +1373,12 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): }, result) -class HighchartsColumnChartTransformerTests(HighchartsBarChartTransformerTests): +class HighChartsColumnChartTransformerTests(HighChartsBarChartTransformerTests): chart_class = HighCharts.ColumnChart chart_type = 'column' -class HighchartsStackedBarChartTransformerTests(HighchartsBarChartTransformerTests): +class HighChartsStackedBarChartTransformerTests(HighChartsBarChartTransformerTests): maxDiff = None chart_class = HighCharts.StackedBarChart @@ -1386,23 +1386,450 @@ class HighchartsStackedBarChartTransformerTests(HighchartsBarChartTransformerTes stacking = "normal" -class HighchartsStackedColumnChartTransformerTests(HighchartsBarChartTransformerTests): +class HighChartsStackedColumnChartTransformerTests(HighChartsBarChartTransformerTests): chart_class = HighCharts.StackedColumnChart chart_type = 'column' stacking = "normal" -class HighchartsAreaChartTransformerTests(HighchartsLineChartTransformerTests): +class HighChartsAreaChartTransformerTests(HighChartsLineChartTransformerTests): chart_class = HighCharts.AreaChart chart_type = 'area' -class HighchartsAreaPercentChartTransformerTests(HighchartsLineChartTransformerTests): +class HighChartsAreaPercentChartTransformerTests(HighChartsLineChartTransformerTests): chart_class = HighCharts.AreaPercentageChart chart_type = 'area' stacking = "normal" -class HighchartsPieChartTransformerTests(TestCase): +class HighChartsPieChartTransformerTests(TestCase): + maxDiff = None + + chart_class = HighCharts.PieChart + chart_type = 'pie' + + @skip('this test has the correct assertions but is failing currently') def test_single_metric_pie_chart(self): - pass + result = HighCharts(title="All Votes", + axes=[self.chart_class([slicer.metrics.votes])]) \ + .transform(single_metric_df, slicer, []) + + self.assertEqual({ + "title": {"text": "All Votes"}, + "chart": {"type": "pie"}, + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "name": "Votes", + "data": [{ + "name": "All", + "y": 111674336, + }], + 'tooltip': { + 'valueDecimals': None, + 'valuePrefix': None, + 'valueSuffix': None + }, + "color": "#DDDF0D", + "marker": {}, + "stacking": None, + }] + }, result) + + @skip + def test_multi_metric_bar_chart(self): + result = HighCharts(title="Votes and Wins", + axes=[self.chart_class([slicer.metrics.votes, + slicer.metrics.wins])]) \ + .transform(multi_metric_df, slicer, []) + + self.assertEqual({ + "title": {"text": "Votes and Wins"}, + "xAxis": { + "type": "category", + "categories": ["All"] + }, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": "#DDDF0D"}} + + }], + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes", + "yAxis": "0", + "data": [111674336], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }, { + "type": self.chart_type, + "name": "Wins", + "yAxis": "0", + "data": [12], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }] + }, result) + + @skip + def test_cat_dim_single_metric_bar_chart(self): + result = HighCharts(title="Votes and Wins", + axes=[self.chart_class([slicer.metrics.votes])]) \ + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + + self.assertEqual({ + "title": {"text": "Votes and Wins"}, + "xAxis": { + "type": "category", + "categories": ["Democrat", "Independent", "Republican"] + }, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}} + + }], + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes", + "yAxis": "0", + "data": [54551568, 1076384, 56046384], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }] + }, result) + + @skip + def test_cat_dim_multi_metric_bar_chart(self): + result = HighCharts(title="Votes and Wins", + axes=[self.chart_class([slicer.metrics.votes, + slicer.metrics.wins])]) \ + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + + self.assertEqual({ + "title": {"text": "Votes and Wins"}, + "xAxis": { + "type": "category", + "categories": ["Democrat", "Independent", "Republican"] + }, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": "#DDDF0D"}} + + }], + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes", + "yAxis": "0", + "data": [54551568, 1076384, 56046384], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }, { + "type": self.chart_type, + "name": "Wins", + "yAxis": "0", + "data": [6, 0, 6], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }] + }, result) + + @skip + def test_cont_uni_dims_single_metric_bar_chart(self): + result = HighCharts(title="Election Votes by State", + axes=[self.chart_class([slicer.metrics.votes])]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + + self.assertEqual({ + "title": {"text": "Election Votes by State"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}} + + }], + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes (Texas)", + "yAxis": "0", + "data": [(820454400000, 5574387), + (946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }, { + "type": self.chart_type, + "name": "Votes (California)", + "yAxis": "0", + "data": [(820454400000, 9646062), + (946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }] + }, result) + + @skip + def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): + result = HighCharts(title="Election Votes by State", + axes=[self.chart_class([slicer.metrics.votes, + slicer.metrics.wins]), ]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + + self.assertEqual({ + "title": {"text": "Election Votes by State"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": "#DDDF0D"}} + + }], + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes (Texas)", + "yAxis": "0", + "data": [(820454400000, 5574387), + (946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }, { + "type": self.chart_type, + "name": "Votes (California)", + "yAxis": "0", + "data": [(820454400000, 9646062), + (946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }, { + "type": self.chart_type, + "name": "Wins (Texas)", + "yAxis": "0", + "data": [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }, { + "type": self.chart_type, + "name": "Wins (California)", + "yAxis": "0", + "data": [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }] + }, result) + + @skip + def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): + result = HighCharts(title="Election Votes by State", + axes=[self.chart_class([slicer.metrics.votes]), + self.chart_class([slicer.metrics.wins]), ]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + + self.assertEqual({ + "title": {"text": "Election Votes by State"}, + "xAxis": {"type": "datetime"}, + "yAxis": [{ + "id": "1", + "title": {"text": None}, + "labels": {"style": {"color": "#55BF3B"}} + }, { + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": "#DDDF0D"}} + + }], + "tooltip": {"shared": True, "useHTML": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes (Texas)", + "yAxis": "0", + "data": [(820454400000, 5574387), + (946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }, { + "type": self.chart_type, + "name": "Votes (California)", + "yAxis": "0", + "data": [(820454400000, 9646062), + (946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }, { + "type": self.chart_type, + "name": "Wins (Texas)", + "yAxis": "1", + "data": [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#55BF3B", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }, { + "type": self.chart_type, + "name": "Wins (California)", + "yAxis": "1", + "data": [(820454400000, 1), + (946684800000, 1), + (1072915200000, 1), + (1199145600000, 1), + (1325376000000, 1), + (1451606400000, 1)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#DF5353", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }] + }, result) diff --git a/fireant/utils.py b/fireant/utils.py index 6f8bcf3c..ecc662cb 100644 --- a/fireant/utils.py +++ b/fireant/utils.py @@ -1,3 +1,4 @@ +from collections import OrderedDict def wrap_list(value): @@ -88,6 +89,27 @@ def ordered_distinct_list_by_attr(l, attr='key'): and not seen.add(getattr(x, attr))] +def groupby(items, by): + """ + Group items using a function to derive a key. + :param items: The items to group + :param by: A lambda function to create a key based on the item + :return: + an Ordered dict + """ + + result = OrderedDict() + for item in items: + key = by(item) + + if key in result: + result[key].append(item) + else: + result[key] = [item] + + return result + + def groupby_first_level(index): seen = set() return [x[1:] From 8fc159921d017ea909dc095862f56306f415165b Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 9 Feb 2018 14:06:09 +0100 Subject: [PATCH 019/123] added fucking pie charts --- fireant/slicer/widgets/datatables.py | 2 +- fireant/slicer/widgets/helpers.py | 9 +- fireant/slicer/widgets/highcharts.py | 32 +++++- .../tests/slicer/widgets/test_highcharts.py | 108 ++++++++---------- 4 files changed, 85 insertions(+), 66 deletions(-) diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index cc811c4e..f5dc6e66 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -215,7 +215,7 @@ def _metric_columns_pivoted(self, references, df_columns, render_column_label): for dimension_values in itertools.product(*dimension_value_sets): for reference in [None] + references: key = reference_key(metric, reference) - title = render_column_label(metric, reference, dimension_values) + title = render_column_label(dimension_values, metric, reference) data = '.'.join([key] + [str(x) for x in dimension_values]) columns.append(dict(title=title, diff --git a/fireant/slicer/widgets/helpers.py b/fireant/slicer/widgets/helpers.py index c7cf2d0b..bfd00b24 100644 --- a/fireant/slicer/widgets/helpers.py +++ b/fireant/slicer/widgets/helpers.py @@ -49,7 +49,7 @@ def dimensional_metric_label(dimensions, dimension_display_values): a callback function which renders a label for a metric, reference, and list of dimension values. """ - def render_series_label(metric, reference, dimension_values): + def render_series_label(dimension_values, metric=None, reference=None): """ Returns a string label for a metric, reference, and set of values for zero or more dimensions. @@ -61,12 +61,17 @@ def render_series_label(metric, reference, dimension_values): a tuple of dimension values. Can be zero-length or longer. :return: """ + used_dimensions = dimensions if metric is None else dimensions[1:] + dimension_values = utils.wrap_list(dimension_values) dimension_labels = [utils.deep_get(dimension_display_values, [dimension.key, dimension_value], dimension_value) if not pd.isnull(dimension_value) else 'Totals' - for dimension, dimension_value in zip(dimensions[1:], dimension_values)] + for dimension, dimension_value in zip(used_dimensions, dimension_values)] + + if metric is None: + return ", ".join(dimension_labels) if dimension_labels: return '{} ({})'.format(reference_label(metric, reference), diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index a67b53f4..2ee6cd95 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -22,7 +22,10 @@ extract_display_values, ) from ..exceptions import MetricRequiredException -from ..references import reference_key +from ..references import ( + reference_key, + reference_label, +) DEFAULT_COLORS = ( "#DDDF0D", @@ -181,6 +184,13 @@ def group_series(keys): colors, series_colors = itertools.tee(colors) axis_color = next(colors) if 1 < total_num_items else None + if isinstance(axis, self.PieChart): + # pie charts suck + for metric in axis.metrics: + for reference in [None] + references: + series += [self._render_pie_series(axis, metric, reference, data_frame, render_series_label)] + continue + # prepend axes, append series, this keeps everything ordered left-to-right y_axes[0:0] = self._render_y_axis(axis_idx, axis_color, @@ -303,7 +313,7 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, "color": series_color, "dashStyle": dash_style, - "name": render_series_label(metric, reference, dimension_values), + "name": render_series_label(dimension_values, metric, reference), "data": self._render_data(group_df, metric_key, is_timeseries), @@ -324,6 +334,24 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, return series + def _render_pie_series(self, axis, metric, reference, data_frame, render_series_label): + pie_chart_df = data_frame[reference_key(metric, reference)] + name = reference_label(metric, reference) + return { + "name": name, + "type": axis.type, + "data": [{ + "name": render_series_label(dimension_values) if dimension_values else name, + "y": metric_value(y), + "color": color, + } for (dimension_values, y), color in zip(pie_chart_df.iteritems(), self.colors)], + 'tooltip': { + 'valueDecimals': metric.precision, + 'valuePrefix': metric.prefix, + 'valueSuffix': metric.suffix, + }, + } + def _render_data(self, group_df, metric_key, is_timeseries): if not is_timeseries: return [metric_value(y) for y in group_df[metric_key].values] diff --git a/fireant/tests/slicer/widgets/test_highcharts.py b/fireant/tests/slicer/widgets/test_highcharts.py index bd529f64..5227db2f 100644 --- a/fireant/tests/slicer/widgets/test_highcharts.py +++ b/fireant/tests/slicer/widgets/test_highcharts.py @@ -1409,7 +1409,6 @@ class HighChartsPieChartTransformerTests(TestCase): chart_class = HighCharts.PieChart chart_type = 'pie' - @skip('this test has the correct assertions but is failing currently') def test_single_metric_pie_chart(self): result = HighCharts(title="All Votes", axes=[self.chart_class([slicer.metrics.votes])]) \ @@ -1417,27 +1416,26 @@ def test_single_metric_pie_chart(self): self.assertEqual({ "title": {"text": "All Votes"}, - "chart": {"type": "pie"}, "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ "name": "Votes", + "type": "pie", "data": [{ - "name": "All", + "name": "Votes", "y": 111674336, + "color": "#DDDF0D", }], 'tooltip': { 'valueDecimals': None, 'valuePrefix': None, 'valueSuffix': None }, - "color": "#DDDF0D", - "marker": {}, - "stacking": None, - }] + }], + 'xAxis': {'categories': ['All'], 'type': 'category'}, + 'yAxis': [], }, result) - @skip def test_multi_metric_bar_chart(self): result = HighCharts(title="Votes and Wins", axes=[self.chart_class([slicer.metrics.votes, @@ -1446,84 +1444,72 @@ def test_multi_metric_bar_chart(self): self.assertEqual({ "title": {"text": "Votes and Wins"}, - "xAxis": { - "type": "category", - "categories": ["All"] - }, - "yAxis": [{ - "id": "0", - "title": {"text": None}, - "labels": {"style": {"color": "#DDDF0D"}} - - }], "tooltip": {"shared": True, "useHTML": True}, "legend": {"useHTML": True}, "series": [{ - "type": self.chart_type, "name": "Votes", - "yAxis": "0", - "data": [111674336], + "type": "pie", + "data": [{ + "name": "Votes", + "y": 111674336, + "color": "#DDDF0D", + }], 'tooltip': { - 'valuePrefix': None, - 'valueSuffix': None, 'valueDecimals': None, + 'valuePrefix': None, + 'valueSuffix': None }, - "color": "#DDDF0D", - "dashStyle": "Solid", - "marker": {}, - "stacking": self.stacking, }, { - "type": self.chart_type, "name": "Wins", - "yAxis": "0", - "data": [12], + "type": "pie", + "data": [{ + "name": "Wins", + "y": 12, + "color": "#DDDF0D", + }], 'tooltip': { - 'valuePrefix': None, - 'valueSuffix': None, 'valueDecimals': None, + 'valuePrefix': None, + 'valueSuffix': None }, - "color": "#55BF3B", - "dashStyle": "Solid", - "marker": {}, - "stacking": self.stacking, - }] + }], + 'xAxis': {'categories': ['All'], 'type': 'category'}, + 'yAxis': [], }, result) - @skip def test_cat_dim_single_metric_bar_chart(self): result = HighCharts(title="Votes and Wins", axes=[self.chart_class([slicer.metrics.votes])]) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) self.assertEqual({ - "title": {"text": "Votes and Wins"}, - "xAxis": { - "type": "category", - "categories": ["Democrat", "Independent", "Republican"] - }, - "yAxis": [{ - "id": "0", - "title": {"text": None}, - "labels": {"style": {"color": None}} - - }], - "tooltip": {"shared": True, "useHTML": True}, - "legend": {"useHTML": True}, - "series": [{ - "type": self.chart_type, - "name": "Votes", - "yAxis": "0", - "data": [54551568, 1076384, 56046384], + 'title': {'text': 'Votes and Wins'}, + 'tooltip': {'useHTML': True, 'shared': True}, + 'legend': {'useHTML': True}, + 'series': [{ + 'name': 'Votes', + 'type': 'pie', + 'data': [{ + 'y': 54551568, + 'name': 'Democrat', + 'color': '#DDDF0D' + }, { + 'y': 1076384, + 'name': 'Independent', + 'color': '#55BF3B' + }, { + 'y': 56046384, + 'name': 'Republican', + 'color': '#DF5353' + }], 'tooltip': { 'valuePrefix': None, 'valueSuffix': None, 'valueDecimals': None, }, - "color": "#DDDF0D", - "dashStyle": "Solid", - "marker": {}, - "stacking": self.stacking, - }] + }], + 'yAxis': [], + 'xAxis': {'categories': ['Democrat', 'Independent', 'Republican'], 'type': 'category'} }, result) @skip From 06ba9876f8f81d404eb0e66ee0935280a0179bc6 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 9 Feb 2018 14:06:35 +0100 Subject: [PATCH 020/123] bumped dev version to 4 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index e2784380..b6fcaa6d 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev3' +__version__ = '1.0.0.dev4' From da1ee228fa2291eb1d933711623b87225129d87b Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 9 Feb 2018 14:34:05 +0100 Subject: [PATCH 021/123] fixed the responses to format infinity --- fireant/__init__.py | 2 +- fireant/slicer/widgets/formats.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index b6fcaa6d..6fbb8f3c 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev4' +__version__ = '1.0.0.dev5' diff --git a/fireant/slicer/widgets/formats.py b/fireant/slicer/widgets/formats.py index c17f3e08..67dad91e 100644 --- a/fireant/slicer/widgets/formats.py +++ b/fireant/slicer/widgets/formats.py @@ -8,6 +8,8 @@ time, ) +INFINITY = "Infinity" + NO_TIME = time(0) epoch = np.datetime64(datetime.utcfromtimestamp(0)) @@ -68,16 +70,18 @@ def metric_value(value): else: return value.strftime('%Y-%m-%dT%H:%M:%S') - if value is None or (isinstance(value, float) and np.isnan(value)) or pd.isnull(value): + if value is None or pd.isnull(value): return None + if isinstance(value, float): + if np.isinf(value): + return INFINITY + return float(value) + if isinstance(value, np.int64): # Cannot transform np.int64 to json return int(value) - if isinstance(value, np.float64): - return float(value) - return value @@ -114,6 +118,9 @@ def metric_display(value, prefix=None, suffix=None, precision=None): float_format = '%d' value = locale.format(float_format, value, grouping=True) + if value is INFINITY: + return "∞" + return '{prefix}{value}{suffix}'.format( prefix=prefix or '', value=str(value), From 5c07ff7ad9404a82007b79a910881c61ee1e5a0f Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 12 Feb 2018 13:39:57 +0100 Subject: [PATCH 022/123] Fixed an issue when querying display options for a unique dimension with no display field --- fireant/slicer/queries/builder.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index c338c425..ab9ad497 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -414,9 +414,12 @@ def fetch(self, limit=None, offset=None, force_include=()) -> pd.Series: str(query), dimensions=self._dimensions) - display_key = getattr(dimension, 'display_key', 'display') + display_key = getattr(dimension, 'display_key') or 'display' if hasattr(dimension, 'display_values'): # Include provided display values data[display_key] = pd.Series(dimension.display_values) + elif getattr(dimension, 'display_key') is None: + data[display_key] = data.index.tolist() + return data[display_key] From addc42abf829d64ae6698ae8eac3737de27ef9e1 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 12 Feb 2018 13:54:45 +0100 Subject: [PATCH 023/123] Bumped version to 1.0.0.dev6 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 6fbb8f3c..843c365b 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev5' +__version__ = '1.0.0.dev6' From ec3f0488aadb68d39439d8c6fa85e70909b1d14f Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 12 Feb 2018 14:14:58 +0100 Subject: [PATCH 024/123] Fixed the display options query to work for categorical dimensions again. --- fireant/slicer/queries/builder.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index ab9ad497..fdecfa83 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -414,12 +414,13 @@ def fetch(self, limit=None, offset=None, force_include=()) -> pd.Series: str(query), dimensions=self._dimensions) - display_key = getattr(dimension, 'display_key') or 'display' + display_key = getattr(dimension, 'display_key', None) + if display_key is not None: + data[display_key] = data.index.tolist() + + display_key = display_key or 'display' if hasattr(dimension, 'display_values'): # Include provided display values - data[display_key] = pd.Series(dimension.display_values) - - elif getattr(dimension, 'display_key') is None: - data[display_key] = data.index.tolist() + data['display'] = pd.Series(dimension.display_values) return data[display_key] From 0a9b19b99e8c5df6a080fcacd763d85393f2b37b Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 12 Feb 2018 16:00:29 +0100 Subject: [PATCH 025/123] Bumped version to 1.0.0.dev7 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 843c365b..5a456ab7 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev6' +__version__ = '1.0.0.dev7' From 91ee1afcada93cd663146a1306c7dabb465d0c61 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 12 Feb 2018 16:35:12 +0100 Subject: [PATCH 026/123] Fixed area percentage charts to using percent stacking --- fireant/slicer/widgets/highcharts.py | 16 +++++++--------- fireant/tests/slicer/widgets/test_highcharts.py | 9 ++++----- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 2ee6cd95..1a351863 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -66,12 +66,12 @@ class ChartWidget(Widget): type = None needs_marker = False - stacked = False + stacking = None - def __init__(self, items=(), name=None, stacked=False): + def __init__(self, items=(), name=None, stacking=None): super(ChartWidget, self).__init__(items) self.name = name - self.stacked = self.stacked or stacked + self.stacking = self.stacking or stacking class ContinuousAxisChartWidget(ChartWidget): @@ -88,7 +88,7 @@ class AreaChart(ContinuousAxisChartWidget): needs_marker = True class AreaPercentageChart(AreaChart): - stacked = True + stacking = "percent" class PieChart(ChartWidget): type = 'pie' @@ -97,13 +97,13 @@ class BarChart(ChartWidget): type = 'bar' class StackedBarChart(BarChart): - stacked = True + stacking = "normal" class ColumnChart(ChartWidget): type = 'column' class StackedColumnChart(ColumnChart): - stacked = True + stacking = "normal" def __init__(self, axes=(), title=None, colors=None): super(HighCharts, self).__init__(axes) @@ -327,9 +327,7 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, if axis.needs_marker else {}), - "stacking": ("normal" - if axis.stacked - else None), + "stacking": axis.stacking, }) return series diff --git a/fireant/tests/slicer/widgets/test_highcharts.py b/fireant/tests/slicer/widgets/test_highcharts.py index 5227db2f..b0e3ac62 100644 --- a/fireant/tests/slicer/widgets/test_highcharts.py +++ b/fireant/tests/slicer/widgets/test_highcharts.py @@ -1383,13 +1383,13 @@ class HighChartsStackedBarChartTransformerTests(HighChartsBarChartTransformerTes chart_class = HighCharts.StackedBarChart chart_type = 'bar' - stacking = "normal" + stacking = 'normal' class HighChartsStackedColumnChartTransformerTests(HighChartsBarChartTransformerTests): chart_class = HighCharts.StackedColumnChart chart_type = 'column' - stacking = "normal" + stacking = 'normal' class HighChartsAreaChartTransformerTests(HighChartsLineChartTransformerTests): @@ -1397,10 +1397,9 @@ class HighChartsAreaChartTransformerTests(HighChartsLineChartTransformerTests): chart_type = 'area' -class HighChartsAreaPercentChartTransformerTests(HighChartsLineChartTransformerTests): +class HighChartsAreaPercentChartTransformerTests(HighChartsAreaChartTransformerTests): chart_class = HighCharts.AreaPercentageChart - chart_type = 'area' - stacking = "normal" + stacking = 'percent' class HighChartsPieChartTransformerTests(TestCase): From 23a365e4c68fe7cae09731060a6a847bbd1201a8 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 12 Feb 2018 18:17:18 +0100 Subject: [PATCH 027/123] Fixed the even worse bug caused by the last commit that changed how display values are handled --- fireant/__init__.py | 2 +- fireant/slicer/queries/builder.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 5a456ab7..f606d9ba 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev7' +__version__ = '1.0.0.dev8' diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index fdecfa83..4e670426 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -416,11 +416,13 @@ def fetch(self, limit=None, offset=None, force_include=()) -> pd.Series: display_key = getattr(dimension, 'display_key', None) if display_key is not None: - data[display_key] = data.index.tolist() + return data[display_key] - display_key = display_key or 'display' + display_key = 'display' if hasattr(dimension, 'display_values'): # Include provided display values - data['display'] = pd.Series(dimension.display_values) + data[display_key] = pd.Series(dimension.display_values) + else: + data[display_key] = data.index.tolist() return data[display_key] From d24bc652df24452307f432d66d7d784b0376b3b0 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 12 Feb 2018 18:18:40 +0100 Subject: [PATCH 028/123] Removed unused import in vertica --- fireant/database/vertica.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/fireant/database/vertica.py b/fireant/database/vertica.py index 9a45fd2c..9c8f15ae 100644 --- a/fireant/database/vertica.py +++ b/fireant/database/vertica.py @@ -4,9 +4,6 @@ terms, ) -from fireant.slicer import ( - weekly, -) from .base import Database From 255fba7b4bc8e8745ae54ca01e3de4c122238d12 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 14 Feb 2018 19:42:24 +0100 Subject: [PATCH 029/123] Fixed a bug where dimension values were not cast to string and bumped version to 1.0.0.dev9 --- fireant/__init__.py | 2 +- fireant/slicer/queries/database.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index f606d9ba..b4d67d36 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev8' +__version__ = '1.0.0.dev9' diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index 84deebb2..bcb100d7 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -1,7 +1,7 @@ import time +from typing import Iterable import pandas as pd -from typing import Iterable from fireant.database.base import Database from .logger import logger @@ -58,7 +58,8 @@ def clean_and_apply_index(data_frame: pd.DataFrame, dimensions: Iterable[Dimensi continue level = dimension.key - data_frame[level] = fill_nans_in_level(data_frame, dimension, dimension_keys[:i]) + data_frame[level] = fill_nans_in_level(data_frame, dimension, dimension_keys[:i]) \ + .apply(lambda x: str(x) if not pd.isnull(x) else None) # Set index on dimension columns return data_frame.set_index(dimension_keys) From daa91455e01e04d92823895d695f58dec57983cf Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 15 Feb 2018 14:55:05 +0100 Subject: [PATCH 030/123] Fixed an issue with toposort where it would try to sort joins when they have equal dependencies --- fireant/slicer/joins.py | 8 +++++ fireant/tests/slicer/mocks.py | 5 ++- fireant/tests/slicer/queries/test_builder.py | 38 +++++++++++++++----- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/fireant/slicer/joins.py b/fireant/slicer/joins.py index 6b463635..c4a18645 100644 --- a/fireant/slicer/joins.py +++ b/fireant/slicer/joins.py @@ -10,3 +10,11 @@ def __init__(self, table, criterion, join_type=JoinType.inner): self.table = table self.criterion = criterion self.join_type = join_type + + def __repr__(self): + return '{type} JOIN {table} ON {criterion}'.format(type=self.join_type, + table=self.table, + criterion=self.criterion) + + def __gt__(self, other): + return self.table.table_name < other.table.table_name \ No newline at end of file diff --git a/fireant/tests/slicer/mocks.py b/fireant/tests/slicer/mocks.py index 4d7bfc3b..774fdae0 100644 --- a/fireant/tests/slicer/mocks.py +++ b/fireant/tests/slicer/mocks.py @@ -39,13 +39,16 @@ def __eq__(self, other): database=test_database, joins=( + Join(table=district_table, + criterion=politicians_table.district_id == district_table.id, + join_type=JoinType.outer), Join(table=district_table, criterion=politicians_table.district_id == district_table.id, join_type=JoinType.outer), Join(table=state_table, criterion=district_table.state_id == state_table.id), Join(table=voters_table, - criterion=district_table.id == voters_table.district_id), + criterion=politicians_table.id == voters_table.politician_id), Join(table=deep_join_table, criterion=deep_join_table.id == state_table.ref_id), ), diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 88fb14a5..25bb143b 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -1,3 +1,4 @@ +from datetime import date from unittest import TestCase from unittest.mock import ( ANY, @@ -5,7 +6,6 @@ patch, ) -from datetime import date from pypika import Order import fireant as f @@ -43,6 +43,7 @@ def test_orderby_is_immutable(self): self.assertIsNot(query1, query2) + # noinspection SqlDialectInspection,SqlNoDataSourceInspection class QueryBuilderMetricTests(TestCase): maxDiff = None @@ -1527,6 +1528,28 @@ def test_dimension_with_join_includes_join_in_query(self): 'GROUP BY "timestamp","district","district_display" ' 'ORDER BY "timestamp","district_display"', str(query)) + def test_dimension_with_multiple_joins_includes_joins_ordered__in_query(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes, + slicer.metrics.voters])) \ + .dimension(slicer.dimensions.timestamp) \ + .dimension(slicer.dimensions.district) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("politician"."timestamp",\'DD\') "timestamp",' + '"politician"."district_id" "district",' + '"district"."district_name" "district_display",' + 'SUM("politician"."votes") "votes",' + 'COUNT("voter"."id") "voters" ' + 'FROM "politics"."politician" ' + 'JOIN "politics"."voter" ' + 'ON "politician"."id"="voter"."politician_id" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'GROUP BY "timestamp","district","district_display" ' + 'ORDER BY "timestamp","district_display"', str(query)) + def test_dimension_with_recursive_join_joins_all_join_tables(self): query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ @@ -1550,20 +1573,17 @@ def test_dimension_with_recursive_join_joins_all_join_tables(self): def test_metric_with_join_includes_join_in_query(self): query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.voters])) \ - .dimension(slicer.dimensions.district) \ + .dimension(slicer.dimensions.political_party) \ .query self.assertEqual('SELECT ' - '"politician"."district_id" "district",' - '"district"."district_name" "district_display",' + '"politician"."political_party" "political_party",' 'COUNT("voter"."id") "voters" ' 'FROM "politics"."politician" ' - 'OUTER JOIN "locations"."district" ' - 'ON "politician"."district_id"="district"."id" ' 'JOIN "politics"."voter" ' - 'ON "district"."id"="voter"."district_id" ' - 'GROUP BY "district","district_display" ' - 'ORDER BY "district_display"', str(query)) + 'ON "politician"."id"="voter"."politician_id" ' + 'GROUP BY "political_party" ' + 'ORDER BY "political_party"', str(query)) def test_dimension_filter_with_join_on_display_definition_does_not_include_join_in_query(self): query = slicer.data \ From bb3a2f7ca390f27f204afc564125218caf3a3c2d Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 15 Feb 2018 14:55:28 +0100 Subject: [PATCH 031/123] Incremented version to 1.0.0.dev10 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index b4d67d36..54641e3b 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev9' +__version__ = '1.0.0.dev10' From fb073d9b391c1e3841823ee1b40d349d1741bef2 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 16 Feb 2018 10:50:19 +0100 Subject: [PATCH 032/123] Fixed the way the data frame is formatted given an int column with nulls and incremented version to 1.0.0.dev11 --- fireant/__init__.py | 2 +- fireant/{slicer/widgets => }/formats.py | 36 +++++++++++-------- fireant/slicer/queries/database.py | 5 +-- fireant/slicer/widgets/datatables.py | 2 +- fireant/slicer/widgets/highcharts.py | 16 ++++----- .../{slicer/widgets => }/test_formats.py | 11 +++--- 6 files changed, 37 insertions(+), 35 deletions(-) rename fireant/{slicer/widgets => }/formats.py (95%) rename fireant/tests/{slicer/widgets => }/test_formats.py (98%) diff --git a/fireant/__init__.py b/fireant/__init__.py index 54641e3b..b502db4d 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev10' +__version__ = '1.0.0.dev11' diff --git a/fireant/slicer/widgets/formats.py b/fireant/formats.py similarity index 95% rename from fireant/slicer/widgets/formats.py rename to fireant/formats.py index 67dad91e..a5db91d7 100644 --- a/fireant/slicer/widgets/formats.py +++ b/fireant/formats.py @@ -1,13 +1,13 @@ import locale - -import numpy as np -import pandas as pd from datetime import ( date, datetime, time, ) +import numpy as np +import pandas as pd + INFINITY = "Infinity" NO_TIME = time(0) @@ -22,6 +22,23 @@ def date_as_millis(value): return int(1000 * value.timestamp()) +def coerce_type(value): + for type_cast in (int, float): + try: + return type_cast(value) + except: + pass + + if 'null' == value: + return None + if 'True' == value: + return True + if 'False' == value: + return False + + return value + + def dimension_value(value): """ Format a dimension value. This will coerce the raw string or date values into a proper primitive value like a @@ -42,18 +59,7 @@ def dimension_value(value): else: return value.strftime('%Y-%m-%d %H:%M:%S') - for type_cast in (int, float): - try: - return type_cast(value) - except: - pass - - if 'True' is value: - return True - if 'False' is value: - return False - - return value + return coerce_type(value) def metric_value(value): diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index bcb100d7..46382dcf 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -74,11 +74,9 @@ def fill_nans_in_level(data_frame, dimension, preceding_dimension_keys): :param data_frame: The data_frame we are replacing values in. - :param level: + :param dimension: The level of the data frame to replace nulls in. This function should be called once per non-conitnuous dimension, in the order of the dimensions. - :param is_rollup: - :param preceding_dimension_keys: :return: The level in the data_frame with the nulls replaced with empty string @@ -100,7 +98,6 @@ def _fill_nan_for_nulls(df): """ Fills the first NaN with a literal string "null" if there are two NaN values, otherwise nothing is filled. - :param df: :return: """ diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index f5dc6e66..c26dff0b 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -5,9 +5,9 @@ from fireant import ( ContinuousDimension, Metric, + formats, utils, ) -from . import formats from .base import ( TransformableWidget, ) diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 1a351863..3cae406f 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -1,22 +1,19 @@ import itertools - -import pandas as pd from datetime import ( datetime, ) +import pandas as pd + from fireant import ( DatetimeDimension, + formats, utils, ) from .base import ( TransformableWidget, Widget, ) -from .formats import ( - date_as_millis, - metric_value, -) from .helpers import ( dimensional_metric_label, extract_display_values, @@ -340,7 +337,7 @@ def _render_pie_series(self, axis, metric, reference, data_frame, render_series_ "type": axis.type, "data": [{ "name": render_series_label(dimension_values) if dimension_values else name, - "y": metric_value(y), + "y": formats.metric_value(y), "color": color, } for (dimension_values, y), color in zip(pie_chart_df.iteritems(), self.colors)], 'tooltip': { @@ -352,7 +349,7 @@ def _render_pie_series(self, axis, metric, reference, data_frame, render_series_ def _render_data(self, group_df, metric_key, is_timeseries): if not is_timeseries: - return [metric_value(y) for y in group_df[metric_key].values] + return [formats.metric_value(y) for y in group_df[metric_key].values] series = [] for dimension_values, y in group_df[metric_key].iteritems(): @@ -362,7 +359,8 @@ def _render_data(self, group_df, metric_key, is_timeseries): # Ignore totals on the x-axis. continue - series.append((date_as_millis(first_dimension_value), metric_value(y))) + series.append((formats.date_as_millis(first_dimension_value), + formats.metric_value(y))) return series diff --git a/fireant/tests/slicer/widgets/test_formats.py b/fireant/tests/test_formats.py similarity index 98% rename from fireant/tests/slicer/widgets/test_formats.py rename to fireant/tests/test_formats.py index fcc5d9e2..1e1aae5d 100644 --- a/fireant/tests/slicer/widgets/test_formats.py +++ b/fireant/tests/test_formats.py @@ -1,14 +1,15 @@ +from datetime import ( + date, + datetime, +) from unittest import ( TestCase, ) import numpy as np import pandas as pd -from datetime import ( - date, - datetime, -) -from fireant.slicer.widgets import formats + +from fireant import formats class FormatMetricValueTests(TestCase): From 2e7a99fa32d888e17ca5bf89b5886e6f42946b7a Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 23 Feb 2018 09:35:33 +0100 Subject: [PATCH 033/123] changed the formatting of infinity to return None --- fireant/formats.py | 2 +- fireant/tests/test_formats.py | 32 +++++++++++++++++++++----------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/fireant/formats.py b/fireant/formats.py index a5db91d7..eea5f5fb 100644 --- a/fireant/formats.py +++ b/fireant/formats.py @@ -81,7 +81,7 @@ def metric_value(value): if isinstance(value, float): if np.isinf(value): - return INFINITY + return None return float(value) if isinstance(value, np.int64): diff --git a/fireant/tests/test_formats.py b/fireant/tests/test_formats.py index 1e1aae5d..4a32643b 100644 --- a/fireant/tests/test_formats.py +++ b/fireant/tests/test_formats.py @@ -1,48 +1,58 @@ -from datetime import ( - date, - datetime, -) from unittest import ( TestCase, ) import numpy as np import pandas as pd +from datetime import ( + date, + datetime, +) from fireant import formats class FormatMetricValueTests(TestCase): - def test_nan_data_point(self): + def test_that_nan_data_point_is_convered_to_none(self): # np.nan is converted to None result = formats.metric_value(np.nan) self.assertIsNone(result) - def test_str_data_point(self): + def test_that_inf_data_point_is_convered_to_none(self): + # np.nan is converted to None + result = formats.metric_value(np.inf) + self.assertIsNone(result) + + def test_that_neg_inf_data_point_is_convered_to_none(self): + # np.nan is converted to None + result = formats.metric_value(-np.inf) + self.assertIsNone(result) + + def test_str_data_point_is_returned_unchanged(self): result = formats.metric_value(u'abc') self.assertEqual('abc', result) - def test_int64_data_point(self): + def test_int64_data_point_is_returned_as_py_int(self): # Needs to be cast to python int result = formats.metric_value(np.int64(1)) self.assertEqual(int(1), result) - def test_date_data_point(self): + def test_data_data_point_is_returned_as_string_iso_no_time(self): # Needs to be converted to milliseconds result = formats.metric_value(date(2000, 1, 1)) self.assertEqual('2000-01-01', result) - def test_datetime_data_point(self): + def test_datatime_data_point_is_returned_as_string_iso_with_time(self): # Needs to be converted to milliseconds result = formats.metric_value(datetime(2000, 1, 1, 1)) self.assertEqual('2000-01-01T01:00:00', result) - def test_ts_date_data_point(self): + def test_timestamp_no_time_data_point_is_returned_as_string_iso_no_time(self): # Needs to be converted to milliseconds result = formats.metric_value(pd.Timestamp(date(2000, 1, 1))) self.assertEqual('2000-01-01', result) - def test_ts_datetime_data_point(self): + def test_timestamp_data_point_is_returned_as_string_iso_no_time(self): # Needs to be converted to milliseconds result = formats.metric_value(pd.Timestamp(datetime(2000, 1, 1, 1))) self.assertEqual('2000-01-01T01:00:00', result) From 562f6ad96a3aa74a4e603a8c9d65ee92a13513cf Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 23 Feb 2018 09:35:57 +0100 Subject: [PATCH 034/123] bumped version to dev12 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index b502db4d..d169855e 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev11' +__version__ = '1.0.0.dev12' From 7af181e0e139745ee1b12bab710dde2987f42cbb Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 26 Feb 2018 19:20:17 +0100 Subject: [PATCH 035/123] Some bug fixes - fixed missing metric formatting in pandas widget - fixed class names in csv widget Bumped version to dev13 --- fireant/__init__.py | 2 +- fireant/slicer/base.py | 2 +- fireant/slicer/dimensions.py | 5 +++++ fireant/slicer/widgets/csv.py | 1 - fireant/slicer/widgets/pandas.py | 8 ++++++++ fireant/tests/slicer/widgets/test_csv.py | 4 ++-- fireant/tests/slicer/widgets/test_pandas.py | 19 +++++++++++++++++++ 7 files changed, 36 insertions(+), 5 deletions(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index d169855e..a1237da9 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev12' +__version__ = '1.0.0.dev13' diff --git a/fireant/slicer/base.py b/fireant/slicer/base.py index ef693781..ddf9ea96 100644 --- a/fireant/slicer/base.py +++ b/fireant/slicer/base.py @@ -26,4 +26,4 @@ def has_display_field(self): return getattr(self, 'display_definition', None) is not None def __hash__(self): - return hash(self.__class__.__name__ + self.key) + return hash('{}({})'.format(self.__class__.__name__, self.definition)) diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 1cf63445..350330f8 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -112,6 +112,11 @@ def __init__(self, key, label=None, definition=None, display_definition=None): if display_definition is not None \ else None + def __hash__(self): + if self.has_display_field: + return hash('{}({},{})'.format(self.__class__.__name__, self.definition, self.display_definition)) + return super(UniqueDimension, self).__hash__() + def isin(self, values, use_display=False): """ Creates a filter to filter a slicer query. diff --git a/fireant/slicer/widgets/csv.py b/fireant/slicer/widgets/csv.py index ade1dc3f..2b5ea4ba 100644 --- a/fireant/slicer/widgets/csv.py +++ b/fireant/slicer/widgets/csv.py @@ -2,7 +2,6 @@ class CSV(Pandas): - def transform(self, data_frame, slicer, dimensions): result_df = super(CSV, self).transform(data_frame, slicer, dimensions) return result_df.to_csv() diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index cdcd088e..4d3c4b83 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -7,6 +7,7 @@ reference_key, reference_label, ) +from ... import formats HARD_MAX_COLUMNS = 24 @@ -31,6 +32,13 @@ def transform(self, data_frame, slicer, dimensions): result = data_frame.copy() references = [] + for metric in self.items: + if any([metric.precision is not None, + metric.prefix is not None, + metric.suffix is not None]): + result[metric.key] = result[metric.key] \ + .apply(lambda x: formats.metric_display(x, metric.prefix, metric.suffix, metric.precision)) + for dimension in dimensions: references += getattr(dimension, 'references', []) diff --git a/fireant/tests/slicer/widgets/test_csv.py b/fireant/tests/slicer/widgets/test_csv.py index 963d68f2..21d7df0f 100644 --- a/fireant/tests/slicer/widgets/test_csv.py +++ b/fireant/tests/slicer/widgets/test_csv.py @@ -2,7 +2,7 @@ import pandas as pd -from fireant.slicer.widgets.csv import CSV +from fireant.slicer.widgets import CSV from fireant.tests.slicer.mocks import ( CumSum, ElectionOverElection, @@ -19,7 +19,7 @@ ) -class DataTablesTransformerTests(TestCase): +class CSVWidgetTests(TestCase): maxDiff = None def test_single_metric(self): diff --git a/fireant/tests/slicer/widgets/test_pandas.py b/fireant/tests/slicer/widgets/test_pandas.py index 00e1c04a..b1802ca2 100644 --- a/fireant/tests/slicer/widgets/test_pandas.py +++ b/fireant/tests/slicer/widgets/test_pandas.py @@ -169,3 +169,22 @@ def test_time_series_ref(self): expected.columns = ['Votes', 'Votes (EoE)'] pandas.testing.assert_frame_equal(result, expected) + + def test_metric_format(self): + import copy + votes = copy.copy(slicer.metrics.votes) + votes.prefix = '$' + votes.suffix = '€' + votes.precision = 2 + + # divide the data frame by 3 to get a repeating decimal so we can check precision + result = Pandas(items=[votes]) \ + .transform(cont_dim_df / 3, slicer, [slicer.dimensions.timestamp]) + + expected = cont_dim_df.copy()[['votes']] + expected['votes'] = ['${0:.2f}€'.format(x) + for x in expected['votes'] / 3] + expected.index.names = ['Timestamp'] + expected.columns = ['Votes'] + + pandas.testing.assert_frame_equal(result, expected) From 6914fe1dc3e6e32d5053886812933929944b4a92 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 6 Mar 2018 15:14:56 +0100 Subject: [PATCH 036/123] Added better support for totals queries by using UNIONs instead of ROLLUP --- fireant/slicer/base.py | 12 +- fireant/slicer/dimensions.py | 25 +- fireant/slicer/queries/builder.py | 313 +++--------- fireant/slicer/queries/finders.py | 147 ++++++ fireant/slicer/queries/makers.py | 471 +++++++++++++++++++ fireant/slicer/queries/references.py | 223 --------- fireant/slicer/references.py | 30 ++ fireant/slicer/widgets/datatables.py | 5 +- fireant/tests/slicer/queries/test_builder.py | 426 +++++++++++------ requirements.txt | 2 +- 10 files changed, 1036 insertions(+), 618 deletions(-) create mode 100644 fireant/slicer/queries/finders.py create mode 100644 fireant/slicer/queries/makers.py delete mode 100644 fireant/slicer/queries/references.py diff --git a/fireant/slicer/base.py b/fireant/slicer/base.py index ddf9ea96..2d379d6d 100644 --- a/fireant/slicer/base.py +++ b/fireant/slicer/base.py @@ -3,7 +3,7 @@ class SlicerElement(object): The `SlicerElement` class represents an element of the slicer, either a metric or dimension, which contains information about such as how to query it from the database. """ - def __init__(self, key, label=None, definition=None): + def __init__(self, key, label=None, definition=None, display_definition=None): """ :param key: The unique identifier of the slicer element, used in the Slicer manager API to reference a defined element. @@ -13,11 +13,21 @@ def __init__(self, key, label=None, definition=None): :param definition: The definition of the element as a PyPika expression which defines how to query it from the database. + + :param display_definition: + The definition of the display field for the element as a PyPika expression. Similar to the definition except + used for querying labels. """ self.key = key self.label = label or key self.definition = definition + self.display_definition = display_definition + + self.display_key = '{}_display'.format(key) \ + if display_definition is not None \ + else None + def __repr__(self): return self.key diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 350330f8..6b0617c3 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -1,6 +1,10 @@ from typing import Iterable from fireant.utils import immutable +from pypika.terms import ( + ValueWrapper, + NullValue, +) from .base import SlicerElement from .exceptions import QueryException from .filters import ( @@ -21,8 +25,8 @@ class Dimension(SlicerElement): The `Dimension` class represents a dimension in the `Slicer` object. """ - def __init__(self, key, label=None, definition=None): - super(Dimension, self).__init__(key, label, definition) + def __init__(self, key, label=None, definition=None, display_definition=None): + super(Dimension, self).__init__(key, label, definition, display_definition) self.is_rollup = False @immutable @@ -104,13 +108,11 @@ class UniqueDimension(Dimension): """ def __init__(self, key, label=None, definition=None, display_definition=None): - super(UniqueDimension, self).__init__(key=key, label=label, definition=definition) - - self.display_definition = display_definition + super(UniqueDimension, self).__init__(key=key, + label=label, + definition=definition, + display_definition=display_definition) - self.display_key = '{}_display'.format(key) \ - if display_definition is not None \ - else None def __hash__(self): if self.has_display_field: @@ -254,3 +256,10 @@ def between(self, start, stop): start and stop. """ return RangeFilter(self.definition, start, stop) + + +class TotalsDimension(Dimension): + def __init__(self, dimension): + totals_definition = NullValue() + display_definition = totals_definition if dimension.has_display_field else None + super(Dimension, self).__init__(dimension.key, dimension.label, totals_definition, display_definition) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 4e670426..5cf209fd 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -1,6 +1,6 @@ -from collections import ( - OrderedDict, - defaultdict, +from typing import ( + Dict, + Iterable, ) import pandas as pd @@ -9,60 +9,21 @@ functions as fn, ) from pypika.enums import SqlTypes -from toposort import ( - CircularDependencyError, - toposort_flatten, -) -from typing import ( - Dict, - Iterable, -) -from fireant.slicer.base import SlicerElement -from fireant.utils import ( - flatten, - groupby, - immutable, - ordered_distinct_list, - ordered_distinct_list_by_attr, -) +from fireant.utils import immutable from .database import fetch_data -from .references import ( - create_container_query, - create_joined_reference_query, - select_reference_metrics, +from .finders import ( + find_and_group_references_for_dimensions, + find_dimensions_with_totals, + find_metrics_for_widgets, + find_operations_for_widgets, ) -from ..exceptions import ( - CircularJoinsException, - MissingTableJoinException, +from .makers import ( + make_orders_for_dimensions, + make_slicer_query, + make_slicer_query_with_totals, ) -from ..filters import DimensionFilter - - -def _build_dimension_definition(dimension, interval_func): - if hasattr(dimension, 'interval'): - return interval_func(dimension.definition, dimension.interval) \ - .as_(dimension.key) - - return dimension.definition.as_(dimension.key) - - -def _select_groups(terms, query, rollup, database): - query = query.select(*terms) - - if not rollup: - return query.groupby(*terms) - - if 1 < len(terms): - # This step packs multiple terms together so that they are rolled up together. This is needed for unique - # dimensions to keep the display field grouped together with the definition field. - terms = [terms] - - return database.totals(query, terms) - - -def is_rolling_up(dimension, rolling_up): - return rolling_up or getattr(dimension, "is_rollup", False) +from ..base import SlicerElement class QueryBuilder(object): @@ -80,103 +41,20 @@ def filter(self, *filters): """ self._filters += filters - @property - def _elements(self): - return flatten([self._dimensions, self._filters]) - - @property - def _tables(self): - """ - Collect all the tables from all of the definitions of all of the elements in the slicer query. This looks - through the metrics, dimensions, and filter included in this slicer query. It also checks both the definition - field of each element as well as the display definition for Unique Dimensions. - - :return: - A collection of tables required to execute a query, - """ - return ordered_distinct_list([table - for element in self._elements - # Need extra for-loop to incl. the `display_definition` from `UniqueDimension` - for attr in [getattr(element, 'definition', None), - getattr(element, 'display_definition', None)] - # ... but then filter Nones since most elements do not have `display_definition` - if attr is not None - for table in attr.tables_]) - - @property - def _joins(self): - """ - Given a set of tables required for a slicer query, this function finds the joins required for the query and - sorts them topologically. - - :return: - A list of joins in the order that they must be joined to the query. - :raises: - MissingTableJoinException - If a table is required but there is no join for that table - CircularJoinsException - If there is a circular dependency between two or more joins - """ - dependencies = defaultdict(set) - slicer_joins = {join.table: join - for join in self.slicer.joins} - tables_to_eval = list(self._tables) - - while tables_to_eval: - table = tables_to_eval.pop() - - if self.table == table: - continue - - if table not in slicer_joins: - raise MissingTableJoinException('Could not find a join for table {table}' - .format(table=str(table))) - - join = slicer_joins[table] - tables_required_for_join = set(join.criterion.tables_) - {self.table, join.table} - - dependencies[join] |= {slicer_joins[table] - for table in tables_required_for_join} - tables_to_eval += tables_required_for_join - {d.table for d in dependencies} - - try: - return toposort_flatten(dependencies) - except CircularDependencyError as e: - raise CircularJoinsException(str(e)) - @property def query(self): """ - WRITEME - """ - query = self.slicer.database.query_cls.from_(self.table) - - # Add joins - for join in self._joins: - query = query.join(join.table, how=join.join_type).on(join.criterion) + Serialize a pypika/SQL query from the slicer builder query. - # Add dimensions - rolling_up = False - for dimension in self._dimensions: - rolling_up = is_rolling_up(dimension, rolling_up) + This is the base implementation shared by two implementations: the query to fetch data for a slicer request and + the query to fetch choices for dimensions. - dimension_definition = _build_dimension_definition(dimension, self.slicer.database.trunc_date) + This function only handles dimensions (select+group by) and filtering (where/having), which is everything needed + for the query to fetch choices for dimensions. - if dimension.has_display_field: - # Add display definition field - dimension_display_definition = dimension.display_definition.as_(dimension.display_key) - fields = [dimension_definition, dimension_display_definition] - - else: - fields = [dimension_definition] - - query = _select_groups(fields, query, rolling_up, self.slicer.database) - - # Add filters - for filter_ in self._filters: - query = query.where(filter_.definition) \ - if isinstance(filter_, DimensionFilter) \ - else query.having(filter_.definition) - - return query + The slicer query extends this with metrics, references, and totals. + """ + raise NotImplementedError() class SlicerQueryBuilder(QueryBuilder): @@ -220,44 +98,6 @@ def orderby(self, element: SlicerElement, orientation=None): """ self._orders += [(element.definition.as_(element.key), orientation)] - @property - def metrics(self): - """ - :return: - an ordered, distinct list of metrics used in all widgets as part of this query. - """ - return ordered_distinct_list_by_attr([metric - for widget in self._widgets - for metric in widget.metrics]) - - @property - def operations(self): - """ - :return: - an ordered, distinct list of metrics used in all widgets as part of this query. - """ - return ordered_distinct_list_by_attr([operation - for widget in self._widgets - for operation in widget.operations]) - - @property - def orders(self): - if self._orders: - return self._orders - - definitions = [dimension.display_definition.as_(dimension.display_key) - if dimension.has_display_field - else dimension.definition.as_(dimension.key) - for dimension in self._dimensions] - - return [(definition, None) - for definition in definitions] - - - @property - def _elements(self): - return flatten([self.metrics, self._dimensions, self._filters]) - @property def query(self): """ @@ -266,82 +106,31 @@ def query(self): normally produced is wrapped in an outer query and a query for each reference is joined based on the referenced dimension shifted. """ + database = self.slicer.database + table = self.table + joins = self.slicer.joins + dimensions = self._dimensions + metrics = find_metrics_for_widgets(self._widgets) + filters = self._filters # Validate for widget in self._widgets: if hasattr(widget, 'validate'): - widget.validate(self._dimensions) - - query = super(SlicerQueryBuilder, self).query + widget.validate(dimensions) - # Add metrics - query = query.select(*[metric.definition.as_(metric.key) - for metric in self.metrics]) + reference_groups = find_and_group_references_for_dimensions(dimensions) + totals_dimensions = find_dimensions_with_totals(dimensions) - # Add references - references_for_dimensions = OrderedDict([(dimension, dimension.references) - for dimension in self._dimensions - if hasattr(dimension, 'references') and dimension.references]) - if references_for_dimensions: - query = self._join_references(query, references_for_dimensions) + query = make_slicer_query_with_totals(database, table, joins, dimensions, metrics, filters, + reference_groups, totals_dimensions) # Add ordering - for (definition, orientation) in self.orders: - query = query.orderby(definition, order=orientation) + orders = (self._orders or make_orders_for_dimensions(dimensions)) + for (term, orientation) in orders: + query = query.orderby(term, order=orientation) return query - def _join_references(self, query, references_for_dimensions): - """ - This converts the pypika query built in `self.query()` into a query that includes references. This is achieved - by wrapping the original query with an outer query using the original query as the FROM clause, then joining - copies of the original query for each reference, with the reference dimension shifted by the appropriate - interval. - - The outer query selects everything from the original query and each metric from the reference query using an - alias constructed from the metric key appended with the reference key. For Delta references, the reference - metric is selected as the difference of the metric from the original query and the reference query. For Delta - Percentage references, the reference metric metric is selected as the difference divided by the reference - metric. - - :param query: - The original query built by `self.query` - :param references_for_dimensions: - An ordered dict with the dimension as the key and a list of references to add for that dimension. - :return: - A new pypika query with the dimensions and metrics included from the original query plus each of the - metrics for each of the references. - """ - metrics = self.metrics - - original_query = query.as_('base') - container_query = create_container_query(original_query, - self.slicer.database.query_cls, - self._dimensions, - metrics) - - # join reference queries - for dimension, references in references_for_dimensions.items(): - get_unit_and_interval = lambda reference: (reference.time_unit, reference.interval) - grouped_references = groupby(references, get_unit_and_interval) - - for (time_unit, interval), references_by_interval in grouped_references.items(): - container_query, ref_query = create_joined_reference_query(time_unit, - interval, - self._dimensions, - dimension, - original_query, - container_query, - self.slicer.database) - - container_query = select_reference_metrics(references_by_interval, - container_query, - original_query, - ref_query, - metrics) - - return container_query - def fetch(self, limit=None, offset=None) -> Iterable[Dict]: """ Fetch the data for this query and transform it into the widgets. @@ -360,7 +149,8 @@ def fetch(self, limit=None, offset=None) -> Iterable[Dict]: dimensions=self._dimensions) # Apply operations - for operation in self.operations: + operations = find_operations_for_widgets(self._widgets) + for operation in operations: data_frame[operation.key] = operation.apply(data_frame) # Apply transformations @@ -370,15 +160,36 @@ def fetch(self, limit=None, offset=None) -> Iterable[Dict]: def __str__(self): return self.query - def __iter__(self): - return iter(self.render()) - class DimensionOptionQueryBuilder(QueryBuilder): + """ + WRITEME + """ def __init__(self, slicer, dimension): super(DimensionOptionQueryBuilder, self).__init__(slicer, slicer.hint_table or slicer.table) self._dimensions.append(dimension) + @property + def query(self): + """ + Serialize a pypika/SQL query from the slicer builder query. + + This is the base implementation shared by two implementations: the query to fetch data for a slicer request and + the query to fetch choices for dimensions. + + This function only handles dimensions (select+group by) and filtering (where/having), which is everything needed + for the query to fetch choices for dimensions. + + The slicer query extends this with metrics, references, and totals. + """ + + return make_slicer_query(query_cls=self.slicer.database.query_cls, + trunc_date=self.slicer.database.trunc_date, + base_table=self.table, + joins=self.slicer.joins, + dimensions=self._dimensions, + filters=self._filters) + def fetch(self, limit=None, offset=None, force_include=()) -> pd.Series: """ Fetch the data for this query and transform it into the widgets. diff --git a/fireant/slicer/queries/finders.py b/fireant/slicer/queries/finders.py new file mode 100644 index 00000000..e6709804 --- /dev/null +++ b/fireant/slicer/queries/finders.py @@ -0,0 +1,147 @@ +from collections import ( + OrderedDict, + defaultdict, + namedtuple, +) + +from toposort import ( + CircularDependencyError, + toposort_flatten, +) + +from fireant.utils import ( + groupby, + ordered_distinct_list, + ordered_distinct_list_by_attr, +) +from ..exceptions import ( + CircularJoinsException, + MissingTableJoinException, +) + +ReferenceGroup = namedtuple('ReferenceGroup', ('dimension', 'time_unit', 'intervals')) + + +def find_required_tables_to_join(elements, base_table): + """ + Collect all the tables required for a given list of slicer elements. This looks through the definition and + display_definition attributes of all elements and + + This looks through the metrics, dimensions, and filter included in this slicer query. It also checks both the + definition + field of each element as well as the display definition for Unique Dimensions. + + :return: + A collection of tables required to execute a query, + """ + return ordered_distinct_list([table + for element in elements + + # Need extra for-loop to incl. the `display_definition` from `UniqueDimension` + for attr in [getattr(element, 'definition', None), + getattr(element, 'display_definition', None)] + + # ... but then filter Nones since most elements do not have `display_definition` + if attr is not None + + for table in attr.tables_ + + # Omit the base table from this list + if base_table != table]) + + +def find_joins_for_tables(joins, base_table, required_tables): + """ + Given a set of tables required for a slicer query, this function finds the joins required for the query and + sorts them topologically. + + :return: + A list of joins in the order that they must be joined to the query. + :raises: + MissingTableJoinException - If a table is required but there is no join for that table + CircularJoinsException - If there is a circular dependency between two or more joins + """ + dependencies = defaultdict(set) + slicer_joins = {join.table: join + for join in joins} + + while required_tables: + table = required_tables.pop() + + if table not in slicer_joins: + raise MissingTableJoinException('Could not find a join for table {}' + .format(str(table))) + + join = slicer_joins[table] + tables_required_for_join = set(join.criterion.tables_) - {base_table, join.table} + + dependencies[join] |= {slicer_joins[table] + for table in tables_required_for_join} + required_tables += tables_required_for_join - {d.table for d in dependencies} + + try: + return toposort_flatten(dependencies) + except CircularDependencyError as e: + raise CircularJoinsException(str(e)) + + +def find_metrics_for_widgets(widgets): + """ + :return: + an ordered, distinct list of metrics used in all widgets as part of this query. + """ + return ordered_distinct_list_by_attr([metric + for widget in widgets + for metric in widget.metrics]) + + +def find_operations_for_widgets(widgets): + """ + :return: + an ordered, distinct list of metrics used in all widgets as part of this query. + """ + return ordered_distinct_list_by_attr([operation + for widget in widgets + for operation in widget.operations]) + + +def find_dimensions_with_totals(dimensions): + return [dimension + for dimension in dimensions + if dimension.is_rollup] + + +def find_and_group_references_for_dimensions(dimensions): + """ + Finds all of the references for dimensions and groups them by dimension, interval unit, number of intervals. + + This structure reflects how the references need to be joined to the slicer query. References of the same + type (WoW, WoW.delta, WoW.delta_percent) can share a join query. + + :param dimensions: + :return: + An `OrderedDict` where the keys are 3-item tuples consisting of "Dimension, interval unit, # of intervals. + + for example: + + { + (Dimension(date_1), 'weeks', 1): [WoW, WoW.delta], + (Dimension(date_1), 'years', 1): [YoY], + (Dimension(date_7), 'days', 1): [DoD, DoD.delta_percent], + } + """ + + def get_unit_and_interval(reference): + return reference.time_unit, reference.interval + + return OrderedDict([(ReferenceGroup(dimension, time_unit, intervals), ordered_distinct_list(group)) + + # group by dimension + for dimension in dimensions + + # filter dimensions with no references + if getattr(dimension, 'references', None) + + # group by time_unit and intervals + for (time_unit, intervals), group in + groupby(dimension.references, get_unit_and_interval).items()]) diff --git a/fireant/slicer/queries/makers.py b/fireant/slicer/queries/makers.py new file mode 100644 index 00000000..a456ae4b --- /dev/null +++ b/fireant/slicer/queries/makers.py @@ -0,0 +1,471 @@ +import copy +from functools import partial + +from typing import ( + Callable, + Iterable, +) + +from fireant.slicer.intervals import weekly +from fireant.slicer.references import ( + YearOverYear, + reference_key, + reference_term, +) +from fireant.utils import flatten +from pypika import JoinType +from pypika.queries import QueryBuilder +from pypika.terms import ( + ComplexCriterion, + Criterion, + NullValue, + Term, +) +from .finders import ( + find_joins_for_tables, + find_required_tables_to_join, +) +from ..dimensions import ( + Dimension, + TotalsDimension, +) +from ..filters import DimensionFilter + + +def make_slicer_query(query_cls, trunc_date, base_table, joins=(), dimensions=(), metrics=(), filters=()): + """ + Creates a pypika/SQL query from a list of slicer elements. + + This is the base implementation shared by two implementations: the query to fetch data for a slicer request and + the query to fetch choices for dimensions. + + This function only handles dimensions (select+group by) and filtering (where/having), which is everything needed + for the query to fetch choices for dimensions. + + The slicer query extends this with metrics, references, and totals. + + :param query_cls: + pypika.Query - The pypika query class to use to create the query + :param trunc_date: + Callable - A function to truncate a date to an interval (database vendor specific) + :param base_table: + pypika.Table - The base table of the query, the one in the FROM clause + :param joins: + Iterable - A collection of joins available in the slicer. This should include all slicer joins. + Only joins required for the query will be used. + :param dimensions: + Iterable - A collection of dimensions to use in the query. + :param metrics: + Iterable - A collection of metircs to use in the query. + :param filters: + Iterable - A collection of filters to apply to the query. + :return: + + """ + query = query_cls.from_(base_table) + + elements = flatten([metrics, dimensions, filters]) + + # Add joins + join_tables_needed_for_query = find_required_tables_to_join(elements, base_table) + for join in find_joins_for_tables(joins, base_table, join_tables_needed_for_query): + query = query.join(join.table, how=join.join_type).on(join.criterion) + + # Add dimensions + for dimension in dimensions: + terms = make_terms_for_dimension(dimension, trunc_date) + query = query.select(*terms) + # Don't group TotalsDimensions + if not isinstance(dimension, TotalsDimension): + query = query.groupby(*terms) + + # Add filters + for filter_ in filters: + query = query.where(filter_.definition) \ + if isinstance(filter_, DimensionFilter) \ + else query.having(filter_.definition) + + # Add metrics + terms = make_terms_for_metrics(metrics) + if terms: + query = query.select(*terms) + + return query + + +def make_slicer_query_with_totals(database, table, joins, dimensions, metrics, filters, + reference_groups, totals_dimensions): + """ + WRITEME + + :param database: + :param table: + :param joins: + :param dimensions: + :param metrics: + :param filters: + :param reference_groups: + :param totals_dimensions: + :return: + """ + query = make_slicer_query_with_references(database, + table, + joins, + dimensions, + metrics, + filters, + reference_groups) + for dimension in totals_dimensions: + totals_dimension_index = dimensions.index(dimension) + grouped_dims, totaled_dims = dimensions[:totals_dimension_index], dimensions[totals_dimension_index:] + + dimensions_with_totals = grouped_dims + [TotalsDimension(dimension) + for dimension in totaled_dims] + + totals_query = make_slicer_query_with_references(database, + table, + joins, + dimensions_with_totals, + metrics, + filters, + reference_groups) + + # UNION ALL + query = query.union_all(totals_query) + + return query + + +def make_slicer_query_with_references(database, base_table, joins, dimensions, metrics, filters, reference_groups): + """ + WRITEME + + :param query_cls: + :param trunc_date: + :param base_table: + :param joins: + :param dimensions: + :param metrics: + :param filters: + :param reference_groups: + :return: + """ + + if not reference_groups: + return make_slicer_query(query_cls=database.query_cls, + trunc_date=database.trunc_date, + base_table=base_table, + joins=joins, + dimensions=dimensions, + metrics=metrics, + filters=filters) + + # Do not include totals dimensions in the reference query (they are selected directly in the container query) + non_totals_dimensions = [dimension + for dimension in dimensions + if not isinstance(dimension, TotalsDimension)] + + query = make_slicer_query(query_cls=database.query_cls, + trunc_date=database.trunc_date, + base_table=base_table, + joins=joins, + dimensions=non_totals_dimensions, + metrics=metrics, + filters=filters) + + original_query = query.as_('base') + container_query = make_reference_container_query(original_query, + database.query_cls, + dimensions, + metrics) + + for (dimension, time_unit, interval), references in reference_groups.items(): + alias = references[0].key[:3] + ref_query, join_criterion = make_reference_query(database, + base_table, + joins, + non_totals_dimensions, + metrics, + filters, + dimension, + time_unit, + interval, + original_query, + alias=alias) + + terms = make_terms_for_references(references, + original_query, + ref_query, + metrics) + + container_query = container_query \ + .join(ref_query, JoinType.full_outer) \ + .on(join_criterion) \ + .select(*terms) + + return container_query + + +def make_reference_container_query(original_query, query_cls, dimensions, metrics): + """ + Creates a container query with the original query used as the FROM clause and selects all of the metrics. The + container query is used as a base for joining reference queries. + + :param original_query: + :param query_cls: + :param dimensions: + :param metrics: + :return: + """ + + def original_query_field(element, display=False): + key = element.display_key if display else element.key + definition = element.display_definition if display else element.definition + + if isinstance(definition, NullValue): + # If an element is a literal NULL, then include the definition directly in the container query. It will be + # omitted from the reference queries + return definition.as_(key) + + return original_query.field(key).as_(key) + + outer_query = query_cls.from_(original_query) + + # Add dimensions + for dimension in dimensions: + outer_query = outer_query.select(original_query_field(dimension)) + + if dimension.has_display_field: + outer_query = outer_query.select(original_query_field(dimension, True)) + + # Add base metrics + return outer_query.select(*[original_query_field(metric) + for metric in metrics]) + + +def make_reference_query(database, base_table, joins, dimensions, metrics, filters, ref_dimension, time_unit, interval, + original_query, alias=None): + """ + WRITEME + + :param alias: + :param database: + :param base_table: + :param joins: + :param dimensions: + :param metrics: + :param filters: + :param references: + :param ref_dimension: + :param time_unit: + :param interval: + :param original_query: + :return: + """ + offset_func = partial(database.date_add, + date_part=time_unit, + interval=interval) + + # for weekly interval dimensions with YoY references, correct the day of week by adding a year, truncating to a + # week, then subtracting a year + offset_for_weekday = weekly == ref_dimension.interval and YearOverYear.time_unit == time_unit + if offset_for_weekday: + def trunc_date(definition, _): + shift_forward = database.date_add(definition, time_unit, interval) + offset = database.trunc_date(shift_forward, weekly.key) + return database.date_add(offset, time_unit, -interval) + + else: + trunc_date = database.trunc_date + + ref_filters = make_reference_filters(filters, + ref_dimension, + offset_func) + + ref_query = make_slicer_query(database.query_cls, + trunc_date, + base_table, + joins, + dimensions, + metrics, + ref_filters) \ + .as_(alias) + + join_criterion = make_reference_join_criterion(ref_dimension, + dimensions, + original_query, + ref_query, + offset_func) + + return ref_query, join_criterion + + +def make_terms_for_metrics(metrics): + return [metric.definition.as_(metric.key) + for metric in metrics] + + +def make_terms_for_dimension(dimension, window=None): + """ + Makes a list of pypika terms for a given slicer definition. + + :param dimension: + A slicer dimension. + :param window: + A window function to apply to the dimension definition if it is a continuous dimension. + :return: + a list of terms required to select and group by in a SQL query given a slicer dimension. This list will contain + either one or two elements. A second element will be included if the dimension has a definition for its display + field. + """ + + # Apply the window function to continuous dimensions only + dimension_definition = ( + window(dimension.definition, dimension.interval) + if window and hasattr(dimension, 'interval') + else dimension.definition + ).as_(dimension.key) + + # Include the display definition if there is one + return [ + dimension_definition, + dimension.display_definition.as_(dimension.display_key) + ] if dimension.has_display_field else [ + dimension_definition + ] + + +def make_terms_for_references(references, original_query, ref_query, metrics): + """ + Makes the terms needed to be selected from a reference query in the container query. + + :param references: + :param original_query: + :param ref_query: + :param metrics: + :return: + """ + seen = set() + terms = [] + for reference in references: + # Don't select duplicate references twice + if reference.key in seen \ + and not seen.add(reference.key): + continue + + # Get function to select reference metrics + ref_metric = reference_term(reference, + original_query, + ref_query) + + terms += [ref_metric(metric).as_(reference_key(metric, reference)) + for metric in metrics] + + return terms + + +def make_reference_filters(filters, ref_dimension, offset_func): + """ + Copies and replaces the reference dimension's definition in all of the filters applied to a slicer query. + + This is used to shift the dimension filters to fit the reference window. + + :param filters: + :param ref_dimension: + :param offset_func: + :return: + """ + offset_ref_dimension_definition = offset_func(ref_dimension.definition) + + reference_filters = [] + for ref_filter in map(copy.deepcopy, filters): + ref_filter.definition = _apply_to_term_in_criterion(ref_dimension.definition, + offset_ref_dimension_definition, + ref_filter.definition) + reference_filters.append(ref_filter) + + return reference_filters + + +def make_reference_join_criterion(ref_dimension: Dimension, + all_dimensions: Iterable[Dimension], + original_query: QueryBuilder, + ref_query: QueryBuilder, + offset_func: Callable): + """ + This creates the criterion for joining a reference query to the base query. It matches the referenced dimension + in the base query to the offset referenced dimension in the reference query and all other dimensions. + + :param ref_dimension: + The referenced dimension. + :param all_dimensions: + All of the dimensions applied to the slicer query. + :param original_query: + The base query, the original query despite the references. + :param ref_query: + The reference query, a copy of the base query with the referenced dimension replaced. + :param offset_func: + The offset function for shifting the referenced dimension. + :return: + pypika.Criterion + """ + join_criterion = original_query.field(ref_dimension.key) == offset_func(ref_query.field(ref_dimension.key)) + + for dimension_ in all_dimensions: + if ref_dimension == dimension_: + continue + + join_criterion &= original_query.field(dimension_.key) == ref_query.field(dimension_.key) + + return join_criterion + + +def make_orders_for_dimensions(dimensions): + """ + Creates a list of ordering for a slicer query based on a list of dimensions. The dimensions's display definition is + used preferably as the ordering term but the definition is used for dimensions that do not have a display + definition. + + :param dimensions: + :return: + a list of tuple pairs like (term, orientation) for ordering a SQL query where the first element is the term + to order by and the second is the orientation of the ordering, ASC or DESC. + """ + + # Use the same function to make the definition terms to force it to be consistent. + # Always take the last element in order to prefer the display definition. + definitions = [make_terms_for_dimension(dimension)[-1] + for dimension in dimensions] + + return [(definition, None) + for definition in definitions] + + +def _apply_to_term_in_criterion(target: Term, + replacement: Term, + criterion: Criterion): + """ + Finds and replaces a term within a criterion. This is necessary for adapting filters used in reference queries + where the reference dimension must be offset by some value. The target term is found inside the criterion and + replaced with the replacement. + + :param target: + The target term to replace in the criterion. It will be replaced in all locations within the criterion with + the func applied to itself. + :param replacement: + The replacement for the term. + :param criterion: + The criterion to replace the term in. + :return: + A criterion identical to the original criterion arg except with the target term replaced by the replacement arg. + """ + if isinstance(criterion, ComplexCriterion): + criterion.left = _apply_to_term_in_criterion(target, replacement, criterion.left) + criterion.right = _apply_to_term_in_criterion(target, replacement, criterion.right) + return criterion + + for attr in ['term', 'left', 'right']: + if hasattr(criterion, attr) and str(getattr(criterion, attr)) == str(target): + setattr(criterion, attr, replacement) + + return criterion diff --git a/fireant/slicer/queries/references.py b/fireant/slicer/queries/references.py deleted file mode 100644 index 89c09963..00000000 --- a/fireant/slicer/queries/references.py +++ /dev/null @@ -1,223 +0,0 @@ -from functools import partial - -from pypika import ( - JoinType, - - functions as fn, -) -from pypika.queries import QueryBuilder -from pypika.terms import ( - ComplexCriterion, - Criterion, - Term, -) -from typing import ( - Callable, - Iterable, -) - -from ..dimensions import ( - DatetimeDimension, - Dimension, -) -from ..intervals import weekly -from ..references import ( - Reference, - YearOverYear, - reference_key, -) - - -def create_container_query(original_query, query_cls, dimensions, metrics): - """ - Creates a container query with the original query used as the FROM clause and selects all of the metrics. The - container query is used as a base for joining reference queries. - - :param original_query: - :param query_cls: - :param dimensions: - :param metrics: - :return: - """ - - def original_query_field(key): - return original_query.field(key).as_(key) - - outer_query = query_cls.from_(original_query) - - # Add dimensions - for dimension in dimensions: - outer_query = outer_query.select(original_query_field(dimension.key)) - - if dimension.has_display_field: - outer_query = outer_query.select(original_query_field(dimension.display_key)) - - # Add base metrics - return outer_query.select(*[original_query_field(metric.key) - for metric in metrics]) - - -def create_joined_reference_query(time_unit: str, - interval: int, - dimensions: Iterable[Dimension], - ref_dimension: DatetimeDimension, - original_query, - container_query, - database): - ref_query = original_query.as_('{}_ref'.format(ref_dimension.key)) - - offset_func = partial(database.date_add, - date_part=time_unit, - interval=interval) - - _hack_fixes_into_the_query(database, interval, offset_func, ref_dimension, ref_query, time_unit) - - # Join inner query - join_criterion = _create_reference_join_criterion(ref_dimension, - dimensions, - original_query, - ref_query, - offset_func) - container_query = container_query \ - .join(ref_query, JoinType.left) \ - .on(join_criterion) - - return container_query, ref_query - - -def _hack_fixes_into_the_query(database, interval, offset_func, ref_dimension, ref_query, time_unit): - # for weekly interval dimensions with YoY references, correct the day of week by adding a year, truncating to a - # week, then subtracting a year - offset_for_weekday = weekly == ref_dimension.interval and YearOverYear.time_unit == time_unit - if offset_for_weekday: - shift_forward = database.date_add(ref_dimension.definition, time_unit, interval) - offset = database.trunc_date(shift_forward, 'week') - shift_back = database.date_add(offset, time_unit, -interval) - - getattr(ref_query, '_selects')[0] = shift_back.as_(ref_dimension.key) - - # need to replace the ref dimension term in all of the filters with the offset - query_wheres = getattr(ref_query, '_wheres') - if query_wheres: - wheres = _apply_to_term_in_criterion(ref_dimension.definition, - offset_func(ref_dimension.definition), - query_wheres) - setattr(ref_query, '_wheres', wheres) - - -def select_reference_metrics(references, container_query, original_query, ref_query, metrics): - """ - Select the metrics for a list of references with a reference query. The list of references is expected to share - the same time unit and interval. - - :param references: - :param container_query: - :param original_query: - :param ref_query: - :param metrics: - :return: - """ - seen = set() - for reference in references: - # Don't select duplicate references twice - if reference.key in seen: - continue - else: - seen.add(reference.key) - - # Get function to select reference metrics - ref_metric = _reference_metric(reference, - original_query, - ref_query) - - # Select metrics - container_query = container_query.select(*[ref_metric(metric).as_(reference_key(metric, reference)) - for metric in metrics]) - return container_query - - -def _apply_to_term_in_criterion(target: Term, - replacement: Term, - criterion: Criterion): - """ - Finds and replaces a term within a criterion. This is necessary for adapting filters used in reference queries - where the reference dimension must be offset by some value. The target term is found inside the criterion and - replaced with the replacement. - - :param target: - The target term to replace in the criterion. It will be replaced in all locations within the criterion with - the func applied to itself. - :param replacement: - The replacement for the term. - :param criterion: - The criterion to replace the term in. - :return: - A criterion identical to the original criterion arg except with the target term replaced by the replacement arg. - """ - if isinstance(criterion, ComplexCriterion): - criterion.left = _apply_to_term_in_criterion(target, replacement, criterion.left) - criterion.right = _apply_to_term_in_criterion(target, replacement, criterion.right) - return criterion - - for attr in ['term', 'left', 'right']: - if hasattr(criterion, attr) and str(getattr(criterion, attr)) == str(target): - setattr(criterion, attr, replacement) - - return criterion - - -def _create_reference_join_criterion(dimension: Dimension, - all_dimensions: Iterable[Dimension], - original_query: QueryBuilder, - ref_query: QueryBuilder, - offset_func: Callable): - """ - This creates the criterion for joining a reference query to the base query. It matches the referenced dimension - in the base query to the offset referenced dimension in the reference query and all other dimensions. - - :param dimension: - The referenced dimension - :param original_query: - The base query, the original query despite the references. - :param ref_query: - The reference query, a copy of the base query with the referenced dimension replaced. - :param offset_func: - The offset function for shifting the referenced dimension. - :return: - pypika.Criterion - """ - join_criterion = original_query.field(dimension.key) == offset_func(ref_query.field(dimension.key)) - - for dimension_ in all_dimensions: - if dimension == dimension_: - continue - - join_criterion &= original_query.field(dimension_.key) == ref_query.field(dimension_.key) - - return join_criterion - - -def _reference_metric(reference: Reference, - original_query: QueryBuilder, - ref_query: QueryBuilder): - """ - WRITEME - - :param reference: - :param original_query: - :param ref_query: - :return: - """ - - def ref_field(metric): - return ref_query.field(metric.key) - - if reference.is_delta: - if reference.is_percent: - return lambda metric: (original_query.field(metric.key) - ref_field(metric)) \ - * \ - (100 / fn.NullIf(ref_field(metric), 0)) - - return lambda metric: original_query.field(metric.key) - ref_field(metric) - - return ref_field diff --git a/fireant/slicer/references.py b/fireant/slicer/references.py index 9c27ef80..5ccef584 100644 --- a/fireant/slicer/references.py +++ b/fireant/slicer/references.py @@ -1,3 +1,7 @@ +from pypika import functions as fn +from pypika.queries import QueryBuilder + + class Reference(object): def __init__(self, key, label, time_unit: str, interval: int, delta=False, percent=False): self.key = key @@ -32,6 +36,32 @@ def __hash__(self): YearOverYear = Reference('yoy', 'YoY', 'year', 1) +def reference_term(reference: Reference, + original_query: QueryBuilder, + ref_query: QueryBuilder): + """ + WRITEME + + :param reference: + :param original_query: + :param ref_query: + :return: + """ + + def ref_field(metric): + return ref_query.field(metric.key) + + if reference.is_delta: + if reference.is_percent: + return lambda metric: (original_query.field(metric.key) - ref_field(metric)) \ + * \ + (100 / fn.NullIf(ref_field(metric), 0)) + + return lambda metric: original_query.field(metric.key) - ref_field(metric) + + return ref_field + + def reference_key(metric, reference): """ Format a metric key for a reference. diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index c26dff0b..a1c81d2e 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -7,6 +7,7 @@ Metric, formats, utils, + UniqueDimension, ) from .base import ( TransformableWidget, @@ -174,7 +175,9 @@ def _dimension_columns(dimensions): render=dict(_='value')) is_cont_dim = isinstance(dimension, ContinuousDimension) - is_uni_dim_no_display = (hasattr(dimension, 'display_definition') and dimension.display_definition is None) + is_uni_dim_no_display = isinstance(dimension, UniqueDimension) \ + and not dimension.has_display_field + if not is_cont_dim and not is_uni_dim_no_display: column['render']['display'] = 'display' diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 25bb143b..1438fede 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -1,4 +1,3 @@ -from datetime import date from unittest import TestCase from unittest.mock import ( ANY, @@ -6,12 +5,13 @@ patch, ) -from pypika import Order +from datetime import date import fireant as f from fireant.slicer.exceptions import ( MetricRequiredException, ) +from pypika import Order from ..matchers import ( DimensionMatcher, ) @@ -277,83 +277,243 @@ def test_build_query_with_multiple_dimensions_and_visualizations(self): # noinspection SqlDialectInspection,SqlNoDataSourceInspection -class QueryBuilderDimensionRollupTests(TestCase): +class QueryBuilderDimensionTotalsTests(TestCase): maxDiff = None - def test_build_query_with_rollup_cat_dimension(self): + def test_build_query_with_totals_cat_dimension(self): query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.political_party.rollup()) \ .query - self.assertEqual('SELECT ' + self.assertEqual('(SELECT ' '"political_party" "political_party",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'GROUP BY ROLLUP("political_party") ' + 'GROUP BY "political_party") ' + + 'UNION ALL ' + + '(SELECT ' + 'NULL "political_party",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician") ' + 'ORDER BY "political_party"', str(query)) - def test_build_query_with_rollup_uni_dimension(self): + def test_build_query_with_totals_uni_dimension(self): query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.candidate.rollup()) \ .query - self.assertEqual('SELECT ' + self.assertEqual('(SELECT ' '"candidate_id" "candidate",' '"candidate_name" "candidate_display",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'GROUP BY ROLLUP(("candidate_id","candidate_name")) ' + 'GROUP BY "candidate","candidate_display") ' + + 'UNION ALL ' + + '(SELECT ' + 'NULL "candidate",' + 'NULL "candidate_display",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician") ' + 'ORDER BY "candidate_display"', str(query)) - def test_rollup_following_non_rolled_up_dimensions(self): + def test_build_query_with_totals_on_dimension_and_subsequent_dimensions(self): query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ .dimension(slicer.dimensions.timestamp, - slicer.dimensions.candidate.rollup()) \ + slicer.dimensions.candidate.rollup(), + slicer.dimensions.political_party) \ .query - self.assertEqual('SELECT ' + self.assertEqual('(SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' '"candidate_id" "candidate",' '"candidate_name" "candidate_display",' + '"political_party" "political_party",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp",ROLLUP(("candidate_id","candidate_name")) ' - 'ORDER BY "timestamp","candidate_display"', str(query)) + 'GROUP BY "timestamp","candidate","candidate_display","political_party") ' + + 'UNION ALL ' + + '(SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'NULL "candidate",' + 'NULL "candidate_display",' + 'NULL "political_party",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp") ' + 'ORDER BY "timestamp","candidate_display","political_party"', str(query)) - def test_force_all_dimensions_following_rollup_to_be_rolled_up(self): + def test_build_query_with_totals_on_multiple_dimensions_dimension(self): query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ - .dimension(slicer.dimensions.political_party.rollup(), - slicer.dimensions.candidate) \ + .dimension(slicer.dimensions.timestamp, + slicer.dimensions.candidate.rollup(), + slicer.dimensions.political_party.rollup()) \ .query - self.assertEqual('SELECT ' + self.assertEqual('(SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + '"candidate_id" "candidate",' + '"candidate_name" "candidate_display",' '"political_party" "political_party",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp","candidate","candidate_display","political_party") ' + + 'UNION ALL ' + + '(SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'NULL "candidate",' + 'NULL "candidate_display",' + 'NULL "political_party",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp") ' + + 'UNION ALL ' + + '(SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' '"candidate_id" "candidate",' '"candidate_name" "candidate_display",' + 'NULL "political_party",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'GROUP BY ROLLUP("political_party",("candidate_id","candidate_name")) ' - 'ORDER BY "political_party","candidate_display"', str(query)) + 'GROUP BY "timestamp","candidate","candidate_display") ' + + 'ORDER BY "timestamp","candidate_display","political_party"', str(query)) - def test_force_all_dimensions_following_rollup_to_be_rolled_up_with_split_dimension_calls(self): + def test_build_query_with_totals_cat_dimension_with_references(self): query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp.reference(f.DayOverDay)) \ .dimension(slicer.dimensions.political_party.rollup()) \ - .dimension(slicer.dimensions.candidate) \ .query - self.assertEqual('SELECT ' + # Important that in reference queries when using totals that the null dimensions are omitted from the nested + # queries and selected in the container query + self.assertEqual('(SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."political_party" "political_party",' + '"base"."votes" "votes",' + '"dod"."votes" "votes_dod" ' + 'FROM (' + + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' '"political_party" "political_party",' - '"candidate_id" "candidate",' - '"candidate_name" "candidate_display",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' - 'GROUP BY ROLLUP("political_party",("candidate_id","candidate_name")) ' - 'ORDER BY "political_party","candidate_display"', str(query)) + 'GROUP BY "timestamp","political_party"' + ') "base" ' + + 'FULL OUTER JOIN (' + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + '"political_party" "political_party",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp","political_party"' + ') "dod" ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' + 'AND "base"."political_party"="dod"."political_party") ' + + 'UNION ALL ' + + '(SELECT ' + '"base"."timestamp" "timestamp",' + 'NULL "political_party",' + '"base"."votes" "votes",' + '"dod"."votes" "votes_dod" ' + 'FROM (' + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' + + 'FULL OUTER JOIN (' + 'SELECT TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "dod" ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp")) ' + 'ORDER BY "timestamp","political_party"', str(query)) + + def test_build_query_with_totals_cat_dimension_with_references_and_date_filters(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .dimension(slicer.dimensions.timestamp.reference(f.DayOverDay)) \ + .dimension(slicer.dimensions.political_party.rollup()) \ + .filter(slicer.dimensions.timestamp.between(date(2018, 1, 1), date(2019, 1, 1))) \ + .query + + self.assertEqual('(SELECT ' + '"base"."timestamp" "timestamp",' + '"base"."political_party" "political_party",' + '"base"."votes" "votes",' + '"dod"."votes" "votes_dod" ' + 'FROM (' + + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + '"political_party" "political_party",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' + 'GROUP BY "timestamp","political_party"' + ') "base" ' + + 'FULL OUTER JOIN (' + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + '"political_party" "political_party",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' + 'GROUP BY "timestamp","political_party"' + ') "dod" ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' + 'AND "base"."political_party"="dod"."political_party") ' + + 'UNION ALL ' + + '(SELECT ' + '"base"."timestamp" "timestamp",' + 'NULL "political_party",' + '"base"."votes" "votes",' + '"dod"."votes" "votes_dod" ' + 'FROM (' + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' + 'GROUP BY "timestamp"' + ') "base" ' + + 'FULL OUTER JOIN (' + 'SELECT TRUNC("timestamp",\'DD\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' + 'GROUP BY "timestamp"' + ') "dod" ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp")) ' + 'ORDER BY "timestamp","political_party"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -673,7 +833,7 @@ def test_dimension_with_single_reference_dod(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_dod" ' + '"dod"."votes" "votes_dod" ' 'FROM ' '(' # nested @@ -684,15 +844,15 @@ def test_dimension_with_single_reference_dod(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_dimension_with_single_reference_wow(self): @@ -707,7 +867,7 @@ def test_dimension_with_single_reference_wow(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_wow" ' + '"wow"."votes" "votes_wow" ' 'FROM ' '(' # nested @@ -718,15 +878,15 @@ def test_dimension_with_single_reference_wow(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "wow" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'week\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'week\',1,"wow"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_dimension_with_single_reference_mom(self): @@ -741,7 +901,7 @@ def test_dimension_with_single_reference_mom(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_mom" ' + '"mom"."votes" "votes_mom" ' 'FROM ' '(' # nested @@ -752,15 +912,15 @@ def test_dimension_with_single_reference_mom(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "mom" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'month\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'month\',1,"mom"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_dimension_with_single_reference_qoq(self): @@ -775,7 +935,7 @@ def test_dimension_with_single_reference_qoq(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_qoq" ' + '"qoq"."votes" "votes_qoq" ' 'FROM ' '(' # nested @@ -786,15 +946,15 @@ def test_dimension_with_single_reference_qoq(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "qoq" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'quarter\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'quarter\',1,"qoq"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_dimension_with_single_reference_yoy(self): @@ -809,7 +969,7 @@ def test_dimension_with_single_reference_yoy(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_yoy" ' + '"yoy"."votes" "votes_yoy" ' 'FROM ' '(' # nested @@ -820,15 +980,15 @@ def test_dimension_with_single_reference_yoy(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_dimension_with_single_reference_as_a_delta(self): @@ -843,7 +1003,7 @@ def test_dimension_with_single_reference_as_a_delta(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"base"."votes"-"sq1"."votes" "votes_dod_delta" ' + '"base"."votes"-"dod"."votes" "votes_dod_delta" ' 'FROM ' '(' # nested @@ -854,15 +1014,15 @@ def test_dimension_with_single_reference_as_a_delta(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_dimension_with_single_reference_as_a_delta_percentage(self): @@ -877,7 +1037,7 @@ def test_dimension_with_single_reference_as_a_delta_percentage(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '("base"."votes"-"sq1"."votes")*100/NULLIF("sq1"."votes",0) "votes_dod_delta_percent" ' + '("base"."votes"-"dod"."votes")*100/NULLIF("dod"."votes",0) "votes_dod_delta_percent" ' 'FROM ' '(' # nested @@ -888,15 +1048,15 @@ def test_dimension_with_single_reference_as_a_delta_percentage(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_reference_on_dimension_with_weekly_interval(self): @@ -911,7 +1071,7 @@ def test_reference_on_dimension_with_weekly_interval(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_dod" ' + '"dod"."votes" "votes_dod" ' 'FROM ' '(' # nested @@ -922,15 +1082,15 @@ def test_reference_on_dimension_with_weekly_interval(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'IW\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_reference_on_dimension_with_monthly_interval(self): @@ -945,7 +1105,7 @@ def test_reference_on_dimension_with_monthly_interval(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_dod" ' + '"dod"."votes" "votes_dod" ' 'FROM ' '(' # nested @@ -956,15 +1116,15 @@ def test_reference_on_dimension_with_monthly_interval(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'MM\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_reference_on_dimension_with_quarterly_interval(self): @@ -979,7 +1139,7 @@ def test_reference_on_dimension_with_quarterly_interval(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_dod" ' + '"dod"."votes" "votes_dod" ' 'FROM ' '(' # nested @@ -990,15 +1150,15 @@ def test_reference_on_dimension_with_quarterly_interval(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'Q\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_reference_on_dimension_with_annual_interval(self): @@ -1013,7 +1173,7 @@ def test_reference_on_dimension_with_annual_interval(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_dod" ' + '"dod"."votes" "votes_dod" ' 'FROM ' '(' # nested @@ -1024,15 +1184,15 @@ def test_reference_on_dimension_with_annual_interval(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'Y\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_dimension_with_multiple_references(self): @@ -1048,8 +1208,8 @@ def test_dimension_with_multiple_references(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_dod",' - '("base"."votes"-"sq2"."votes")*100/NULLIF("sq2"."votes",0) "votes_yoy_delta_percent" ' + '"dod"."votes" "votes_dod",' + '("base"."votes"-"yoy"."votes")*100/NULLIF("yoy"."votes",0) "votes_yoy_delta_percent" ' 'FROM ' '(' # nested @@ -1060,25 +1220,25 @@ def test_dimension_with_multiple_references(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq2" ' # end-nested + ') "yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq2"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_reference_joins_nested_query_on_dimensions(self): @@ -1095,7 +1255,7 @@ def test_reference_joins_nested_query_on_dimensions(self): '"base"."timestamp" "timestamp",' '"base"."political_party" "political_party",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_yoy" ' + '"yoy"."votes" "votes_yoy" ' 'FROM ' '(' # nested @@ -1107,17 +1267,17 @@ def test_reference_joins_nested_query_on_dimensions(self): 'GROUP BY "timestamp","political_party"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' '"political_party" "political_party",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp","political_party"' - ') "sq1" ' # end-nested + ') "yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' - 'AND "base"."political_party"="sq1"."political_party" ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' + 'AND "base"."political_party"="yoy"."political_party" ' 'ORDER BY "timestamp","political_party"', str(query)) def test_reference_with_unique_dimension_includes_display_definition(self): @@ -1135,7 +1295,7 @@ def test_reference_with_unique_dimension_includes_display_definition(self): '"base"."candidate" "candidate",' '"base"."candidate_display" "candidate_display",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_yoy" ' + '"yoy"."votes" "votes_yoy" ' 'FROM ' '(' # nested @@ -1148,7 +1308,7 @@ def test_reference_with_unique_dimension_includes_display_definition(self): 'GROUP BY "timestamp","candidate","candidate_display"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' '"candidate_id" "candidate",' @@ -1156,10 +1316,10 @@ def test_reference_with_unique_dimension_includes_display_definition(self): 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp","candidate","candidate_display"' - ') "sq1" ' # end-nested + ') "yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' - 'AND "base"."candidate"="sq1"."candidate" ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' + 'AND "base"."candidate"="yoy"."candidate" ' 'ORDER BY "timestamp","candidate_display"', str(query)) def test_adjust_reference_dimension_filters_in_reference_query(self): @@ -1176,7 +1336,7 @@ def test_adjust_reference_dimension_filters_in_reference_query(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_dod" ' + '"dod"."votes" "votes_dod" ' 'FROM ' '(' # nested @@ -1188,16 +1348,16 @@ def test_adjust_reference_dimension_filters_in_reference_query(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_filters(self): @@ -1216,7 +1376,7 @@ def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_fil self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_dod" ' + '"dod"."votes" "votes_dod" ' 'FROM ' '(' # nested @@ -1229,7 +1389,7 @@ def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_fil 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' @@ -1237,9 +1397,9 @@ def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_fil 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' 'AND "political_party" IN (\'d\') ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_reference(self): @@ -1254,7 +1414,7 @@ def test_adapt_dow_for_leap_year_for_yoy_reference(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_yoy" ' + '"yoy"."votes" "votes_yoy" ' 'FROM ' '(' # nested @@ -1265,15 +1425,15 @@ def test_adapt_dow_for_leap_year_for_yoy_reference(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): @@ -1288,7 +1448,7 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"base"."votes"-"sq1"."votes" "votes_yoy_delta" ' + '"base"."votes"-"yoy"."votes" "votes_yoy_delta" ' 'FROM ' '(' # nested @@ -1299,15 +1459,15 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' 'ORDER BY "timestamp"', str(query)) @@ -1323,7 +1483,7 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '("base"."votes"-"sq1"."votes")*100/NULLIF("sq1"."votes",0) "votes_yoy_delta_percent" ' + '("base"."votes"-"yoy"."votes")*100/NULLIF("yoy"."votes",0) "votes_yoy_delta_percent" ' 'FROM ' '(' # nested @@ -1334,15 +1494,15 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_reference_with_date_filter(self): @@ -1358,7 +1518,7 @@ def test_adapt_dow_for_leap_year_for_yoy_reference_with_date_filter(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_yoy" ' + '"yoy"."votes" "votes_yoy" ' 'FROM ' '(' # nested @@ -1370,16 +1530,16 @@ def test_adapt_dow_for_leap_year_for_yoy_reference_with_date_filter(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'year\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_adding_duplicate_reference_does_not_join_more_queries(self): @@ -1395,7 +1555,7 @@ def test_adding_duplicate_reference_does_not_join_more_queries(self): self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_dod" ' + '"dod"."votes" "votes_dod" ' 'FROM ' '(' # nested @@ -1406,15 +1566,15 @@ def test_adding_duplicate_reference_does_not_join_more_queries(self): 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension(self): @@ -1431,9 +1591,9 @@ def test_use_same_nested_query_for_joining_references_with_same_period_and_dimen self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_dod",' - '"base"."votes"-"sq1"."votes" "votes_dod_delta",' - '("base"."votes"-"sq1"."votes")*100/NULLIF("sq1"."votes",0) "votes_dod_delta_percent" ' + '"dod"."votes" "votes_dod",' + '"base"."votes"-"dod"."votes" "votes_dod_delta",' + '("base"."votes"-"dod"."votes")*100/NULLIF("dod"."votes",0) "votes_dod_delta_percent" ' 'FROM ' '(' # nested @@ -1444,15 +1604,15 @@ def test_use_same_nested_query_for_joining_references_with_same_period_and_dimen 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' 'ORDER BY "timestamp"', str(query)) def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension_with_different_periods(self): @@ -1470,10 +1630,10 @@ def test_use_same_nested_query_for_joining_references_with_same_period_and_dimen self.assertEqual('SELECT ' '"base"."timestamp" "timestamp",' '"base"."votes" "votes",' - '"sq1"."votes" "votes_dod",' - '"base"."votes"-"sq1"."votes" "votes_dod_delta",' - '"sq2"."votes" "votes_yoy",' - '"base"."votes"-"sq2"."votes" "votes_yoy_delta" ' + '"dod"."votes" "votes_dod",' + '"base"."votes"-"dod"."votes" "votes_dod_delta",' + '"yoy"."votes" "votes_yoy",' + '"base"."votes"-"yoy"."votes" "votes_yoy_delta" ' 'FROM ' '(' # nested @@ -1484,25 +1644,25 @@ def test_use_same_nested_query_for_joining_references_with_same_period_and_dimen 'GROUP BY "timestamp"' ') "base" ' # end-nested - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq1" ' # end-nested + ') "dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"sq1"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'LEFT JOIN (' # nested + 'FULL OUTER JOIN (' # nested 'SELECT ' 'TRUNC("timestamp",\'DD\') "timestamp",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' 'GROUP BY "timestamp"' - ') "sq2" ' # end-nested + ') "yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"sq2"."timestamp") ' + 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' 'ORDER BY "timestamp"', str(query)) diff --git a/requirements.txt b/requirements.txt index 9336b688..86d1574b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ six pandas==0.22.0 -pypika==0.10.4 +pypika==0.10.5 pymysql==0.8.0 vertica-python==0.7.3 psycopg2==2.7.3.2 From 18b644cc6f1d9813419d11d35d3af3bf96ce89d3 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 6 Mar 2018 15:15:37 +0100 Subject: [PATCH 037/123] Bumped version to 1.0.0.dev14 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index a1237da9..5796cd11 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev13' +__version__ = '1.0.0.dev14' From 2c2e9d8f4c69337647e4b2b29724dd77e204d164 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 7 Mar 2018 16:18:41 +0100 Subject: [PATCH 038/123] Fixed references to both work without selecting the referenced dimension in the query and also in case there is data missing for days in the base query but not in the reference query --- fireant/__init__.py | 2 +- fireant/slicer/base.py | 7 +- fireant/slicer/dimensions.py | 61 ++- fireant/slicer/intervals.py | 3 + fireant/slicer/queries/builder.py | 54 ++- fireant/slicer/queries/finders.py | 43 +- fireant/slicer/queries/makers.py | 416 +++++------------- fireant/slicer/queries/references.py | 239 ++++++++++ fireant/slicer/references.py | 108 +++-- fireant/slicer/widgets/base.py | 2 +- fireant/slicer/widgets/csv.py | 4 +- fireant/slicer/widgets/datatables.py | 6 +- fireant/slicer/widgets/highcharts.py | 12 +- fireant/slicer/widgets/matplotlib.py | 2 +- fireant/slicer/widgets/pandas.py | 6 +- fireant/tests/slicer/mocks.py | 12 +- fireant/tests/slicer/queries/test_builder.py | 286 ++++++++---- fireant/tests/slicer/widgets/test_csv.py | 34 +- .../tests/slicer/widgets/test_datatables.py | 45 +- .../tests/slicer/widgets/test_highcharts.py | 70 +-- fireant/tests/slicer/widgets/test_pandas.py | 35 +- 21 files changed, 834 insertions(+), 613 deletions(-) create mode 100644 fireant/slicer/queries/references.py diff --git a/fireant/__init__.py b/fireant/__init__.py index 5796cd11..86c8d8b6 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev14' +__version__ = '1.0.0.dev15' diff --git a/fireant/slicer/base.py b/fireant/slicer/base.py index 2d379d6d..c32665e9 100644 --- a/fireant/slicer/base.py +++ b/fireant/slicer/base.py @@ -35,5 +35,10 @@ def __repr__(self): def has_display_field(self): return getattr(self, 'display_definition', None) is not None + def __eq__(self, other): + return isinstance(other, self.__class__) \ + and self.key == other.key \ + and str(self.definition) == str(other.definition) + def __hash__(self): - return hash('{}({})'.format(self.__class__.__name__, self.definition)) + return hash('{}({})'.format(self.__class__.__name__, self.key, self.definition)) diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 6b0617c3..25ddd632 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -2,7 +2,6 @@ from fireant.utils import immutable from pypika.terms import ( - ValueWrapper, NullValue, ) from .base import SlicerElement @@ -46,9 +45,9 @@ class BooleanDimension(Dimension): """ def __init__(self, key, label=None, definition=None): - super(BooleanDimension, self).__init__(key=key, - label=label, - definition=definition) + super(BooleanDimension, self).__init__(key, + label, + definition) def is_(self, value: bool): """ @@ -69,9 +68,9 @@ class CategoricalDimension(Dimension): """ def __init__(self, key, label=None, definition=None, display_values=()): - super(CategoricalDimension, self).__init__(key=key, - label=label, - definition=definition) + super(CategoricalDimension, self).__init__(key, + label, + definition) self.display_values = dict(display_values) def isin(self, values: Iterable): @@ -108,11 +107,10 @@ class UniqueDimension(Dimension): """ def __init__(self, key, label=None, definition=None, display_definition=None): - super(UniqueDimension, self).__init__(key=key, - label=label, - definition=definition, - display_definition=display_definition) - + super(UniqueDimension, self).__init__(key, + label, + definition, + display_definition) def __hash__(self): if self.has_display_field: @@ -182,7 +180,9 @@ class DisplayDimension(Dimension): """ def __init__(self, dimension): - super(DisplayDimension, self).__init__(dimension.display_key, dimension.label, dimension.display_definition) + super(DisplayDimension, self).__init__(dimension.display_key, + dimension.label, + dimension.display_definition) class ContinuousDimension(Dimension): @@ -192,9 +192,9 @@ class ContinuousDimension(Dimension): """ def __init__(self, key, label=None, definition=None, default_interval=NumericInterval(1, 0)): - super(ContinuousDimension, self).__init__(key=key, - label=label, - definition=definition) + super(ContinuousDimension, self).__init__(key, + label, + definition) self.interval = default_interval @@ -206,11 +206,10 @@ class DatetimeDimension(ContinuousDimension): """ def __init__(self, key, label=None, definition=None, default_interval=daily): - super(DatetimeDimension, self).__init__(key=key, - label=label, - definition=definition, + super(DatetimeDimension, self).__init__(key, + label, + definition, default_interval=default_interval) - self.references = [] @immutable def __call__(self, interval): @@ -231,18 +230,6 @@ def __call__(self, interval): """ self.interval = interval - @immutable - def reference(self, reference): - """ - Add a reference to this dimension when building a slicer query. - - :param reference: - A reference to add to the query - :return: - A copy of the dimension with the reference added. - """ - self.references.append(reference) - def between(self, start, stop): """ Creates a filter to filter a slicer query. @@ -261,5 +248,11 @@ def between(self, start, stop): class TotalsDimension(Dimension): def __init__(self, dimension): totals_definition = NullValue() - display_definition = totals_definition if dimension.has_display_field else None - super(Dimension, self).__init__(dimension.key, dimension.label, totals_definition, display_definition) + display_definition = totals_definition \ + if dimension.has_display_field \ + else None + + super(Dimension, self).__init__(dimension.key, + dimension.label, + totals_definition, + display_definition) diff --git a/fireant/slicer/intervals.py b/fireant/slicer/intervals.py index e7020fff..9e173e31 100644 --- a/fireant/slicer/intervals.py +++ b/fireant/slicer/intervals.py @@ -29,6 +29,9 @@ def __eq__(self, other): def __str__(self): return self.key + def __repr__(self): + return 'DatetimeInterval(\'{}\')'.format(self.key) + hourly = DatetimeInterval('hour') daily = DatetimeInterval('day') diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 5cf209fd..847005fd 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -1,19 +1,19 @@ +import pandas as pd from typing import ( Dict, Iterable, ) -import pandas as pd +from fireant.utils import immutable from pypika import ( Order, functions as fn, ) from pypika.enums import SqlTypes - -from fireant.utils import immutable from .database import fetch_data from .finders import ( find_and_group_references_for_dimensions, + find_and_replace_reference_dimensions, find_dimensions_with_totals, find_metrics_for_widgets, find_operations_for_widgets, @@ -21,7 +21,7 @@ from .makers import ( make_orders_for_dimensions, make_slicer_query, - make_slicer_query_with_totals, + make_slicer_query_with_references_and_totals, ) from ..base import SlicerElement @@ -32,6 +32,7 @@ def __init__(self, slicer, table): self.table = table self._dimensions = [] self._filters = [] + self._references = [] @immutable def filter(self, *filters): @@ -87,6 +88,18 @@ def dimension(self, *dimensions): for dimension in dimensions if dimension not in self._dimensions] + @immutable + def reference(self, *references): + """ + Add a reference for a dimension when building a slicer query. + + :param references: + References to add to the query + :return: + A copy of the dimension with the reference added. + """ + self._references += references + @immutable def orderby(self, element: SlicerElement, orientation=None): """ @@ -106,26 +119,26 @@ def query(self): normally produced is wrapped in an outer query and a query for each reference is joined based on the referenced dimension shifted. """ - database = self.slicer.database - table = self.table - joins = self.slicer.joins - dimensions = self._dimensions - metrics = find_metrics_for_widgets(self._widgets) - filters = self._filters - - # Validate + # First run validation for the query on all widgets for widget in self._widgets: if hasattr(widget, 'validate'): - widget.validate(dimensions) + widget.validate(self._dimensions) - reference_groups = find_and_group_references_for_dimensions(dimensions) - totals_dimensions = find_dimensions_with_totals(dimensions) + self._references = find_and_replace_reference_dimensions(self._references, self._dimensions) + reference_groups = find_and_group_references_for_dimensions(self._references) + totals_dimensions = find_dimensions_with_totals(self._dimensions) - query = make_slicer_query_with_totals(database, table, joins, dimensions, metrics, filters, - reference_groups, totals_dimensions) + query = make_slicer_query_with_references_and_totals(self.slicer.database, + self.table, + self.slicer.joins, + self._dimensions, + find_metrics_for_widgets(self._widgets), + self._filters, + reference_groups, + totals_dimensions) # Add ordering - orders = (self._orders or make_orders_for_dimensions(dimensions)) + orders = (self._orders or make_orders_for_dimensions(self._dimensions)) for (term, orientation) in orders: query = query.orderby(term, order=orientation) @@ -154,7 +167,7 @@ def fetch(self, limit=None, offset=None) -> Iterable[Dict]: data_frame[operation.key] = operation.apply(data_frame) # Apply transformations - return [widget.transform(data_frame, self.slicer, self._dimensions) + return [widget.transform(data_frame, self.slicer, self._dimensions, self._references) for widget in self._widgets] def __str__(self): @@ -183,8 +196,7 @@ def query(self): The slicer query extends this with metrics, references, and totals. """ - return make_slicer_query(query_cls=self.slicer.database.query_cls, - trunc_date=self.slicer.database.trunc_date, + return make_slicer_query(database=self.slicer.database, base_table=self.table, joins=self.slicer.joins, dimensions=self._dimensions, diff --git a/fireant/slicer/queries/finders.py b/fireant/slicer/queries/finders.py index e6709804..1ff942bb 100644 --- a/fireant/slicer/queries/finders.py +++ b/fireant/slicer/queries/finders.py @@ -1,3 +1,4 @@ +import copy from collections import ( OrderedDict, defaultdict, @@ -111,14 +112,35 @@ def find_dimensions_with_totals(dimensions): if dimension.is_rollup] -def find_and_group_references_for_dimensions(dimensions): +def find_and_replace_reference_dimensions(references, dimensions): + """ + Finds the dimension for a reference in the query if there is one and replaces it. This is to force the reference to + use the same modifiers with a dimension if it is selected in the query. + + :param references: + :param dimensions: + :return: + """ + dimensions_by_key = {dimension.key: dimension + for dimension in dimensions} + + reference_copies = [] + for reference in map(copy.deepcopy, references): + dimension = dimensions_by_key.get(reference.dimension.key) + if dimension is not None: + reference.dimension = dimension + reference_copies.append(reference) + return reference_copies + + +def find_and_group_references_for_dimensions(references): """ Finds all of the references for dimensions and groups them by dimension, interval unit, number of intervals. This structure reflects how the references need to be joined to the slicer query. References of the same type (WoW, WoW.delta, WoW.delta_percent) can share a join query. - :param dimensions: + :param references: :return: An `OrderedDict` where the keys are 3-item tuples consisting of "Dimension, interval unit, # of intervals. @@ -131,17 +153,8 @@ def find_and_group_references_for_dimensions(dimensions): } """ - def get_unit_and_interval(reference): - return reference.time_unit, reference.interval - - return OrderedDict([(ReferenceGroup(dimension, time_unit, intervals), ordered_distinct_list(group)) - - # group by dimension - for dimension in dimensions - - # filter dimensions with no references - if getattr(dimension, 'references', None) + def get_dimension_time_unit_and_interval(reference): + return reference.dimension, reference.time_unit, reference.interval - # group by time_unit and intervals - for (time_unit, intervals), group in - groupby(dimension.references, get_unit_and_interval).items()]) + distinct_references = ordered_distinct_list(references) + return groupby(distinct_references, get_dimension_time_unit_and_interval) diff --git a/fireant/slicer/queries/makers.py b/fireant/slicer/queries/makers.py index a456ae4b..9fff7e64 100644 --- a/fireant/slicer/queries/makers.py +++ b/fireant/slicer/queries/makers.py @@ -1,38 +1,26 @@ -import copy from functools import partial -from typing import ( - Callable, - Iterable, -) - -from fireant.slicer.intervals import weekly -from fireant.slicer.references import ( - YearOverYear, - reference_key, - reference_term, -) from fireant.utils import flatten from pypika import JoinType -from pypika.queries import QueryBuilder -from pypika.terms import ( - ComplexCriterion, - Criterion, - NullValue, - Term, -) from .finders import ( find_joins_for_tables, find_required_tables_to_join, ) -from ..dimensions import ( - Dimension, - TotalsDimension, +from .references import ( + make_dimension_terms_for_reference_container_query, + make_metric_terms_for_reference_container_query, + make_reference_filters, + make_reference_join_criterion, + make_terms_for_references, + monkey_patch_align_weekdays, ) +from ..dimensions import TotalsDimension from ..filters import DimensionFilter +from ..intervals import weekly +from ..references import YearOverYear -def make_slicer_query(query_cls, trunc_date, base_table, joins=(), dimensions=(), metrics=(), filters=()): +def make_slicer_query(database, base_table, joins=(), dimensions=(), metrics=(), filters=()): """ Creates a pypika/SQL query from a list of slicer elements. @@ -44,10 +32,8 @@ def make_slicer_query(query_cls, trunc_date, base_table, joins=(), dimensions=() The slicer query extends this with metrics, references, and totals. - :param query_cls: - pypika.Query - The pypika query class to use to create the query - :param trunc_date: - Callable - A function to truncate a date to an interval (database vendor specific) + :param database: + :param base_table: pypika.Table - The base table of the query, the one in the FROM clause :param joins: @@ -60,9 +46,8 @@ def make_slicer_query(query_cls, trunc_date, base_table, joins=(), dimensions=() :param filters: Iterable - A collection of filters to apply to the query. :return: - """ - query = query_cls.from_(base_table) + query = database.query_cls.from_(base_table) elements = flatten([metrics, dimensions, filters]) @@ -73,7 +58,7 @@ def make_slicer_query(query_cls, trunc_date, base_table, joins=(), dimensions=() # Add dimensions for dimension in dimensions: - terms = make_terms_for_dimension(dimension, trunc_date) + terms = make_terms_for_dimension(dimension, database.trunc_date) query = query.select(*terms) # Don't group TotalsDimensions if not isinstance(dimension, TotalsDimension): @@ -93,8 +78,8 @@ def make_slicer_query(query_cls, trunc_date, base_table, joins=(), dimensions=() return query -def make_slicer_query_with_totals(database, table, joins, dimensions, metrics, filters, - reference_groups, totals_dimensions): +def make_slicer_query_with_references_and_totals(database, table, joins, dimensions, metrics, filters, + reference_groups, totals_dimensions): """ WRITEME @@ -108,27 +93,32 @@ def make_slicer_query_with_totals(database, table, joins, dimensions, metrics, f :param totals_dimensions: :return: """ - query = make_slicer_query_with_references(database, - table, - joins, - dimensions, - metrics, - filters, - reference_groups) + make_query = partial(make_slicer_query_with_references, reference_groups=reference_groups) \ + if reference_groups else \ + make_slicer_query + + query = make_query(database, + table, + joins, + dimensions, + metrics, + filters) + for dimension in totals_dimensions: - totals_dimension_index = dimensions.index(dimension) - grouped_dims, totaled_dims = dimensions[:totals_dimension_index], dimensions[totals_dimension_index:] + index = dimensions.index(dimension) + grouped_dims, totaled_dims = dimensions[:index], dimensions[index:] + # Replace the dimension and all following dimensions with a TotalsDimension. This will prevent those dimensions + # from being grouped on in the totals UNION query, selecting NULL instead dimensions_with_totals = grouped_dims + [TotalsDimension(dimension) for dimension in totaled_dims] - totals_query = make_slicer_query_with_references(database, - table, - joins, - dimensions_with_totals, - metrics, - filters, - reference_groups) + totals_query = make_query(database, + table, + joins, + dimensions_with_totals, + metrics, + filters) # UNION ALL query = query.union_all(totals_query) @@ -140,8 +130,7 @@ def make_slicer_query_with_references(database, base_table, joins, dimensions, m """ WRITEME - :param query_cls: - :param trunc_date: + :param database: :param base_table: :param joins: :param dimensions: @@ -150,154 +139,81 @@ def make_slicer_query_with_references(database, base_table, joins, dimensions, m :param reference_groups: :return: """ - - if not reference_groups: - return make_slicer_query(query_cls=database.query_cls, - trunc_date=database.trunc_date, - base_table=base_table, - joins=joins, - dimensions=dimensions, - metrics=metrics, - filters=filters) - # Do not include totals dimensions in the reference query (they are selected directly in the container query) non_totals_dimensions = [dimension for dimension in dimensions if not isinstance(dimension, TotalsDimension)] - query = make_slicer_query(query_cls=database.query_cls, - trunc_date=database.trunc_date, - base_table=base_table, - joins=joins, - dimensions=non_totals_dimensions, - metrics=metrics, - filters=filters) - - original_query = query.as_('base') - container_query = make_reference_container_query(original_query, - database.query_cls, - dimensions, - metrics) - - for (dimension, time_unit, interval), references in reference_groups.items(): - alias = references[0].key[:3] - ref_query, join_criterion = make_reference_query(database, - base_table, - joins, - non_totals_dimensions, - metrics, - filters, - dimension, - time_unit, - interval, - original_query, - alias=alias) - - terms = make_terms_for_references(references, - original_query, - ref_query, - metrics) - - container_query = container_query \ - .join(ref_query, JoinType.full_outer) \ - .on(join_criterion) \ - .select(*terms) - - return container_query - - -def make_reference_container_query(original_query, query_cls, dimensions, metrics): - """ - Creates a container query with the original query used as the FROM clause and selects all of the metrics. The - container query is used as a base for joining reference queries. - - :param original_query: - :param query_cls: - :param dimensions: - :param metrics: - :return: - """ - - def original_query_field(element, display=False): - key = element.display_key if display else element.key - definition = element.display_definition if display else element.definition - - if isinstance(definition, NullValue): - # If an element is a literal NULL, then include the definition directly in the container query. It will be - # omitted from the reference queries - return definition.as_(key) - - return original_query.field(key).as_(key) - - outer_query = query_cls.from_(original_query) - - # Add dimensions - for dimension in dimensions: - outer_query = outer_query.select(original_query_field(dimension)) - - if dimension.has_display_field: - outer_query = outer_query.select(original_query_field(dimension, True)) - - # Add base metrics - return outer_query.select(*[original_query_field(metric) - for metric in metrics]) - - -def make_reference_query(database, base_table, joins, dimensions, metrics, filters, ref_dimension, time_unit, interval, - original_query, alias=None): - """ - WRITEME - - :param alias: - :param database: - :param base_table: - :param joins: - :param dimensions: - :param metrics: - :param filters: - :param references: - :param ref_dimension: - :param time_unit: - :param interval: - :param original_query: - :return: - """ - offset_func = partial(database.date_add, - date_part=time_unit, - interval=interval) - - # for weekly interval dimensions with YoY references, correct the day of week by adding a year, truncating to a - # week, then subtracting a year - offset_for_weekday = weekly == ref_dimension.interval and YearOverYear.time_unit == time_unit - if offset_for_weekday: - def trunc_date(definition, _): - shift_forward = database.date_add(definition, time_unit, interval) - offset = database.trunc_date(shift_forward, weekly.key) - return database.date_add(offset, time_unit, -interval) - - else: - trunc_date = database.trunc_date - - ref_filters = make_reference_filters(filters, - ref_dimension, - offset_func) - - ref_query = make_slicer_query(database.query_cls, - trunc_date, - base_table, - joins, - dimensions, - metrics, - ref_filters) \ - .as_(alias) - - join_criterion = make_reference_join_criterion(ref_dimension, - dimensions, - original_query, - ref_query, - offset_func) - - return ref_query, join_criterion + original_query = make_slicer_query(database=database, + base_table=base_table, + joins=joins, + dimensions=non_totals_dimensions, + metrics=metrics, + filters=filters) \ + .as_('base') + + container_query = database.query_cls.from_(original_query) + + ref_dimension_definitions, ref_terms = [], [] + for (ref_dimension, time_unit, interval), references in reference_groups.items(): + offset_func = partial(database.date_add, + date_part=time_unit, + interval=interval) + + # NOTE: In the case of weekly intervals with YoY references, the trunc date function needs to adjust for weekday + # to keep things aligned. To do this, the date is first shifted forward a year before being truncated by week + # and then shifted back. + offset_for_weekday = weekly == ref_dimension.interval and YearOverYear.time_unit == time_unit + ref_database = monkey_patch_align_weekdays(database, time_unit, interval) \ + if offset_for_weekday \ + else database + + ref_filters = make_reference_filters(filters, + ref_dimension, + offset_func) + + # Grab the first reference out of the list since all of the references in the group must share the same + # reference type + alias = references[0].reference_type.key + ref_query = make_slicer_query(ref_database, + base_table, + joins, + non_totals_dimensions, + metrics, + ref_filters) \ + .as_(alias) + + join_criterion = make_reference_join_criterion(ref_dimension, + non_totals_dimensions, + original_query, + ref_query, + offset_func) + + if join_criterion: + container_query = container_query \ + .join(ref_query, JoinType.full_outer) \ + .on(join_criterion) + else: + container_query = container_query.from_(ref_query) + + ref_dimension_definitions.append([offset_func(ref_query.field(dimension.key)) + if ref_dimension == dimension + else ref_query.field(dimension.key) + for dimension in dimensions]) + + ref_terms += make_terms_for_references(references, + original_query, + ref_query, + metrics) + + dimension_terms = make_dimension_terms_for_reference_container_query(original_query, + dimensions, + ref_dimension_definitions) + metric_terms = make_metric_terms_for_reference_container_query(original_query, + metrics) + + all_terms = dimension_terms + metric_terms + ref_terms + return container_query.select(*all_terms) def make_terms_for_metrics(metrics): @@ -335,91 +251,6 @@ def make_terms_for_dimension(dimension, window=None): ] -def make_terms_for_references(references, original_query, ref_query, metrics): - """ - Makes the terms needed to be selected from a reference query in the container query. - - :param references: - :param original_query: - :param ref_query: - :param metrics: - :return: - """ - seen = set() - terms = [] - for reference in references: - # Don't select duplicate references twice - if reference.key in seen \ - and not seen.add(reference.key): - continue - - # Get function to select reference metrics - ref_metric = reference_term(reference, - original_query, - ref_query) - - terms += [ref_metric(metric).as_(reference_key(metric, reference)) - for metric in metrics] - - return terms - - -def make_reference_filters(filters, ref_dimension, offset_func): - """ - Copies and replaces the reference dimension's definition in all of the filters applied to a slicer query. - - This is used to shift the dimension filters to fit the reference window. - - :param filters: - :param ref_dimension: - :param offset_func: - :return: - """ - offset_ref_dimension_definition = offset_func(ref_dimension.definition) - - reference_filters = [] - for ref_filter in map(copy.deepcopy, filters): - ref_filter.definition = _apply_to_term_in_criterion(ref_dimension.definition, - offset_ref_dimension_definition, - ref_filter.definition) - reference_filters.append(ref_filter) - - return reference_filters - - -def make_reference_join_criterion(ref_dimension: Dimension, - all_dimensions: Iterable[Dimension], - original_query: QueryBuilder, - ref_query: QueryBuilder, - offset_func: Callable): - """ - This creates the criterion for joining a reference query to the base query. It matches the referenced dimension - in the base query to the offset referenced dimension in the reference query and all other dimensions. - - :param ref_dimension: - The referenced dimension. - :param all_dimensions: - All of the dimensions applied to the slicer query. - :param original_query: - The base query, the original query despite the references. - :param ref_query: - The reference query, a copy of the base query with the referenced dimension replaced. - :param offset_func: - The offset function for shifting the referenced dimension. - :return: - pypika.Criterion - """ - join_criterion = original_query.field(ref_dimension.key) == offset_func(ref_query.field(ref_dimension.key)) - - for dimension_ in all_dimensions: - if ref_dimension == dimension_: - continue - - join_criterion &= original_query.field(dimension_.key) == ref_query.field(dimension_.key) - - return join_criterion - - def make_orders_for_dimensions(dimensions): """ Creates a list of ordering for a slicer query based on a list of dimensions. The dimensions's display definition is @@ -440,32 +271,3 @@ def make_orders_for_dimensions(dimensions): return [(definition, None) for definition in definitions] - -def _apply_to_term_in_criterion(target: Term, - replacement: Term, - criterion: Criterion): - """ - Finds and replaces a term within a criterion. This is necessary for adapting filters used in reference queries - where the reference dimension must be offset by some value. The target term is found inside the criterion and - replaced with the replacement. - - :param target: - The target term to replace in the criterion. It will be replaced in all locations within the criterion with - the func applied to itself. - :param replacement: - The replacement for the term. - :param criterion: - The criterion to replace the term in. - :return: - A criterion identical to the original criterion arg except with the target term replaced by the replacement arg. - """ - if isinstance(criterion, ComplexCriterion): - criterion.left = _apply_to_term_in_criterion(target, replacement, criterion.left) - criterion.right = _apply_to_term_in_criterion(target, replacement, criterion.right) - return criterion - - for attr in ['term', 'left', 'right']: - if hasattr(criterion, attr) and str(getattr(criterion, attr)) == str(target): - setattr(criterion, attr, replacement) - - return criterion diff --git a/fireant/slicer/queries/references.py b/fireant/slicer/queries/references.py new file mode 100644 index 00000000..0f535ddf --- /dev/null +++ b/fireant/slicer/queries/references.py @@ -0,0 +1,239 @@ +import copy + +from typing import ( + Callable, + Iterable, +) + +from fireant.slicer.references import ( + reference_key, + reference_term, +) +from pypika import functions as fn +from pypika.queries import QueryBuilder +from pypika.terms import ( + ComplexCriterion, + Criterion, + NullValue, + Term, +) +from ..dimensions import Dimension +from ..intervals import weekly + + +def make_terms_for_references(references, original_query, ref_query, metrics): + """ + Makes the terms needed to be selected from a reference query in the container query. + + :param references: + :param original_query: + :param ref_query: + :param metrics: + :return: + """ + seen = set() + terms = [] + for reference in references: + # Don't select duplicate references twice + if reference.key in seen \ + and not seen.add(reference.key): + continue + + # Get function to select reference metrics + ref_metric = reference_term(reference, + original_query, + ref_query) + + terms += [ref_metric(metric).as_(reference_key(metric, reference)) + for metric in metrics] + + return terms + + +def make_dimension_terms_for_reference_container_query(original_query, + dimensions, + ref_dimension_definitions): + """ + Creates a list of terms that will be selected from the reference container query based on the selected dimensions + and all of the reference queries. Concretely, this will return one term per dimension where the term is a COALESCE + function call with the dimension defintion as the base value and the reference dimension definitions as the default + values (COALESCE takes a variable number of args). + + Consequently, the reference dimension definitions should be shifted using the reference's offset function. This + function expects that to have already been done in the arg ref_dimension_definitions. + + :param original_query: + :param dimensions: + :param ref_dimension_definitions: + :return: + """ + + # Zip these so they are one row per dimension + ref_dimension_definitions = list(zip(*ref_dimension_definitions)) + + terms = [] + for dimension, ref_dimension_definition in zip(dimensions, ref_dimension_definitions): + term = _select_for_reference_container_query(dimension.key, + dimension.definition, + original_query, + ref_dimension_definition) + terms.append(term) + + if not dimension.has_display_field: + continue + + # Select the display definitions as a field from the ref query + ref_display_definition = [definition.table.field(dimension.display_key) + for definition in ref_dimension_definition] + display_term = _select_for_reference_container_query(dimension.display_key, + dimension.display_definition, + original_query, + ref_display_definition) + terms.append(display_term) + + return terms + + +def make_metric_terms_for_reference_container_query(original_query, metrics): + return [_select_for_reference_container_query(metric.key, metric.definition, original_query) + for metric in metrics] + + +def _select_for_reference_container_query(element_key, element_definition, query, extra_defintions=()): + """ + + :param element_key: + :param element_definition: + :param query: + :param extra_defintions: + :return: + """ + if isinstance(element_definition, NullValue): + # If an element is a literal NULL, then include the definition directly in the container query. It will be + # omitted from the reference queries + return element_definition.as_(element_key) + + term = query.field(element_key) + + if extra_defintions: + term = fn.Coalesce(term, *extra_defintions) + + return term.as_(element_key) + + +def make_reference_filters(filters, ref_dimension, offset_func): + """ + Copies and replaces the reference dimension's definition in all of the filters applied to a slicer query. + + This is used to shift the dimension filters to fit the reference window. + + :param filters: + :param ref_dimension: + :param offset_func: + :return: + """ + offset_ref_dimension_definition = offset_func(ref_dimension.definition) + + reference_filters = [] + for ref_filter in map(copy.deepcopy, filters): + ref_filter.definition = _apply_to_term_in_criterion(ref_dimension.definition, + offset_ref_dimension_definition, + ref_filter.definition) + reference_filters.append(ref_filter) + + return reference_filters + + +def make_reference_join_criterion(ref_dimension: Dimension, + all_dimensions: Iterable[Dimension], + original_query: QueryBuilder, + ref_query: QueryBuilder, + offset_func: Callable): + """ + This creates the criterion for joining a reference query to the base query. It matches the referenced dimension + in the base query to the offset referenced dimension in the reference query and all other dimensions. + + :param ref_dimension: + The referenced dimension. + :param all_dimensions: + All of the dimensions applied to the slicer query. + :param original_query: + The base query, the original query despite the references. + :param ref_query: + The reference query, a copy of the base query with the referenced dimension replaced. + :param offset_func: + The offset function for shifting the referenced dimension. + :return: + pypika.Criterion + A Criterion that compares all of the dimensions using their terms in the original query and the reference + query. If the reference dimension is found in the list of dimensions, then it will be shifted using the + offset function. + + Examples: + original.date == DATE_ADD(reference.date) AND original.dim1 == reference.dim1 + + None + if there are no dimensions. In that case there's nothing to join on and the reference queries should be + added to the FROM clause of the container query. + + """ + join_criterion = None + + for dimension in all_dimensions: + ref_query_field = ref_query.field(dimension.key) + + # If this is the reference dimension, it needs to be offset by the reference interval + if ref_dimension == dimension: + ref_query_field = offset_func(ref_query_field) + + next_criterion = original_query.field(dimension.key) == ref_query_field + + join_criterion = next_criterion \ + if join_criterion is None \ + else join_criterion & next_criterion + + return join_criterion + + +def monkey_patch_align_weekdays(database, time_unit, interval): + def trunc_date(definition, _): + shift_forward = database.date_add(definition, time_unit, interval) + # Since we're going to monkey patch the trunc_date function on the database, this needs to call the original + # function + offset = database.__class__.trunc_date(database, shift_forward, weekly.key) + return database.date_add(offset, time_unit, -interval) + + # Copy the database to avoid side effects then monkey patch the trunc date function with the correction for weekday + database = copy.deepcopy(database) + database.trunc_date = trunc_date + return database + + +def _apply_to_term_in_criterion(target: Term, + replacement: Term, + criterion: Criterion): + """ + Finds and replaces a term within a criterion. This is necessary for adapting filters used in reference queries + where the reference dimension must be offset by some value. The target term is found inside the criterion and + replaced with the replacement. + + :param target: + The target term to replace in the criterion. It will be replaced in all locations within the criterion with + the func applied to itself. + :param replacement: + The replacement for the term. + :param criterion: + The criterion to replace the term in. + :return: + A criterion identical to the original criterion arg except with the target term replaced by the replacement arg. + """ + if isinstance(criterion, ComplexCriterion): + criterion.left = _apply_to_term_in_criterion(target, replacement, criterion.left) + criterion.right = _apply_to_term_in_criterion(target, replacement, criterion.right) + return criterion + + for attr in ['term', 'left', 'right']: + if hasattr(criterion, attr) and str(getattr(criterion, attr)) == str(target): + setattr(criterion, attr, replacement) + + return criterion diff --git a/fireant/slicer/references.py b/fireant/slicer/references.py index 5ccef584..f40866d2 100644 --- a/fireant/slicer/references.py +++ b/fireant/slicer/references.py @@ -3,63 +3,62 @@ class Reference(object): - def __init__(self, key, label, time_unit: str, interval: int, delta=False, percent=False): - self.key = key - self.label = label + def __init__(self, dimension, reference_type, delta=False, delta_percent=False): + self.dimension = dimension - self.time_unit = time_unit - self.interval = interval + self.reference_type = reference_type + self.key = reference_type.key + '_delta_percent' \ + if delta_percent \ + else reference_type.key + '_delta' \ + if delta \ + else reference_type.key - self.is_delta = delta - self.is_percent = percent + self.label = reference_type.label + ' Δ%' \ + if delta_percent \ + else reference_type.label + ' Δ' \ + if delta \ + else reference_type.label - def delta(self, percent=False): - key = self.key + '_delta_percent' if percent else self.key + '_delta' - label = self.label + ' Δ%' if percent else self.label + ' Δ' - return Reference(key, label, self.time_unit, self.interval, delta=True, percent=percent) + self.time_unit = reference_type.time_unit + self.interval = reference_type.interval + + self.delta = delta_percent or delta + self.delta_percent = delta_percent def __eq__(self, other): return isinstance(self, Reference) \ - and self.time_unit == other.time_unit \ - and self.interval == other.interval \ - and self.is_delta == other.is_delta \ - and self.is_percent == other.is_percent + and self.dimension == other.dimension \ + and self.key == other.key def __hash__(self): - return hash('reference' + self.key) + return hash('reference{}{}'.format(self.key, self.dimension)) -DayOverDay = Reference('dod', 'DoD', 'day', 1) -WeekOverWeek = Reference('wow', 'WoW', 'week', 1) -MonthOverMonth = Reference('mom', 'MoM', 'month', 1) -QuarterOverQuarter = Reference('qoq', 'QoQ', 'quarter', 1) -YearOverYear = Reference('yoy', 'YoY', 'year', 1) - +class ReferenceType(object): + def __init__(self, key, label, time_unit: str, interval: int): + self.key = key + self.label = label -def reference_term(reference: Reference, - original_query: QueryBuilder, - ref_query: QueryBuilder): - """ - WRITEME + self.time_unit = time_unit + self.interval = interval - :param reference: - :param original_query: - :param ref_query: - :return: - """ + def __call__(self, dimension, delta=False, delta_percent=False): + return Reference(dimension, self, delta=delta, delta_percent=delta_percent) - def ref_field(metric): - return ref_query.field(metric.key) + def __eq__(self, other): + return isinstance(self, ReferenceType) \ + and self.time_unit == other.time_unit \ + and self.interval == other.interval - if reference.is_delta: - if reference.is_percent: - return lambda metric: (original_query.field(metric.key) - ref_field(metric)) \ - * \ - (100 / fn.NullIf(ref_field(metric), 0)) + def __hash__(self): + return hash('reference' + self.key) - return lambda metric: original_query.field(metric.key) - ref_field(metric) - return ref_field +DayOverDay = ReferenceType('dod', 'DoD', 'day', 1) +WeekOverWeek = ReferenceType('wow', 'WoW', 'week', 1) +MonthOverMonth = ReferenceType('mom', 'MoM', 'month', 1) +QuarterOverQuarter = ReferenceType('qoq', 'QoQ', 'quarter', 1) +YearOverYear = ReferenceType('yoy', 'YoY', 'year', 1) def reference_key(metric, reference): @@ -90,3 +89,30 @@ def reference_label(metric, reference): return '{} ({})'.format(label, reference.label) return label + + +def reference_term(reference: Reference, + original_query: QueryBuilder, + ref_query: QueryBuilder): + """ + Part of query building. Given a reference, the original slicer query, and the ref query, creates the pypika for + the reference that should be selected in the reference container query. + + :param reference: + :param original_query: + :param ref_query: + :return: + """ + + def ref_field(metric): + return ref_query.field(metric.key) + + if reference.delta: + if reference.delta_percent: + return lambda metric: (original_query.field(metric.key) - ref_field(metric)) \ + * \ + (100 / fn.NullIf(ref_field(metric), 0)) + + return lambda metric: original_query.field(metric.key) - ref_field(metric) + + return ref_field diff --git a/fireant/slicer/widgets/base.py b/fireant/slicer/widgets/base.py index a77a45c6..7e46cddf 100644 --- a/fireant/slicer/widgets/base.py +++ b/fireant/slicer/widgets/base.py @@ -32,5 +32,5 @@ def __repr__(self): class TransformableWidget(Widget): - def transform(self, data_frame, slicer, dimensions): + def transform(self, data_frame, slicer, dimensions, references): raise NotImplementedError() diff --git a/fireant/slicer/widgets/csv.py b/fireant/slicer/widgets/csv.py index 2b5ea4ba..c8f62109 100644 --- a/fireant/slicer/widgets/csv.py +++ b/fireant/slicer/widgets/csv.py @@ -2,6 +2,6 @@ class CSV(Pandas): - def transform(self, data_frame, slicer, dimensions): - result_df = super(CSV, self).transform(data_frame, slicer, dimensions) + def transform(self, data_frame, slicer, dimensions, references): + result_df = super(CSV, self).transform(data_frame, slicer, dimensions, references) return result_df.to_csv() diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index a1c81d2e..7e9b63eb 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -112,7 +112,7 @@ def __init__(self, items=(), pivot=False, max_columns=None): if max_columns is not None \ else HARD_MAX_COLUMNS - def transform(self, data_frame, slicer, dimensions): + def transform(self, data_frame, slicer, dimensions, references): """ WRITEME @@ -123,10 +123,6 @@ def transform(self, data_frame, slicer, dimensions): """ dimension_display_values = extract_display_values(dimensions, data_frame) - references = [reference - for dimension in dimensions - for reference in getattr(dimension, 'references', ())] - metric_keys = [reference_key(metric, reference) for metric in self.items for reference in [None] + references] diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 3cae406f..a9f51e5b 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -139,7 +139,7 @@ def operations(self): for item in self.items for operation in item.operations]) - def transform(self, data_frame, slicer, dimensions): + def transform(self, data_frame, slicer, dimensions, references): """ - Main entry point - @@ -153,6 +153,8 @@ def transform(self, data_frame, slicer, dimensions): The slicer that is in use. :param dimensions: A list of dimensions that are being rendered. + :param references: + A list of references that are being rendered. :return: A dict meant to be dumped as JSON. """ @@ -170,10 +172,6 @@ def group_series(keys): dimension_display_values = extract_display_values(dimensions, data_frame) render_series_label = dimensional_metric_label(dimensions, dimension_display_values) - references = [reference - for dimension in dimensions - for reference in getattr(dimension, 'references', ())] - total_num_items = sum([len(axis.items) for axis in self.items]) y_axes, series = [], [] @@ -268,7 +266,7 @@ def _render_y_axis(axis_idx, color, references): "labels": {"style": {"color": color}} } for reference in references - if reference.is_delta] + if reference.delta] return y_axes @@ -317,7 +315,7 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, "tooltip": self._render_tooltip(metric), "yAxis": ("{}_{}".format(axis_idx, reference.key) - if reference is not None and reference.is_delta + if reference is not None and reference.delta else str(axis_idx)), "marker": ({"symbol": symbol, "fillColor": axis_color or series_color} diff --git a/fireant/slicer/widgets/matplotlib.py b/fireant/slicer/widgets/matplotlib.py index 1d695047..b78fb1d8 100644 --- a/fireant/slicer/widgets/matplotlib.py +++ b/fireant/slicer/widgets/matplotlib.py @@ -2,5 +2,5 @@ class Matplotlib(TransformableWidget): - def transform(self, data_frame, slicer, dimensions): + def transform(self, data_frame, slicer, dimensions, references): raise NotImplementedError() diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index 4d3c4b83..0a79ca59 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -20,17 +20,17 @@ def __init__(self, items=(), pivot=False, max_columns=None): if max_columns is not None \ else HARD_MAX_COLUMNS - def transform(self, data_frame, slicer, dimensions): + def transform(self, data_frame, slicer, dimensions, references): """ WRITEME :param data_frame: :param slicer: :param dimensions: + :param references: :return: """ result = data_frame.copy() - references = [] for metric in self.items: if any([metric.precision is not None, @@ -40,8 +40,6 @@ def transform(self, data_frame, slicer, dimensions): .apply(lambda x: formats.metric_display(x, metric.prefix, metric.suffix, metric.precision)) for dimension in dimensions: - references += getattr(dimension, 'references', []) - if dimension.has_display_field: result = result.set_index(dimension.display_key, append=True) result = result.reset_index(dimension.key, drop=True) diff --git a/fireant/tests/slicer/mocks.py b/fireant/tests/slicer/mocks.py index 774fdae0..3160d0f8 100644 --- a/fireant/tests/slicer/mocks.py +++ b/fireant/tests/slicer/mocks.py @@ -2,21 +2,21 @@ OrderedDict, namedtuple, ) +from unittest.mock import Mock + +import pandas as pd from datetime import ( datetime, ) -from unittest.mock import Mock -import pandas as pd +from fireant import * +from fireant.slicer.references import ReferenceType from pypika import ( JoinType, Table, functions as fn, ) -from fireant import * -from fireant import VerticaDatabase - class TestDatabase(VerticaDatabase): # Vertica client that uses the vertica_python driver. @@ -346,4 +346,4 @@ def _totals(df): cont_uni_dim_totals_df = totals(cont_uni_dim_df, ['state'], _columns) cont_uni_dim_all_totals_df = totals(cont_uni_dim_df, ['timestamp', 'state'], _columns) -ElectionOverElection = Reference('eoe', 'EoE', 'year', 4) \ No newline at end of file +ElectionOverElection = ReferenceType('eoe', 'EoE', 'year', 4) diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 1438fede..a89a080b 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -397,15 +397,16 @@ def test_build_query_with_totals_on_multiple_dimensions_dimension(self): def test_build_query_with_totals_cat_dimension_with_references(self): query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ - .dimension(slicer.dimensions.timestamp.reference(f.DayOverDay)) \ - .dimension(slicer.dimensions.political_party.rollup()) \ + .dimension(slicer.dimensions.timestamp, + slicer.dimensions.political_party.rollup()) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .query # Important that in reference queries when using totals that the null dimensions are omitted from the nested # queries and selected in the container query self.assertEqual('(SELECT ' - '"base"."timestamp" "timestamp",' - '"base"."political_party" "political_party",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' + 'COALESCE("base"."political_party","dod"."political_party") "political_party",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod" ' 'FROM (' @@ -432,7 +433,7 @@ def test_build_query_with_totals_cat_dimension_with_references(self): 'UNION ALL ' '(SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' 'NULL "political_party",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod" ' @@ -456,14 +457,15 @@ def test_build_query_with_totals_cat_dimension_with_references(self): def test_build_query_with_totals_cat_dimension_with_references_and_date_filters(self): query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ - .dimension(slicer.dimensions.timestamp.reference(f.DayOverDay)) \ + .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.political_party.rollup()) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .filter(slicer.dimensions.timestamp.between(date(2018, 1, 1), date(2019, 1, 1))) \ .query self.assertEqual('(SELECT ' - '"base"."timestamp" "timestamp",' - '"base"."political_party" "political_party",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' + 'COALESCE("base"."political_party","dod"."political_party") "political_party",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod" ' 'FROM (' @@ -492,7 +494,7 @@ def test_build_query_with_totals_cat_dimension_with_references_and_date_filters( 'UNION ALL ' '(SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' 'NULL "political_party",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod" ' @@ -821,17 +823,76 @@ def test_build_query_with_cummean_operation(self): class QueryBuilderDatetimeReferenceTests(TestCase): maxDiff = None + def test_single_reference_dod_with_no_dimension_uses_multiple_from_clauses_instead_of_joins(self): + query = slicer.data \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + [slicer.metrics.votes])])) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + '"base"."votes" "votes",' + '"dod"."votes" "votes_dod" ' + + 'FROM (' + 'SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician"' + ') "base",(' + 'SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician"' + ') "dod"', str(query)) + + def test_single_reference_dod_with_dimension_but_not_reference_dimension_in_query_using_filter(self): + query = slicer.data \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + [slicer.metrics.votes])])) \ + .dimension(slicer.dimensions.political_party) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .filter(slicer.dimensions.timestamp.between(date(2000, 1, 1), date(2000, 3, 1))) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("base"."political_party","dod"."political_party") "political_party",' + '"base"."votes" "votes",' + '"dod"."votes" "votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + '"political_party" "political_party",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' + 'GROUP BY "political_party"' + ') "base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + '"political_party" "political_party",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' + 'GROUP BY "political_party"' + ') "dod" ' # end-nested + + 'ON "base"."political_party"="dod"."political_party" ' + 'ORDER BY "political_party"', str(query)) + def test_dimension_with_single_reference_dod(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.DayOverDay)) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod" ' 'FROM ' @@ -860,12 +921,12 @@ def test_dimension_with_single_reference_wow(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.WeekOverWeek)) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.WeekOverWeek(slicer.dimensions.timestamp)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'week\',1,"wow"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"wow"."votes" "votes_wow" ' 'FROM ' @@ -894,12 +955,12 @@ def test_dimension_with_single_reference_mom(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.MonthOverMonth)) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.MonthOverMonth(slicer.dimensions.timestamp)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'month\',1,"mom"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"mom"."votes" "votes_mom" ' 'FROM ' @@ -928,12 +989,12 @@ def test_dimension_with_single_reference_qoq(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.QuarterOverQuarter)) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.QuarterOverQuarter(slicer.dimensions.timestamp)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'quarter\',1,"qoq"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"qoq"."votes" "votes_qoq" ' 'FROM ' @@ -962,12 +1023,12 @@ def test_dimension_with_single_reference_yoy(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.YearOverYear)) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"yoy"."votes" "votes_yoy" ' 'FROM ' @@ -996,12 +1057,12 @@ def test_dimension_with_single_reference_as_a_delta(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.DayOverDay.delta())) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp, delta=True)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"base"."votes"-"dod"."votes" "votes_dod_delta" ' 'FROM ' @@ -1030,12 +1091,12 @@ def test_dimension_with_single_reference_as_a_delta_percentage(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.DayOverDay.delta(percent=True))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp, delta_percent=True)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' '"base"."votes" "votes",' '("base"."votes"-"dod"."votes")*100/NULLIF("dod"."votes",0) "votes_dod_delta_percent" ' 'FROM ' @@ -1060,16 +1121,51 @@ def test_dimension_with_single_reference_as_a_delta_percentage(self): 'ORDER BY "timestamp"', str(query)) def test_reference_on_dimension_with_weekly_interval(self): + weekly_timestamp = slicer.dimensions.timestamp(f.weekly) + query = slicer.data \ + .widget(f.HighCharts( + axes=[f.HighCharts.LineChart( + [slicer.metrics.votes])])) \ + .dimension(weekly_timestamp) \ + .reference(f.DayOverDay(weekly_timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' + '"base"."votes" "votes",' + '"dod"."votes" "votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "timestamp",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "timestamp"' + ') "dod" ' # end-nested + + 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' + 'ORDER BY "timestamp"', str(query)) + + def test_reference_on_dimension_with_weekly_interval_no_interval_on_reference(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp(f.weekly) - .reference(f.DayOverDay)) \ + .dimension(slicer.dimensions.timestamp(f.weekly)) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod" ' 'FROM ' @@ -1098,12 +1194,12 @@ def test_reference_on_dimension_with_monthly_interval(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp(f.monthly) - .reference(f.DayOverDay)) \ + .dimension(slicer.dimensions.timestamp(f.monthly)) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod" ' 'FROM ' @@ -1132,12 +1228,12 @@ def test_reference_on_dimension_with_quarterly_interval(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp(f.quarterly) - .reference(f.DayOverDay)) \ + .dimension(slicer.dimensions.timestamp(f.quarterly)) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod" ' 'FROM ' @@ -1166,12 +1262,12 @@ def test_reference_on_dimension_with_annual_interval(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp(f.annually) - .reference(f.DayOverDay)) \ + .dimension(slicer.dimensions.timestamp(f.annually)) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod" ' 'FROM ' @@ -1200,13 +1296,19 @@ def test_dimension_with_multiple_references(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.DayOverDay) - .reference(f.YearOverYear.delta(percent=True))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp, delta_percent=True)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + + 'COALESCE(' + '"base"."timestamp",' + 'TIMESTAMPADD(\'day\',1,"dod"."timestamp"),' + 'TIMESTAMPADD(\'year\',1,"yoy"."timestamp")' + ') "timestamp",' + '"base"."votes" "votes",' '"dod"."votes" "votes_dod",' '("base"."votes"-"yoy"."votes")*100/NULLIF("yoy"."votes",0) "votes_yoy_delta_percent" ' @@ -1246,14 +1348,14 @@ def test_reference_joins_nested_query_on_dimensions(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.YearOverYear)) \ + .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.political_party) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' - '"base"."political_party" "political_party",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' + 'COALESCE("base"."political_party","yoy"."political_party") "political_party",' '"base"."votes" "votes",' '"yoy"."votes" "votes_yoy" ' 'FROM ' @@ -1285,15 +1387,15 @@ def test_reference_with_unique_dimension_includes_display_definition(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.YearOverYear)) \ + .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.candidate) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' - '"base"."candidate" "candidate",' - '"base"."candidate_display" "candidate_display",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' + 'COALESCE("base"."candidate","yoy"."candidate") "candidate",' + 'COALESCE("base"."candidate_display","yoy"."candidate_display") "candidate_display",' '"base"."votes" "votes",' '"yoy"."votes" "votes_yoy" ' 'FROM ' @@ -1327,14 +1429,14 @@ def test_adjust_reference_dimension_filters_in_reference_query(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.DayOverDay)) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .filter(slicer.dimensions.timestamp .between(date(2018, 1, 1), date(2018, 1, 31))) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod" ' 'FROM ' @@ -1365,8 +1467,8 @@ def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_fil .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.DayOverDay)) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .filter(slicer.dimensions.timestamp .between(date(2018, 1, 1), date(2018, 1, 31))) \ .filter(slicer.dimensions.political_party @@ -1374,7 +1476,7 @@ def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_fil .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod" ' 'FROM ' @@ -1407,12 +1509,12 @@ def test_adapt_dow_for_leap_year_for_yoy_reference(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp(f.weekly) - .reference(f.YearOverYear)) \ + .dimension(slicer.dimensions.timestamp(f.weekly)) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"yoy"."votes" "votes_yoy" ' 'FROM ' @@ -1441,12 +1543,12 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp(f.weekly) - .reference(f.YearOverYear.delta())) \ + .dimension(slicer.dimensions.timestamp(f.weekly)) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp, delta=True)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"base"."votes"-"yoy"."votes" "votes_yoy_delta" ' 'FROM ' @@ -1470,18 +1572,17 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' 'ORDER BY "timestamp"', str(query)) - def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): query = slicer.data \ .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp(f.weekly) - .reference(f.YearOverYear.delta(True))) \ + .dimension(slicer.dimensions.timestamp(f.weekly)) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp, delta_percent=True)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' '"base"."votes" "votes",' '("base"."votes"-"yoy"."votes")*100/NULLIF("yoy"."votes",0) "votes_yoy_delta_percent" ' 'FROM ' @@ -1510,13 +1611,13 @@ def test_adapt_dow_for_leap_year_for_yoy_reference_with_date_filter(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp(f.weekly) - .reference(f.YearOverYear)) \ + .dimension(slicer.dimensions.timestamp(f.weekly)) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ .filter(slicer.dimensions.timestamp.between(date(2018, 1, 1), date(2018, 1, 31))) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"yoy"."votes" "votes_yoy" ' 'FROM ' @@ -1547,13 +1648,13 @@ def test_adding_duplicate_reference_does_not_join_more_queries(self): .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.DayOverDay) - .reference(f.DayOverDay)) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp), + f.DayOverDay(slicer.dimensions.timestamp)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod" ' 'FROM ' @@ -1582,14 +1683,14 @@ def test_use_same_nested_query_for_joining_references_with_same_period_and_dimen .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.DayOverDay) - .reference(f.DayOverDay.delta()) - .reference(f.DayOverDay.delta(percent=True))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp), + f.DayOverDay(slicer.dimensions.timestamp, delta=True), + f.DayOverDay(slicer.dimensions.timestamp, delta_percent=True)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod",' '"base"."votes"-"dod"."votes" "votes_dod_delta",' @@ -1620,15 +1721,19 @@ def test_use_same_nested_query_for_joining_references_with_same_period_and_dimen .widget(f.HighCharts( axes=[f.HighCharts.LineChart( [slicer.metrics.votes])])) \ - .dimension(slicer.dimensions.timestamp - .reference(f.DayOverDay) - .reference(f.DayOverDay.delta()) - .reference(f.YearOverYear) - .reference(f.YearOverYear.delta())) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp), + f.DayOverDay(slicer.dimensions.timestamp, delta=True), + f.YearOverYear(slicer.dimensions.timestamp), + f.YearOverYear(slicer.dimensions.timestamp, delta=True)) \ .query self.assertEqual('SELECT ' - '"base"."timestamp" "timestamp",' + 'COALESCE(' + '"base"."timestamp",' + 'TIMESTAMPADD(\'day\',1,"dod"."timestamp"),' + 'TIMESTAMPADD(\'year\',1,"yoy"."timestamp")' + ') "timestamp",' '"base"."votes" "votes",' '"dod"."votes" "votes_dod",' '"base"."votes"-"dod"."votes" "votes_dod_delta",' @@ -2119,7 +2224,8 @@ def test_call_transform_on_widget(self, mock_fetch_data: Mock): mock_widget.transform.assert_called_once_with(mock_fetch_data.return_value, slicer, - DimensionMatcher(slicer.dimensions.timestamp)) + DimensionMatcher(slicer.dimensions.timestamp), + []) def test_returns_results_from_widget_transform(self, mock_fetch_data: Mock): mock_widget = f.Widget([slicer.metrics.votes]) diff --git a/fireant/tests/slicer/widgets/test_csv.py b/fireant/tests/slicer/widgets/test_csv.py index 21d7df0f..fdb5c67b 100644 --- a/fireant/tests/slicer/widgets/test_csv.py +++ b/fireant/tests/slicer/widgets/test_csv.py @@ -24,7 +24,7 @@ class CSVWidgetTests(TestCase): def test_single_metric(self): result = CSV(items=[slicer.metrics.votes]) \ - .transform(single_metric_df, slicer, []) + .transform(single_metric_df, slicer, [], []) expected = single_metric_df.copy()[['votes']] expected.columns = ['Votes'] @@ -33,7 +33,7 @@ def test_single_metric(self): def test_multiple_metrics(self): result = CSV(items=[slicer.metrics.votes, slicer.metrics.wins]) \ - .transform(multi_metric_df, slicer, []) + .transform(multi_metric_df, slicer, [], []) expected = multi_metric_df.copy()[['votes', 'wins']] expected.columns = ['Votes', 'Wins'] @@ -42,7 +42,7 @@ def test_multiple_metrics(self): def test_multiple_metrics_reversed(self): result = CSV(items=[slicer.metrics.wins, slicer.metrics.votes]) \ - .transform(multi_metric_df, slicer, []) + .transform(multi_metric_df, slicer, [], []) expected = multi_metric_df.copy()[['wins', 'votes']] expected.columns = ['Wins', 'Votes'] @@ -51,7 +51,7 @@ def test_multiple_metrics_reversed(self): def test_time_series_dim(self): result = CSV(items=[slicer.metrics.wins]) \ - .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) expected = cont_dim_df.copy()[['wins']] expected.index.names = ['Timestamp'] @@ -61,7 +61,7 @@ def test_time_series_dim(self): def test_time_series_dim_with_operation(self): result = CSV(items=[CumSum(slicer.metrics.votes)]) \ - .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp]) + .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp], []) expected = cont_dim_operation_df.copy()[['cumsum(votes)']] expected.index.names = ['Timestamp'] @@ -71,7 +71,7 @@ def test_time_series_dim_with_operation(self): def test_cat_dim(self): result = CSV(items=[slicer.metrics.wins]) \ - .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) expected = cat_dim_df.copy()[['wins']] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') @@ -81,7 +81,7 @@ def test_cat_dim(self): def test_uni_dim(self): result = CSV(items=[slicer.metrics.wins]) \ - .transform(uni_dim_df, slicer, [slicer.dimensions.candidate]) + .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) expected = uni_dim_df.copy() \ .set_index('candidate_display', append=True) \ @@ -102,7 +102,7 @@ def test_uni_dim_no_display_definition(self): del uni_dim_df_copy[slicer.dimensions.candidate.display_key] result = CSV(items=[slicer.metrics.wins]) \ - .transform(uni_dim_df_copy, slicer, [candidate]) + .transform(uni_dim_df_copy, slicer, [candidate], []) expected = uni_dim_df_copy.copy()[['wins']] expected.index.names = ['Candidate'] @@ -112,7 +112,7 @@ def test_uni_dim_no_display_definition(self): def test_multi_dims_time_series_and_uni(self): result = CSV(items=[slicer.metrics.wins]) \ - .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ .set_index('state_display', append=True) \ @@ -124,7 +124,7 @@ def test_multi_dims_time_series_and_uni(self): def test_pivoted_single_dimension_no_effect(self): result = CSV(items=[slicer.metrics.wins], pivot=True) \ - .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) expected = cat_dim_df.copy()[['wins']] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') @@ -134,7 +134,7 @@ def test_pivoted_single_dimension_no_effect(self): def test_pivoted_multi_dims_time_series_and_cat(self): result = CSV(items=[slicer.metrics.wins], pivot=True) \ - .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party]) + .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party], []) expected = cont_cat_dim_df.copy()[['wins']] expected.index.names = ['Timestamp', 'Party'] @@ -145,7 +145,7 @@ def test_pivoted_multi_dims_time_series_and_cat(self): def test_pivoted_multi_dims_time_series_and_uni(self): result = CSV(items=[slicer.metrics.votes], pivot=True) \ - .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ .set_index('state_display', append=True) \ @@ -158,8 +158,14 @@ def test_pivoted_multi_dims_time_series_and_uni(self): def test_time_series_ref(self): result = CSV(items=[slicer.metrics.votes]) \ - .transform(cont_uni_dim_ref_df, slicer, [slicer.dimensions.timestamp.reference(ElectionOverElection), - slicer.dimensions.state]) + .transform(cont_uni_dim_ref_df, + slicer, + [ + slicer.dimensions.timestamp, + slicer.dimensions.state + ], [ + ElectionOverElection(slicer.dimensions.timestamp) + ]) expected = cont_uni_dim_ref_df.copy() \ .set_index('state_display', append=True) \ diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py index 55783755..b53724cd 100644 --- a/fireant/tests/slicer/widgets/test_datatables.py +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -1,8 +1,8 @@ -from datetime import date from unittest import TestCase from unittest.mock import Mock import pandas as pd +from datetime import date from fireant.slicer.widgets.datatables import ( DataTablesJS, @@ -13,9 +13,9 @@ ElectionOverElection, cat_dim_df, cont_cat_dim_df, - cont_uni_dim_all_totals_df, cont_dim_df, cont_dim_operation_df, + cont_uni_dim_all_totals_df, cont_uni_dim_df, cont_uni_dim_ref_df, cont_uni_dim_totals_df, @@ -31,7 +31,7 @@ class DataTablesTransformerTests(TestCase): def test_single_metric(self): result = DataTablesJS(items=[slicer.metrics.votes]) \ - .transform(single_metric_df, slicer, []) + .transform(single_metric_df, slicer, [], []) self.assertEqual({ 'columns': [{ @@ -46,7 +46,7 @@ def test_single_metric(self): def test_multiple_metrics(self): result = DataTablesJS(items=[slicer.metrics.votes, slicer.metrics.wins]) \ - .transform(multi_metric_df, slicer, []) + .transform(multi_metric_df, slicer, [], []) self.assertEqual({ 'columns': [{ @@ -66,7 +66,7 @@ def test_multiple_metrics(self): def test_multiple_metrics_reversed(self): result = DataTablesJS(items=[slicer.metrics.wins, slicer.metrics.votes]) \ - .transform(multi_metric_df, slicer, []) + .transform(multi_metric_df, slicer, [], []) self.assertEqual({ 'columns': [{ @@ -86,7 +86,7 @@ def test_multiple_metrics_reversed(self): def test_time_series_dim(self): result = DataTablesJS(items=[slicer.metrics.wins]) \ - .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ 'columns': [{ @@ -121,7 +121,7 @@ def test_time_series_dim(self): def test_time_series_dim_with_operation(self): result = DataTablesJS(items=[CumSum(slicer.metrics.votes)]) \ - .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp]) + .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ 'columns': [{ @@ -156,7 +156,7 @@ def test_time_series_dim_with_operation(self): def test_cat_dim(self): result = DataTablesJS(items=[slicer.metrics.wins]) \ - .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) self.assertEqual({ 'columns': [{ @@ -182,7 +182,7 @@ def test_cat_dim(self): def test_uni_dim(self): result = DataTablesJS(items=[slicer.metrics.wins]) \ - .transform(uni_dim_df, slicer, [slicer.dimensions.candidate]) + .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) self.assertEqual({ 'columns': [{ @@ -240,7 +240,7 @@ def test_uni_dim_no_display_definition(self): del uni_dim_df_copy[slicer.dimensions.candidate.display_key] result = DataTablesJS(items=[slicer.metrics.wins]) \ - .transform(uni_dim_df_copy, slicer, [candidate]) + .transform(uni_dim_df_copy, slicer, [candidate], []) self.assertEqual({ 'columns': [{ @@ -290,7 +290,7 @@ def test_uni_dim_no_display_definition(self): def test_multi_dims_time_series_and_uni(self): result = DataTablesJS(items=[slicer.metrics.wins]) \ - .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ 'columns': [{ @@ -359,7 +359,8 @@ def test_multi_dims_time_series_and_uni(self): def test_multi_dims_with_one_level_totals(self): result = DataTablesJS(items=[slicer.metrics.wins]) \ - .transform(cont_uni_dim_totals_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state.rollup()]) + .transform(cont_uni_dim_totals_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state.rollup()], + []) self.assertEqual({ 'columns': [{ @@ -453,7 +454,7 @@ def test_multi_dims_with_one_level_totals(self): def test_multi_dims_with_all_levels_totals(self): result = DataTablesJS(items=[slicer.metrics.wins]) \ .transform(cont_uni_dim_all_totals_df, slicer, [slicer.dimensions.timestamp.rollup(), - slicer.dimensions.state.rollup()]) + slicer.dimensions.state.rollup()], []) self.assertEqual({ 'columns': [{ @@ -550,7 +551,7 @@ def test_multi_dims_with_all_levels_totals(self): def test_pivoted_single_dimension_no_effect(self): result = DataTablesJS(items=[slicer.metrics.wins], pivot=True) \ - .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) self.assertEqual({ 'columns': [{ @@ -576,7 +577,7 @@ def test_pivoted_single_dimension_no_effect(self): def test_pivoted_multi_dims_time_series_and_cat(self): result = DataTablesJS(items=[slicer.metrics.wins], pivot=True) \ - .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party]) + .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party], []) self.assertEqual({ 'columns': [{ @@ -643,7 +644,7 @@ def test_pivoted_multi_dims_time_series_and_cat(self): def test_pivoted_multi_dims_time_series_and_uni(self): result = DataTablesJS(items=[slicer.metrics.votes], pivot=True) \ - .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ 'columns': [{ @@ -700,8 +701,16 @@ def test_pivoted_multi_dims_time_series_and_uni(self): def test_time_series_ref(self): result = DataTablesJS(items=[slicer.metrics.votes]) \ - .transform(cont_uni_dim_ref_df, slicer, [slicer.dimensions.timestamp.reference(ElectionOverElection), - slicer.dimensions.state]) + .transform(cont_uni_dim_ref_df, + slicer, + [ + slicer.dimensions.timestamp, + slicer.dimensions.state + ], [ + ElectionOverElection(slicer.dimensions.timestamp) + ]) + + print(result) self.assertEqual({ 'columns': [{ diff --git a/fireant/tests/slicer/widgets/test_highcharts.py b/fireant/tests/slicer/widgets/test_highcharts.py index b0e3ac62..5f9370d6 100644 --- a/fireant/tests/slicer/widgets/test_highcharts.py +++ b/fireant/tests/slicer/widgets/test_highcharts.py @@ -32,7 +32,7 @@ class HighChartsLineChartTransformerTests(TestCase): def test_single_metric_line_chart(self): result = HighCharts(title="Time Series, Single Metric", axes=[self.chart_class([slicer.metrics.votes])]) \ - .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ "title": {"text": "Time Series, Single Metric"}, @@ -71,7 +71,7 @@ def test_metric_prefix_line_chart(self): votes.prefix = '$' result = HighCharts(title="Time Series, Single Metric", axes=[self.chart_class([votes])]) \ - .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ "title": {"text": "Time Series, Single Metric"}, @@ -110,7 +110,7 @@ def test_metric_suffix_line_chart(self): votes.suffix = '%' result = HighCharts(title="Time Series, Single Metric", axes=[self.chart_class([votes])]) \ - .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ "title": {"text": "Time Series, Single Metric"}, @@ -149,7 +149,7 @@ def test_metric_precision_line_chart(self): votes.precision = 2 result = HighCharts(title="Time Series, Single Metric", axes=[self.chart_class([votes])]) \ - .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ "title": {"text": "Time Series, Single Metric"}, @@ -186,7 +186,7 @@ def test_metric_precision_line_chart(self): def test_single_operation_line_chart(self): result = HighCharts(title="Time Series, Single Metric", axes=[self.chart_class([CumSum(slicer.metrics.votes)])]) \ - .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp]) + .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ "title": {"text": "Time Series, Single Metric"}, @@ -224,7 +224,7 @@ def test_single_metric_with_uni_dim_line_chart(self): result = HighCharts(title="Time Series with Unique Dimension and Single Metric", axes=[self.chart_class([slicer.metrics.votes])]) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, - slicer.dimensions.state]) + slicer.dimensions.state], []) self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Single Metric"}, @@ -282,7 +282,7 @@ def test_multi_metrics_single_axis_line_chart(self): axes=[self.chart_class([slicer.metrics.votes, slicer.metrics.wins])]) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, - slicer.dimensions.state]) + slicer.dimensions.state], []) self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Multiple Metrics"}, @@ -378,7 +378,7 @@ def test_multi_metrics_multi_axis_line_chart(self): axes=[self.chart_class([slicer.metrics.votes]), self.chart_class([slicer.metrics.wins]), ]) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, - slicer.dimensions.state]) + slicer.dimensions.state], []) self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Multiple Metrics, Multi-Axis"}, @@ -478,7 +478,7 @@ def test_multi_dim_with_totals_line_chart(self): axes=[self.chart_class([slicer.metrics.votes]), self.chart_class([slicer.metrics.wins]), ]) \ .transform(cont_uni_dim_totals_df, slicer, [slicer.dimensions.timestamp.rollup(), - slicer.dimensions.state.rollup()]) + slicer.dimensions.state.rollup()], []) self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Multiple Metrics, Multi-Axis"}, @@ -616,7 +616,7 @@ def test_multi_dim_with_totals_on_first_dim_line_chart(self): axes=[self.chart_class([slicer.metrics.votes]), self.chart_class([slicer.metrics.wins]), ]) \ .transform(cont_uni_dim_all_totals_df, slicer, [slicer.dimensions.timestamp.rollup(), - slicer.dimensions.state.rollup()]) + slicer.dimensions.state.rollup()], []) self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Multiple Metrics, Multi-Axis"}, @@ -752,8 +752,14 @@ def test_multi_dim_with_totals_on_first_dim_line_chart(self): def test_uni_dim_with_ref_line_chart(self): result = HighCharts(title="Time Series with Unique Dimension and Reference", axes=[self.chart_class([slicer.metrics.votes])]) \ - .transform(cont_uni_dim_ref_df, slicer, [slicer.dimensions.timestamp.reference(ElectionOverElection), - slicer.dimensions.state]) + .transform(cont_uni_dim_ref_df, + slicer, + [ + slicer.dimensions.timestamp, + slicer.dimensions.state + ], [ + ElectionOverElection(slicer.dimensions.timestamp) + ]) self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Reference"}, @@ -845,8 +851,12 @@ def test_uni_dim_with_ref_delta_line_chart(self): axes=[self.chart_class([slicer.metrics.votes])]) \ .transform(cont_uni_dim_ref_delta_df, slicer, - [slicer.dimensions.timestamp.reference(ElectionOverElection.delta()), - slicer.dimensions.state]) + [ + slicer.dimensions.timestamp, + slicer.dimensions.state + ], [ + ElectionOverElection(slicer.dimensions.timestamp, delta=True) + ]) self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Delta Reference"}, @@ -949,7 +959,7 @@ class HighChartsBarChartTransformerTests(TestCase): def test_single_metric_bar_chart(self): result = HighCharts(title="All Votes", axes=[self.chart_class([slicer.metrics.votes])]) \ - .transform(single_metric_df, slicer, []) + .transform(single_metric_df, slicer, [], []) self.assertEqual({ "title": {"text": "All Votes"}, @@ -985,7 +995,7 @@ def test_multi_metric_bar_chart(self): result = HighCharts(title="Votes and Wins", axes=[self.chart_class([slicer.metrics.votes, slicer.metrics.wins])]) \ - .transform(multi_metric_df, slicer, []) + .transform(multi_metric_df, slicer, [], []) self.assertEqual({ "title": {"text": "Votes and Wins"}, @@ -1035,7 +1045,7 @@ def test_multi_metric_bar_chart(self): def test_cat_dim_single_metric_bar_chart(self): result = HighCharts(title="Votes and Wins", axes=[self.chart_class([slicer.metrics.votes])]) \ - .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) self.assertEqual({ "title": {"text": "Votes and Wins"}, @@ -1072,7 +1082,7 @@ def test_cat_dim_multi_metric_bar_chart(self): result = HighCharts(title="Votes and Wins", axes=[self.chart_class([slicer.metrics.votes, slicer.metrics.wins])]) \ - .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) self.assertEqual({ "title": {"text": "Votes and Wins"}, @@ -1122,7 +1132,7 @@ def test_cat_dim_multi_metric_bar_chart(self): def test_cont_uni_dims_single_metric_bar_chart(self): result = HighCharts(title="Election Votes by State", axes=[self.chart_class([slicer.metrics.votes])]) \ - .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ "title": {"text": "Election Votes by State"}, @@ -1179,8 +1189,8 @@ def test_cont_uni_dims_single_metric_bar_chart(self): def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): result = HighCharts(title="Election Votes by State", axes=[self.chart_class([slicer.metrics.votes, - slicer.metrics.wins]), ]) \ - .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + slicer.metrics.wins])]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ "title": {"text": "Election Votes by State"}, @@ -1275,8 +1285,8 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): result = HighCharts(title="Election Votes by State", axes=[self.chart_class([slicer.metrics.votes]), - self.chart_class([slicer.metrics.wins]), ]) \ - .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + self.chart_class([slicer.metrics.wins])]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ "title": {"text": "Election Votes by State"}, @@ -1411,7 +1421,7 @@ class HighChartsPieChartTransformerTests(TestCase): def test_single_metric_pie_chart(self): result = HighCharts(title="All Votes", axes=[self.chart_class([slicer.metrics.votes])]) \ - .transform(single_metric_df, slicer, []) + .transform(single_metric_df, slicer, [], []) self.assertEqual({ "title": {"text": "All Votes"}, @@ -1439,7 +1449,7 @@ def test_multi_metric_bar_chart(self): result = HighCharts(title="Votes and Wins", axes=[self.chart_class([slicer.metrics.votes, slicer.metrics.wins])]) \ - .transform(multi_metric_df, slicer, []) + .transform(multi_metric_df, slicer, [], []) self.assertEqual({ "title": {"text": "Votes and Wins"}, @@ -1479,7 +1489,7 @@ def test_multi_metric_bar_chart(self): def test_cat_dim_single_metric_bar_chart(self): result = HighCharts(title="Votes and Wins", axes=[self.chart_class([slicer.metrics.votes])]) \ - .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) self.assertEqual({ 'title': {'text': 'Votes and Wins'}, @@ -1516,7 +1526,7 @@ def test_cat_dim_multi_metric_bar_chart(self): result = HighCharts(title="Votes and Wins", axes=[self.chart_class([slicer.metrics.votes, slicer.metrics.wins])]) \ - .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) self.assertEqual({ "title": {"text": "Votes and Wins"}, @@ -1567,7 +1577,7 @@ def test_cat_dim_multi_metric_bar_chart(self): def test_cont_uni_dims_single_metric_bar_chart(self): result = HighCharts(title="Election Votes by State", axes=[self.chart_class([slicer.metrics.votes])]) \ - .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ "title": {"text": "Election Votes by State"}, @@ -1626,7 +1636,7 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): result = HighCharts(title="Election Votes by State", axes=[self.chart_class([slicer.metrics.votes, slicer.metrics.wins]), ]) \ - .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ "title": {"text": "Election Votes by State"}, @@ -1723,7 +1733,7 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): result = HighCharts(title="Election Votes by State", axes=[self.chart_class([slicer.metrics.votes]), self.chart_class([slicer.metrics.wins]), ]) \ - .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ "title": {"text": "Election Votes by State"}, diff --git a/fireant/tests/slicer/widgets/test_pandas.py b/fireant/tests/slicer/widgets/test_pandas.py index b1802ca2..37c3ac79 100644 --- a/fireant/tests/slicer/widgets/test_pandas.py +++ b/fireant/tests/slicer/widgets/test_pandas.py @@ -25,7 +25,7 @@ class DataTablesTransformerTests(TestCase): def test_single_metric(self): result = Pandas(items=[slicer.metrics.votes]) \ - .transform(single_metric_df, slicer, []) + .transform(single_metric_df, slicer, [], []) expected = single_metric_df.copy()[['votes']] expected.columns = ['Votes'] @@ -34,7 +34,7 @@ def test_single_metric(self): def test_multiple_metrics(self): result = Pandas(items=[slicer.metrics.votes, slicer.metrics.wins]) \ - .transform(multi_metric_df, slicer, []) + .transform(multi_metric_df, slicer, [], []) expected = multi_metric_df.copy()[['votes', 'wins']] expected.columns = ['Votes', 'Wins'] @@ -43,7 +43,7 @@ def test_multiple_metrics(self): def test_multiple_metrics_reversed(self): result = Pandas(items=[slicer.metrics.wins, slicer.metrics.votes]) \ - .transform(multi_metric_df, slicer, []) + .transform(multi_metric_df, slicer, [], []) expected = multi_metric_df.copy()[['wins', 'votes']] expected.columns = ['Wins', 'Votes'] @@ -52,7 +52,7 @@ def test_multiple_metrics_reversed(self): def test_time_series_dim(self): result = Pandas(items=[slicer.metrics.wins]) \ - .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp]) + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) expected = cont_dim_df.copy()[['wins']] expected.index.names = ['Timestamp'] @@ -62,7 +62,7 @@ def test_time_series_dim(self): def test_time_series_dim_with_operation(self): result = Pandas(items=[CumSum(slicer.metrics.votes)]) \ - .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp]) + .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp], []) expected = cont_dim_operation_df.copy()[['cumsum(votes)']] expected.index.names = ['Timestamp'] @@ -72,7 +72,7 @@ def test_time_series_dim_with_operation(self): def test_cat_dim(self): result = Pandas(items=[slicer.metrics.wins]) \ - .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) expected = cat_dim_df.copy()[['wins']] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') @@ -82,7 +82,7 @@ def test_cat_dim(self): def test_uni_dim(self): result = Pandas(items=[slicer.metrics.wins]) \ - .transform(uni_dim_df, slicer, [slicer.dimensions.candidate]) + .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) expected = uni_dim_df.copy() \ .set_index('candidate_display', append=True) \ @@ -103,7 +103,7 @@ def test_uni_dim_no_display_definition(self): del uni_dim_df_copy[slicer.dimensions.candidate.display_key] result = Pandas(items=[slicer.metrics.wins]) \ - .transform(uni_dim_df_copy, slicer, [candidate]) + .transform(uni_dim_df_copy, slicer, [candidate], []) expected = uni_dim_df_copy.copy()[['wins']] expected.index.names = ['Candidate'] @@ -113,7 +113,7 @@ def test_uni_dim_no_display_definition(self): def test_multi_dims_time_series_and_uni(self): result = Pandas(items=[slicer.metrics.wins]) \ - .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ .set_index('state_display', append=True) \ @@ -125,7 +125,7 @@ def test_multi_dims_time_series_and_uni(self): def test_pivoted_single_dimension_no_effect(self): result = Pandas(items=[slicer.metrics.wins], pivot=True) \ - .transform(cat_dim_df, slicer, [slicer.dimensions.political_party]) + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) expected = cat_dim_df.copy()[['wins']] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') @@ -135,7 +135,7 @@ def test_pivoted_single_dimension_no_effect(self): def test_pivoted_multi_dims_time_series_and_cat(self): result = Pandas(items=[slicer.metrics.wins], pivot=True) \ - .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party]) + .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party], []) expected = cont_cat_dim_df.copy()[['wins']] expected.index.names = ['Timestamp', 'Party'] @@ -146,7 +146,7 @@ def test_pivoted_multi_dims_time_series_and_cat(self): def test_pivoted_multi_dims_time_series_and_uni(self): result = Pandas(items=[slicer.metrics.votes], pivot=True) \ - .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state]) + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ .set_index('state_display', append=True) \ @@ -159,8 +159,13 @@ def test_pivoted_multi_dims_time_series_and_uni(self): def test_time_series_ref(self): result = Pandas(items=[slicer.metrics.votes]) \ - .transform(cont_uni_dim_ref_df, slicer, [slicer.dimensions.timestamp.reference(ElectionOverElection), - slicer.dimensions.state]) + .transform(cont_uni_dim_ref_df, slicer, + [ + slicer.dimensions.timestamp, + slicer.dimensions.state + ], [ + ElectionOverElection(slicer.dimensions.timestamp) + ]) expected = cont_uni_dim_ref_df.copy() \ .set_index('state_display', append=True) \ @@ -179,7 +184,7 @@ def test_metric_format(self): # divide the data frame by 3 to get a repeating decimal so we can check precision result = Pandas(items=[votes]) \ - .transform(cont_dim_df / 3, slicer, [slicer.dimensions.timestamp]) + .transform(cont_dim_df / 3, slicer, [slicer.dimensions.timestamp], []) expected = cont_dim_df.copy()[['votes']] expected['votes'] = ['${0:.2f}€'.format(x) From d50d25525e75912197503d01fd2804384455ccb8 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 7 Mar 2018 16:42:49 +0100 Subject: [PATCH 039/123] Bumped pypika dependency to 0.10.6 and version to dev16 --- fireant/__init__.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 86c8d8b6..f1b32374 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev15' +__version__ = '1.0.0.dev16' diff --git a/requirements.txt b/requirements.txt index 86d1574b..fe62d1c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ six pandas==0.22.0 -pypika==0.10.5 +pypika==0.10.6 pymysql==0.8.0 vertica-python==0.7.3 psycopg2==2.7.3.2 From 09853f604dd88e862f3f9872ed0616bbac131b56 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 7 Mar 2018 17:54:52 +0100 Subject: [PATCH 040/123] Fixed the setup.py pypika dependency --- fireant/slicer/widgets/highcharts.py | 2 +- fireant/tests/slicer/widgets/test_datatables.py | 2 -- requirements.txt | 8 +++++--- setup.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index a9f51e5b..3005cb49 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -233,7 +233,7 @@ def _render_x_axis(data_frame, dimensions, dimension_display_values): if isinstance(first_level, pd.RangeIndex) \ else [utils.deep_get(dimension_display_values, [first_level.name, dimension_value], - dimension_value) + dimension_value or 'Totals') for dimension_value in first_level] return { diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py index b53724cd..ebf53c9f 100644 --- a/fireant/tests/slicer/widgets/test_datatables.py +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -710,8 +710,6 @@ def test_time_series_ref(self): ElectionOverElection(slicer.dimensions.timestamp) ]) - print(result) - self.assertEqual({ 'columns': [{ 'data': 'timestamp', diff --git a/requirements.txt b/requirements.txt index fe62d1c7..488a8871 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,12 @@ six pandas==0.22.0 -pypika==0.10.6 +pypika==0.10.7 pymysql==0.8.0 +toposort==1.5 +typing==3.6.2 + vertica-python==0.7.3 psycopg2==2.7.3.2 -toposort==1.5 matplotlib + mock -typing==3.6.2 \ No newline at end of file diff --git a/setup.py b/setup.py index e6795a21..573446d6 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ def find_version(*file_paths): install_requires=[ 'six', 'pandas==0.22.0', - 'pypika==0.10.4', + 'pypika==0.10.7', 'toposort==1.5', 'typing==3.6.2', ], From 73c1b21c802a3214973b8e782b22afe9fd9ec331 Mon Sep 17 00:00:00 2001 From: Michael England Date: Thu, 8 Mar 2018 15:56:13 +0100 Subject: [PATCH 041/123] Added slow query warning log for queries over 15s --- fireant/slicer/queries/database.py | 19 +++-- fireant/slicer/queries/logger.py | 6 +- fireant/tests/slicer/queries/test_database.py | 72 ++++++++++++++----- 3 files changed, 72 insertions(+), 25 deletions(-) diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index 46382dcf..5695862f 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -1,10 +1,14 @@ import time -from typing import Iterable import pandas as pd +from typing import Iterable from fireant.database.base import Database -from .logger import logger +from .logger import ( + SLOW_QUERY_LOG_MIN_DURATION, + query_logger, + slow_query_logger, +) from ..dimensions import ( ContinuousDimension, Dimension, @@ -26,13 +30,16 @@ def fetch_data(database: Database, query: str, dimensions: Iterable[Dimension]): :return: `pd.DataFrame` constructed from the result of the query """ start_time = time.time() - logger.debug(query) + query_logger.debug(query) data_frame = database.fetch_data(str(query)) - logger.info('[{duration} seconds]: {query}' - .format(duration=round(time.time() - start_time, 4), - query=query)) + duration = round(time.time() - start_time, 4) + query_log_msg = '[{duration} seconds]: {query}'.format(duration=duration, query=query) + query_logger.info(query_log_msg) + + if duration >= SLOW_QUERY_LOG_MIN_DURATION: + slow_query_logger.warning(query_log_msg) return clean_and_apply_index(data_frame, dimensions) diff --git a/fireant/slicer/queries/logger.py b/fireant/slicer/queries/logger.py index 75d9f56b..5443cb61 100644 --- a/fireant/slicer/queries/logger.py +++ b/fireant/slicer/queries/logger.py @@ -1,3 +1,7 @@ import logging -logger = logging.getLogger('fireant.query_log$') +SLOW_QUERY_LOG_MIN_DURATION = 15 + +query_logger = logging.getLogger('fireant.query_log') + +slow_query_logger = logging.getLogger('fireant.slow_query_log') diff --git a/fireant/tests/slicer/queries/test_database.py b/fireant/tests/slicer/queries/test_database.py index 7dfecd0d..7a5ab6a3 100644 --- a/fireant/tests/slicer/queries/test_database.py +++ b/fireant/tests/slicer/queries/test_database.py @@ -1,3 +1,4 @@ +import time from unittest import TestCase from unittest.mock import ( MagicMock, @@ -23,34 +24,69 @@ class FetchDataTests(TestCase): - @classmethod - def setUpClass(cls): - cls.mock_database = Mock(name='database') - cls.mock_data_frame = cls.mock_database.fetch_data.return_value = MagicMock(name='data_frame') - cls.mock_query = 'SELECT *' - cls.mock_dimensions = [Mock(), Mock()] - cls.mock_dimensions[0].is_rollup = False - cls.mock_dimensions[1].is_rollup = True - cls.result = fetch_data(cls.mock_database, cls.mock_query, cls.mock_dimensions) + def setUp(self): + self.mock_database = Mock(name='database') + self.mock_data_frame = self.mock_database.fetch_data.return_value = MagicMock(name='data_frame') + self.mock_query = 'SELECT *' + self.mock_dimensions = [Mock(), Mock()] + self.mock_dimensions[0].is_rollup = False + self.mock_dimensions[1].is_rollup = True def test_fetch_data_called_on_database(self): + fetch_data(self.mock_database, self.mock_query, self.mock_dimensions) + self.mock_database.fetch_data.assert_called_once_with(self.mock_query) def test_index_set_on_data_frame_result(self): + fetch_data(self.mock_database, self.mock_query, self.mock_dimensions) + self.mock_data_frame.set_index.assert_called_once_with([d.key for d in self.mock_dimensions]) + @patch('fireant.slicer.queries.database.query_logger.debug') + def test_debug_query_log_called_with_query(self, mock_logger): + fetch_data(self.mock_database, self.mock_query, self.mock_dimensions) + + mock_logger.assert_called_once_with('SELECT *') + + @patch.object(time, 'time', return_value=1520520255.0) + @patch('fireant.slicer.queries.database.query_logger.info') + def test_info_query_log_called_with_query_and_duration(self, mock_logger, mock_time): + fetch_data(self.mock_database, self.mock_query, self.mock_dimensions) + + mock_logger.assert_called_once_with('[0.0 seconds]: SELECT *') + + @patch.object(time, 'time') + @patch('fireant.slicer.queries.database.slow_query_logger.warning') + def test_warning_slow_query_logger_called_with_duration_and_query_if_over_slow_query_limit(self, + mock_logger, + mock_time): + mock_time.side_effect = [1520520255.0, 1520520277.0] + fetch_data(self.mock_database, self.mock_query, self.mock_dimensions) + + mock_logger.assert_called_once_with('[22.0 seconds]: SELECT *') + + @patch.object(time, 'time') + @patch('fireant.slicer.queries.database.slow_query_logger.warning') + def test_warning_slow_query_logger_not_called_with_duration_and_query_if_not_over_slow_query_limit(self, + mock_logger, + mock_time): + mock_time.side_effect = [1520520763.0, 1520520764.0] + fetch_data(self.mock_database, self.mock_query, self.mock_dimensions) + + mock_logger.assert_not_called() + cat_dim_nans_df = cat_dim_df.append( - pd.DataFrame([[300, 2]], - columns=cat_dim_df.columns, - index=pd.Index([None], - name=cat_dim_df.index.name))) + pd.DataFrame([[300, 2]], + columns=cat_dim_df.columns, + index=pd.Index([None], + name=cat_dim_df.index.name))) uni_dim_nans_df = uni_dim_df.append( - pd.DataFrame([[None, 300, 2]], - columns=uni_dim_df.columns, - index=pd.Index([None], - name=uni_dim_df.index.name))) + pd.DataFrame([[None, 300, 2]], + columns=uni_dim_df.columns, + index=pd.Index([None], + name=uni_dim_df.index.name))) def add_nans(df): @@ -71,7 +107,7 @@ def totals(df): cont_uni_dim_nans_totals_df = cont_uni_dim_nans_df \ - .append(cont_uni_dim_nans_df.groupby(level='timestamp').apply(totals))\ + .append(cont_uni_dim_nans_df.groupby(level='timestamp').apply(totals)) \ .sort_index() \ .sort_index(level=[0, 1], ascending=False) # This sorts the DF so that the first instance of NaN is the totals From 04908aedac34b6515ff9c6076b08abfe6714c0cb Mon Sep 17 00:00:00 2001 From: Michael England Date: Fri, 9 Mar 2018 09:51:08 +0100 Subject: [PATCH 042/123] Moved slow query log min duration constant to be an attribute of database --- fireant/database/base.py | 2 ++ fireant/slicer/queries/database.py | 3 +-- fireant/slicer/queries/logger.py | 2 -- fireant/tests/slicer/queries/test_database.py | 1 + 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fireant/database/base.py b/fireant/database/base.py index c3df135d..42c727d0 100644 --- a/fireant/database/base.py +++ b/fireant/database/base.py @@ -13,6 +13,8 @@ class Database(object): # The pypika query class to use for constructing queries query_cls = Query + SLOW_QUERY_LOG_MIN_DURATION = 15 + def connect(self): raise NotImplementedError diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index 5695862f..fe64e3a4 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -5,7 +5,6 @@ from fireant.database.base import Database from .logger import ( - SLOW_QUERY_LOG_MIN_DURATION, query_logger, slow_query_logger, ) @@ -38,7 +37,7 @@ def fetch_data(database: Database, query: str, dimensions: Iterable[Dimension]): query_log_msg = '[{duration} seconds]: {query}'.format(duration=duration, query=query) query_logger.info(query_log_msg) - if duration >= SLOW_QUERY_LOG_MIN_DURATION: + if duration >= database.SLOW_QUERY_LOG_MIN_DURATION: slow_query_logger.warning(query_log_msg) return clean_and_apply_index(data_frame, dimensions) diff --git a/fireant/slicer/queries/logger.py b/fireant/slicer/queries/logger.py index 5443cb61..85f81ff6 100644 --- a/fireant/slicer/queries/logger.py +++ b/fireant/slicer/queries/logger.py @@ -1,7 +1,5 @@ import logging -SLOW_QUERY_LOG_MIN_DURATION = 15 - query_logger = logging.getLogger('fireant.query_log') slow_query_logger = logging.getLogger('fireant.slow_query_log') diff --git a/fireant/tests/slicer/queries/test_database.py b/fireant/tests/slicer/queries/test_database.py index 7a5ab6a3..9ef308ce 100644 --- a/fireant/tests/slicer/queries/test_database.py +++ b/fireant/tests/slicer/queries/test_database.py @@ -27,6 +27,7 @@ class FetchDataTests(TestCase): def setUp(self): self.mock_database = Mock(name='database') + self.mock_database.SLOW_QUERY_LOG_MIN_DURATION = 15 self.mock_data_frame = self.mock_database.fetch_data.return_value = MagicMock(name='data_frame') self.mock_query = 'SELECT *' self.mock_dimensions = [Mock(), Mock()] From 5906ebbffe79faa16ea5eb9e2373c96111add761 Mon Sep 17 00:00:00 2001 From: Michael England Date: Fri, 9 Mar 2018 09:54:21 +0100 Subject: [PATCH 043/123] Lower cased the slow query log attribute --- fireant/database/base.py | 2 +- fireant/slicer/queries/database.py | 2 +- fireant/tests/slicer/queries/test_database.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fireant/database/base.py b/fireant/database/base.py index 42c727d0..7571365a 100644 --- a/fireant/database/base.py +++ b/fireant/database/base.py @@ -13,7 +13,7 @@ class Database(object): # The pypika query class to use for constructing queries query_cls = Query - SLOW_QUERY_LOG_MIN_DURATION = 15 + slow_query_log_min_seconds = 15 def connect(self): raise NotImplementedError diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index fe64e3a4..8ae4e371 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -37,7 +37,7 @@ def fetch_data(database: Database, query: str, dimensions: Iterable[Dimension]): query_log_msg = '[{duration} seconds]: {query}'.format(duration=duration, query=query) query_logger.info(query_log_msg) - if duration >= database.SLOW_QUERY_LOG_MIN_DURATION: + if duration >= database.slow_query_log_min_seconds: slow_query_logger.warning(query_log_msg) return clean_and_apply_index(data_frame, dimensions) diff --git a/fireant/tests/slicer/queries/test_database.py b/fireant/tests/slicer/queries/test_database.py index 9ef308ce..fd8def54 100644 --- a/fireant/tests/slicer/queries/test_database.py +++ b/fireant/tests/slicer/queries/test_database.py @@ -27,7 +27,7 @@ class FetchDataTests(TestCase): def setUp(self): self.mock_database = Mock(name='database') - self.mock_database.SLOW_QUERY_LOG_MIN_DURATION = 15 + self.mock_database.slow_query_log_min_seconds = 15 self.mock_data_frame = self.mock_database.fetch_data.return_value = MagicMock(name='data_frame') self.mock_query = 'SELECT *' self.mock_dimensions = [Mock(), Mock()] From 7e65fb7f3d4ba0013700e3dbe10b87c68b23b4f1 Mon Sep 17 00:00:00 2001 From: Michael England Date: Fri, 9 Mar 2018 14:50:36 +0100 Subject: [PATCH 044/123] Bump version to 1.0.0.dev17 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index f1b32374..d0da2bdc 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev16' +__version__ = '1.0.0.dev17' From b1dbb75dc890f5d6e81e637c5d905423dfd20ead Mon Sep 17 00:00:00 2001 From: Michael England Date: Fri, 9 Mar 2018 15:39:34 +0100 Subject: [PATCH 045/123] Added NOT LIKE support and renamed WildcardFilter to LikeFilter --- fireant/slicer/dimensions.py | 31 +++++++++++++++----- fireant/slicer/filters.py | 10 +++++-- fireant/tests/slicer/queries/test_builder.py | 25 +++++++++++++--- requirements.txt | 2 +- setup.py | 2 +- 5 files changed, 54 insertions(+), 16 deletions(-) diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 25ddd632..991fb941 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -1,17 +1,18 @@ -from typing import Iterable - -from fireant.utils import immutable from pypika.terms import ( NullValue, ) +from typing import Iterable + +from fireant.utils import immutable from .base import SlicerElement from .exceptions import QueryException from .filters import ( BooleanFilter, ContainsFilter, ExcludesFilter, + LikeFilter, + NotLikeFilter, RangeFilter, - WildcardFilter, ) from .intervals import ( NumericInterval, @@ -153,21 +154,35 @@ def notin(self, values, use_display=False): filter_field = self.display_definition if use_display else self.definition return ExcludesFilter(filter_field, values) - def wildcard(self, pattern): + def like(self, pattern): + """ + Creates a filter to filter a slicer query. + + :param pattern: + A pattern to match against the dimension's display definition. This pattern is used in the SQL query as + the `LIKE` expression. + :return: + A slicer query filter used to filter a slicer query to results where this dimension's display definition + matches the pattern. + """ + if self.display_definition is None: + raise QueryException('No value set for display_definition.') + return LikeFilter(self.display_definition, pattern) + + def not_like(self, pattern): """ Creates a filter to filter a slicer query. :param pattern: A pattern to match against the dimension's display definition. This pattern is used in the SQL query as - the - `LIKE` expression. + the `NOT LIKE` expression. :return: A slicer query filter used to filter a slicer query to results where this dimension's display definition matches the pattern. """ if self.display_definition is None: raise QueryException('No value set for display_definition.') - return WildcardFilter(self.display_definition, pattern) + return NotLikeFilter(self.display_definition, pattern) @property def display(self): diff --git a/fireant/slicer/filters.py b/fireant/slicer/filters.py index 385f3a12..3cea3ebc 100644 --- a/fireant/slicer/filters.py +++ b/fireant/slicer/filters.py @@ -59,7 +59,13 @@ def __init__(self, dimension_definition, start, stop): super(RangeFilter, self).__init__(definition) -class WildcardFilter(DimensionFilter): +class LikeFilter(DimensionFilter): def __init__(self, dimension_definition, pattern): definition = dimension_definition.like(pattern) - super(WildcardFilter, self).__init__(definition) + super(LikeFilter, self).__init__(definition) + + +class NotLikeFilter(DimensionFilter): + def __init__(self, dimension_definition, pattern): + definition = dimension_definition.not_like(pattern) + super(NotLikeFilter, self).__init__(definition) diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index a89a080b..c7c657d1 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -588,10 +588,10 @@ def test_build_query_with_filter_notin_unique_dim_display(self): 'FROM "politics"."politician" ' 'WHERE "candidate_name" NOT IN (\'Donald Trump\')', str(query)) - def test_build_query_with_filter_wildcard_unique_dim(self): + def test_build_query_with_filter_like_unique_dim(self): query = slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ - .filter(slicer.dimensions.candidate.wildcard('%Trump')) \ + .filter(slicer.dimensions.candidate.like('%Trump')) \ .query self.assertEqual('SELECT ' @@ -599,6 +599,17 @@ def test_build_query_with_filter_wildcard_unique_dim(self): 'FROM "politics"."politician" ' 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) + def test_build_query_with_filter_not_like_unique_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.candidate.not_like('%Trump')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" NOT LIKE \'%Trump\'', str(query)) + def test_build_query_with_filter_isin_raise_exception_when_display_definition_undefined(self): with self.assertRaises(f.QueryException): slicer.data \ @@ -611,11 +622,17 @@ def test_build_query_with_filter_notin_raise_exception_when_display_definition_u .widget(f.DataTablesJS([slicer.metrics.votes])) \ .filter(slicer.dimensions.deepjoin.notin([1], use_display=True)) - def test_build_query_with_filter_wildcard_raise_exception_when_display_definition_undefined(self): + def test_build_query_with_filter_like_raise_exception_when_display_definition_undefined(self): + with self.assertRaises(f.QueryException): + slicer.data \ + .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .filter(slicer.dimensions.deepjoin.like('test')) + + def test_build_query_with_filter_not_like_raise_exception_when_display_definition_undefined(self): with self.assertRaises(f.QueryException): slicer.data \ .widget(f.DataTablesJS([slicer.metrics.votes])) \ - .filter(slicer.dimensions.deepjoin.wildcard('test')) + .filter(slicer.dimensions.deepjoin.not_like('test')) def test_build_query_with_filter_range_datetime_dimension(self): query = slicer.data \ diff --git a/requirements.txt b/requirements.txt index 488a8871..a64ca9a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ six pandas==0.22.0 -pypika==0.10.7 +pypika==0.10.8 pymysql==0.8.0 toposort==1.5 typing==3.6.2 diff --git a/setup.py b/setup.py index 573446d6..c1a74f8b 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ def find_version(*file_paths): install_requires=[ 'six', 'pandas==0.22.0', - 'pypika==0.10.7', + 'pypika==0.10.8', 'toposort==1.5', 'typing==3.6.2', ], From cabfc95c93b547b8e44e3b33d9411205149a7aed Mon Sep 17 00:00:00 2001 From: Michael England Date: Fri, 9 Mar 2018 15:43:34 +0100 Subject: [PATCH 046/123] Bumped version to 1.0.0.dev18 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index d0da2bdc..8276c87b 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev17' +__version__ = '1.0.0.dev18' From 79776cfe1cdf052f8b460d9f28d40e6fd2624305 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 12 Mar 2018 15:25:20 +0100 Subject: [PATCH 047/123] Added stacked area charts --- fireant/slicer/widgets/highcharts.py | 7 +++++-- fireant/tests/slicer/widgets/test_highcharts.py | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 3005cb49..41c19689 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -1,10 +1,10 @@ import itertools + +import pandas as pd from datetime import ( datetime, ) -import pandas as pd - from fireant import ( DatetimeDimension, formats, @@ -84,6 +84,9 @@ class AreaChart(ContinuousAxisChartWidget): type = 'area' needs_marker = True + class AreaStackedChart(AreaChart): + stacking = "normal" + class AreaPercentageChart(AreaChart): stacking = "percent" diff --git a/fireant/tests/slicer/widgets/test_highcharts.py b/fireant/tests/slicer/widgets/test_highcharts.py index 5f9370d6..c50b243e 100644 --- a/fireant/tests/slicer/widgets/test_highcharts.py +++ b/fireant/tests/slicer/widgets/test_highcharts.py @@ -1407,6 +1407,11 @@ class HighChartsAreaChartTransformerTests(HighChartsLineChartTransformerTests): chart_type = 'area' +class HighChartsAreaStackedChartTransformerTests(HighChartsAreaChartTransformerTests): + chart_class = HighCharts.AreaStackedChart + stacking = 'normal' + + class HighChartsAreaPercentChartTransformerTests(HighChartsAreaChartTransformerTests): chart_class = HighCharts.AreaPercentageChart stacking = 'percent' From 15d844281f6029415f2eacb93dc36e47fa1e08ed Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 12 Mar 2018 15:27:24 +0100 Subject: [PATCH 048/123] bumped version to dev19 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 8276c87b..9328d560 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev18' +__version__ = '1.0.0.dev19' From 14dfeacecf3ff8c20dfbe5d09ce07d6672f92965 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 19 Mar 2018 18:26:50 +0100 Subject: [PATCH 049/123] Added options to highcharts transformers to enable/disable axis labels and tooltips --- fireant/slicer/widgets/highcharts.py | 37 +- .../tests/slicer/widgets/test_highcharts.py | 437 ++++++++++++++---- 2 files changed, 380 insertions(+), 94 deletions(-) diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 41c19689..7806898c 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -1,10 +1,10 @@ import itertools - -import pandas as pd from datetime import ( datetime, ) +import pandas as pd + from fireant import ( DatetimeDimension, formats, @@ -65,10 +65,11 @@ class ChartWidget(Widget): needs_marker = False stacking = None - def __init__(self, items=(), name=None, stacking=None): + def __init__(self, items=(), name=None, stacking=None, y_axis_visible=True): super(ChartWidget, self).__init__(items) self.name = name self.stacking = self.stacking or stacking + self.y_axis_visible = y_axis_visible class ContinuousAxisChartWidget(ChartWidget): @@ -105,10 +106,12 @@ class ColumnChart(ChartWidget): class StackedColumnChart(ColumnChart): stacking = "normal" - def __init__(self, axes=(), title=None, colors=None): + def __init__(self, axes=(), title=None, colors=None, x_axis_visible=True, tooltip_visible=True): super(HighCharts, self).__init__(axes) self.title = title self.colors = colors or DEFAULT_COLORS + self.x_axis_visible = x_axis_visible + self.tooltip_visible = tooltip_visible @utils.immutable def axis(self, axis: ChartWidget): @@ -210,16 +213,15 @@ def group_series(keys): "xAxis": x_axis, "yAxis": y_axes, "series": series, - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": self.tooltip_visible}, "legend": {"useHTML": True}, } - @staticmethod - def _render_x_axis(data_frame, dimensions, dimension_display_values): + def _render_x_axis(self, data_frame, dimensions, dimension_display_values): """ - Renders the xAxis configuraiton. + Renders the xAxis configuration. - https://api.highcharts.com/highcharts/yAxis + https://api.highcharts.com/highcharts/xAxis :param data_frame: :param dimension_display_values: @@ -230,7 +232,10 @@ def _render_x_axis(data_frame, dimensions, dimension_display_values): else data_frame.index if dimensions and isinstance(dimensions[0], DatetimeDimension): - return {"type": "datetime"} + return { + "type": "datetime", + "visible": self.x_axis_visible, + } categories = ["All"] \ if isinstance(first_level, pd.RangeIndex) \ @@ -242,10 +247,10 @@ def _render_x_axis(data_frame, dimensions, dimension_display_values): return { "type": "category", "categories": categories, + "visible": self.x_axis_visible, } - @staticmethod - def _render_y_axis(axis_idx, color, references): + def _render_y_axis(self, axis_idx, color, references): """ Renders the yAxis configuration. @@ -256,17 +261,21 @@ def _render_y_axis(axis_idx, color, references): :param references: :return: """ + axis = self.items[axis_idx] + y_axes = [{ "id": str(axis_idx), "title": {"text": None}, - "labels": {"style": {"color": color}} + "labels": {"style": {"color": color}}, + "visible": axis.y_axis_visible, }] y_axes += [{ "id": "{}_{}".format(axis_idx, reference.key), "title": {"text": reference.label}, "opposite": True, - "labels": {"style": {"color": color}} + "labels": {"style": {"color": color}}, + "visible": axis.y_axis_visible, } for reference in references if reference.delta] diff --git a/fireant/tests/slicer/widgets/test_highcharts.py b/fireant/tests/slicer/widgets/test_highcharts.py index c50b243e..dd1c5938 100644 --- a/fireant/tests/slicer/widgets/test_highcharts.py +++ b/fireant/tests/slicer/widgets/test_highcharts.py @@ -36,13 +36,17 @@ def test_single_metric_line_chart(self): self.assertEqual({ "title": {"text": "Time Series, Single Metric"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": None}} + "labels": {"style": {"color": None}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -75,13 +79,17 @@ def test_metric_prefix_line_chart(self): self.assertEqual({ "title": {"text": "Time Series, Single Metric"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": None}} + "labels": {"style": {"color": None}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -114,13 +122,17 @@ def test_metric_suffix_line_chart(self): self.assertEqual({ "title": {"text": "Time Series, Single Metric"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": None}} + "labels": {"style": {"color": None}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -153,13 +165,17 @@ def test_metric_precision_line_chart(self): self.assertEqual({ "title": {"text": "Time Series, Single Metric"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": None}} + "labels": {"style": {"color": None}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -190,13 +206,17 @@ def test_single_operation_line_chart(self): self.assertEqual({ "title": {"text": "Time Series, Single Metric"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": None}} + "labels": {"style": {"color": None}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -228,13 +248,17 @@ def test_single_metric_with_uni_dim_line_chart(self): self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Single Metric"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": None}} + "labels": {"style": {"color": None}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -286,13 +310,17 @@ def test_multi_metrics_single_axis_line_chart(self): self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Multiple Metrics"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": "#DDDF0D"}} + "labels": {"style": {"color": "#DDDF0D"}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -382,17 +410,22 @@ def test_multi_metrics_multi_axis_line_chart(self): self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Multiple Metrics, Multi-Axis"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "1", "title": {"text": None}, - "labels": {"style": {"color": "#55BF3B"}} + "labels": {"style": {"color": "#55BF3B"}}, + "visible": True, }, { "id": "0", "title": {"text": None}, - "labels": {"style": {"color": "#DDDF0D"}} + "labels": {"style": {"color": "#DDDF0D"}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -482,17 +515,22 @@ def test_multi_dim_with_totals_line_chart(self): self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Multiple Metrics, Multi-Axis"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "1", "title": {"text": None}, - "labels": {"style": {"color": "#55BF3B"}} + "labels": {"style": {"color": "#55BF3B"}}, + "visible": True, }, { "id": "0", "title": {"text": None}, - "labels": {"style": {"color": "#DDDF0D"}} + "labels": {"style": {"color": "#DDDF0D"}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ 'color': '#DDDF0D', @@ -620,17 +658,22 @@ def test_multi_dim_with_totals_on_first_dim_line_chart(self): self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Multiple Metrics, Multi-Axis"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "1", "title": {"text": None}, - "labels": {"style": {"color": "#55BF3B"}} + "labels": {"style": {"color": "#55BF3B"}}, + "visible": True, }, { "id": "0", "title": {"text": None}, - "labels": {"style": {"color": "#DDDF0D"}} + "labels": {"style": {"color": "#DDDF0D"}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ 'color': '#DDDF0D', @@ -763,13 +806,17 @@ def test_uni_dim_with_ref_line_chart(self): self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Reference"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": None}} + "labels": {"style": {"color": None}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -860,18 +907,173 @@ def test_uni_dim_with_ref_delta_line_chart(self): self.assertEqual({ "title": {"text": "Time Series with Unique Dimension and Delta Reference"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": None}} + "labels": {"style": {"color": None}}, + "visible": True, }, { "id": "0_eoe_delta", "title": {"text": "EoE Δ"}, "opposite": True, - "labels": {"style": {"color": None}} + "labels": {"style": {"color": None}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes (Texas)", + "yAxis": "0", + "data": [(946684800000, 6233385), + (1072915200000, 7359621), + (1199145600000, 8007961), + (1325376000000, 7877967), + (1451606400000, 5072915)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + }, { + "type": self.chart_type, + "name": "Votes (EoE Δ) (Texas)", + "yAxis": "0_eoe_delta", + "data": [(946684800000, -658998), + (1072915200000, -1126236), + (1199145600000, -648340), + (1325376000000, 129994), + (1451606400000, 2805052)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Dash", + "stacking": self.stacking, + }, { + "type": self.chart_type, + "name": "Votes (California)", + "yAxis": "0", + "data": [(946684800000, 10428632), + (1072915200000, 12255311), + (1199145600000, 13286254), + (1325376000000, 12694243), + (1451606400000, 13237598)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#55BF3B", + "marker": {"symbol": "square", "fillColor": "#55BF3B"}, + "dashStyle": "Solid", + "stacking": self.stacking, + }, { + "type": self.chart_type, + "name": "Votes (EoE Δ) (California)", + "yAxis": "0_eoe_delta", + "data": [(946684800000, -782570), + (1072915200000, -1826679), + (1199145600000, -1030943), + (1325376000000, 592011), + (1451606400000, -543355)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#55BF3B", + "marker": {"symbol": "square", "fillColor": "#55BF3B"}, + "dashStyle": "Dash", + "stacking": self.stacking, + }] + }, result) + + def test_invisible_y_axis(self): + result = HighCharts(title="Time Series, Single Metric", + axes=[self.chart_class([slicer.metrics.votes], + y_axis_visible=False)]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) + + self.assertEqual({ + "title": {"text": "Time Series, Single Metric"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}}, + "visible": False, + }], + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes", + "yAxis": "0", + "data": [(820454400000, 15220449), + (946684800000, 16662017), + (1072915200000, 19614932), + (1199145600000, 21294215), + (1325376000000, 20572210), + (1451606400000, 18310513)], + 'tooltip': { + 'valuePrefix': None, + 'valueSuffix': None, + 'valueDecimals': None, + }, + "color": "#DDDF0D", + "marker": {"symbol": "circle", "fillColor": "#DDDF0D"}, + "dashStyle": "Solid", + "stacking": self.stacking, + }] + }, result) + + def test_ref_axes_set_to_same_visibility_as_parent_axis(self): + result = HighCharts(title="Time Series with Unique Dimension and Delta Reference", + axes=[self.chart_class([slicer.metrics.votes], + y_axis_visible=False)]) \ + .transform(cont_uni_dim_ref_delta_df, + slicer, + [ + slicer.dimensions.timestamp, + slicer.dimensions.state + ], [ + ElectionOverElection(slicer.dimensions.timestamp, delta=True) + ]) + + self.assertEqual({ + "title": {"text": "Time Series with Unique Dimension and Delta Reference"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}}, + "visible": False, + }, { + "id": "0_eoe_delta", + "title": {"text": "EoE Δ"}, + "opposite": True, + "labels": {"style": {"color": None}}, + "visible": False, + }], + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -965,14 +1167,16 @@ def test_single_metric_bar_chart(self): "title": {"text": "All Votes"}, "xAxis": { "type": "category", - "categories": ["All"] + "categories": ["All"], + 'visible': True, }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": None}} + "labels": {"style": {"color": None}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -1001,15 +1205,16 @@ def test_multi_metric_bar_chart(self): "title": {"text": "Votes and Wins"}, "xAxis": { "type": "category", - "categories": ["All"] + "categories": ["All"], + 'visible': True, }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": "#DDDF0D"}} - + "labels": {"style": {"color": "#DDDF0D"}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -1051,15 +1256,16 @@ def test_cat_dim_single_metric_bar_chart(self): "title": {"text": "Votes and Wins"}, "xAxis": { "type": "category", - "categories": ["Democrat", "Independent", "Republican"] + "categories": ["Democrat", "Independent", "Republican"], + 'visible': True, }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": None}} - + "labels": {"style": {"color": None}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -1088,15 +1294,16 @@ def test_cat_dim_multi_metric_bar_chart(self): "title": {"text": "Votes and Wins"}, "xAxis": { "type": "category", - "categories": ["Democrat", "Independent", "Republican"] + "categories": ["Democrat", "Independent", "Republican"], + 'visible': True, }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": "#DDDF0D"}} - + "labels": {"style": {"color": "#DDDF0D"}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -1136,14 +1343,18 @@ def test_cont_uni_dims_single_metric_bar_chart(self): self.assertEqual({ "title": {"text": "Election Votes by State"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": None}} + "labels": {"style": {"color": None}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -1194,14 +1405,17 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): self.assertEqual({ "title": {"text": "Election Votes by State"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "0", "title": {"text": None}, - "labels": {"style": {"color": "#DDDF0D"}} - + "labels": {"style": {"color": "#DDDF0D"}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -1290,18 +1504,22 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): self.assertEqual({ "title": {"text": "Election Votes by State"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "1", "title": {"text": None}, - "labels": {"style": {"color": "#55BF3B"}} + "labels": {"style": {"color": "#55BF3B"}}, + "visible": True, }, { "id": "0", "title": {"text": None}, - "labels": {"style": {"color": "#DDDF0D"}} - + "labels": {"style": {"color": "#DDDF0D"}}, + "visible": True, }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -1382,6 +1600,44 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): }] }, result) + def test_invisible_y_axis(self): + result = HighCharts(title="All Votes", + axes=[self.chart_class([slicer.metrics.votes], + y_axis_visible=False)]) \ + .transform(single_metric_df, slicer, [], []) + + self.assertEqual({ + "title": {"text": "All Votes"}, + "xAxis": { + "type": "category", + "categories": ["All"], + 'visible': True, + }, + "yAxis": [{ + "id": "0", + "title": {"text": None}, + "labels": {"style": {"color": None}}, + "visible": False, + }], + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, + "legend": {"useHTML": True}, + "series": [{ + "type": self.chart_type, + "name": "Votes", + "yAxis": "0", + "data": [111674336], + 'tooltip': { + 'valueDecimals': None, + 'valuePrefix': None, + 'valueSuffix': None + }, + "color": "#DDDF0D", + "dashStyle": "Solid", + "marker": {}, + "stacking": self.stacking, + }] + }, result) + class HighChartsColumnChartTransformerTests(HighChartsBarChartTransformerTests): chart_class = HighCharts.ColumnChart @@ -1430,7 +1686,7 @@ def test_single_metric_pie_chart(self): self.assertEqual({ "title": {"text": "All Votes"}, - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "name": "Votes", @@ -1446,7 +1702,11 @@ def test_single_metric_pie_chart(self): 'valueSuffix': None }, }], - 'xAxis': {'categories': ['All'], 'type': 'category'}, + 'xAxis': { + 'type': 'category', + 'categories': ['All'], + 'visible': True, + }, 'yAxis': [], }, result) @@ -1458,7 +1718,7 @@ def test_multi_metric_bar_chart(self): self.assertEqual({ "title": {"text": "Votes and Wins"}, - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "name": "Votes", @@ -1487,7 +1747,11 @@ def test_multi_metric_bar_chart(self): 'valueSuffix': None }, }], - 'xAxis': {'categories': ['All'], 'type': 'category'}, + 'xAxis': { + 'type': 'category', + 'categories': ['All'], + 'visible': True, + }, 'yAxis': [], }, result) @@ -1498,7 +1762,7 @@ def test_cat_dim_single_metric_bar_chart(self): self.assertEqual({ 'title': {'text': 'Votes and Wins'}, - 'tooltip': {'useHTML': True, 'shared': True}, + 'tooltip': {'useHTML': True, 'shared': True, 'enabled': True}, 'legend': {'useHTML': True}, 'series': [{ 'name': 'Votes', @@ -1523,7 +1787,11 @@ def test_cat_dim_single_metric_bar_chart(self): }, }], 'yAxis': [], - 'xAxis': {'categories': ['Democrat', 'Independent', 'Republican'], 'type': 'category'} + 'xAxis': { + 'type': 'category', + 'categories': ['Democrat', 'Independent', 'Republican'], + 'visible': True, + } }, result) @skip @@ -1545,7 +1813,7 @@ def test_cat_dim_multi_metric_bar_chart(self): "labels": {"style": {"color": "#DDDF0D"}} }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -1586,14 +1854,17 @@ def test_cont_uni_dims_single_metric_bar_chart(self): self.assertEqual({ "title": {"text": "Election Votes by State"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "0", "title": {"text": None}, "labels": {"style": {"color": None}} }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -1645,14 +1916,17 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): self.assertEqual({ "title": {"text": "Election Votes by State"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "0", "title": {"text": None}, "labels": {"style": {"color": "#DDDF0D"}} }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, @@ -1742,7 +2016,10 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): self.assertEqual({ "title": {"text": "Election Votes by State"}, - "xAxis": {"type": "datetime"}, + "xAxis": { + "type": "datetime", + "visible": True, + }, "yAxis": [{ "id": "1", "title": {"text": None}, @@ -1753,7 +2030,7 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): "labels": {"style": {"color": "#DDDF0D"}} }], - "tooltip": {"shared": True, "useHTML": True}, + "tooltip": {"shared": True, "useHTML": True, "enabled": True}, "legend": {"useHTML": True}, "series": [{ "type": self.chart_type, From 8450caff8f81a76e7713e7eab4bfc953f88b303e Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 19 Mar 2018 18:27:22 +0100 Subject: [PATCH 050/123] bumped version to dev20 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 9328d560..30c1f9f5 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev19' +__version__ = '1.0.0.dev20' From 8d07093db1628f7f300f2949c4aa195ccb96fbcd Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 22 Mar 2018 15:47:03 +0100 Subject: [PATCH 051/123] Fixes #105 Added a VerticaQueryBuilder class with support for a query hint --- fireant/database/vertica.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fireant/database/vertica.py b/fireant/database/vertica.py index 9c8f15ae..136b0ae2 100644 --- a/fireant/database/vertica.py +++ b/fireant/database/vertica.py @@ -53,6 +53,9 @@ def connect(self): user=self.user, password=self.password, read_timeout=self.read_timeout) + def fetch(self, query): + return super(VerticaDatabase, self).fetch(query) + def trunc_date(self, field, interval): trunc_date_interval = self.DATETIME_INTERVALS.get(str(interval), 'DD') return Trunc(field, trunc_date_interval) From fd56d9422085284f4f058dfb3020bece0b6a3ef5 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 23 Mar 2018 11:37:11 +0100 Subject: [PATCH 052/123] added kwarg to fetch which allows a query hint to be used for supporting db vendors --- fireant/slicer/queries/builder.py | 13 ++++++++++--- requirements.txt | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 847005fd..bb690d17 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -1,15 +1,16 @@ -import pandas as pd from typing import ( Dict, Iterable, ) +import pandas as pd from fireant.utils import immutable from pypika import ( Order, functions as fn, ) from pypika.enums import SqlTypes + from .database import fetch_data from .finders import ( find_and_group_references_for_dimensions, @@ -144,7 +145,7 @@ def query(self): return query - def fetch(self, limit=None, offset=None) -> Iterable[Dict]: + def fetch(self, limit=None, offset=None, hint=None) -> Iterable[Dict]: """ Fetch the data for this query and transform it into the widgets. @@ -152,10 +153,14 @@ def fetch(self, limit=None, offset=None) -> Iterable[Dict]: A limit on the number of database rows returned. :param offset: A offset on the number of database rows returned. + :param hint: + A query hint label used with database vendors which support it. Adds a label comment to the query. :return: A list of dict (JSON) objects containing the widget configurations. """ query = self.query.limit(limit).offset(offset) + if hint and hasattr(query, 'hint'): + query = query.hint(hint) data_frame = fetch_data(self.slicer.database, str(query), @@ -202,7 +207,7 @@ def query(self): dimensions=self._dimensions, filters=self._filters) - def fetch(self, limit=None, offset=None, force_include=()) -> pd.Series: + def fetch(self, limit=None, offset=None, hint=None, force_include=()) -> pd.Series: """ Fetch the data for this query and transform it into the widgets. @@ -217,6 +222,8 @@ def fetch(self, limit=None, offset=None, force_include=()) -> pd.Series: A list of dict (JSON) objects containing the widget configurations. """ query = self.query + if hint and hasattr(query, 'hint'): + query = query.hint(hint) dimension = self._dimensions[0] definition = dimension.display_definition.as_(dimension.display_key) \ diff --git a/requirements.txt b/requirements.txt index a64ca9a5..4d979173 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ six pandas==0.22.0 -pypika==0.10.8 +pypika==0.11.2 pymysql==0.8.0 toposort==1.5 typing==3.6.2 From bf27790cc12cfd757261fbcdf3aff89ba582df4c Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 23 Mar 2018 13:28:34 +0100 Subject: [PATCH 053/123] bumped version to dev21 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 30c1f9f5..2b230da2 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev20' +__version__ = '1.0.0.dev21' From e620d1102520d94426f988af51f93b57c42d6f59 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 23 Mar 2018 15:17:11 +0100 Subject: [PATCH 054/123] Fixed the hint query to prevent it from being used unless it actually exists and is a callable --- fireant/database/base.py | 6 +++++- fireant/database/mysql.py | 7 ++++++- fireant/slicer/queries/builder.py | 6 +++--- fireant/tests/database/test_databases.py | 6 ++++++ fireant/tests/database/test_mysql.py | 9 +++++++-- requirements.txt | 2 +- 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/fireant/database/base.py b/fireant/database/base.py index 7571365a..10b07750 100644 --- a/fireant/database/base.py +++ b/fireant/database/base.py @@ -1,7 +1,8 @@ import pandas as pd - from pypika import ( Query, + enums, + functions as fn, terms, ) @@ -31,6 +32,9 @@ def fetch(self, query): cursor.execute(query) return cursor.fetchall() + def to_char(self, definition): + return fn.Cast(definition, enums.SqlTypes.VARCHAR) + def fetch_data(self, query): with self.connect() as connection: return pd.read_sql(query, connection, coerce_float=True, parse_dates=True) diff --git a/fireant/database/mysql.py b/fireant/database/mysql.py index 7b25284b..1d7129ea 100644 --- a/fireant/database/mysql.py +++ b/fireant/database/mysql.py @@ -1,10 +1,12 @@ import pandas as pd - from pypika import ( Dialects, MySQLQuery, + enums, + functions as fn, terms, ) + from .base import Database @@ -66,6 +68,9 @@ def fetch_data(self, query): def trunc_date(self, field, interval): return Trunc(field, str(interval)) + def to_char(self, definition): + return fn.Cast(definition, enums.SqlTypes.CHAR) + def date_add(self, field, date_part, interval): # adding an extra 's' as MySQL's interval doesn't work with 'year', 'week' etc, it expects a plural interval_term = terms.Interval(**{'{}s'.format(str(date_part)): interval, 'dialect': Dialects.MYSQL}) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index bb690d17..5f67d150 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -159,7 +159,7 @@ def fetch(self, limit=None, offset=None, hint=None) -> Iterable[Dict]: A list of dict (JSON) objects containing the widget configurations. """ query = self.query.limit(limit).offset(offset) - if hint and hasattr(query, 'hint'): + if hint and hasattr(query, 'hint') and callable(query.hint): query = query.hint(hint) data_frame = fetch_data(self.slicer.database, @@ -222,7 +222,7 @@ def fetch(self, limit=None, offset=None, hint=None, force_include=()) -> pd.Seri A list of dict (JSON) objects containing the widget configurations. """ query = self.query - if hint and hasattr(query, 'hint'): + if hint and hasattr(query, 'hint') and callable(query.hint): query = query.hint(hint) dimension = self._dimensions[0] @@ -231,7 +231,7 @@ def fetch(self, limit=None, offset=None, hint=None, force_include=()) -> pd.Seri else dimension.definition.as_(dimension.key) if force_include: - include = fn.Cast(dimension.definition, SqlTypes.VARCHAR) \ + include = self.slicer.database.to_char(dimension.definition) \ .isin([str(x) for x in force_include]) # Ensure that these values are included diff --git a/fireant/tests/database/test_databases.py b/fireant/tests/database/test_databases.py index c76e6372..a6b0ddc0 100644 --- a/fireant/tests/database/test_databases.py +++ b/fireant/tests/database/test_databases.py @@ -40,3 +40,9 @@ def test_database_api(self): with self.assertRaises(NotImplementedError): db.trunc_date(Field('abc'), 'day') + + def test_to_char(self): + db = Database() + + to_char = db.to_char(Field('field')) + self.assertEqual(str(to_char), 'CAST("field" AS VARCHAR)') diff --git a/fireant/tests/database/test_mysql.py b/fireant/tests/database/test_mysql.py index 1f24fe8a..cfd9f129 100644 --- a/fireant/tests/database/test_mysql.py +++ b/fireant/tests/database/test_mysql.py @@ -5,8 +5,6 @@ patch, ) -from pypika import Field - from fireant import ( annually, daily, @@ -16,6 +14,7 @@ weekly, ) from fireant.database import MySQLDatabase +from pypika import Field class TestMySQLDatabase(TestCase): @@ -104,3 +103,9 @@ def test_date_add_year(self): result = self.mysql.date_add(Field('date'), 'year', 1) self.assertEqual('DATE_ADD("date",INTERVAL 1 YEAR)', str(result)) + + def test_to_char(self): + db = MySQLDatabase() + + to_char = db.to_char(Field('field')) + self.assertEqual(str(to_char), 'CAST("field" AS CHAR)') diff --git a/requirements.txt b/requirements.txt index 4d979173..3715871a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ six pandas==0.22.0 -pypika==0.11.2 +pypika==0.11.3 pymysql==0.8.0 toposort==1.5 typing==3.6.2 From e64d0017b31c212c0a7c35fa077bc7f89a80cfd2 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 23 Mar 2018 15:19:53 +0100 Subject: [PATCH 055/123] bumped to dev22 and updated reqs in setup.py --- fireant/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 2b230da2..e35827fe 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev21' +__version__ = '1.0.0.dev22' diff --git a/setup.py b/setup.py index c1a74f8b..be027acb 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ def find_version(*file_paths): install_requires=[ 'six', 'pandas==0.22.0', - 'pypika==0.10.8', + 'pypika==0.11.3', 'toposort==1.5', 'typing==3.6.2', ], From 0d7847abd3650af8914e1ab7702d2fe1d1b41c18 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 10 Apr 2018 16:08:58 +0200 Subject: [PATCH 056/123] Changed the API of highcharts to include a series level in between axis and metric os that multiple chart types can be rendered on the same axis --- fireant/slicer/__init__.py | 1 + fireant/slicer/dimensions.py | 41 ++- fireant/slicer/metrics.py | 3 + fireant/slicer/queries/__init__.py | 2 +- fireant/slicer/queries/builder.py | 19 +- fireant/slicer/references.py | 3 + fireant/slicer/slicers.py | 7 +- fireant/slicer/widgets/base.py | 3 +- fireant/slicer/widgets/datatables.py | 11 +- fireant/slicer/widgets/highcharts.py | 111 +++--- fireant/slicer/widgets/pandas.py | 5 +- fireant/tests/slicer/queries/test_builder.py | 334 ++++++++---------- ...n_options.py => test_dimension_choices.py} | 26 +- fireant/tests/slicer/widgets/test_csv.py | 26 +- .../tests/slicer/widgets/test_datatables.py | 30 +- .../tests/slicer/widgets/test_highcharts.py | 191 +++++----- fireant/tests/slicer/widgets/test_pandas.py | 28 +- fireant/tests/slicer/widgets/test_widgets.py | 6 +- requirements.txt | 2 +- setup.py | 2 +- 20 files changed, 447 insertions(+), 404 deletions(-) rename fireant/tests/slicer/queries/{test_dimension_options.py => test_dimension_choices.py} (77%) diff --git a/fireant/slicer/__init__.py b/fireant/slicer/__init__.py index ed3ef65a..8b95da4e 100644 --- a/fireant/slicer/__init__.py +++ b/fireant/slicer/__init__.py @@ -4,6 +4,7 @@ ContinuousDimension, DatetimeDimension, Dimension, + DisplayDimension, UniqueDimension, ) from .exceptions import ( diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 991fb941..3da84d13 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -38,6 +38,9 @@ def rollup(self): """ self.is_rollup = True + def __repr__(self): + return "slicer.dimensions.{}".format(self.key) + class BooleanDimension(Dimension): """ @@ -101,23 +104,7 @@ def notin(self, values): return ExcludesFilter(self.definition, values) -class UniqueDimension(Dimension): - """ - This is a dimension that represents a field in a database which is a unique identifier, such as a primary/foreign - key. It provides support for a display value field which is selected and used in the results. - """ - - def __init__(self, key, label=None, definition=None, display_definition=None): - super(UniqueDimension, self).__init__(key, - label, - definition, - display_definition) - - def __hash__(self): - if self.has_display_field: - return hash('{}({},{})'.format(self.__class__.__name__, self.definition, self.display_definition)) - return super(UniqueDimension, self).__hash__() - +class _UniqueDimensionBase(Dimension): def isin(self, values, use_display=False): """ Creates a filter to filter a slicer query. @@ -184,12 +171,30 @@ def not_like(self, pattern): raise QueryException('No value set for display_definition.') return NotLikeFilter(self.display_definition, pattern) + +class UniqueDimension(_UniqueDimensionBase): + """ + This is a dimension that represents a field in a database which is a unique identifier, such as a primary/foreign + key. It provides support for a display value field which is selected and used in the results. + """ + + def __init__(self, key, label=None, definition=None, display_definition=None): + super(UniqueDimension, self).__init__(key, + label, + definition, + display_definition) + + def __hash__(self): + if self.has_display_field: + return hash('{}({},{})'.format(self.__class__.__name__, self.definition, self.display_definition)) + return super(UniqueDimension, self).__hash__() + @property def display(self): return self -class DisplayDimension(Dimension): +class DisplayDimension(_UniqueDimensionBase): """ WRITEME """ diff --git a/fireant/slicer/metrics.py b/fireant/slicer/metrics.py index 85bb84f6..4efd1ecc 100644 --- a/fireant/slicer/metrics.py +++ b/fireant/slicer/metrics.py @@ -30,3 +30,6 @@ def __lt__(self, other): def __le__(self, other): return ComparatorFilter(self.definition, ComparatorFilter.Operator.lte, other) + + def __repr__(self): + return "slicer.metrics.{}".format(self.key) \ No newline at end of file diff --git a/fireant/slicer/queries/__init__.py b/fireant/slicer/queries/__init__.py index dd8b109a..4a2c5dc8 100644 --- a/fireant/slicer/queries/__init__.py +++ b/fireant/slicer/queries/__init__.py @@ -1,4 +1,4 @@ from .builder import ( - DimensionOptionQueryBuilder, + DimensionChoicesQueryBuilder, SlicerQueryBuilder, ) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 5f67d150..af8ce8c1 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -7,9 +7,7 @@ from fireant.utils import immutable from pypika import ( Order, - functions as fn, ) -from pypika.enums import SqlTypes from .database import fetch_data from .finders import ( @@ -176,15 +174,26 @@ def fetch(self, limit=None, offset=None, hint=None) -> Iterable[Dict]: for widget in self._widgets] def __str__(self): - return self.query + return str(self.query) + def __repr__(self): + return ".".join(["slicer", "data"] + + ["widget({})".format(repr(widget)) + for widget in self._widgets] + + ["dimension({})".format(repr(dimension)) + for dimension in self._dimensions] + + ["filter({})".format(repr(filter)) + for filter in self._filters] + + ["reference({})".format(repr(reference)) + for reference in self._references]) -class DimensionOptionQueryBuilder(QueryBuilder): + +class DimensionChoicesQueryBuilder(QueryBuilder): """ WRITEME """ def __init__(self, slicer, dimension): - super(DimensionOptionQueryBuilder, self).__init__(slicer, slicer.hint_table or slicer.table) + super(DimensionChoicesQueryBuilder, self).__init__(slicer, slicer.hint_table or slicer.table) self._dimensions.append(dimension) @property diff --git a/fireant/slicer/references.py b/fireant/slicer/references.py index f40866d2..5ead27b3 100644 --- a/fireant/slicer/references.py +++ b/fireant/slicer/references.py @@ -33,6 +33,9 @@ def __eq__(self, other): def __hash__(self): return hash('reference{}{}'.format(self.key, self.dimension)) + def __repr__(self): + return '{}({})'.format(self.key, self.dimension.key) + class ReferenceType(object): def __init__(self, key, label, time_unit: str, interval: int): diff --git a/fireant/slicer/slicers.py b/fireant/slicer/slicers.py index 3f250e59..967cab3d 100644 --- a/fireant/slicer/slicers.py +++ b/fireant/slicer/slicers.py @@ -6,7 +6,7 @@ UniqueDimension, ) from .queries import ( - DimensionOptionQueryBuilder, + DimensionChoicesQueryBuilder, SlicerQueryBuilder, ) @@ -40,6 +40,9 @@ def __iter__(self): def __getitem__(self, item): return getattr(self, item) + def __contains__(self, item): + return hasattr(self, item) + def __eq__(self, other): """ Checks if the other object is an instance of _Container and has the same number of items with matching keys. @@ -102,7 +105,7 @@ def __init__(self, table, database, joins=(), dimensions=(), metrics=(), hint_ta for dimension in dimensions: if not isinstance(dimension, (UniqueDimension, CategoricalDimension)): continue - dimension.options = DimensionOptionQueryBuilder(self, dimension) + dimension.choices = DimensionChoicesQueryBuilder(self, dimension) def __eq__(self, other): return isinstance(other, Slicer) \ diff --git a/fireant/slicer/widgets/base.py b/fireant/slicer/widgets/base.py index 7e46cddf..3e3bff9f 100644 --- a/fireant/slicer/widgets/base.py +++ b/fireant/slicer/widgets/base.py @@ -1,10 +1,11 @@ +from fireant import Metric from fireant.slicer.exceptions import MetricRequiredException from fireant.utils import immutable from ..operations import Operation class Widget: - def __init__(self, items=()): + def __init__(self, *items: Metric): self.items = list(items) @immutable diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index 7e9b63eb..6e6e8db5 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -5,9 +5,9 @@ from fireant import ( ContinuousDimension, Metric, + UniqueDimension, formats, utils, - UniqueDimension, ) from .base import ( TransformableWidget, @@ -105,13 +105,18 @@ def _format_metric_cell(value, metric): class DataTablesJS(TransformableWidget): - def __init__(self, items=(), pivot=False, max_columns=None): - super(DataTablesJS, self).__init__(items) + def __init__(self, metric, *metrics: Metric, pivot=False, max_columns=None): + super(DataTablesJS, self).__init__(metric, *metrics) self.pivot = pivot self.max_columns = min(max_columns, HARD_MAX_COLUMNS) \ if max_columns is not None \ else HARD_MAX_COLUMNS + def __repr__(self): + return '{}({},pivot={})'.format(self.__class__.__name__, + ','.join(str(m) for m in self.items), + self.pivot) + def transform(self, data_frame, slicer, dimensions, references): """ WRITEME diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 7806898c..07ceb0d7 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -1,18 +1,23 @@ import itertools + +import pandas as pd from datetime import ( datetime, ) - -import pandas as pd +from typing import ( + Iterable, + Union, +) from fireant import ( DatetimeDimension, + Metric, + Operation, formats, utils, ) from .base import ( TransformableWidget, - Widget, ) from .helpers import ( dimensional_metric_label, @@ -60,16 +65,19 @@ ) -class ChartWidget(Widget): +class ChartWidget: type = None needs_marker = False stacking = None - def __init__(self, items=(), name=None, stacking=None, y_axis_visible=True): - super(ChartWidget, self).__init__(items) + def __init__(self, metric: Union[Metric, Operation], name=None, stacking=None): + self.metric = metric self.name = name self.stacking = self.stacking or stacking - self.y_axis_visible = y_axis_visible + + def __repr__(self): + return "{}({})".format(self.__class__.__name__, + repr(self.metric)) class ContinuousAxisChartWidget(ChartWidget): @@ -77,6 +85,20 @@ class ContinuousAxisChartWidget(ChartWidget): class HighCharts(TransformableWidget): + class Axis: + def __init__(self, series: Iterable[ChartWidget], y_axis_visible=True): + self._series = series or [] + self.y_axis_visible = y_axis_visible + + def __iter__(self): + return iter(self._series) + + def __len__(self): + return len(self._series) + + def __repr__(self): + return "axis({})".format(", ".join(map(repr, self))) + class LineChart(ContinuousAxisChartWidget): type = 'line' needs_marker = True @@ -106,15 +128,18 @@ class ColumnChart(ChartWidget): class StackedColumnChart(ColumnChart): stacking = "normal" - def __init__(self, axes=(), title=None, colors=None, x_axis_visible=True, tooltip_visible=True): - super(HighCharts, self).__init__(axes) + def __init__(self, title=None, colors=None, x_axis_visible=True, tooltip_visible=True): + super(HighCharts, self).__init__() self.title = title self.colors = colors or DEFAULT_COLORS self.x_axis_visible = x_axis_visible self.tooltip_visible = tooltip_visible + def __repr__(self): + return ".".join(["HighCharts()"] + [repr(axis) for axis in self.items]) + @utils.immutable - def axis(self, axis: ChartWidget): + def axis(self, *series: ChartWidget, **kwargs): """ (Immutable) Adds an axis to the Chart. @@ -122,7 +147,7 @@ def axis(self, axis: ChartWidget): :return: """ - self.items.append(axis) + self.items.append(self.Axis(series, **kwargs)) @property def metrics(self): @@ -134,10 +159,10 @@ def metrics(self): raise MetricRequiredException(str(self)) seen = set() - return [metric + return [series.metric for axis in self.items - for metric in axis.metrics - if not (metric.key in seen or seen.add(metric.key))] + for series in axis + if not (series.metric.key in seen or seen.add(series.metric.key))] @property def operations(self): @@ -178,19 +203,13 @@ def group_series(keys): dimension_display_values = extract_display_values(dimensions, data_frame) render_series_label = dimensional_metric_label(dimensions, dimension_display_values) - total_num_items = sum([len(axis.items) for axis in self.items]) + total_num_series = sum([len(axis) + for axis in self.items]) y_axes, series = [], [] for axis_idx, axis in enumerate(self.items): colors, series_colors = itertools.tee(colors) - axis_color = next(colors) if 1 < total_num_items else None - - if isinstance(axis, self.PieChart): - # pie charts suck - for metric in axis.metrics: - for reference in [None] + references: - series += [self._render_pie_series(axis, metric, reference, data_frame, render_series_label)] - continue + axis_color = next(colors) if 1 < total_num_series else None # prepend axes, append series, this keeps everything ordered left-to-right y_axes[0:0] = self._render_y_axis(axis_idx, @@ -299,57 +318,67 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, :param is_timeseries: :return: """ - has_multi_metric = 1 < len(axis.items) + has_multi_axis = 1 < len(axis) - series = [] - for metric in axis.items: + hc_series = [] + for series in axis: symbols = itertools.cycle(MARKER_SYMBOLS) - series_color = next(colors) if has_multi_metric else None + series_color = next(colors) if has_multi_axis else None for (dimension_values, group_df), symbol in zip(data_frame_groups, symbols): dimension_values = utils.wrap_list(dimension_values) - if not has_multi_metric: + if isinstance(series, self.PieChart): + # pie charts suck + for reference in [None] + references: + hc_series += [self._render_pie_series(series, + reference, + dimension_values, + group_df, + render_series_label)] + continue + + if not has_multi_axis: series_color = next(colors) for reference, dash_style in zip([None] + references, itertools.cycle(DASH_STYLES)): - metric_key = reference_key(metric, reference) + metric_key = reference_key(series.metric, reference) - series.append({ - "type": axis.type, + hc_series.append({ + "type": series.type, "color": series_color, "dashStyle": dash_style, - "name": render_series_label(dimension_values, metric, reference), + "name": render_series_label(dimension_values, series.metric, reference), "data": self._render_data(group_df, metric_key, is_timeseries), - "tooltip": self._render_tooltip(metric), + "tooltip": self._render_tooltip(series.metric), "yAxis": ("{}_{}".format(axis_idx, reference.key) if reference is not None and reference.delta else str(axis_idx)), "marker": ({"symbol": symbol, "fillColor": axis_color or series_color} - if axis.needs_marker + if series.needs_marker else {}), - "stacking": axis.stacking, + "stacking": series.stacking, }) - return series + return hc_series - def _render_pie_series(self, axis, metric, reference, data_frame, render_series_label): - pie_chart_df = data_frame[reference_key(metric, reference)] + def _render_pie_series(self, series, reference, dimension_values, data_frame, render_series_label): + metric = series.metric name = reference_label(metric, reference) return { "name": name, - "type": axis.type, + "type": series.type, + "colors": list(self.colors), "data": [{ "name": render_series_label(dimension_values) if dimension_values else name, "y": formats.metric_value(y), - "color": color, - } for (dimension_values, y), color in zip(pie_chart_df.iteritems(), self.colors)], + } for dimension_values, y in data_frame[series.metric.key].iteritems()], 'tooltip': { 'valueDecimals': metric.precision, 'valuePrefix': metric.prefix, diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index 0a79ca59..22e5cd2e 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -1,5 +1,6 @@ import pandas as pd +from fireant import Metric from .base import ( TransformableWidget, ) @@ -13,8 +14,8 @@ class Pandas(TransformableWidget): - def __init__(self, items=(), pivot=False, max_columns=None): - super(Pandas, self).__init__(items) + def __init__(self, metric, *metrics: Metric, pivot=False, max_columns=None): + super(Pandas, self).__init__(metric, *metrics) self.pivot = pivot self.max_columns = min(max_columns, HARD_MAX_COLUMNS) \ if max_columns is not None \ diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index c7c657d1..e4ffbb3a 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -1,3 +1,4 @@ +from datetime import date from unittest import TestCase from unittest.mock import ( ANY, @@ -5,13 +6,12 @@ patch, ) -from datetime import date +from pypika import Order import fireant as f from fireant.slicer.exceptions import ( MetricRequiredException, ) -from pypika import Order from ..matchers import ( DimensionMatcher, ) @@ -21,7 +21,7 @@ class QueryBuilderTests(TestCase): def test_widget_is_immutable(self): query1 = slicer.data - query2 = query1.widget(f.DataTablesJS([slicer.metrics.votes])) + query2 = query1.widget(f.DataTablesJS(slicer.metrics.votes)) self.assertIsNot(query1, query2) @@ -50,7 +50,7 @@ class QueryBuilderMetricTests(TestCase): def test_build_query_with_single_metric(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .query self.assertEqual('SELECT ' @@ -59,7 +59,7 @@ def test_build_query_with_single_metric(self): def test_build_query_with_multiple_metrics(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes, slicer.metrics.wins])) \ + .widget(f.DataTablesJS(slicer.metrics.votes, slicer.metrics.wins)) \ .query self.assertEqual('SELECT ' @@ -69,8 +69,8 @@ def test_build_query_with_multiple_metrics(self): def test_build_query_with_multiple_visualizations(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ - .widget(f.DataTablesJS([slicer.metrics.wins])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .widget(f.DataTablesJS(slicer.metrics.wins)) \ .query self.assertEqual('SELECT ' @@ -80,10 +80,8 @@ def test_build_query_with_multiple_visualizations(self): def test_build_query_for_chart_visualization_with_single_axis(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[ - f.HighCharts.PieChart([slicer.metrics.votes]) - ])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .query self.assertEqual('SELECT ' @@ -93,8 +91,8 @@ def test_build_query_for_chart_visualization_with_single_axis(self): def test_build_query_for_chart_visualization_with_multiple_axes(self): query = slicer.data \ .widget(f.HighCharts() - .axis(f.HighCharts.PieChart([slicer.metrics.votes])) - .axis(f.HighCharts.PieChart([slicer.metrics.wins]))) \ + .axis(f.HighCharts.LineChart(slicer.metrics.votes)) + .axis(f.HighCharts.LineChart(slicer.metrics.wins))) \ .query self.assertEqual('SELECT ' @@ -109,7 +107,7 @@ class QueryBuilderDimensionTests(TestCase): def test_build_query_with_datetime_dimension(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .query @@ -122,7 +120,7 @@ def test_build_query_with_datetime_dimension(self): def test_build_query_with_datetime_dimension_hourly(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp(f.hourly)) \ .query @@ -135,7 +133,7 @@ def test_build_query_with_datetime_dimension_hourly(self): def test_build_query_with_datetime_dimension_daily(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp(f.daily)) \ .query @@ -148,7 +146,7 @@ def test_build_query_with_datetime_dimension_daily(self): def test_build_query_with_datetime_dimension_weekly(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp(f.weekly)) \ .query @@ -161,7 +159,7 @@ def test_build_query_with_datetime_dimension_weekly(self): def test_build_query_with_datetime_dimension_monthly(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp(f.monthly)) \ .query @@ -174,7 +172,7 @@ def test_build_query_with_datetime_dimension_monthly(self): def test_build_query_with_datetime_dimension_quarterly(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp(f.quarterly)) \ .query @@ -187,7 +185,7 @@ def test_build_query_with_datetime_dimension_quarterly(self): def test_build_query_with_datetime_dimension_annually(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp(f.annually)) \ .query @@ -200,7 +198,7 @@ def test_build_query_with_datetime_dimension_annually(self): def test_build_query_with_boolean_dimension(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.winner) \ .query @@ -213,7 +211,7 @@ def test_build_query_with_boolean_dimension(self): def test_build_query_with_categorical_dimension(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.political_party) \ .query @@ -226,7 +224,7 @@ def test_build_query_with_categorical_dimension(self): def test_build_query_with_unique_dimension(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.election) \ .query @@ -240,7 +238,7 @@ def test_build_query_with_unique_dimension(self): def test_build_query_with_multiple_dimensions(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.candidate) \ .query @@ -256,12 +254,10 @@ def test_build_query_with_multiple_dimensions(self): def test_build_query_with_multiple_dimensions_and_visualizations(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes, slicer.metrics.wins])) \ - .widget(f.HighCharts( - axes=[ - f.HighCharts.PieChart([slicer.metrics.votes]), - f.HighCharts.ColumnChart([slicer.metrics.wins]), - ])) \ + .widget(f.DataTablesJS(slicer.metrics.votes, slicer.metrics.wins)) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes)) + .axis(f.HighCharts.LineChart(slicer.metrics.wins))) \ .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.political_party) \ .query @@ -282,7 +278,7 @@ class QueryBuilderDimensionTotalsTests(TestCase): def test_build_query_with_totals_cat_dimension(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.political_party.rollup()) \ .query @@ -303,7 +299,7 @@ def test_build_query_with_totals_cat_dimension(self): def test_build_query_with_totals_uni_dimension(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.candidate.rollup()) \ .query @@ -326,7 +322,7 @@ def test_build_query_with_totals_uni_dimension(self): def test_build_query_with_totals_on_dimension_and_subsequent_dimensions(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp, slicer.dimensions.candidate.rollup(), slicer.dimensions.political_party) \ @@ -355,7 +351,7 @@ def test_build_query_with_totals_on_dimension_and_subsequent_dimensions(self): def test_build_query_with_totals_on_multiple_dimensions_dimension(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp, slicer.dimensions.candidate.rollup(), slicer.dimensions.political_party.rollup()) \ @@ -396,7 +392,7 @@ def test_build_query_with_totals_on_multiple_dimensions_dimension(self): def test_build_query_with_totals_cat_dimension_with_references(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp, slicer.dimensions.political_party.rollup()) \ .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ @@ -456,7 +452,7 @@ def test_build_query_with_totals_cat_dimension_with_references(self): def test_build_query_with_totals_cat_dimension_with_references_and_date_filters(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.political_party.rollup()) \ .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ @@ -524,7 +520,7 @@ class QueryBuilderDimensionFilterTests(TestCase): def test_build_query_with_filter_isin_categorical_dim(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.political_party.isin(['d'])) \ .query @@ -535,7 +531,7 @@ def test_build_query_with_filter_isin_categorical_dim(self): def test_build_query_with_filter_notin_categorical_dim(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.political_party.notin(['d'])) \ .query @@ -546,7 +542,7 @@ def test_build_query_with_filter_notin_categorical_dim(self): def test_build_query_with_filter_isin_unique_dim(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.candidate.isin([1])) \ .query @@ -557,7 +553,7 @@ def test_build_query_with_filter_isin_unique_dim(self): def test_build_query_with_filter_notin_unique_dim(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.candidate.notin([1])) \ .query @@ -568,7 +564,7 @@ def test_build_query_with_filter_notin_unique_dim(self): def test_build_query_with_filter_isin_unique_dim_display(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.candidate.isin(['Donald Trump'], use_display=True)) \ .query @@ -579,7 +575,7 @@ def test_build_query_with_filter_isin_unique_dim_display(self): def test_build_query_with_filter_notin_unique_dim_display(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.candidate.notin(['Donald Trump'], use_display=True)) \ .query @@ -590,7 +586,7 @@ def test_build_query_with_filter_notin_unique_dim_display(self): def test_build_query_with_filter_like_unique_dim(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.candidate.like('%Trump')) \ .query @@ -601,7 +597,7 @@ def test_build_query_with_filter_like_unique_dim(self): def test_build_query_with_filter_not_like_unique_dim(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.candidate.not_like('%Trump')) \ .query @@ -613,30 +609,30 @@ def test_build_query_with_filter_not_like_unique_dim(self): def test_build_query_with_filter_isin_raise_exception_when_display_definition_undefined(self): with self.assertRaises(f.QueryException): slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.deepjoin.isin([1], use_display=True)) def test_build_query_with_filter_notin_raise_exception_when_display_definition_undefined(self): with self.assertRaises(f.QueryException): slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.deepjoin.notin([1], use_display=True)) def test_build_query_with_filter_like_raise_exception_when_display_definition_undefined(self): with self.assertRaises(f.QueryException): slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.deepjoin.like('test')) def test_build_query_with_filter_not_like_raise_exception_when_display_definition_undefined(self): with self.assertRaises(f.QueryException): slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.deepjoin.not_like('test')) def test_build_query_with_filter_range_datetime_dimension(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.timestamp.between(date(2009, 1, 20), date(2017, 1, 20))) \ .query @@ -647,7 +643,7 @@ def test_build_query_with_filter_range_datetime_dimension(self): def test_build_query_with_filter_boolean_true(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.winner.is_(True)) \ .query @@ -658,7 +654,7 @@ def test_build_query_with_filter_boolean_true(self): def test_build_query_with_filter_boolean_false(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.winner.is_(False)) \ .query @@ -674,7 +670,7 @@ class QueryBuilderMetricFilterTests(TestCase): def test_build_query_with_metric_filter_eq(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.metrics.votes == 5) \ .query @@ -685,7 +681,7 @@ def test_build_query_with_metric_filter_eq(self): def test_build_query_with_metric_filter_eq_left(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(5 == slicer.metrics.votes) \ .query @@ -696,7 +692,7 @@ def test_build_query_with_metric_filter_eq_left(self): def test_build_query_with_metric_filter_ne(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.metrics.votes != 5) \ .query @@ -707,7 +703,7 @@ def test_build_query_with_metric_filter_ne(self): def test_build_query_with_metric_filter_ne_left(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(5 != slicer.metrics.votes) \ .query @@ -718,7 +714,7 @@ def test_build_query_with_metric_filter_ne_left(self): def test_build_query_with_metric_filter_gt(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.metrics.votes > 5) \ .query @@ -729,7 +725,7 @@ def test_build_query_with_metric_filter_gt(self): def test_build_query_with_metric_filter_gt_left(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(5 < slicer.metrics.votes) \ .query @@ -740,7 +736,7 @@ def test_build_query_with_metric_filter_gt_left(self): def test_build_query_with_metric_filter_gte(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.metrics.votes >= 5) \ .query @@ -751,7 +747,7 @@ def test_build_query_with_metric_filter_gte(self): def test_build_query_with_metric_filter_gte_left(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(5 <= slicer.metrics.votes) \ .query @@ -762,7 +758,7 @@ def test_build_query_with_metric_filter_gte_left(self): def test_build_query_with_metric_filter_lt(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.metrics.votes < 5) \ .query @@ -773,7 +769,7 @@ def test_build_query_with_metric_filter_lt(self): def test_build_query_with_metric_filter_lt_left(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(5 > slicer.metrics.votes) \ .query @@ -784,7 +780,7 @@ def test_build_query_with_metric_filter_lt_left(self): def test_build_query_with_metric_filter_lte(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.metrics.votes <= 5) \ .query @@ -795,7 +791,7 @@ def test_build_query_with_metric_filter_lte(self): def test_build_query_with_metric_filter_lte_left(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(5 >= slicer.metrics.votes) \ .query @@ -811,7 +807,7 @@ class QueryBuilderOperationTests(TestCase): def test_build_query_with_cumsum_operation(self): query = slicer.data \ - .widget(f.DataTablesJS([f.CumSum(slicer.metrics.votes)])) \ + .widget(f.DataTablesJS(f.CumSum(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .query @@ -824,7 +820,7 @@ def test_build_query_with_cumsum_operation(self): def test_build_query_with_cummean_operation(self): query = slicer.data \ - .widget(f.DataTablesJS([f.CumMean(slicer.metrics.votes)])) \ + .widget(f.DataTablesJS(f.CumMean(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .query @@ -842,9 +838,8 @@ class QueryBuilderDatetimeReferenceTests(TestCase): def test_single_reference_dod_with_no_dimension_uses_multiple_from_clauses_instead_of_joins(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .query @@ -864,9 +859,8 @@ def test_single_reference_dod_with_no_dimension_uses_multiple_from_clauses_inste def test_single_reference_dod_with_dimension_but_not_reference_dimension_in_query_using_filter(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.political_party) \ .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .filter(slicer.dimensions.timestamp.between(date(2000, 1, 1), date(2000, 3, 1))) \ @@ -901,9 +895,8 @@ def test_single_reference_dod_with_dimension_but_not_reference_dimension_in_quer def test_dimension_with_single_reference_dod(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .query @@ -935,9 +928,8 @@ def test_dimension_with_single_reference_dod(self): def test_dimension_with_single_reference_wow(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .reference(f.WeekOverWeek(slicer.dimensions.timestamp)) \ .query @@ -969,9 +961,8 @@ def test_dimension_with_single_reference_wow(self): def test_dimension_with_single_reference_mom(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .reference(f.MonthOverMonth(slicer.dimensions.timestamp)) \ .query @@ -1003,9 +994,8 @@ def test_dimension_with_single_reference_mom(self): def test_dimension_with_single_reference_qoq(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .reference(f.QuarterOverQuarter(slicer.dimensions.timestamp)) \ .query @@ -1037,9 +1027,8 @@ def test_dimension_with_single_reference_qoq(self): def test_dimension_with_single_reference_yoy(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ .query @@ -1071,9 +1060,8 @@ def test_dimension_with_single_reference_yoy(self): def test_dimension_with_single_reference_as_a_delta(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .reference(f.DayOverDay(slicer.dimensions.timestamp, delta=True)) \ .query @@ -1105,9 +1093,8 @@ def test_dimension_with_single_reference_as_a_delta(self): def test_dimension_with_single_reference_as_a_delta_percentage(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .reference(f.DayOverDay(slicer.dimensions.timestamp, delta_percent=True)) \ .query @@ -1140,9 +1127,8 @@ def test_dimension_with_single_reference_as_a_delta_percentage(self): def test_reference_on_dimension_with_weekly_interval(self): weekly_timestamp = slicer.dimensions.timestamp(f.weekly) query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(weekly_timestamp) \ .reference(f.DayOverDay(weekly_timestamp)) \ .query @@ -1174,9 +1160,8 @@ def test_reference_on_dimension_with_weekly_interval(self): def test_reference_on_dimension_with_weekly_interval_no_interval_on_reference(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp(f.weekly)) \ .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .query @@ -1208,9 +1193,8 @@ def test_reference_on_dimension_with_weekly_interval_no_interval_on_reference(se def test_reference_on_dimension_with_monthly_interval(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp(f.monthly)) \ .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .query @@ -1242,9 +1226,8 @@ def test_reference_on_dimension_with_monthly_interval(self): def test_reference_on_dimension_with_quarterly_interval(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp(f.quarterly)) \ .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .query @@ -1276,9 +1259,8 @@ def test_reference_on_dimension_with_quarterly_interval(self): def test_reference_on_dimension_with_annual_interval(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp(f.annually)) \ .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .query @@ -1310,9 +1292,8 @@ def test_reference_on_dimension_with_annual_interval(self): def test_dimension_with_multiple_references(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .reference(f.YearOverYear(slicer.dimensions.timestamp, delta_percent=True)) \ @@ -1362,9 +1343,8 @@ def test_dimension_with_multiple_references(self): def test_reference_joins_nested_query_on_dimensions(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.political_party) \ .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ @@ -1401,9 +1381,8 @@ def test_reference_joins_nested_query_on_dimensions(self): def test_reference_with_unique_dimension_includes_display_definition(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.candidate) \ .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ @@ -1443,9 +1422,8 @@ def test_reference_with_unique_dimension_includes_display_definition(self): def test_adjust_reference_dimension_filters_in_reference_query(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .filter(slicer.dimensions.timestamp @@ -1481,9 +1459,8 @@ def test_adjust_reference_dimension_filters_in_reference_query(self): def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_filters(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ .filter(slicer.dimensions.timestamp @@ -1523,9 +1500,8 @@ def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_fil def test_adapt_dow_for_leap_year_for_yoy_reference(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp(f.weekly)) \ .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ .query @@ -1557,9 +1533,8 @@ def test_adapt_dow_for_leap_year_for_yoy_reference(self): def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp(f.weekly)) \ .reference(f.YearOverYear(slicer.dimensions.timestamp, delta=True)) \ .query @@ -1591,9 +1566,8 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp(f.weekly)) \ .reference(f.YearOverYear(slicer.dimensions.timestamp, delta_percent=True)) \ .query @@ -1625,9 +1599,8 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): def test_adapt_dow_for_leap_year_for_yoy_reference_with_date_filter(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp(f.weekly)) \ .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ .filter(slicer.dimensions.timestamp.between(date(2018, 1, 1), date(2018, 1, 31))) \ @@ -1662,9 +1635,8 @@ def test_adapt_dow_for_leap_year_for_yoy_reference_with_date_filter(self): def test_adding_duplicate_reference_does_not_join_more_queries(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .reference(f.DayOverDay(slicer.dimensions.timestamp), f.DayOverDay(slicer.dimensions.timestamp)) \ @@ -1697,9 +1669,8 @@ def test_adding_duplicate_reference_does_not_join_more_queries(self): def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .reference(f.DayOverDay(slicer.dimensions.timestamp), f.DayOverDay(slicer.dimensions.timestamp, delta=True), @@ -1735,9 +1706,8 @@ def test_use_same_nested_query_for_joining_references_with_same_period_and_dimen def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension_with_different_periods(self): query = slicer.data \ - .widget(f.HighCharts( - axes=[f.HighCharts.LineChart( - [slicer.metrics.votes])])) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ .dimension(slicer.dimensions.timestamp) \ .reference(f.DayOverDay(slicer.dimensions.timestamp), f.DayOverDay(slicer.dimensions.timestamp, delta=True), @@ -1794,7 +1764,7 @@ class QueryBuilderJoinTests(TestCase): def test_dimension_with_join_includes_join_in_query(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.district) \ .query @@ -1812,8 +1782,8 @@ def test_dimension_with_join_includes_join_in_query(self): def test_dimension_with_multiple_joins_includes_joins_ordered__in_query(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes, - slicer.metrics.voters])) \ + .widget(f.DataTablesJS(slicer.metrics.votes, + slicer.metrics.voters)) \ .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.district) \ .query @@ -1834,7 +1804,7 @@ def test_dimension_with_multiple_joins_includes_joins_ordered__in_query(self): def test_dimension_with_recursive_join_joins_all_join_tables(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.state) \ .query @@ -1854,7 +1824,7 @@ def test_dimension_with_recursive_join_joins_all_join_tables(self): def test_metric_with_join_includes_join_in_query(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.voters])) \ + .widget(f.DataTablesJS(slicer.metrics.voters)) \ .dimension(slicer.dimensions.political_party) \ .query @@ -1869,7 +1839,7 @@ def test_metric_with_join_includes_join_in_query(self): def test_dimension_filter_with_join_on_display_definition_does_not_include_join_in_query(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.district.isin([1])) \ .query @@ -1880,7 +1850,7 @@ def test_dimension_filter_with_join_on_display_definition_does_not_include_join_ def test_dimension_filter_display_field_with_join_includes_join_in_query(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.district.isin(['District 4'], use_display=True)) \ .query @@ -1893,7 +1863,7 @@ def test_dimension_filter_display_field_with_join_includes_join_in_query(self): def test_dimension_filter_with_recursive_join_includes_join_in_query(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.state.isin([1])) \ .query @@ -1906,7 +1876,7 @@ def test_dimension_filter_with_recursive_join_includes_join_in_query(self): def test_dimension_filter_with_deep_recursive_join_includes_joins_in_query(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .filter(slicer.dimensions.deepjoin.isin([1])) \ .query @@ -1927,7 +1897,7 @@ class QueryBuilderOrderTests(TestCase): def test_build_query_order_by_dimension(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .orderby(slicer.dimensions.timestamp) \ .query @@ -1941,7 +1911,7 @@ def test_build_query_order_by_dimension(self): def test_build_query_order_by_dimension_display(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.candidate) \ .orderby(slicer.dimensions.candidate_display) \ .query @@ -1956,7 +1926,7 @@ def test_build_query_order_by_dimension_display(self): def test_build_query_order_by_dimension_asc(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .orderby(slicer.dimensions.timestamp, orientation=Order.asc) \ .query @@ -1970,7 +1940,7 @@ def test_build_query_order_by_dimension_asc(self): def test_build_query_order_by_dimension_desc(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .orderby(slicer.dimensions.timestamp, orientation=Order.desc) \ .query @@ -1984,7 +1954,7 @@ def test_build_query_order_by_dimension_desc(self): def test_build_query_order_by_metric(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .orderby(slicer.metrics.votes) \ .query @@ -1998,7 +1968,7 @@ def test_build_query_order_by_metric(self): def test_build_query_order_by_metric_asc(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .orderby(slicer.metrics.votes, orientation=Order.asc) \ .query @@ -2012,7 +1982,7 @@ def test_build_query_order_by_metric_asc(self): def test_build_query_order_by_metric_desc(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .orderby(slicer.metrics.votes, orientation=Order.desc) \ .query @@ -2026,7 +1996,7 @@ def test_build_query_order_by_metric_desc(self): def test_build_query_order_by_multiple_dimensions(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp, slicer.dimensions.candidate) \ .orderby(slicer.dimensions.timestamp) \ .orderby(slicer.dimensions.candidate) \ @@ -2043,7 +2013,7 @@ def test_build_query_order_by_multiple_dimensions(self): def test_build_query_order_by_multiple_dimensions_with_different_orientations(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp, slicer.dimensions.candidate) \ .orderby(slicer.dimensions.timestamp, orientation=Order.desc) \ .orderby(slicer.dimensions.candidate, orientation=Order.asc) \ @@ -2060,7 +2030,7 @@ def test_build_query_order_by_multiple_dimensions_with_different_orientations(se def test_build_query_order_by_metrics_and_dimensions(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .orderby(slicer.dimensions.timestamp) \ .orderby(slicer.metrics.votes) \ @@ -2075,7 +2045,7 @@ def test_build_query_order_by_metrics_and_dimensions(self): def test_build_query_order_by_metrics_and_dimensions_with_different_orientations(self): query = slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .orderby(slicer.dimensions.timestamp, orientation=Order.asc) \ .orderby(slicer.metrics.votes, orientation=Order.desc) \ @@ -2093,7 +2063,7 @@ def test_build_query_order_by_metrics_and_dimensions_with_different_orientations class QueryBuildPaginationTests(TestCase): def test_set_limit(self, mock_fetch_data: Mock): slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .fetch(limit=20) @@ -2108,7 +2078,7 @@ def test_set_limit(self, mock_fetch_data: Mock): def test_set_offset(self, mock_fetch_data: Mock): slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .fetch(offset=20) @@ -2124,7 +2094,7 @@ def test_set_offset(self, mock_fetch_data: Mock): def test_set_limit_and_offset(self, mock_fetch_data: Mock): slicer.data \ - .widget(f.DataTablesJS([slicer.metrics.votes])) \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ .fetch(limit=20, offset=20) @@ -2147,29 +2117,21 @@ class QueryBuilderValidationTests(TestCase): def test_highcharts_requires_at_least_one_axis(self): with self.assertRaises(MetricRequiredException): slicer.data \ - .widget(f.HighCharts([])) \ - .dimension(slicer.dimensions.timestamp) \ - .query - - def test_highcharts_axis_requires_at_least_one_metric(self): - with self.assertRaises(MetricRequiredException): - slicer.data \ - .widget(f.HighCharts([f.HighCharts.LineChart([])])) \ + .widget(f.HighCharts()) \ .dimension(slicer.dimensions.timestamp) \ .query def test_datatablesjs_requires_at_least_one_metric(self): - with self.assertRaises(MetricRequiredException): + with self.assertRaises(TypeError): slicer.data \ - .widget(f.DataTablesJS([])) \ - .query + .widget(f.DataTablesJS()) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @patch('fireant.slicer.queries.builder.fetch_data') class QueryBuilderRenderTests(TestCase): def test_pass_slicer_database_as_arg(self, mock_fetch_data: Mock): - mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget = f.Widget(slicer.metrics.votes) mock_widget.transform = Mock() slicer.data \ @@ -2181,7 +2143,7 @@ def test_pass_slicer_database_as_arg(self, mock_fetch_data: Mock): dimensions=ANY) def test_pass_query_from_builder_as_arg(self, mock_fetch_data: Mock): - mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget = f.Widget(slicer.metrics.votes) mock_widget.transform = Mock() slicer.data \ @@ -2194,7 +2156,7 @@ def test_pass_query_from_builder_as_arg(self, mock_fetch_data: Mock): dimensions=ANY) def test_builder_dimensions_as_arg_with_zero_dimensions(self, mock_fetch_data: Mock): - mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget = f.Widget(slicer.metrics.votes) mock_widget.transform = Mock() slicer.data \ @@ -2204,7 +2166,7 @@ def test_builder_dimensions_as_arg_with_zero_dimensions(self, mock_fetch_data: M mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=[]) def test_builder_dimensions_as_arg_with_one_dimension(self, mock_fetch_data: Mock): - mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget = f.Widget(slicer.metrics.votes) mock_widget.transform = Mock() dimensions = [slicer.dimensions.state] @@ -2217,7 +2179,7 @@ def test_builder_dimensions_as_arg_with_one_dimension(self, mock_fetch_data: Moc mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=DimensionMatcher(*dimensions)) def test_builder_dimensions_as_arg_with_multiple_dimensions(self, mock_fetch_data: Mock): - mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget = f.Widget(slicer.metrics.votes) mock_widget.transform = Mock() dimensions = slicer.dimensions.timestamp, slicer.dimensions.state, slicer.dimensions.political_party @@ -2230,7 +2192,7 @@ def test_builder_dimensions_as_arg_with_multiple_dimensions(self, mock_fetch_dat mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=DimensionMatcher(*dimensions)) def test_call_transform_on_widget(self, mock_fetch_data: Mock): - mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget = f.Widget(slicer.metrics.votes) mock_widget.transform = Mock() # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work @@ -2245,7 +2207,7 @@ def test_call_transform_on_widget(self, mock_fetch_data: Mock): []) def test_returns_results_from_widget_transform(self, mock_fetch_data: Mock): - mock_widget = f.Widget([slicer.metrics.votes]) + mock_widget = f.Widget(slicer.metrics.votes) mock_widget.transform = Mock() # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work @@ -2260,7 +2222,7 @@ def test_operations_evaluated(self, mock_fetch_data: Mock): mock_operation = Mock(name='mock_operation ', spec=f.Operation) mock_operation.key, mock_operation.definition = 'mock_operation', slicer.table.abc - mock_widget = f.Widget([mock_operation]) + mock_widget = f.Widget(mock_operation) mock_widget.transform = Mock() mock_df = {} @@ -2278,7 +2240,7 @@ def test_operations_results_stored_in_data_frame(self, mock_fetch_data: Mock): mock_operation = Mock(name='mock_operation ', spec=f.Operation) mock_operation.key, mock_operation.definition = 'mock_operation', slicer.table.abc - mock_widget = f.Widget([mock_operation]) + mock_widget = f.Widget(mock_operation) mock_widget.transform = Mock() mock_df = {} diff --git a/fireant/tests/slicer/queries/test_dimension_options.py b/fireant/tests/slicer/queries/test_dimension_choices.py similarity index 77% rename from fireant/tests/slicer/queries/test_dimension_options.py rename to fireant/tests/slicer/queries/test_dimension_choices.py index c1729f6f..168f36cb 100644 --- a/fireant/tests/slicer/queries/test_dimension_options.py +++ b/fireant/tests/slicer/queries/test_dimension_choices.py @@ -4,12 +4,12 @@ # noinspection SqlDialectInspection,SqlNoDataSourceInspection -class DimensionsOptionsQueryBuilderTests(TestCase): +class DimensionsChoicesQueryBuilderTests(TestCase): maxDiff = None - def test_query_options_for_cat_dimension(self): + def test_query_choices_for_cat_dimension(self): query = slicer.dimensions.political_party \ - .options \ + .choices \ .query self.assertEqual('SELECT ' @@ -17,9 +17,9 @@ def test_query_options_for_cat_dimension(self): 'FROM "politics"."politician" ' 'GROUP BY "political_party"', str(query)) - def test_query_options_for_uni_dimension(self): + def test_query_choices_for_uni_dimension(self): query = slicer.dimensions.candidate \ - .options \ + .choices \ .query self.assertEqual('SELECT ' @@ -28,9 +28,9 @@ def test_query_options_for_uni_dimension(self): 'FROM "politics"."politician" ' 'GROUP BY "candidate","candidate_display"', str(query)) - def test_query_options_for_uni_dimension_with_join(self): + def test_query_choices_for_uni_dimension_with_join(self): query = slicer.dimensions.district \ - .options \ + .choices \ .query self.assertEqual('SELECT ' @@ -41,17 +41,17 @@ def test_query_options_for_uni_dimension_with_join(self): 'ON "politician"."district_id"="district"."id" ' 'GROUP BY "district","district_display"', str(query)) - def test_no_options_attr_for_datetime_dimension(self): + def test_no_choices_attr_for_datetime_dimension(self): with self.assertRaises(AttributeError): - slicer.dimensions.timestamp.options + slicer.dimensions.timestamp.choices - def test_no_options_attr_for_boolean_dimension(self): + def test_no_choices_attr_for_boolean_dimension(self): with self.assertRaises(AttributeError): - slicer.dimensions.winner.options + slicer.dimensions.winner.choices - def test_filter_options(self): + def test_filter_choices(self): query = slicer.dimensions.candidate \ - .options \ + .choices \ .filter(slicer.dimensions.political_party.isin(['d', 'r'])) \ .query diff --git a/fireant/tests/slicer/widgets/test_csv.py b/fireant/tests/slicer/widgets/test_csv.py index fdb5c67b..44faaf8b 100644 --- a/fireant/tests/slicer/widgets/test_csv.py +++ b/fireant/tests/slicer/widgets/test_csv.py @@ -23,7 +23,7 @@ class CSVWidgetTests(TestCase): maxDiff = None def test_single_metric(self): - result = CSV(items=[slicer.metrics.votes]) \ + result = CSV(slicer.metrics.votes) \ .transform(single_metric_df, slicer, [], []) expected = single_metric_df.copy()[['votes']] @@ -32,7 +32,7 @@ def test_single_metric(self): self.assertEqual(result, expected.to_csv()) def test_multiple_metrics(self): - result = CSV(items=[slicer.metrics.votes, slicer.metrics.wins]) \ + result = CSV(slicer.metrics.votes, slicer.metrics.wins) \ .transform(multi_metric_df, slicer, [], []) expected = multi_metric_df.copy()[['votes', 'wins']] @@ -41,7 +41,7 @@ def test_multiple_metrics(self): self.assertEqual(result, expected.to_csv()) def test_multiple_metrics_reversed(self): - result = CSV(items=[slicer.metrics.wins, slicer.metrics.votes]) \ + result = CSV(slicer.metrics.wins, slicer.metrics.votes) \ .transform(multi_metric_df, slicer, [], []) expected = multi_metric_df.copy()[['wins', 'votes']] @@ -50,7 +50,7 @@ def test_multiple_metrics_reversed(self): self.assertEqual(result, expected.to_csv()) def test_time_series_dim(self): - result = CSV(items=[slicer.metrics.wins]) \ + result = CSV(slicer.metrics.wins) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) expected = cont_dim_df.copy()[['wins']] @@ -60,7 +60,7 @@ def test_time_series_dim(self): self.assertEqual(result, expected.to_csv()) def test_time_series_dim_with_operation(self): - result = CSV(items=[CumSum(slicer.metrics.votes)]) \ + result = CSV(CumSum(slicer.metrics.votes)) \ .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp], []) expected = cont_dim_operation_df.copy()[['cumsum(votes)']] @@ -70,7 +70,7 @@ def test_time_series_dim_with_operation(self): self.assertEqual(result, expected.to_csv()) def test_cat_dim(self): - result = CSV(items=[slicer.metrics.wins]) \ + result = CSV(slicer.metrics.wins) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) expected = cat_dim_df.copy()[['wins']] @@ -80,7 +80,7 @@ def test_cat_dim(self): self.assertEqual(result, expected.to_csv()) def test_uni_dim(self): - result = CSV(items=[slicer.metrics.wins]) \ + result = CSV(slicer.metrics.wins) \ .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) expected = uni_dim_df.copy() \ @@ -101,7 +101,7 @@ def test_uni_dim_no_display_definition(self): uni_dim_df_copy = uni_dim_df.copy() del uni_dim_df_copy[slicer.dimensions.candidate.display_key] - result = CSV(items=[slicer.metrics.wins]) \ + result = CSV(slicer.metrics.wins) \ .transform(uni_dim_df_copy, slicer, [candidate], []) expected = uni_dim_df_copy.copy()[['wins']] @@ -111,7 +111,7 @@ def test_uni_dim_no_display_definition(self): self.assertEqual(result, expected.to_csv()) def test_multi_dims_time_series_and_uni(self): - result = CSV(items=[slicer.metrics.wins]) \ + result = CSV(slicer.metrics.wins) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ @@ -123,7 +123,7 @@ def test_multi_dims_time_series_and_uni(self): self.assertEqual(result, expected.to_csv()) def test_pivoted_single_dimension_no_effect(self): - result = CSV(items=[slicer.metrics.wins], pivot=True) \ + result = CSV(slicer.metrics.wins, pivot=True) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) expected = cat_dim_df.copy()[['wins']] @@ -133,7 +133,7 @@ def test_pivoted_single_dimension_no_effect(self): self.assertEqual(result, expected.to_csv()) def test_pivoted_multi_dims_time_series_and_cat(self): - result = CSV(items=[slicer.metrics.wins], pivot=True) \ + result = CSV(slicer.metrics.wins, pivot=True) \ .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party], []) expected = cont_cat_dim_df.copy()[['wins']] @@ -144,7 +144,7 @@ def test_pivoted_multi_dims_time_series_and_cat(self): self.assertEqual(result, expected.to_csv()) def test_pivoted_multi_dims_time_series_and_uni(self): - result = CSV(items=[slicer.metrics.votes], pivot=True) \ + result = CSV(slicer.metrics.votes, pivot=True) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ @@ -157,7 +157,7 @@ def test_pivoted_multi_dims_time_series_and_uni(self): self.assertEqual(result, expected.to_csv()) def test_time_series_ref(self): - result = CSV(items=[slicer.metrics.votes]) \ + result = CSV(slicer.metrics.votes) \ .transform(cont_uni_dim_ref_df, slicer, [ diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py index ebf53c9f..194da18c 100644 --- a/fireant/tests/slicer/widgets/test_datatables.py +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -30,7 +30,7 @@ class DataTablesTransformerTests(TestCase): maxDiff = None def test_single_metric(self): - result = DataTablesJS(items=[slicer.metrics.votes]) \ + result = DataTablesJS(slicer.metrics.votes) \ .transform(single_metric_df, slicer, [], []) self.assertEqual({ @@ -45,7 +45,7 @@ def test_single_metric(self): }, result) def test_multiple_metrics(self): - result = DataTablesJS(items=[slicer.metrics.votes, slicer.metrics.wins]) \ + result = DataTablesJS(slicer.metrics.votes, slicer.metrics.wins) \ .transform(multi_metric_df, slicer, [], []) self.assertEqual({ @@ -65,7 +65,7 @@ def test_multiple_metrics(self): }, result) def test_multiple_metrics_reversed(self): - result = DataTablesJS(items=[slicer.metrics.wins, slicer.metrics.votes]) \ + result = DataTablesJS(slicer.metrics.wins, slicer.metrics.votes) \ .transform(multi_metric_df, slicer, [], []) self.assertEqual({ @@ -85,7 +85,7 @@ def test_multiple_metrics_reversed(self): }, result) def test_time_series_dim(self): - result = DataTablesJS(items=[slicer.metrics.wins]) \ + result = DataTablesJS(slicer.metrics.wins) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ @@ -120,7 +120,7 @@ def test_time_series_dim(self): }, result) def test_time_series_dim_with_operation(self): - result = DataTablesJS(items=[CumSum(slicer.metrics.votes)]) \ + result = DataTablesJS(CumSum(slicer.metrics.votes)) \ .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ @@ -155,7 +155,7 @@ def test_time_series_dim_with_operation(self): }, result) def test_cat_dim(self): - result = DataTablesJS(items=[slicer.metrics.wins]) \ + result = DataTablesJS(slicer.metrics.wins) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) self.assertEqual({ @@ -181,7 +181,7 @@ def test_cat_dim(self): }, result) def test_uni_dim(self): - result = DataTablesJS(items=[slicer.metrics.wins]) \ + result = DataTablesJS(slicer.metrics.wins) \ .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) self.assertEqual({ @@ -239,7 +239,7 @@ def test_uni_dim_no_display_definition(self): uni_dim_df_copy = uni_dim_df.copy() del uni_dim_df_copy[slicer.dimensions.candidate.display_key] - result = DataTablesJS(items=[slicer.metrics.wins]) \ + result = DataTablesJS(slicer.metrics.wins) \ .transform(uni_dim_df_copy, slicer, [candidate], []) self.assertEqual({ @@ -289,7 +289,7 @@ def test_uni_dim_no_display_definition(self): }, result) def test_multi_dims_time_series_and_uni(self): - result = DataTablesJS(items=[slicer.metrics.wins]) \ + result = DataTablesJS(slicer.metrics.wins) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ @@ -358,7 +358,7 @@ def test_multi_dims_time_series_and_uni(self): }, result) def test_multi_dims_with_one_level_totals(self): - result = DataTablesJS(items=[slicer.metrics.wins]) \ + result = DataTablesJS(slicer.metrics.wins) \ .transform(cont_uni_dim_totals_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state.rollup()], []) @@ -452,7 +452,7 @@ def test_multi_dims_with_one_level_totals(self): }, result) def test_multi_dims_with_all_levels_totals(self): - result = DataTablesJS(items=[slicer.metrics.wins]) \ + result = DataTablesJS(slicer.metrics.wins) \ .transform(cont_uni_dim_all_totals_df, slicer, [slicer.dimensions.timestamp.rollup(), slicer.dimensions.state.rollup()], []) @@ -550,7 +550,7 @@ def test_multi_dims_with_all_levels_totals(self): }, result) def test_pivoted_single_dimension_no_effect(self): - result = DataTablesJS(items=[slicer.metrics.wins], pivot=True) \ + result = DataTablesJS(slicer.metrics.wins, pivot=True) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) self.assertEqual({ @@ -576,7 +576,7 @@ def test_pivoted_single_dimension_no_effect(self): }, result) def test_pivoted_multi_dims_time_series_and_cat(self): - result = DataTablesJS(items=[slicer.metrics.wins], pivot=True) \ + result = DataTablesJS(slicer.metrics.wins, pivot=True) \ .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party], []) self.assertEqual({ @@ -643,7 +643,7 @@ def test_pivoted_multi_dims_time_series_and_cat(self): }, result) def test_pivoted_multi_dims_time_series_and_uni(self): - result = DataTablesJS(items=[slicer.metrics.votes], pivot=True) \ + result = DataTablesJS(slicer.metrics.votes, pivot=True) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ @@ -700,7 +700,7 @@ def test_pivoted_multi_dims_time_series_and_uni(self): }, result) def test_time_series_ref(self): - result = DataTablesJS(items=[slicer.metrics.votes]) \ + result = DataTablesJS(slicer.metrics.votes) \ .transform(cont_uni_dim_ref_df, slicer, [ diff --git a/fireant/tests/slicer/widgets/test_highcharts.py b/fireant/tests/slicer/widgets/test_highcharts.py index dd1c5938..9b8e6642 100644 --- a/fireant/tests/slicer/widgets/test_highcharts.py +++ b/fireant/tests/slicer/widgets/test_highcharts.py @@ -30,8 +30,8 @@ class HighChartsLineChartTransformerTests(TestCase): stacking = None def test_single_metric_line_chart(self): - result = HighCharts(title="Time Series, Single Metric", - axes=[self.chart_class([slicer.metrics.votes])]) \ + result = HighCharts(title="Time Series, Single Metric") \ + .axis(self.chart_class(slicer.metrics.votes)) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ @@ -73,8 +73,8 @@ def test_single_metric_line_chart(self): def test_metric_prefix_line_chart(self): votes = copy.copy(slicer.metrics.votes) votes.prefix = '$' - result = HighCharts(title="Time Series, Single Metric", - axes=[self.chart_class([votes])]) \ + result = HighCharts(title="Time Series, Single Metric") \ + .axis(self.chart_class(votes)) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ @@ -116,8 +116,8 @@ def test_metric_prefix_line_chart(self): def test_metric_suffix_line_chart(self): votes = copy.copy(slicer.metrics.votes) votes.suffix = '%' - result = HighCharts(title="Time Series, Single Metric", - axes=[self.chart_class([votes])]) \ + result = HighCharts(title="Time Series, Single Metric") \ + .axis(self.chart_class(votes)) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ @@ -159,8 +159,8 @@ def test_metric_suffix_line_chart(self): def test_metric_precision_line_chart(self): votes = copy.copy(slicer.metrics.votes) votes.precision = 2 - result = HighCharts(title="Time Series, Single Metric", - axes=[self.chart_class([votes])]) \ + result = HighCharts(title="Time Series, Single Metric") \ + .axis(self.chart_class(votes)) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ @@ -200,8 +200,8 @@ def test_metric_precision_line_chart(self): }, result) def test_single_operation_line_chart(self): - result = HighCharts(title="Time Series, Single Metric", - axes=[self.chart_class([CumSum(slicer.metrics.votes)])]) \ + result = HighCharts(title="Time Series, Single Metric") \ + .axis(self.chart_class(CumSum(slicer.metrics.votes))) \ .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ @@ -241,8 +241,8 @@ def test_single_operation_line_chart(self): }, result) def test_single_metric_with_uni_dim_line_chart(self): - result = HighCharts(title="Time Series with Unique Dimension and Single Metric", - axes=[self.chart_class([slicer.metrics.votes])]) \ + result = HighCharts(title="Time Series with Unique Dimension and Single Metric") \ + .axis(self.chart_class(slicer.metrics.votes)) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) @@ -302,9 +302,9 @@ def test_single_metric_with_uni_dim_line_chart(self): }, result) def test_multi_metrics_single_axis_line_chart(self): - result = HighCharts(title="Time Series with Unique Dimension and Multiple Metrics", - axes=[self.chart_class([slicer.metrics.votes, - slicer.metrics.wins])]) \ + result = HighCharts(title="Time Series with Unique Dimension and Multiple Metrics") \ + .axis(self.chart_class(slicer.metrics.votes), + self.chart_class(slicer.metrics.wins)) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) @@ -402,9 +402,9 @@ def test_multi_metrics_single_axis_line_chart(self): }, result) def test_multi_metrics_multi_axis_line_chart(self): - result = HighCharts(title="Time Series with Unique Dimension and Multiple Metrics, Multi-Axis", - axes=[self.chart_class([slicer.metrics.votes]), - self.chart_class([slicer.metrics.wins]), ]) \ + result = HighCharts(title="Time Series with Unique Dimension and Multiple Metrics, Multi-Axis") \ + .axis(self.chart_class(slicer.metrics.votes)) \ + .axis(self.chart_class(slicer.metrics.wins)) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) @@ -507,9 +507,9 @@ def test_multi_metrics_multi_axis_line_chart(self): }, result) def test_multi_dim_with_totals_line_chart(self): - result = HighCharts(title="Time Series with Unique Dimension and Multiple Metrics, Multi-Axis", - axes=[self.chart_class([slicer.metrics.votes]), - self.chart_class([slicer.metrics.wins]), ]) \ + result = HighCharts(title="Time Series with Unique Dimension and Multiple Metrics, Multi-Axis") \ + .axis(self.chart_class(slicer.metrics.votes)) \ + .axis(self.chart_class(slicer.metrics.wins)) \ .transform(cont_uni_dim_totals_df, slicer, [slicer.dimensions.timestamp.rollup(), slicer.dimensions.state.rollup()], []) @@ -650,9 +650,9 @@ def test_multi_dim_with_totals_line_chart(self): }, result) def test_multi_dim_with_totals_on_first_dim_line_chart(self): - result = HighCharts(title="Time Series with Unique Dimension and Multiple Metrics, Multi-Axis", - axes=[self.chart_class([slicer.metrics.votes]), - self.chart_class([slicer.metrics.wins]), ]) \ + result = HighCharts(title="Time Series with Unique Dimension and Multiple Metrics, Multi-Axis") \ + .axis(self.chart_class(slicer.metrics.votes)) \ + .axis(self.chart_class(slicer.metrics.wins)) \ .transform(cont_uni_dim_all_totals_df, slicer, [slicer.dimensions.timestamp.rollup(), slicer.dimensions.state.rollup()], []) @@ -793,8 +793,8 @@ def test_multi_dim_with_totals_on_first_dim_line_chart(self): }, result) def test_uni_dim_with_ref_line_chart(self): - result = HighCharts(title="Time Series with Unique Dimension and Reference", - axes=[self.chart_class([slicer.metrics.votes])]) \ + result = HighCharts(title="Time Series with Unique Dimension and Reference") \ + .axis(self.chart_class(slicer.metrics.votes)) \ .transform(cont_uni_dim_ref_df, slicer, [ @@ -894,8 +894,8 @@ def test_uni_dim_with_ref_line_chart(self): }, result) def test_uni_dim_with_ref_delta_line_chart(self): - result = HighCharts(title="Time Series with Unique Dimension and Delta Reference", - axes=[self.chart_class([slicer.metrics.votes])]) \ + result = HighCharts(title="Time Series with Unique Dimension and Delta Reference") \ + .axis(self.chart_class(slicer.metrics.votes)) \ .transform(cont_uni_dim_ref_delta_df, slicer, [ @@ -1001,9 +1001,8 @@ def test_uni_dim_with_ref_delta_line_chart(self): }, result) def test_invisible_y_axis(self): - result = HighCharts(title="Time Series, Single Metric", - axes=[self.chart_class([slicer.metrics.votes], - y_axis_visible=False)]) \ + result = HighCharts(title="Time Series, Single Metric") \ + .axis(self.chart_class(slicer.metrics.votes), y_axis_visible=False) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) self.assertEqual({ @@ -1043,9 +1042,8 @@ def test_invisible_y_axis(self): }, result) def test_ref_axes_set_to_same_visibility_as_parent_axis(self): - result = HighCharts(title="Time Series with Unique Dimension and Delta Reference", - axes=[self.chart_class([slicer.metrics.votes], - y_axis_visible=False)]) \ + result = HighCharts(title="Time Series with Unique Dimension and Delta Reference") \ + .axis(self.chart_class(slicer.metrics.votes), y_axis_visible=False) \ .transform(cont_uni_dim_ref_delta_df, slicer, [ @@ -1159,8 +1157,8 @@ class HighChartsBarChartTransformerTests(TestCase): stacking = None def test_single_metric_bar_chart(self): - result = HighCharts(title="All Votes", - axes=[self.chart_class([slicer.metrics.votes])]) \ + result = HighCharts(title="All Votes") \ + .axis(self.chart_class(slicer.metrics.votes)) \ .transform(single_metric_df, slicer, [], []) self.assertEqual({ @@ -1196,9 +1194,9 @@ def test_single_metric_bar_chart(self): }, result) def test_multi_metric_bar_chart(self): - result = HighCharts(title="Votes and Wins", - axes=[self.chart_class([slicer.metrics.votes, - slicer.metrics.wins])]) \ + result = HighCharts(title="Votes and Wins") \ + .axis(self.chart_class(slicer.metrics.votes), + self.chart_class(slicer.metrics.wins)) \ .transform(multi_metric_df, slicer, [], []) self.assertEqual({ @@ -1248,8 +1246,8 @@ def test_multi_metric_bar_chart(self): }, result) def test_cat_dim_single_metric_bar_chart(self): - result = HighCharts(title="Votes and Wins", - axes=[self.chart_class([slicer.metrics.votes])]) \ + result = HighCharts("Votes and Wins") \ + .axis(self.chart_class(slicer.metrics.votes)) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) self.assertEqual({ @@ -1285,9 +1283,9 @@ def test_cat_dim_single_metric_bar_chart(self): }, result) def test_cat_dim_multi_metric_bar_chart(self): - result = HighCharts(title="Votes and Wins", - axes=[self.chart_class([slicer.metrics.votes, - slicer.metrics.wins])]) \ + result = HighCharts("Votes and Wins") \ + .axis(self.chart_class(slicer.metrics.votes), + self.chart_class(slicer.metrics.wins)) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) self.assertEqual({ @@ -1337,8 +1335,8 @@ def test_cat_dim_multi_metric_bar_chart(self): }, result) def test_cont_uni_dims_single_metric_bar_chart(self): - result = HighCharts(title="Election Votes by State", - axes=[self.chart_class([slicer.metrics.votes])]) \ + result = HighCharts("Election Votes by State") \ + .axis(self.chart_class(slicer.metrics.votes)) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ @@ -1398,9 +1396,9 @@ def test_cont_uni_dims_single_metric_bar_chart(self): }, result) def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): - result = HighCharts(title="Election Votes by State", - axes=[self.chart_class([slicer.metrics.votes, - slicer.metrics.wins])]) \ + result = HighCharts(title="Election Votes by State") \ + .axis(self.chart_class(slicer.metrics.votes), + self.chart_class(slicer.metrics.wins)) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ @@ -1497,9 +1495,9 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): }, result) def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): - result = HighCharts(title="Election Votes by State", - axes=[self.chart_class([slicer.metrics.votes]), - self.chart_class([slicer.metrics.wins])]) \ + result = HighCharts(title="Election Votes by State") \ + .axis(self.chart_class(slicer.metrics.votes)) \ + .axis(self.chart_class(slicer.metrics.wins)) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ @@ -1601,9 +1599,9 @@ def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): }, result) def test_invisible_y_axis(self): - result = HighCharts(title="All Votes", - axes=[self.chart_class([slicer.metrics.votes], - y_axis_visible=False)]) \ + result = HighCharts(title="All Votes") \ + .axis(self.chart_class(slicer.metrics.votes), + y_axis_visible=False) \ .transform(single_metric_df, slicer, [], []) self.assertEqual({ @@ -1679,11 +1677,14 @@ class HighChartsPieChartTransformerTests(TestCase): chart_class = HighCharts.PieChart chart_type = 'pie' - def test_single_metric_pie_chart(self): - result = HighCharts(title="All Votes", - axes=[self.chart_class([slicer.metrics.votes])]) \ + def test_single_metric_chart(self): + result = HighCharts(title="All Votes") \ + .axis(self.chart_class(slicer.metrics.votes)) \ .transform(single_metric_df, slicer, [], []) + import json + print(json.dumps(result)) + self.assertEqual({ "title": {"text": "All Votes"}, "tooltip": {"shared": True, "useHTML": True, "enabled": True}, @@ -1694,8 +1695,9 @@ def test_single_metric_pie_chart(self): "data": [{ "name": "Votes", "y": 111674336, - "color": "#DDDF0D", }], + 'colors': ['#DDDF0D', '#55BF3B', '#DF5353', '#7798BF', '#AAEEEE', '#FF0066', '#EEAAEE', '#DF5353', + '#7798BF', '#AAEEEE'], 'tooltip': { 'valueDecimals': None, 'valuePrefix': None, @@ -1707,15 +1709,23 @@ def test_single_metric_pie_chart(self): 'categories': ['All'], 'visible': True, }, - 'yAxis': [], + 'yAxis': [{ + 'id': '0', + 'labels': {'style': {'color': None}}, + 'title': {'text': None}, + 'visible': True + }], }, result) - def test_multi_metric_bar_chart(self): - result = HighCharts(title="Votes and Wins", - axes=[self.chart_class([slicer.metrics.votes, - slicer.metrics.wins])]) \ + def test_multi_metric_chart(self): + result = HighCharts(title="Votes and Wins") \ + .axis(self.chart_class(slicer.metrics.votes), + self.chart_class(slicer.metrics.wins)) \ .transform(multi_metric_df, slicer, [], []) + import json + print(json.dumps(result)) + self.assertEqual({ "title": {"text": "Votes and Wins"}, "tooltip": {"shared": True, "useHTML": True, "enabled": True}, @@ -1726,8 +1736,9 @@ def test_multi_metric_bar_chart(self): "data": [{ "name": "Votes", "y": 111674336, - "color": "#DDDF0D", }], + 'colors': ['#DDDF0D', '#55BF3B', '#DF5353', '#7798BF', '#AAEEEE', '#FF0066', '#EEAAEE', '#DF5353', + '#7798BF', '#AAEEEE'], 'tooltip': { 'valueDecimals': None, 'valuePrefix': None, @@ -1739,8 +1750,9 @@ def test_multi_metric_bar_chart(self): "data": [{ "name": "Wins", "y": 12, - "color": "#DDDF0D", }], + 'colors': ['#DDDF0D', '#55BF3B', '#DF5353', '#7798BF', '#AAEEEE', '#FF0066', '#EEAAEE', '#DF5353', + '#7798BF', '#AAEEEE'], 'tooltip': { 'valueDecimals': None, 'valuePrefix': None, @@ -1752,12 +1764,17 @@ def test_multi_metric_bar_chart(self): 'categories': ['All'], 'visible': True, }, - 'yAxis': [], + 'yAxis': [{ + 'id': '0', + 'labels': {'style': {'color': '#DDDF0D'}}, + 'title': {'text': None}, + 'visible': True + }], }, result) - def test_cat_dim_single_metric_bar_chart(self): - result = HighCharts(title="Votes and Wins", - axes=[self.chart_class([slicer.metrics.votes])]) \ + def test_cat_dim_single_metric_chart(self): + result = HighCharts("Votes and Wins") \ + .axis(self.chart_class(slicer.metrics.votes)) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) self.assertEqual({ @@ -1770,23 +1787,27 @@ def test_cat_dim_single_metric_bar_chart(self): 'data': [{ 'y': 54551568, 'name': 'Democrat', - 'color': '#DDDF0D' }, { 'y': 1076384, 'name': 'Independent', - 'color': '#55BF3B' }, { 'y': 56046384, 'name': 'Republican', - 'color': '#DF5353' }], + 'colors': ['#DDDF0D', '#55BF3B', '#DF5353', '#7798BF', '#AAEEEE', '#FF0066', '#EEAAEE', '#DF5353', + '#7798BF', '#AAEEEE'], 'tooltip': { 'valuePrefix': None, 'valueSuffix': None, 'valueDecimals': None, }, }], - 'yAxis': [], + 'yAxis': [{ + 'id': '0', + 'labels': {'style': {'color': None}}, + 'title': {'text': None}, + 'visible': True + }], 'xAxis': { 'type': 'category', 'categories': ['Democrat', 'Independent', 'Republican'], @@ -1796,9 +1817,9 @@ def test_cat_dim_single_metric_bar_chart(self): @skip def test_cat_dim_multi_metric_bar_chart(self): - result = HighCharts(title="Votes and Wins", - axes=[self.chart_class([slicer.metrics.votes, - slicer.metrics.wins])]) \ + result = HighCharts(title="Votes and Wins") \ + .axis(self.chart_class(slicer.metrics.votes), + self.chart_class(slicer.metrics.wins)) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) self.assertEqual({ @@ -1848,8 +1869,8 @@ def test_cat_dim_multi_metric_bar_chart(self): @skip def test_cont_uni_dims_single_metric_bar_chart(self): - result = HighCharts(title="Election Votes by State", - axes=[self.chart_class([slicer.metrics.votes])]) \ + result = HighCharts(title="Election Votes by State") \ + .axis(self.chart_class(slicer.metrics.votes)) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ @@ -1909,9 +1930,9 @@ def test_cont_uni_dims_single_metric_bar_chart(self): @skip def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): - result = HighCharts(title="Election Votes by State", - axes=[self.chart_class([slicer.metrics.votes, - slicer.metrics.wins]), ]) \ + result = HighCharts(title="Election Votes by State") \ + .axis(self.chart_class(slicer.metrics.votes), + self.chart_class(slicer.metrics.wins)) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ @@ -2009,9 +2030,9 @@ def test_cont_uni_dims_multi_metric_single_axis_bar_chart(self): @skip def test_cont_uni_dims_multi_metric_multi_axis_bar_chart(self): - result = HighCharts(title="Election Votes by State", - axes=[self.chart_class([slicer.metrics.votes]), - self.chart_class([slicer.metrics.wins]), ]) \ + result = HighCharts(title="Election Votes by State") \ + .axis(self.chart_class(slicer.metrics.votes), + self.chart_class(slicer.metrics.wins)) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) self.assertEqual({ diff --git a/fireant/tests/slicer/widgets/test_pandas.py b/fireant/tests/slicer/widgets/test_pandas.py index 37c3ac79..3bc9bb28 100644 --- a/fireant/tests/slicer/widgets/test_pandas.py +++ b/fireant/tests/slicer/widgets/test_pandas.py @@ -24,7 +24,7 @@ class DataTablesTransformerTests(TestCase): maxDiff = None def test_single_metric(self): - result = Pandas(items=[slicer.metrics.votes]) \ + result = Pandas(slicer.metrics.votes) \ .transform(single_metric_df, slicer, [], []) expected = single_metric_df.copy()[['votes']] @@ -33,7 +33,7 @@ def test_single_metric(self): pandas.testing.assert_frame_equal(result, expected) def test_multiple_metrics(self): - result = Pandas(items=[slicer.metrics.votes, slicer.metrics.wins]) \ + result = Pandas(slicer.metrics.votes, slicer.metrics.wins) \ .transform(multi_metric_df, slicer, [], []) expected = multi_metric_df.copy()[['votes', 'wins']] @@ -42,7 +42,7 @@ def test_multiple_metrics(self): pandas.testing.assert_frame_equal(result, expected) def test_multiple_metrics_reversed(self): - result = Pandas(items=[slicer.metrics.wins, slicer.metrics.votes]) \ + result = Pandas(slicer.metrics.wins, slicer.metrics.votes) \ .transform(multi_metric_df, slicer, [], []) expected = multi_metric_df.copy()[['wins', 'votes']] @@ -51,7 +51,7 @@ def test_multiple_metrics_reversed(self): pandas.testing.assert_frame_equal(result, expected) def test_time_series_dim(self): - result = Pandas(items=[slicer.metrics.wins]) \ + result = Pandas(slicer.metrics.wins) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) expected = cont_dim_df.copy()[['wins']] @@ -61,7 +61,7 @@ def test_time_series_dim(self): pandas.testing.assert_frame_equal(result, expected) def test_time_series_dim_with_operation(self): - result = Pandas(items=[CumSum(slicer.metrics.votes)]) \ + result = Pandas(CumSum(slicer.metrics.votes)) \ .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp], []) expected = cont_dim_operation_df.copy()[['cumsum(votes)']] @@ -71,7 +71,7 @@ def test_time_series_dim_with_operation(self): pandas.testing.assert_frame_equal(result, expected) def test_cat_dim(self): - result = Pandas(items=[slicer.metrics.wins]) \ + result = Pandas(slicer.metrics.wins) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) expected = cat_dim_df.copy()[['wins']] @@ -81,7 +81,7 @@ def test_cat_dim(self): pandas.testing.assert_frame_equal(result, expected) def test_uni_dim(self): - result = Pandas(items=[slicer.metrics.wins]) \ + result = Pandas(slicer.metrics.wins) \ .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) expected = uni_dim_df.copy() \ @@ -102,7 +102,7 @@ def test_uni_dim_no_display_definition(self): uni_dim_df_copy = uni_dim_df.copy() del uni_dim_df_copy[slicer.dimensions.candidate.display_key] - result = Pandas(items=[slicer.metrics.wins]) \ + result = Pandas(slicer.metrics.wins) \ .transform(uni_dim_df_copy, slicer, [candidate], []) expected = uni_dim_df_copy.copy()[['wins']] @@ -112,7 +112,7 @@ def test_uni_dim_no_display_definition(self): pandas.testing.assert_frame_equal(result, expected) def test_multi_dims_time_series_and_uni(self): - result = Pandas(items=[slicer.metrics.wins]) \ + result = Pandas(slicer.metrics.wins) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ @@ -124,7 +124,7 @@ def test_multi_dims_time_series_and_uni(self): pandas.testing.assert_frame_equal(result, expected) def test_pivoted_single_dimension_no_effect(self): - result = Pandas(items=[slicer.metrics.wins], pivot=True) \ + result = Pandas(slicer.metrics.wins, pivot=True) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) expected = cat_dim_df.copy()[['wins']] @@ -134,7 +134,7 @@ def test_pivoted_single_dimension_no_effect(self): pandas.testing.assert_frame_equal(result, expected) def test_pivoted_multi_dims_time_series_and_cat(self): - result = Pandas(items=[slicer.metrics.wins], pivot=True) \ + result = Pandas(slicer.metrics.wins, pivot=True) \ .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party], []) expected = cont_cat_dim_df.copy()[['wins']] @@ -145,7 +145,7 @@ def test_pivoted_multi_dims_time_series_and_cat(self): pandas.testing.assert_frame_equal(result, expected) def test_pivoted_multi_dims_time_series_and_uni(self): - result = Pandas(items=[slicer.metrics.votes], pivot=True) \ + result = Pandas(slicer.metrics.votes, pivot=True) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ @@ -158,7 +158,7 @@ def test_pivoted_multi_dims_time_series_and_uni(self): pandas.testing.assert_frame_equal(result, expected) def test_time_series_ref(self): - result = Pandas(items=[slicer.metrics.votes]) \ + result = Pandas(slicer.metrics.votes) \ .transform(cont_uni_dim_ref_df, slicer, [ slicer.dimensions.timestamp, @@ -183,7 +183,7 @@ def test_metric_format(self): votes.precision = 2 # divide the data frame by 3 to get a repeating decimal so we can check precision - result = Pandas(items=[votes]) \ + result = Pandas(votes) \ .transform(cont_dim_df / 3, slicer, [slicer.dimensions.timestamp], []) expected = cont_dim_df.copy()[['votes']] diff --git a/fireant/tests/slicer/widgets/test_widgets.py b/fireant/tests/slicer/widgets/test_widgets.py index 40d0fb7b..e0c774c4 100644 --- a/fireant/tests/slicer/widgets/test_widgets.py +++ b/fireant/tests/slicer/widgets/test_widgets.py @@ -8,15 +8,15 @@ class BaseWidgetTests(TestCase): def test_create_widget_with_items(self): - widget = Widget(items=[0, 1, 2]) + widget = Widget(0, 1, 2) self.assertListEqual(widget.items, [0, 1, 2]) def test_add_widget_to_items(self): - widget = Widget(items=[0, 1, 2]).item(3) + widget = Widget(0, 1, 2).item(3) self.assertListEqual(widget.items, [0, 1, 2, 3]) def test_item_func_immuatable(self): - widget1 = Widget(items=[0, 1, 2]) + widget1 = Widget(0, 1, 2) widget2 = widget1.item(3) self.assertIsNot(widget1, widget2) diff --git a/requirements.txt b/requirements.txt index 3715871a..923edbe2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ six pandas==0.22.0 -pypika==0.11.3 +pypika==0.14.0 pymysql==0.8.0 toposort==1.5 typing==3.6.2 diff --git a/setup.py b/setup.py index be027acb..c1454048 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ def find_version(*file_paths): install_requires=[ 'six', 'pandas==0.22.0', - 'pypika==0.11.3', + 'pypika==0.14.0', 'toposort==1.5', 'typing==3.6.2', ], From da79bbd5b5972e170b89802ada3fe70fc7304462 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 10 Apr 2018 16:09:40 +0200 Subject: [PATCH 057/123] bumped to dev version dev23 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index e35827fe..f85d93ef 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev22' +__version__ = '1.0.0.dev23' From 4e3e0939683e9c32543f5f28e4035579a04518c4 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 31 May 2018 13:24:06 +0200 Subject: [PATCH 058/123] Fixed 'operations' in Highcharts to ignore items without operations --- fireant/slicer/widgets/highcharts.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 07ceb0d7..e500f00a 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -1,6 +1,4 @@ import itertools - -import pandas as pd from datetime import ( datetime, ) @@ -9,6 +7,8 @@ Union, ) +import pandas as pd + from fireant import ( DatetimeDimension, Metric, @@ -168,6 +168,7 @@ def metrics(self): def operations(self): return utils.ordered_distinct_list_by_attr([operation for item in self.items + if hasattr(item, 'operations') for operation in item.operations]) def transform(self, data_frame, slicer, dimensions, references): From e1a4064f2e5f3156d0649ccc160d9452fa02802d Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 31 May 2018 13:24:49 +0200 Subject: [PATCH 059/123] bumped version to dev24 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index f85d93ef..0f9fd2bd 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev23' +__version__ = '1.0.0.dev24' From 1652894009577fac651fb790b26325b8dd92f7f7 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 31 May 2018 18:02:49 +0200 Subject: [PATCH 060/123] Added like and not_like functions to display dimension --- fireant/slicer/dimensions.py | 26 ++++++++++++++++++++ fireant/tests/slicer/queries/test_builder.py | 11 +++++++++ 2 files changed, 37 insertions(+) diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 3da84d13..1da3931a 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -204,6 +204,32 @@ def __init__(self, dimension): dimension.label, dimension.display_definition) + def like(self, pattern): + """ + Creates a filter to filter a slicer query. + + :param pattern: + A pattern to match against the dimension's display definition. This pattern is used in the SQL query as + the `LIKE` expression. + :return: + A slicer query filter used to filter a slicer query to results where this dimension's display definition + matches the pattern. + """ + return LikeFilter(self.definition, pattern) + + def not_like(self, pattern): + """ + Creates a filter to filter a slicer query. + + :param pattern: + A pattern to match against the dimension's display definition. This pattern is used in the SQL query as + the `NOT LIKE` expression. + :return: + A slicer query filter used to filter a slicer query to results where this dimension's display definition + matches the pattern. + """ + return NotLikeFilter(self.definition, pattern) + class ContinuousDimension(Dimension): """ diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index e4ffbb3a..62423bb1 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -595,6 +595,17 @@ def test_build_query_with_filter_like_unique_dim(self): 'FROM "politics"."politician" ' 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) + def test_build_query_with_filter_like_display_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate_display.like('%Trump')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) + def test_build_query_with_filter_not_like_unique_dim(self): query = slicer.data \ .widget(f.DataTablesJS(slicer.metrics.votes)) \ From 6205d6bf4eb8d1209d39840c909f2586613bd9f3 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 31 May 2018 18:03:05 +0200 Subject: [PATCH 061/123] bumped version to dev25 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 0f9fd2bd..31224e36 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev24' +__version__ = '1.0.0.dev25' From ff9f16dbef26696c4e49d38a4ce4709fc1417311 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 1 Jun 2018 13:30:03 +0200 Subject: [PATCH 062/123] Fixed operations and metrics functions in highcharts transformer --- fireant/slicer/operations.py | 11 +++++++++++ fireant/slicer/widgets/highcharts.py | 12 +++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/fireant/slicer/operations.py b/fireant/slicer/operations.py index 419009a0..9ad352ac 100644 --- a/fireant/slicer/operations.py +++ b/fireant/slicer/operations.py @@ -18,6 +18,10 @@ class Operation(object): def apply(self, data_frame): raise NotImplementedError() + @property + def operations(self): + return [] + class _Cumulative(Operation): def __init__(self, arg): @@ -46,6 +50,13 @@ def metrics(self): for metric in [self.arg] if isinstance(metric, Metric)] + @property + def operations(self): + return [op_and_children + for operation in [self.arg] + if isinstance(operation, Operation) + for op_and_children in [operation] + operation.operations] + def apply(self, data_frame): raise NotImplementedError() diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index e500f00a..702154e3 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -159,17 +159,19 @@ def metrics(self): raise MetricRequiredException(str(self)) seen = set() - return [series.metric + return [metric for axis in self.items for series in axis - if not (series.metric.key in seen or seen.add(series.metric.key))] + for metric in getattr(series.metric, 'metrics', [series.metric]) + if not (metric.key in seen or seen.add(metric.key))] @property def operations(self): return utils.ordered_distinct_list_by_attr([operation - for item in self.items - if hasattr(item, 'operations') - for operation in item.operations]) + for axis in self.items + for series in axis + if isinstance(series.metric, Operation) + for operation in [series.metric] + series.metric.operations]) def transform(self, data_frame, slicer, dimensions, references): """ From 5db523eeb763140df69ffdf988b0267daa6922b9 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 1 Jun 2018 13:30:19 +0200 Subject: [PATCH 063/123] bumped version to dev26 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 31224e36..e4dcb07b 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev25' +__version__ = '1.0.0.dev26' From 2c3b915946ef8576bc8ac0d2e09766813cb043d2 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 26 Jun 2018 17:15:25 +0200 Subject: [PATCH 064/123] Added a pattern dimension type --- fireant/slicer/__init__.py | 2 + fireant/slicer/dimensions.py | 38 ++++++++++ fireant/slicer/operations.py | 70 +++++++++++++++---- fireant/tests/slicer/mocks.py | 9 ++- fireant/tests/slicer/queries/test_builder.py | 29 ++++++++ fireant/tests/slicer/test_operations.py | 32 +++++++++ fireant/tests/slicer/test_slicer.py | 2 +- .../tests/slicer/widgets/test_highcharts.py | 5 -- 8 files changed, 165 insertions(+), 22 deletions(-) diff --git a/fireant/slicer/__init__.py b/fireant/slicer/__init__.py index 8b95da4e..e1d2ec0c 100644 --- a/fireant/slicer/__init__.py +++ b/fireant/slicer/__init__.py @@ -5,6 +5,7 @@ DatetimeDimension, Dimension, DisplayDimension, + PatternDimension, UniqueDimension, ) from .exceptions import ( @@ -28,6 +29,7 @@ CumProd, CumSum, Operation, + RollingMean, ) from .references import ( DayOverDay, diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 1da3931a..6028aaf5 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -1,5 +1,7 @@ from pypika.terms import ( NullValue, + Case, + ValueWrapper, ) from typing import Iterable @@ -291,6 +293,42 @@ def between(self, start, stop): return RangeFilter(self.definition, start, stop) +class PatternDimension(Dimension): + """ + This is a dimension that represents a boolean true/false value. The expression should always result in a boolean + value. + """ + + def __init__(self, key, label=None, definition=None): + super(PatternDimension, self).__init__(key, + label, + ValueWrapper('No Group')) + self.field = definition + + @immutable + def __call__(self, groups): + """ + When calling a datetime dimension an interval can be supplied: + + ``` + from fireant import weekly + + my_slicer.dimensions.date # Daily interval used as default + my_slicer.dimensions.date(weekly) # Daily interval used as default + ``` + + :param interval: + An interval to use with the dimension. See `fireant.intervals`. + :return: + A copy of the dimension with the interval set. + """ + cases = Case() + for group in groups: + cases = cases.when(self.field.like(group), group) + + self.definition = cases + + class TotalsDimension(Dimension): def __init__(self, dimension): totals_definition = NullValue() diff --git a/fireant/slicer/operations.py b/fireant/slicer/operations.py index 9ad352ac..9f0a1fdb 100644 --- a/fireant/slicer/operations.py +++ b/fireant/slicer/operations.py @@ -23,16 +23,13 @@ def operations(self): return [] -class _Cumulative(Operation): - def __init__(self, arg): - self.arg = arg - self.key = '{}({})'.format(self.__class__.__name__.lower(), - getattr(arg, 'key', arg)) - self.label = '{}({})'.format(self.__class__.__name__, - getattr(arg, 'label', arg)) - self.prefix = getattr(arg, 'prefix') - self.suffix = getattr(arg, 'suffix') - self.precision = getattr(arg, 'precision') +class _BaseOperation(Operation): + def __init__(self, key, label, prefix=None, suffix=None, precision=None): + self.key = key + self.label = label + self.prefix = prefix + self.suffix = suffix + self.precision = precision def _group_levels(self, index): """ @@ -44,6 +41,21 @@ def _group_levels(self, index): """ return index.names[1:] + +class _Cumulative(_BaseOperation): + def __init__(self, arg): + super(_Cumulative, self).__init__( + key='{}({})'.format(self.__class__.__name__.lower(), + getattr(arg, 'key', arg)), + label='{}({})'.format(self.__class__.__name__, + getattr(arg, 'label', arg)), + prefix=getattr(arg, 'prefix'), + suffix=getattr(arg, 'suffix'), + precision=getattr(arg, 'precision'), + ) + + self.arg = arg + @property def metrics(self): return [metric @@ -57,9 +69,6 @@ def operations(self): if isinstance(operation, Operation) for op_and_children in [operation] + operation.operations] - def apply(self, data_frame): - raise NotImplementedError() - def __repr__(self): return self.key @@ -102,3 +111,38 @@ def apply(self, data_frame): .apply(self.cummean) return self.cummean(data_frame[self.arg.key]) + + +class _Rolling(_BaseOperation): + def apply(self, data_frame): + raise NotImplementedError() + + def __init__(self, arg, window, min_periods=None): + super(_Rolling, self).__init__( + key='{}({})'.format(self.__class__.__name__.lower(), + getattr(arg, 'key', arg)), + label='{}({})'.format(self.__class__.__name__, + getattr(arg, 'label', arg)), + prefix=getattr(arg, 'prefix'), + suffix=getattr(arg, 'suffix'), + precision=getattr(arg, 'precision'), + ) + + self.arg = arg + self.window = window + self.min_periods = min_periods + + +class RollingMean(_Rolling): + def rolling_mean(self, x): + return x.rolling(self.window, self.min_periods).mean() + + def apply(self, data_frame): + if isinstance(data_frame.index, pd.MultiIndex): + levels = self._group_levels(data_frame.index) + + return data_frame[self.arg.key] \ + .groupby(level=levels) \ + .apply(self.rolling_mean) + + return self.rolling_mean(data_frame[self.arg.key]) diff --git a/fireant/tests/slicer/mocks.py b/fireant/tests/slicer/mocks.py index 3160d0f8..41629246 100644 --- a/fireant/tests/slicer/mocks.py +++ b/fireant/tests/slicer/mocks.py @@ -8,15 +8,15 @@ from datetime import ( datetime, ) - -from fireant import * -from fireant.slicer.references import ReferenceType from pypika import ( JoinType, Table, functions as fn, ) +from fireant import * +from fireant.slicer.references import ReferenceType + class TestDatabase(VerticaDatabase): # Vertica client that uses the vertica_python driver. @@ -88,6 +88,9 @@ def __eq__(self, other): definition=politicians_table.is_winner), UniqueDimension('deepjoin', definition=deep_join_table.id), + PatternDimension('pattern', + label='Pattern', + definition=politicians_table.pattern), ), metrics=( diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 62423bb1..fb40eb26 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -236,6 +236,35 @@ def test_build_query_with_unique_dimension(self): 'GROUP BY "election","election_display" ' 'ORDER BY "election_display"', str(query)) + def test_build_query_with_pattern_dimension(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.pattern(['groupA%', 'groupB%'])) \ + .query + + self.assertEqual('SELECT ' + 'CASE ' + 'WHEN "pattern" LIKE \'groupA%\' THEN \'groupA%\' ' + 'WHEN "pattern" LIKE \'groupB%\' THEN \'groupB%\' ' + 'END "pattern",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "pattern" ' + 'ORDER BY "pattern"', str(query)) + + def test_build_query_with_pattern_no_values(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.pattern) \ + .query + + self.assertEqual('SELECT ' + '\'No Group\' "pattern",' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "pattern" ' + 'ORDER BY "pattern"', str(query)) + def test_build_query_with_multiple_dimensions(self): query = slicer.data \ .widget(f.DataTablesJS(slicer.metrics.votes)) \ diff --git a/fireant/tests/slicer/test_operations.py b/fireant/tests/slicer/test_operations.py index 772e8c4b..9ba4164e 100644 --- a/fireant/tests/slicer/test_operations.py +++ b/fireant/tests/slicer/test_operations.py @@ -1,11 +1,14 @@ from unittest import TestCase +import numpy as np import pandas as pd import pandas.testing + from fireant import ( CumMean, CumProd, CumSum, + RollingMean, ) from fireant.tests.slicer.mocks import ( cont_dim_df, @@ -102,3 +105,32 @@ def test_apply_to_timeseries_with_uni_dim_and_ref(self): name='votes', index=cont_uni_dim_ref_df.index) pandas.testing.assert_series_equal(result, expected) + + +class RollingMeanTests(TestCase): + def test_apply_to_timeseries(self): + rolling_mean = RollingMean(slicer.metrics.wins, 3) + result = rolling_mean.apply(cont_dim_df) + + expected = pd.Series([np.nan, np.nan, 2.0, 2.0, 2.0, 2.0], + name='wins', + index=cont_dim_df.index) + pandas.testing.assert_series_equal(result, expected) + + def test_apply_to_timeseries_with_uni_dim(self): + rolling_mean = RollingMean(slicer.metrics.wins, 3) + result = rolling_mean.apply(cont_uni_dim_df) + + expected = pd.Series([np.nan, np.nan, np.nan, np.nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + name='wins', + index=cont_uni_dim_df.index) + pandas.testing.assert_series_equal(result, expected) + + def test_apply_to_timeseries_with_uni_dim_and_ref(self): + rolling_mean = RollingMean(slicer.metrics.wins, 3) + result = rolling_mean.apply(cont_uni_dim_ref_df) + + expected = pd.Series([np.nan, np.nan, np.nan, np.nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + name='wins', + index=cont_uni_dim_ref_df.index) + pandas.testing.assert_series_equal(result, expected) diff --git a/fireant/tests/slicer/test_slicer.py b/fireant/tests/slicer/test_slicer.py index e93adc04..0ceb160f 100644 --- a/fireant/tests/slicer/test_slicer.py +++ b/fireant/tests/slicer/test_slicer.py @@ -46,4 +46,4 @@ def test_iter_metrics(self): def test_iter_dimensions(self): dimension_keys = [dimension.key for dimension in slicer.dimensions] self.assertListEqual(dimension_keys, ['timestamp', 'political_party', 'candidate', 'election', 'district', - 'state', 'winner', 'deepjoin']) + 'state', 'winner', 'deepjoin', 'pattern']) diff --git a/fireant/tests/slicer/widgets/test_highcharts.py b/fireant/tests/slicer/widgets/test_highcharts.py index 9b8e6642..9ac40578 100644 --- a/fireant/tests/slicer/widgets/test_highcharts.py +++ b/fireant/tests/slicer/widgets/test_highcharts.py @@ -1682,8 +1682,6 @@ def test_single_metric_chart(self): .axis(self.chart_class(slicer.metrics.votes)) \ .transform(single_metric_df, slicer, [], []) - import json - print(json.dumps(result)) self.assertEqual({ "title": {"text": "All Votes"}, @@ -1723,9 +1721,6 @@ def test_multi_metric_chart(self): self.chart_class(slicer.metrics.wins)) \ .transform(multi_metric_df, slicer, [], []) - import json - print(json.dumps(result)) - self.assertEqual({ "title": {"text": "Votes and Wins"}, "tooltip": {"shared": True, "useHTML": True, "enabled": True}, From b787343728bf9b017827d99a9d4af0267465a00c Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 26 Jun 2018 17:40:05 +0200 Subject: [PATCH 065/123] Bumped version to dev27 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index e4dcb07b..887d79c5 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev26' +__version__ = '1.0.0.dev27' From 00b93f047388d1a7e144da9bceedce7e66b743b7 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 27 Jun 2018 18:26:08 +0200 Subject: [PATCH 066/123] added like and not_like filters to categorical dimension --- fireant/slicer/dimensions.py | 28 +++++++++++++++++++- fireant/tests/slicer/queries/test_builder.py | 22 +++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 6028aaf5..99573e5f 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -1,6 +1,6 @@ from pypika.terms import ( - NullValue, Case, + NullValue, ValueWrapper, ) from typing import Iterable @@ -105,6 +105,32 @@ def notin(self, values): """ return ExcludesFilter(self.definition, values) + def like(self, pattern): + """ + Creates a filter to filter a slicer query. + + :param pattern: + A pattern to match against the dimension's display definition. This pattern is used in the SQL query as + the `LIKE` expression. + :return: + A slicer query filter used to filter a slicer query to results where this dimension's display definition + matches the pattern. + """ + return LikeFilter(self.definition, pattern) + + def not_like(self, pattern): + """ + Creates a filter to filter a slicer query. + + :param pattern: + A pattern to match against the dimension's display definition. This pattern is used in the SQL query as + the `NOT LIKE` expression. + :return: + A slicer query filter used to filter a slicer query to results where this dimension's display definition + matches the pattern. + """ + return NotLikeFilter(self.definition, pattern) + class _UniqueDimensionBase(Dimension): def isin(self, values, use_display=False): diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index fb40eb26..113977d5 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -569,6 +569,28 @@ def test_build_query_with_filter_notin_categorical_dim(self): 'FROM "politics"."politician" ' 'WHERE "political_party" NOT IN (\'d\')', str(query)) + def test_build_query_with_filter_like_categorical_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.political_party.like('Rep%')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "political_party" LIKE \'Rep%\'', str(query)) + + def test_build_query_with_filter_not_like_categorical_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.political_party.not_like('Rep%')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "political_party" NOT LIKE \'Rep%\'', str(query)) + def test_build_query_with_filter_isin_unique_dim(self): query = slicer.data \ .widget(f.DataTablesJS(slicer.metrics.votes)) \ From 2070a9caf23a1151ba49f47ad1ec57a24f281c6e Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 27 Jun 2018 18:40:24 +0200 Subject: [PATCH 067/123] bumped version to dev28 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 887d79c5..2d61de53 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev27' +__version__ = '1.0.0.dev28' From 1fa41c61084add28e4411524def80051a70f18fa Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 28 Jun 2018 22:18:32 +0200 Subject: [PATCH 068/123] Fixed the pattern dimension --- fireant/slicer/dimensions.py | 6 ++++-- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 99573e5f..b44ee9a8 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -325,10 +325,12 @@ class PatternDimension(Dimension): value. """ + _DEFAULT = ValueWrapper('No Group') + def __init__(self, key, label=None, definition=None): super(PatternDimension, self).__init__(key, label, - ValueWrapper('No Group')) + self._DEFAULT) self.field = definition @immutable @@ -352,7 +354,7 @@ def __call__(self, groups): for group in groups: cases = cases.when(self.field.like(group), group) - self.definition = cases + self.definition = cases.else_(self._DEFAULT) class TotalsDimension(Dimension): diff --git a/requirements.txt b/requirements.txt index 923edbe2..4a9eb3d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ six pandas==0.22.0 -pypika==0.14.0 +pypika==0.14.5 pymysql==0.8.0 toposort==1.5 typing==3.6.2 diff --git a/setup.py b/setup.py index c1454048..a6470b5b 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ def find_version(*file_paths): install_requires=[ 'six', 'pandas==0.22.0', - 'pypika==0.14.0', + 'pypika==0.14.5', 'toposort==1.5', 'typing==3.6.2', ], From 15a2846a78fd361f09a16d44480cbf70a4c27cb1 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 28 Jun 2018 22:29:25 +0200 Subject: [PATCH 069/123] fixing codacy issues --- fireant/database/base.py | 3 --- fireant/database/mysql.py | 3 --- fireant/database/postgresql.py | 3 --- fireant/database/redshift.py | 1 + fireant/database/vertica.py | 3 --- fireant/slicer/dimensions.py | 8 ++++---- fireant/slicer/joins.py | 2 +- fireant/slicer/metrics.py | 3 ++- fireant/slicer/operations.py | 9 ++++++--- fireant/slicer/queries/finders.py | 1 - fireant/slicer/queries/references.py | 18 +++++++++--------- fireant/tests/slicer/queries/test_builder.py | 5 +++-- 12 files changed, 26 insertions(+), 33 deletions(-) diff --git a/fireant/database/base.py b/fireant/database/base.py index 10b07750..ae6e9aee 100644 --- a/fireant/database/base.py +++ b/fireant/database/base.py @@ -38,6 +38,3 @@ def to_char(self, definition): def fetch_data(self, query): with self.connect() as connection: return pd.read_sql(query, connection, coerce_float=True, parse_dates=True) - - def totals(self, query, terms): - raise NotImplementedError diff --git a/fireant/database/mysql.py b/fireant/database/mysql.py index 1d7129ea..d5d8fbee 100644 --- a/fireant/database/mysql.py +++ b/fireant/database/mysql.py @@ -75,6 +75,3 @@ def date_add(self, field, date_part, interval): # adding an extra 's' as MySQL's interval doesn't work with 'year', 'week' etc, it expects a plural interval_term = terms.Interval(**{'{}s'.format(str(date_part)): interval, 'dialect': Dialects.MYSQL}) return DateAdd(field, interval_term) - - def totals(self, query, terms): - raise NotImplementedError diff --git a/fireant/database/postgresql.py b/fireant/database/postgresql.py index 48a9ff49..b0646f23 100644 --- a/fireant/database/postgresql.py +++ b/fireant/database/postgresql.py @@ -56,6 +56,3 @@ def trunc_date(self, field, interval): def date_add(self, field, date_part, interval): return fn.DateAdd(str(date_part), interval, field) - - def totals(self, query, terms): - raise NotImplementedError diff --git a/fireant/database/redshift.py b/fireant/database/redshift.py index 65e4d9cf..c1341d29 100644 --- a/fireant/database/redshift.py +++ b/fireant/database/redshift.py @@ -7,6 +7,7 @@ class RedshiftDatabase(PostgreSQLDatabase): """ Redshift client that uses the psycopg module. """ + # The pypika query class to use for constructing queries query_cls = RedshiftQuery diff --git a/fireant/database/vertica.py b/fireant/database/vertica.py index 136b0ae2..b376094a 100644 --- a/fireant/database/vertica.py +++ b/fireant/database/vertica.py @@ -62,6 +62,3 @@ def trunc_date(self, field, interval): def date_add(self, field, date_part, interval): return fn.TimestampAdd(str(date_part), interval, field) - - def totals(self, query, terms): - return query.rollup(*terms) diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index b44ee9a8..6cf6c714 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -364,7 +364,7 @@ def __init__(self, dimension): if dimension.has_display_field \ else None - super(Dimension, self).__init__(dimension.key, - dimension.label, - totals_definition, - display_definition) + super(TotalsDimension, self).__init__(dimension.key, + dimension.label, + totals_definition, + display_definition) diff --git a/fireant/slicer/joins.py b/fireant/slicer/joins.py index c4a18645..14cbd306 100644 --- a/fireant/slicer/joins.py +++ b/fireant/slicer/joins.py @@ -17,4 +17,4 @@ def __repr__(self): criterion=self.criterion) def __gt__(self, other): - return self.table.table_name < other.table.table_name \ No newline at end of file + return self.table.table_name < other.table.table_name diff --git a/fireant/slicer/metrics.py b/fireant/slicer/metrics.py index 4efd1ecc..5fc0d88a 100644 --- a/fireant/slicer/metrics.py +++ b/fireant/slicer/metrics.py @@ -32,4 +32,5 @@ def __le__(self, other): return ComparatorFilter(self.definition, ComparatorFilter.Operator.lte, other) def __repr__(self): - return "slicer.metrics.{}".format(self.key) \ No newline at end of file + return "slicer.metrics.{}".format(self.key) + \ No newline at end of file diff --git a/fireant/slicer/operations.py b/fireant/slicer/operations.py index 9f0a1fdb..61948ada 100644 --- a/fireant/slicer/operations.py +++ b/fireant/slicer/operations.py @@ -69,6 +69,9 @@ def operations(self): if isinstance(operation, Operation) for op_and_children in [operation] + operation.operations] + def apply(self, data_frame): + raise NotImplementedError() + def __repr__(self): return self.key @@ -114,9 +117,6 @@ def apply(self, data_frame): class _Rolling(_BaseOperation): - def apply(self, data_frame): - raise NotImplementedError() - def __init__(self, arg, window, min_periods=None): super(_Rolling, self).__init__( key='{}({})'.format(self.__class__.__name__.lower(), @@ -132,6 +132,9 @@ def __init__(self, arg, window, min_periods=None): self.window = window self.min_periods = min_periods + def apply(self, data_frame): + raise NotImplementedError() + class RollingMean(_Rolling): def rolling_mean(self, x): diff --git a/fireant/slicer/queries/finders.py b/fireant/slicer/queries/finders.py index 1ff942bb..0fe94e75 100644 --- a/fireant/slicer/queries/finders.py +++ b/fireant/slicer/queries/finders.py @@ -1,6 +1,5 @@ import copy from collections import ( - OrderedDict, defaultdict, namedtuple, ) diff --git a/fireant/slicer/queries/references.py b/fireant/slicer/queries/references.py index 0f535ddf..40f832b5 100644 --- a/fireant/slicer/queries/references.py +++ b/fireant/slicer/queries/references.py @@ -1,5 +1,13 @@ import copy +from pypika import functions as fn +from pypika.queries import QueryBuilder +from pypika.terms import ( + ComplexCriterion, + Criterion, + NullValue, + Term, +) from typing import ( Callable, Iterable, @@ -9,14 +17,6 @@ reference_key, reference_term, ) -from pypika import functions as fn -from pypika.queries import QueryBuilder -from pypika.terms import ( - ComplexCriterion, - Criterion, - NullValue, - Term, -) from ..dimensions import Dimension from ..intervals import weekly @@ -171,7 +171,7 @@ def make_reference_join_criterion(ref_dimension: Dimension, Examples: original.date == DATE_ADD(reference.date) AND original.dim1 == reference.dim1 - + None if there are no dimensions. In that case there's nothing to join on and the reference queries should be added to the FROM clause of the container query. diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 113977d5..39492534 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -1,4 +1,3 @@ -from datetime import date from unittest import TestCase from unittest.mock import ( ANY, @@ -6,6 +5,7 @@ patch, ) +from datetime import date from pypika import Order import fireant as f @@ -246,6 +246,7 @@ def test_build_query_with_pattern_dimension(self): 'CASE ' 'WHEN "pattern" LIKE \'groupA%\' THEN \'groupA%\' ' 'WHEN "pattern" LIKE \'groupB%\' THEN \'groupB%\' ' + 'ELSE \'No Group\' ' 'END "pattern",' 'SUM("votes") "votes" ' 'FROM "politics"."politician" ' @@ -1845,7 +1846,7 @@ def test_dimension_with_join_includes_join_in_query(self): def test_dimension_with_multiple_joins_includes_joins_ordered__in_query(self): query = slicer.data \ .widget(f.DataTablesJS(slicer.metrics.votes, - slicer.metrics.voters)) \ + slicer.metrics.voters)) \ .dimension(slicer.dimensions.timestamp) \ .dimension(slicer.dimensions.district) \ .query From 5c74af31f2d4b1c6744bcd4d3e264fa26511bba6 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 28 Jun 2018 22:30:19 +0200 Subject: [PATCH 070/123] Updated version to dev29 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 2d61de53..bee6d36f 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev28' +__version__ = '1.0.0.dev29' From cb1691863f81790ec85abb4f9493e5f031bebcad Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 3 Jul 2018 16:43:05 +0200 Subject: [PATCH 071/123] Changed like and not_like filters to support multiple patterns --- fireant/slicer/dimensions.py | 140 +++++++------------ fireant/slicer/filters.py | 20 +-- fireant/slicer/joins.py | 2 +- fireant/slicer/queries/finders.py | 5 +- fireant/tests/slicer/queries/test_builder.py | 107 ++++++++++++++ 5 files changed, 176 insertions(+), 98 deletions(-) diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 6cf6c714..1fdd407a 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -12,8 +12,8 @@ BooleanFilter, ContainsFilter, ExcludesFilter, - LikeFilter, NotLikeFilter, + PatternFilter, RangeFilter, ) from .intervals import ( @@ -67,7 +67,44 @@ def is_(self, value: bool): return BooleanFilter(self.definition, value) -class CategoricalDimension(Dimension): +class PatternFilterableMixin: + definition = None + pattern_definition_attribute = 'definition' + + def like(self, pattern, *patterns): + """ + Creates a filter to filter a slicer query. + + :param pattern: + A pattern to match against the dimension's display definition. This pattern is used in the SQL query as + the `LIKE` expression. + :param patterns: + Additional patterns. This is the same as the pattern argument. The function signature is intended to + syntactically require at least one pattern. + :return: + A slicer query filter used to filter a slicer query to results where this dimension's display definition + matches the pattern. + """ + return PatternFilter(getattr(self, self.pattern_definition_attribute), pattern, *patterns) + + def not_like(self, pattern, *patterns): + """ + Creates a filter to filter a slicer query. + + :param pattern: + A pattern to match against the dimension's display definition. This pattern is used in the SQL query as + the `NOT LIKE` expression. + :param patterns: + Additional patterns. This is the same as the pattern argument. The function signature is intended to + syntactically require at least one pattern. + :return: + A slicer query filter used to filter a slicer query to results where this dimension's display definition + matches the pattern. + """ + return NotLikeFilter(getattr(self, self.pattern_definition_attribute), pattern, *patterns) + + +class CategoricalDimension(PatternFilterableMixin, Dimension): """ This is a dimension that represents an enum-like database field, with a finite list of options to chose from. It provides support for configuring a display value for each of the possible values. @@ -105,34 +142,8 @@ def notin(self, values): """ return ExcludesFilter(self.definition, values) - def like(self, pattern): - """ - Creates a filter to filter a slicer query. - :param pattern: - A pattern to match against the dimension's display definition. This pattern is used in the SQL query as - the `LIKE` expression. - :return: - A slicer query filter used to filter a slicer query to results where this dimension's display definition - matches the pattern. - """ - return LikeFilter(self.definition, pattern) - - def not_like(self, pattern): - """ - Creates a filter to filter a slicer query. - - :param pattern: - A pattern to match against the dimension's display definition. This pattern is used in the SQL query as - the `NOT LIKE` expression. - :return: - A slicer query filter used to filter a slicer query to results where this dimension's display definition - matches the pattern. - """ - return NotLikeFilter(self.definition, pattern) - - -class _UniqueDimensionBase(Dimension): +class _UniqueDimensionBase(PatternFilterableMixin, Dimension): def isin(self, values, use_display=False): """ Creates a filter to filter a slicer query. @@ -169,42 +180,13 @@ def notin(self, values, use_display=False): filter_field = self.display_definition if use_display else self.definition return ExcludesFilter(filter_field, values) - def like(self, pattern): - """ - Creates a filter to filter a slicer query. - - :param pattern: - A pattern to match against the dimension's display definition. This pattern is used in the SQL query as - the `LIKE` expression. - :return: - A slicer query filter used to filter a slicer query to results where this dimension's display definition - matches the pattern. - """ - if self.display_definition is None: - raise QueryException('No value set for display_definition.') - return LikeFilter(self.display_definition, pattern) - - def not_like(self, pattern): - """ - Creates a filter to filter a slicer query. - - :param pattern: - A pattern to match against the dimension's display definition. This pattern is used in the SQL query as - the `NOT LIKE` expression. - :return: - A slicer query filter used to filter a slicer query to results where this dimension's display definition - matches the pattern. - """ - if self.display_definition is None: - raise QueryException('No value set for display_definition.') - return NotLikeFilter(self.display_definition, pattern) - class UniqueDimension(_UniqueDimensionBase): """ This is a dimension that represents a field in a database which is a unique identifier, such as a primary/foreign key. It provides support for a display value field which is selected and used in the results. """ + pattern_definition_attribute = 'display_definition' def __init__(self, key, label=None, definition=None, display_definition=None): super(UniqueDimension, self).__init__(key, @@ -221,6 +203,16 @@ def __hash__(self): def display(self): return self + def like(self, pattern, *patterns): + if self.display_definition is None: + raise QueryException('No value set for display_definition.') + return super(UniqueDimension, self).like(pattern, *patterns) + + def not_like(self, pattern, *patterns): + if self.display_definition is None: + raise QueryException('No value set for display_definition.') + return super(UniqueDimension, self).not_like(pattern, *patterns) + class DisplayDimension(_UniqueDimensionBase): """ @@ -232,32 +224,6 @@ def __init__(self, dimension): dimension.label, dimension.display_definition) - def like(self, pattern): - """ - Creates a filter to filter a slicer query. - - :param pattern: - A pattern to match against the dimension's display definition. This pattern is used in the SQL query as - the `LIKE` expression. - :return: - A slicer query filter used to filter a slicer query to results where this dimension's display definition - matches the pattern. - """ - return LikeFilter(self.definition, pattern) - - def not_like(self, pattern): - """ - Creates a filter to filter a slicer query. - - :param pattern: - A pattern to match against the dimension's display definition. This pattern is used in the SQL query as - the `NOT LIKE` expression. - :return: - A slicer query filter used to filter a slicer query to results where this dimension's display definition - matches the pattern. - """ - return NotLikeFilter(self.definition, pattern) - class ContinuousDimension(Dimension): """ @@ -319,12 +285,12 @@ def between(self, start, stop): return RangeFilter(self.definition, start, stop) -class PatternDimension(Dimension): +class PatternDimension(PatternFilterableMixin, Dimension): """ This is a dimension that represents a boolean true/false value. The expression should always result in a boolean value. """ - + pattern_definition_attribute = 'field' _DEFAULT = ValueWrapper('No Group') def __init__(self, key, label=None, definition=None): diff --git a/fireant/slicer/filters.py b/fireant/slicer/filters.py index 3cea3ebc..1c2d04b0 100644 --- a/fireant/slicer/filters.py +++ b/fireant/slicer/filters.py @@ -59,13 +59,17 @@ def __init__(self, dimension_definition, start, stop): super(RangeFilter, self).__init__(definition) -class LikeFilter(DimensionFilter): - def __init__(self, dimension_definition, pattern): - definition = dimension_definition.like(pattern) - super(LikeFilter, self).__init__(definition) +class PatternFilter(DimensionFilter): + def _apply(self, dimension_definition, pattern): + return dimension_definition.like(pattern) + def __init__(self, dimension_definition, pattern, *patterns): + definition = self._apply(dimension_definition, pattern) + for extra_pattern in patterns: + definition |= self._apply(dimension_definition, extra_pattern) + super(PatternFilter, self).__init__(definition) -class NotLikeFilter(DimensionFilter): - def __init__(self, dimension_definition, pattern): - definition = dimension_definition.not_like(pattern) - super(NotLikeFilter, self).__init__(definition) + +class NotLikeFilter(PatternFilter): + def _apply(self, dimension_definition, pattern): + return dimension_definition.not_like(pattern) diff --git a/fireant/slicer/joins.py b/fireant/slicer/joins.py index 14cbd306..51ec34ac 100644 --- a/fireant/slicer/joins.py +++ b/fireant/slicer/joins.py @@ -17,4 +17,4 @@ def __repr__(self): criterion=self.criterion) def __gt__(self, other): - return self.table.table_name < other.table.table_name + return self.table._table_name < other.table._table_name diff --git a/fireant/slicer/queries/finders.py b/fireant/slicer/queries/finders.py index 0fe94e75..6cdfa636 100644 --- a/fireant/slicer/queries/finders.py +++ b/fireant/slicer/queries/finders.py @@ -1,7 +1,8 @@ import copy from collections import ( - defaultdict, + OrderedDict, namedtuple, + defaultdict, ) from toposort import ( @@ -80,7 +81,7 @@ def find_joins_for_tables(joins, base_table, required_tables): required_tables += tables_required_for_join - {d.table for d in dependencies} try: - return toposort_flatten(dependencies) + return toposort_flatten(dependencies, sort=True) except CircularDependencyError as e: raise CircularJoinsException(str(e)) diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 39492534..4c336a62 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -669,6 +669,113 @@ def test_build_query_with_filter_not_like_unique_dim(self): 'FROM "politics"."politician" ' 'WHERE "candidate_name" NOT LIKE \'%Trump\'', str(query)) + def test_build_query_with_filter_not_like_display_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate_display.not_like('%Trump')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" NOT LIKE \'%Trump\'', str(query)) + + def test_build_query_with_filter_like_categorical_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.political_party.like('Rep%', 'Dem%')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "political_party" LIKE \'Rep%\' ' + 'OR "political_party" LIKE \'Dem%\'', str(query)) + + def test_build_query_with_filter_not_like_categorical_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.political_party.not_like('Rep%', 'Dem%')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "political_party" NOT LIKE \'Rep%\' ' + 'OR "political_party" NOT LIKE \'Dem%\'', str(query)) + + def test_build_query_with_filter_like_pattern_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.pattern.like('a%', 'b%')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "pattern" LIKE \'a%\' ' + 'OR "pattern" LIKE \'b%\'', str(query)) + + def test_build_query_with_filter_not_like_pattern_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.pattern.not_like('a%', 'b%')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "pattern" NOT LIKE \'a%\' ' + 'OR "pattern" NOT LIKE \'b%\'', str(query)) + + def test_build_query_with_filter_like_unique_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate.like('%Trump', '%Clinton')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" LIKE \'%Trump\' ' + 'OR "candidate_name" LIKE \'%Clinton\'', str(query)) + + def test_build_query_with_filter_like_display_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate_display.like('%Trump', '%Clinton')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" LIKE \'%Trump\' ' + 'OR "candidate_name" LIKE \'%Clinton\'', str(query)) + + def test_build_query_with_filter_not_like_unique_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate.not_like('%Trump', '%Clinton')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" NOT LIKE \'%Trump\' ' + 'OR "candidate_name" NOT LIKE \'%Clinton\'', str(query)) + + def test_build_query_with_filter_not_like_display_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate_display.not_like('%Trump', '%Clinton')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" NOT LIKE \'%Trump\' ' + 'OR "candidate_name" NOT LIKE \'%Clinton\'', str(query)) + def test_build_query_with_filter_isin_raise_exception_when_display_definition_undefined(self): with self.assertRaises(f.QueryException): slicer.data \ From 09b6264c9e476af06f5aed16345b471a9802978a Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 3 Jul 2018 16:56:35 +0200 Subject: [PATCH 072/123] bumped version to dev30 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index bee6d36f..f7dc1a96 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev29' +__version__ = '1.0.0.dev30' From ceee6bb1bf4363015c73a32acf12830603f6d83d Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 3 Jul 2018 18:02:57 +0200 Subject: [PATCH 073/123] Fixed coerce type to allow literal NAN and INF strings --- fireant/formats.py | 4 ++++ fireant/tests/test_formats.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/fireant/formats.py b/fireant/formats.py index eea5f5fb..86128a38 100644 --- a/fireant/formats.py +++ b/fireant/formats.py @@ -23,6 +23,10 @@ def date_as_millis(value): def coerce_type(value): + # Should never be any real NaNs or INFs at this point, so if that's the value, it's meant to be that. + if value.lower() in ['nan', 'inf']: + return value + for type_cast in (int, float): try: return type_cast(value) diff --git a/fireant/tests/test_formats.py b/fireant/tests/test_formats.py index 4a32643b..e1ae49bd 100644 --- a/fireant/tests/test_formats.py +++ b/fireant/tests/test_formats.py @@ -86,3 +86,20 @@ def test_prefix(self): def test_suffix(self): result = formats.metric_display(0.12, suffix='€') self.assertEqual('0.12€', result) + + +class CoerceTypeTests(TestCase): + def allow_literal_nan(self): + result = formats.coerce_type('nan') + self.assertEqual('nan', result) + + def allow_literal_nan_upper(self): + result = formats.coerce_type('NAN') + self.assertEqual('NAN', result) + def allow_literal_inf(self): + result = formats.coerce_type('inf') + self.assertEqual('inf', result) + + def allow_literal_inf_upper(self): + result = formats.coerce_type('INF') + self.assertEqual('INF', result) From 71f8f569a174c7b079d05de54d5fc1af751c1afa Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 3 Jul 2018 18:03:30 +0200 Subject: [PATCH 074/123] bumped version to dev31 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index f7dc1a96..a2cc8978 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev30' +__version__ = '1.0.0.dev31' From 3bd0f98e25c0067f20b7b4eea374f7648d3523b9 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 5 Jul 2018 12:36:49 +0200 Subject: [PATCH 075/123] Fixes for the generated aliases, prepending $ to make them distinct --- fireant/formats.py | 3 +- fireant/slicer/operations.py | 25 +- fireant/slicer/queries/builder.py | 18 +- fireant/slicer/queries/database.py | 14 +- fireant/slicer/queries/makers.py | 21 +- fireant/slicer/queries/references.py | 15 +- fireant/slicer/references.py | 10 +- fireant/slicer/widgets/datatables.py | 12 +- fireant/slicer/widgets/helpers.py | 6 +- fireant/slicer/widgets/highcharts.py | 4 +- fireant/slicer/widgets/pandas.py | 20 +- fireant/tests/slicer/mocks.py | 68 +- fireant/tests/slicer/queries/test_builder.py | 1342 ++++++++--------- fireant/tests/slicer/queries/test_database.py | 13 +- .../slicer/queries/test_dimension_choices.py | 22 +- fireant/tests/slicer/test_operations.py | 24 +- fireant/tests/slicer/widgets/test_csv.py | 38 +- .../tests/slicer/widgets/test_datatables.py | 3 +- fireant/tests/slicer/widgets/test_pandas.py | 45 +- fireant/utils.py | 6 + requirements.txt | 2 +- setup.py | 2 +- 22 files changed, 882 insertions(+), 831 deletions(-) diff --git a/fireant/formats.py b/fireant/formats.py index 86128a38..58cada1c 100644 --- a/fireant/formats.py +++ b/fireant/formats.py @@ -9,6 +9,7 @@ import pandas as pd INFINITY = "Infinity" +NULL_VALUE = 'null' NO_TIME = time(0) @@ -33,7 +34,7 @@ def coerce_type(value): except: pass - if 'null' == value: + if NULL_VALUE == value: return None if 'True' == value: return True diff --git a/fireant/slicer/operations.py b/fireant/slicer/operations.py index 61948ada..07325705 100644 --- a/fireant/slicer/operations.py +++ b/fireant/slicer/operations.py @@ -1,6 +1,7 @@ import numpy as np import pandas as pd +from fireant.utils import format_key from .metrics import Metric @@ -78,26 +79,30 @@ def __repr__(self): class CumSum(_Cumulative): def apply(self, data_frame): + df_key = format_key(self.arg.key) + if isinstance(data_frame.index, pd.MultiIndex): levels = self._group_levels(data_frame.index) - return data_frame[self.arg.key] \ + return data_frame[df_key] \ .groupby(level=levels) \ .cumsum() - return data_frame[self.arg.key].cumsum() + return data_frame[df_key].cumsum() class CumProd(_Cumulative): def apply(self, data_frame): + df_key = format_key(self.arg.key) + if isinstance(data_frame.index, pd.MultiIndex): levels = self._group_levels(data_frame.index) - return data_frame[self.arg.key] \ + return data_frame[df_key] \ .groupby(level=levels) \ .cumprod() - return data_frame[self.arg.key].cumprod() + return data_frame[df_key].cumprod() class CumMean(_Cumulative): @@ -106,14 +111,16 @@ def cummean(x): return x.cumsum() / np.arange(1, len(x) + 1) def apply(self, data_frame): + df_key = format_key(self.arg.key) + if isinstance(data_frame.index, pd.MultiIndex): levels = self._group_levels(data_frame.index) - return data_frame[self.arg.key] \ + return data_frame[df_key] \ .groupby(level=levels) \ .apply(self.cummean) - return self.cummean(data_frame[self.arg.key]) + return self.cummean(data_frame[df_key]) class _Rolling(_BaseOperation): @@ -141,11 +148,13 @@ def rolling_mean(self, x): return x.rolling(self.window, self.min_periods).mean() def apply(self, data_frame): + df_key = format_key(self.arg.key) + if isinstance(data_frame.index, pd.MultiIndex): levels = self._group_levels(data_frame.index) - return data_frame[self.arg.key] \ + return data_frame[df_key] \ .groupby(level=levels) \ .apply(self.rolling_mean) - return self.rolling_mean(data_frame[self.arg.key]) + return self.rolling_mean(data_frame[df_key]) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index af8ce8c1..eddec28a 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -1,14 +1,16 @@ +import pandas as pd +from pypika import ( + Order, +) from typing import ( Dict, Iterable, ) -import pandas as pd -from fireant.utils import immutable -from pypika import ( - Order, +from fireant.utils import ( + format_key, + immutable, ) - from .database import fetch_data from .finders import ( find_and_group_references_for_dimensions, @@ -108,7 +110,7 @@ def orderby(self, element: SlicerElement, orientation=None): The directionality to order by, either ascending or descending. :return: """ - self._orders += [(element.definition.as_(element.key), orientation)] + self._orders += [(element.definition.as_(format_key(element.key)), orientation)] @property def query(self): @@ -235,9 +237,9 @@ def fetch(self, limit=None, offset=None, hint=None, force_include=()) -> pd.Seri query = query.hint(hint) dimension = self._dimensions[0] - definition = dimension.display_definition.as_(dimension.display_key) \ + definition = dimension.display_definition.as_(format_key(dimension.display_key)) \ if dimension.has_display_field \ - else dimension.definition.as_(dimension.key) + else dimension.definition.as_(format_key(dimension.key)) if force_include: include = self.slicer.database.to_char(dimension.definition) \ diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index 8ae4e371..ef0b4670 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -1,9 +1,10 @@ -import time - import pandas as pd +import time from typing import Iterable from fireant.database.base import Database +from fireant.formats import NULL_VALUE +from fireant.utils import format_key from .logger import ( query_logger, slow_query_logger, @@ -13,8 +14,6 @@ Dimension, ) -NULL_VALUE = 'null' - def fetch_data(database: Database, query: str, dimensions: Iterable[Dimension]): """ @@ -55,7 +54,8 @@ def clean_and_apply_index(data_frame: pd.DataFrame, dimensions: Iterable[Dimensi if not dimensions: return data_frame - dimension_keys = [d.key for d in dimensions] + dimension_keys = [format_key(d.key) + for d in dimensions] for i, dimension in enumerate(dimensions): if isinstance(dimension, ContinuousDimension): @@ -63,7 +63,7 @@ def clean_and_apply_index(data_frame: pd.DataFrame, dimensions: Iterable[Dimensi # With that in mind, we leave the NaNs in them to represent Totals. continue - level = dimension.key + level = format_key(dimension.key) data_frame[level] = fill_nans_in_level(data_frame, dimension, dimension_keys[:i]) \ .apply(lambda x: str(x) if not pd.isnull(x) else None) @@ -87,7 +87,7 @@ def fill_nans_in_level(data_frame, dimension, preceding_dimension_keys): :return: The level in the data_frame with the nulls replaced with empty string """ - level = dimension.key + level = format_key(dimension.key) if dimension.is_rollup: if preceding_dimension_keys: diff --git a/fireant/slicer/queries/makers.py b/fireant/slicer/queries/makers.py index 9fff7e64..990851f1 100644 --- a/fireant/slicer/queries/makers.py +++ b/fireant/slicer/queries/makers.py @@ -1,7 +1,10 @@ from functools import partial - -from fireant.utils import flatten from pypika import JoinType + +from fireant.utils import ( + flatten, + format_key, +) from .finders import ( find_joins_for_tables, find_required_tables_to_join, @@ -150,7 +153,7 @@ def make_slicer_query_with_references(database, base_table, joins, dimensions, m dimensions=non_totals_dimensions, metrics=metrics, filters=filters) \ - .as_('base') + .as_('$base') container_query = database.query_cls.from_(original_query) @@ -181,7 +184,7 @@ def make_slicer_query_with_references(database, base_table, joins, dimensions, m non_totals_dimensions, metrics, ref_filters) \ - .as_(alias) + .as_(format_key(alias)) join_criterion = make_reference_join_criterion(ref_dimension, non_totals_dimensions, @@ -196,9 +199,9 @@ def make_slicer_query_with_references(database, base_table, joins, dimensions, m else: container_query = container_query.from_(ref_query) - ref_dimension_definitions.append([offset_func(ref_query.field(dimension.key)) + ref_dimension_definitions.append([offset_func(ref_query.field(format_key(dimension.key))) if ref_dimension == dimension - else ref_query.field(dimension.key) + else ref_query.field(format_key(dimension.key)) for dimension in dimensions]) ref_terms += make_terms_for_references(references, @@ -217,7 +220,7 @@ def make_slicer_query_with_references(database, base_table, joins, dimensions, m def make_terms_for_metrics(metrics): - return [metric.definition.as_(metric.key) + return [metric.definition.as_(format_key(metric.key)) for metric in metrics] @@ -240,12 +243,12 @@ def make_terms_for_dimension(dimension, window=None): window(dimension.definition, dimension.interval) if window and hasattr(dimension, 'interval') else dimension.definition - ).as_(dimension.key) + ).as_(format_key(dimension.key)) # Include the display definition if there is one return [ dimension_definition, - dimension.display_definition.as_(dimension.display_key) + dimension.display_definition.as_(format_key(dimension.display_key)) ] if dimension.has_display_field else [ dimension_definition ] diff --git a/fireant/slicer/queries/references.py b/fireant/slicer/queries/references.py index 40f832b5..e0e990bf 100644 --- a/fireant/slicer/queries/references.py +++ b/fireant/slicer/queries/references.py @@ -17,6 +17,7 @@ reference_key, reference_term, ) +from fireant.utils import format_key from ..dimensions import Dimension from ..intervals import weekly @@ -44,7 +45,7 @@ def make_terms_for_references(references, original_query, ref_query, metrics): original_query, ref_query) - terms += [ref_metric(metric).as_(reference_key(metric, reference)) + terms += [ref_metric(metric).as_(format_key(reference_key(metric, reference))) for metric in metrics] return terms @@ -73,7 +74,7 @@ def make_dimension_terms_for_reference_container_query(original_query, terms = [] for dimension, ref_dimension_definition in zip(dimensions, ref_dimension_definitions): - term = _select_for_reference_container_query(dimension.key, + term = _select_for_reference_container_query(format_key(dimension.key), dimension.definition, original_query, ref_dimension_definition) @@ -83,9 +84,9 @@ def make_dimension_terms_for_reference_container_query(original_query, continue # Select the display definitions as a field from the ref query - ref_display_definition = [definition.table.field(dimension.display_key) + ref_display_definition = [definition.table.field(format_key(dimension.display_key)) for definition in ref_dimension_definition] - display_term = _select_for_reference_container_query(dimension.display_key, + display_term = _select_for_reference_container_query(format_key(dimension.display_key), dimension.display_definition, original_query, ref_display_definition) @@ -95,7 +96,7 @@ def make_dimension_terms_for_reference_container_query(original_query, def make_metric_terms_for_reference_container_query(original_query, metrics): - return [_select_for_reference_container_query(metric.key, metric.definition, original_query) + return [_select_for_reference_container_query(format_key(metric.key), metric.definition, original_query) for metric in metrics] @@ -180,13 +181,13 @@ def make_reference_join_criterion(ref_dimension: Dimension, join_criterion = None for dimension in all_dimensions: - ref_query_field = ref_query.field(dimension.key) + ref_query_field = ref_query.field(format_key(dimension.key)) # If this is the reference dimension, it needs to be offset by the reference interval if ref_dimension == dimension: ref_query_field = offset_func(ref_query_field) - next_criterion = original_query.field(dimension.key) == ref_query_field + next_criterion = original_query.field(format_key(dimension.key)) == ref_query_field join_criterion = next_criterion \ if join_criterion is None \ diff --git a/fireant/slicer/references.py b/fireant/slicer/references.py index 5ead27b3..c06dbcc6 100644 --- a/fireant/slicer/references.py +++ b/fireant/slicer/references.py @@ -1,5 +1,6 @@ from pypika import functions as fn from pypika.queries import QueryBuilder +from fireant import utils class Reference(object): @@ -107,15 +108,18 @@ def reference_term(reference: Reference, :return: """ + def original_field(metric): + return original_query.field(utils.format_key(metric.key)) + def ref_field(metric): - return ref_query.field(metric.key) + return ref_query.field(utils.format_key(metric.key)) if reference.delta: if reference.delta_percent: - return lambda metric: (original_query.field(metric.key) - ref_field(metric)) \ + return lambda metric: (original_field(metric) - ref_field(metric)) \ * \ (100 / fn.NullIf(ref_field(metric), 0)) - return lambda metric: original_query.field(metric.key) - ref_field(metric) + return lambda metric: original_field(metric) - ref_field(metric) return ref_field diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index 6e6e8db5..7e542920 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -9,6 +9,7 @@ formats, utils, ) +from fireant.utils import format_key from .base import ( TransformableWidget, ) @@ -70,9 +71,10 @@ def _render_dimensional_metric_cell(row_data: pd.Series, metric: Metric): for key, next_row in row_data.groupby(level=-1): next_row.reset_index(level=-1, drop=True, inplace=True) + df_key = format_key(metric.key) level[key] = _render_dimensional_metric_cell(next_row, metric) \ if isinstance(next_row.index, pd.MultiIndex) \ - else _format_metric_cell(next_row[metric.key], metric) + else _format_metric_cell(next_row[df_key], metric) return level @@ -128,7 +130,7 @@ def transform(self, data_frame, slicer, dimensions, references): """ dimension_display_values = extract_display_values(dimensions, data_frame) - metric_keys = [reference_key(metric, reference) + metric_keys = [format_key(reference_key(metric, reference)) for metric in self.items for reference in [None] + references] data_frame = data_frame[metric_keys] @@ -241,14 +243,16 @@ def _data_row(self, dimensions, dimension_values, dimension_display_values, refe row = {} for dimension, dimension_value in zip(dimensions, utils.wrap_list(dimension_values)): - row[dimension.key] = _render_dimension_cell(dimension_value, dimension_display_values.get(dimension.key)) + df_key = format_key(dimension.key) + row[dimension.key] = _render_dimension_cell(dimension_value, dimension_display_values.get(df_key)) for metric in self.items: for reference in [None] + references: key = reference_key(metric, reference) + df_key = format_key(key) row[key] = _render_dimensional_metric_cell(row_data, metric) \ if isinstance(row_data.index, pd.MultiIndex) \ - else _format_metric_cell(row_data[key], metric) + else _format_metric_cell(row_data[df_key], metric) return row diff --git a/fireant/slicer/widgets/helpers.py b/fireant/slicer/widgets/helpers.py index bfd00b24..26aabfb2 100644 --- a/fireant/slicer/widgets/helpers.py +++ b/fireant/slicer/widgets/helpers.py @@ -23,13 +23,13 @@ def extract_display_values(dimensions, data_frame): display_values = {} for dimension in dimensions: - key = dimension.key + key = utils.format_key(dimension.key) if hasattr(dimension, 'display_values'): display_values[key] = dimension.display_values elif getattr(dimension, 'display_key', None): - display_values[key] = data_frame[dimension.display_key] \ + display_values[key] = data_frame[utils.format_key(dimension.display_key)] \ .groupby(level=key) \ .first() @@ -64,7 +64,7 @@ def render_series_label(dimension_values, metric=None, reference=None): used_dimensions = dimensions if metric is None else dimensions[1:] dimension_values = utils.wrap_list(dimension_values) dimension_labels = [utils.deep_get(dimension_display_values, - [dimension.key, dimension_value], + [utils.format_key(dimension.key), dimension_value], dimension_value) if not pd.isnull(dimension_value) else 'Totals' diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 702154e3..0a07e95f 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -345,7 +345,7 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, series_color = next(colors) for reference, dash_style in zip([None] + references, itertools.cycle(DASH_STYLES)): - metric_key = reference_key(series.metric, reference) + metric_key = utils.format_key(reference_key(series.metric, reference)) hc_series.append({ "type": series.type, @@ -381,7 +381,7 @@ def _render_pie_series(self, series, reference, dimension_values, data_frame, re "data": [{ "name": render_series_label(dimension_values) if dimension_values else name, "y": formats.metric_value(y), - } for dimension_values, y in data_frame[series.metric.key].iteritems()], + } for dimension_values, y in data_frame[utils.format_key(series.metric.key)].iteritems()], 'tooltip': { 'valueDecimals': metric.precision, 'valuePrefix': metric.prefix, diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index 22e5cd2e..45600366 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -1,6 +1,7 @@ import pandas as pd from fireant import Metric +from fireant.utils import format_key from .base import ( TransformableWidget, ) @@ -37,26 +38,28 @@ def transform(self, data_frame, slicer, dimensions, references): if any([metric.precision is not None, metric.prefix is not None, metric.suffix is not None]): - result[metric.key] = result[metric.key] \ + df_key = format_key(metric.key) + + result[df_key] = result[df_key] \ .apply(lambda x: formats.metric_display(x, metric.prefix, metric.suffix, metric.precision)) for dimension in dimensions: if dimension.has_display_field: - result = result.set_index(dimension.display_key, append=True) - result = result.reset_index(dimension.key, drop=True) + result = result.set_index(format_key(dimension.display_key), append=True) + result = result.reset_index(format_key(dimension.key), drop=True) if hasattr(dimension, 'display_values'): self._replace_display_values_in_index(dimension, result) if isinstance(data_frame.index, pd.MultiIndex): - index_levels = [dimension.display_key + index_levels = [format_key(dimension.display_key) if dimension.has_display_field - else dimension.key + else format_key(dimension.key) for dimension in dimensions] result = result.reorder_levels(index_levels) - result = result[[reference_key(item, reference) + result = result[[format_key(reference_key(item, reference)) for reference in [None] + references for item in self.items]] @@ -79,9 +82,10 @@ def _replace_display_values_in_index(self, dimension, result): Replaces the raw values of a (categorical) dimension in the index with their corresponding display values. """ if isinstance(result.index, pd.MultiIndex): + df_key = format_key(dimension.key) values = [dimension.display_values.get(x, x) - for x in result.index.get_level_values(dimension.key)] - result.index.set_levels(level=dimension.key, levels=values) + for x in result.index.get_level_values(df_key)] + result.index.set_levels(level=df_key, levels=values) return result values = [dimension.display_values.get(x, x) diff --git a/fireant/tests/slicer/mocks.py b/fireant/tests/slicer/mocks.py index 41629246..bbffd705 100644 --- a/fireant/tests/slicer/mocks.py +++ b/fireant/tests/slicer/mocks.py @@ -1,6 +1,5 @@ from collections import ( OrderedDict, - namedtuple, ) from unittest.mock import Mock @@ -16,6 +15,7 @@ from fireant import * from fireant.slicer.references import ReferenceType +from fireant.utils import format_key as f class TestDatabase(VerticaDatabase): @@ -209,15 +209,22 @@ def __eq__(self, other): (6, 11): False, } -df_columns = ['timestamp', - 'candidate', 'candidate_display', - 'political_party', - 'election', 'election_display', - 'state', 'state_display', - 'winner', - 'votes', - 'wins'] -PoliticsRow = namedtuple('PoliticsRow', df_columns) +df_columns = [f('timestamp'), + f('candidate'), f('candidate_display'), + f('political_party'), + f('election'), f('election_display'), + f('state'), f('state_display'), + f('winner'), + f('votes'), + f('wins')] + + +def PoliticsRow(timestamp, candidate, candidate_display, political_party, election, election_display, state, + state_display, winner, votes, wins): + return ( + timestamp, candidate, candidate_display, political_party, election, election_display, state, state_display, + winner, votes, wins + ) records = [] for (election_id, candidate_id, state_id), votes in election_candidate_state_votes.items(): @@ -236,36 +243,39 @@ def __eq__(self, other): mock_politics_database = pd.DataFrame.from_records(records, columns=df_columns) -single_metric_df = pd.DataFrame(mock_politics_database[['votes']] +single_metric_df = pd.DataFrame(mock_politics_database[[f('votes')]] .sum()).T -multi_metric_df = pd.DataFrame(mock_politics_database[['votes', 'wins']] +multi_metric_df = pd.DataFrame(mock_politics_database[[f('votes'), f('wins')]] .sum()).T -cont_dim_df = mock_politics_database[['timestamp', 'votes', 'wins']] \ - .groupby('timestamp') \ +cont_dim_df = mock_politics_database[[f('timestamp'), f('votes'), f('wins')]] \ + .groupby(f('timestamp')) \ .sum() -cat_dim_df = mock_politics_database[['political_party', 'votes', 'wins']] \ - .groupby('political_party') \ +cat_dim_df = mock_politics_database[[f('political_party'), f('votes'), f('wins')]] \ + .groupby(f('political_party')) \ .sum() -uni_dim_df = mock_politics_database[['candidate', 'candidate_display', 'votes', 'wins']] \ - .groupby(['candidate', 'candidate_display']) \ +uni_dim_df = mock_politics_database[[f('candidate'), f('candidate_display'), f('votes'), f('wins')]] \ + .groupby([f('candidate'), f('candidate_display')]) \ .sum() \ - .reset_index('candidate_display') + .reset_index(f('candidate_display')) -cont_cat_dim_df = mock_politics_database[['timestamp', 'political_party', 'votes', 'wins']] \ - .groupby(['timestamp', 'political_party']) \ +cont_cat_dim_df = mock_politics_database[[f('timestamp'), f('political_party'), f('votes'), f('wins')]] \ + .groupby([f('timestamp'), f('political_party')]) \ .sum() -cont_uni_dim_df = mock_politics_database[['timestamp', 'state', 'state_display', 'votes', 'wins']] \ - .groupby(['timestamp', 'state', 'state_display']) \ +cont_uni_dim_df = mock_politics_database[[f('timestamp'), f('state'), f('state_display'), f('votes'), f('wins')]] \ + .groupby([f('timestamp'), f('state'), f('state_display')]) \ .sum() \ - .reset_index('state_display') + .reset_index(f('state_display')) cont_dim_operation_df = cont_dim_df.copy() -cont_dim_operation_df['cumsum(votes)'] = cont_dim_df['votes'].cumsum() + +operation_key = f('cumsum(votes)') +cont_dim_operation_df[operation_key] = cont_dim_df[f('votes')].cumsum() + def ref(data_frame, columns): ref_cols = {column: '%s_eoe' % column @@ -292,7 +302,7 @@ def ref_delta(ref_data_frame, columns): return ref_data_frame.join(delta_data_frame) -_columns = ['votes', 'wins'] +_columns = [f('votes'), f('wins')] cont_uni_dim_ref_df = ref(cont_uni_dim_df, _columns) cont_uni_dim_ref_delta_df = ref_delta(cont_uni_dim_ref_df, _columns) @@ -345,8 +355,8 @@ def _totals(df): elif not isinstance(l.index, (pd.DatetimeIndex, pd.RangeIndex)): l.index = l.index.astype('str') -cont_cat_dim_totals_df = totals(cont_cat_dim_df, ['political_party'], _columns) -cont_uni_dim_totals_df = totals(cont_uni_dim_df, ['state'], _columns) -cont_uni_dim_all_totals_df = totals(cont_uni_dim_df, ['timestamp', 'state'], _columns) +cont_cat_dim_totals_df = totals(cont_cat_dim_df, [f('political_party')], _columns) +cont_uni_dim_totals_df = totals(cont_uni_dim_df, [f('state')], _columns) +cont_uni_dim_all_totals_df = totals(cont_uni_dim_df, [f('timestamp'), f('state')], _columns) ElectionOverElection = ReferenceType('eoe', 'EoE', 'year', 4) diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 4c336a62..99e3618e 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -54,7 +54,7 @@ def test_build_query_with_single_metric(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician"', str(query)) def test_build_query_with_multiple_metrics(self): @@ -63,8 +63,8 @@ def test_build_query_with_multiple_metrics(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes",' - 'SUM("is_winner") "wins" ' + 'SUM("votes") "$votes",' + 'SUM("is_winner") "$wins" ' 'FROM "politics"."politician"', str(query)) def test_build_query_with_multiple_visualizations(self): @@ -74,8 +74,8 @@ def test_build_query_with_multiple_visualizations(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes",' - 'SUM("is_winner") "wins" ' + 'SUM("votes") "$votes",' + 'SUM("is_winner") "$wins" ' 'FROM "politics"."politician"', str(query)) def test_build_query_for_chart_visualization_with_single_axis(self): @@ -85,7 +85,7 @@ def test_build_query_for_chart_visualization_with_single_axis(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician"', str(query)) def test_build_query_for_chart_visualization_with_multiple_axes(self): @@ -96,8 +96,8 @@ def test_build_query_for_chart_visualization_with_multiple_axes(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes",' - 'SUM("is_winner") "wins" ' + 'SUM("votes") "$votes",' + 'SUM("is_winner") "$wins" ' 'FROM "politics"."politician"', str(query)) @@ -112,11 +112,11 @@ def test_build_query_with_datetime_dimension(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp"', str(query)) def test_build_query_with_datetime_dimension_hourly(self): query = slicer.data \ @@ -125,11 +125,11 @@ def test_build_query_with_datetime_dimension_hourly(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'HH\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'HH\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp"', str(query)) def test_build_query_with_datetime_dimension_daily(self): query = slicer.data \ @@ -138,11 +138,11 @@ def test_build_query_with_datetime_dimension_daily(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp"', str(query)) def test_build_query_with_datetime_dimension_weekly(self): query = slicer.data \ @@ -151,11 +151,11 @@ def test_build_query_with_datetime_dimension_weekly(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'IW\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'IW\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp"', str(query)) def test_build_query_with_datetime_dimension_monthly(self): query = slicer.data \ @@ -164,11 +164,11 @@ def test_build_query_with_datetime_dimension_monthly(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'MM\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'MM\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp"', str(query)) def test_build_query_with_datetime_dimension_quarterly(self): query = slicer.data \ @@ -177,11 +177,11 @@ def test_build_query_with_datetime_dimension_quarterly(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'Q\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'Q\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp"', str(query)) def test_build_query_with_datetime_dimension_annually(self): query = slicer.data \ @@ -190,11 +190,11 @@ def test_build_query_with_datetime_dimension_annually(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'Y\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'Y\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp"', str(query)) def test_build_query_with_boolean_dimension(self): query = slicer.data \ @@ -203,11 +203,11 @@ def test_build_query_with_boolean_dimension(self): .query self.assertEqual('SELECT ' - '"is_winner" "winner",' - 'SUM("votes") "votes" ' + '"is_winner" "$winner",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "winner" ' - 'ORDER BY "winner"', str(query)) + 'GROUP BY "$winner" ' + 'ORDER BY "$winner"', str(query)) def test_build_query_with_categorical_dimension(self): query = slicer.data \ @@ -216,11 +216,11 @@ def test_build_query_with_categorical_dimension(self): .query self.assertEqual('SELECT ' - '"political_party" "political_party",' - 'SUM("votes") "votes" ' + '"political_party" "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "political_party" ' - 'ORDER BY "political_party"', str(query)) + 'GROUP BY "$political_party" ' + 'ORDER BY "$political_party"', str(query)) def test_build_query_with_unique_dimension(self): query = slicer.data \ @@ -229,12 +229,12 @@ def test_build_query_with_unique_dimension(self): .query self.assertEqual('SELECT ' - '"election_id" "election",' - '"election_year" "election_display",' - 'SUM("votes") "votes" ' + '"election_id" "$election",' + '"election_year" "$election_display",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "election","election_display" ' - 'ORDER BY "election_display"', str(query)) + 'GROUP BY "$election","$election_display" ' + 'ORDER BY "$election_display"', str(query)) def test_build_query_with_pattern_dimension(self): query = slicer.data \ @@ -247,11 +247,11 @@ def test_build_query_with_pattern_dimension(self): 'WHEN "pattern" LIKE \'groupA%\' THEN \'groupA%\' ' 'WHEN "pattern" LIKE \'groupB%\' THEN \'groupB%\' ' 'ELSE \'No Group\' ' - 'END "pattern",' - 'SUM("votes") "votes" ' + 'END "$pattern",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "pattern" ' - 'ORDER BY "pattern"', str(query)) + 'GROUP BY "$pattern" ' + 'ORDER BY "$pattern"', str(query)) def test_build_query_with_pattern_no_values(self): query = slicer.data \ @@ -260,11 +260,11 @@ def test_build_query_with_pattern_no_values(self): .query self.assertEqual('SELECT ' - '\'No Group\' "pattern",' - 'SUM("votes") "votes" ' + '\'No Group\' "$pattern",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "pattern" ' - 'ORDER BY "pattern"', str(query)) + 'GROUP BY "$pattern" ' + 'ORDER BY "$pattern"', str(query)) def test_build_query_with_multiple_dimensions(self): query = slicer.data \ @@ -274,13 +274,13 @@ def test_build_query_with_multiple_dimensions(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"candidate_id" "candidate",' - '"candidate_name" "candidate_display",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"candidate_id" "$candidate",' + '"candidate_name" "$candidate_display",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp","candidate","candidate_display" ' - 'ORDER BY "timestamp","candidate_display"', str(query)) + 'GROUP BY "$timestamp","$candidate","$candidate_display" ' + 'ORDER BY "$timestamp","$candidate_display"', str(query)) def test_build_query_with_multiple_dimensions_and_visualizations(self): query = slicer.data \ @@ -293,13 +293,13 @@ def test_build_query_with_multiple_dimensions_and_visualizations(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"political_party" "political_party",' - 'SUM("votes") "votes",' - 'SUM("is_winner") "wins" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"political_party" "$political_party",' + 'SUM("votes") "$votes",' + 'SUM("is_winner") "$wins" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp","political_party" ' - 'ORDER BY "timestamp","political_party"', str(query)) + 'GROUP BY "$timestamp","$political_party" ' + 'ORDER BY "$timestamp","$political_party"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -313,19 +313,19 @@ def test_build_query_with_totals_cat_dimension(self): .query self.assertEqual('(SELECT ' - '"political_party" "political_party",' - 'SUM("votes") "votes" ' + '"political_party" "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "political_party") ' + 'GROUP BY "$political_party") ' 'UNION ALL ' '(SELECT ' - 'NULL "political_party",' - 'SUM("votes") "votes" ' + 'NULL "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician") ' - 'ORDER BY "political_party"', str(query)) + 'ORDER BY "$political_party"', str(query)) def test_build_query_with_totals_uni_dimension(self): query = slicer.data \ @@ -334,21 +334,21 @@ def test_build_query_with_totals_uni_dimension(self): .query self.assertEqual('(SELECT ' - '"candidate_id" "candidate",' - '"candidate_name" "candidate_display",' - 'SUM("votes") "votes" ' + '"candidate_id" "$candidate",' + '"candidate_name" "$candidate_display",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "candidate","candidate_display") ' + 'GROUP BY "$candidate","$candidate_display") ' 'UNION ALL ' '(SELECT ' - 'NULL "candidate",' - 'NULL "candidate_display",' - 'SUM("votes") "votes" ' + 'NULL "$candidate",' + 'NULL "$candidate_display",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician") ' - 'ORDER BY "candidate_display"', str(query)) + 'ORDER BY "$candidate_display"', str(query)) def test_build_query_with_totals_on_dimension_and_subsequent_dimensions(self): query = slicer.data \ @@ -359,25 +359,25 @@ def test_build_query_with_totals_on_dimension_and_subsequent_dimensions(self): .query self.assertEqual('(SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"candidate_id" "candidate",' - '"candidate_name" "candidate_display",' - '"political_party" "political_party",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"candidate_id" "$candidate",' + '"candidate_name" "$candidate_display",' + '"political_party" "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp","candidate","candidate_display","political_party") ' + 'GROUP BY "$timestamp","$candidate","$candidate_display","$political_party") ' 'UNION ALL ' '(SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'NULL "candidate",' - 'NULL "candidate_display",' - 'NULL "political_party",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'NULL "$candidate",' + 'NULL "$candidate_display",' + 'NULL "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp") ' - 'ORDER BY "timestamp","candidate_display","political_party"', str(query)) + 'GROUP BY "$timestamp") ' + 'ORDER BY "$timestamp","$candidate_display","$political_party"', str(query)) def test_build_query_with_totals_on_multiple_dimensions_dimension(self): query = slicer.data \ @@ -388,37 +388,37 @@ def test_build_query_with_totals_on_multiple_dimensions_dimension(self): .query self.assertEqual('(SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"candidate_id" "candidate",' - '"candidate_name" "candidate_display",' - '"political_party" "political_party",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"candidate_id" "$candidate",' + '"candidate_name" "$candidate_display",' + '"political_party" "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp","candidate","candidate_display","political_party") ' + 'GROUP BY "$timestamp","$candidate","$candidate_display","$political_party") ' 'UNION ALL ' '(SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'NULL "candidate",' - 'NULL "candidate_display",' - 'NULL "political_party",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'NULL "$candidate",' + 'NULL "$candidate_display",' + 'NULL "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp") ' + 'GROUP BY "$timestamp") ' 'UNION ALL ' '(SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"candidate_id" "candidate",' - '"candidate_name" "candidate_display",' - 'NULL "political_party",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"candidate_id" "$candidate",' + '"candidate_name" "$candidate_display",' + 'NULL "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp","candidate","candidate_display") ' + 'GROUP BY "$timestamp","$candidate","$candidate_display") ' - 'ORDER BY "timestamp","candidate_display","political_party"', str(query)) + 'ORDER BY "$timestamp","$candidate_display","$political_party"', str(query)) def test_build_query_with_totals_cat_dimension_with_references(self): query = slicer.data \ @@ -431,54 +431,54 @@ def test_build_query_with_totals_cat_dimension_with_references(self): # Important that in reference queries when using totals that the null dimensions are omitted from the nested # queries and selected in the container query self.assertEqual('(SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - 'COALESCE("base"."political_party","dod"."political_party") "political_party",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + 'COALESCE("$base"."$political_party","$dod"."$political_party") "$political_party",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM (' 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"political_party" "political_party",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"political_party" "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp","political_party"' - ') "base" ' + 'GROUP BY "$timestamp","$political_party"' + ') "$base" ' 'FULL OUTER JOIN (' 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"political_party" "political_party",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"political_party" "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp","political_party"' - ') "dod" ' - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'AND "base"."political_party"="dod"."political_party") ' + 'GROUP BY "$timestamp","$political_party"' + ') "$dod" ' + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'AND "$base"."$political_party"="$dod"."$political_party") ' 'UNION ALL ' '(SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - 'NULL "political_party",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + 'NULL "$political_party",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM (' 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' + 'GROUP BY "$timestamp"' + ') "$base" ' 'FULL OUTER JOIN (' - 'SELECT TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'SELECT TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "dod" ' - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp")) ' - 'ORDER BY "timestamp","political_party"', str(query)) + 'GROUP BY "$timestamp"' + ') "$dod" ' + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) ' + 'ORDER BY "$timestamp","$political_party"', str(query)) def test_build_query_with_totals_cat_dimension_with_references_and_date_filters(self): query = slicer.data \ @@ -490,58 +490,58 @@ def test_build_query_with_totals_cat_dimension_with_references_and_date_filters( .query self.assertEqual('(SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - 'COALESCE("base"."political_party","dod"."political_party") "political_party",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + 'COALESCE("$base"."$political_party","$dod"."$political_party") "$political_party",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM (' 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"political_party" "political_party",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"political_party" "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' - 'GROUP BY "timestamp","political_party"' - ') "base" ' + 'GROUP BY "$timestamp","$political_party"' + ') "$base" ' 'FULL OUTER JOIN (' 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"political_party" "political_party",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"political_party" "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' - 'GROUP BY "timestamp","political_party"' - ') "dod" ' - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'AND "base"."political_party"="dod"."political_party") ' + 'GROUP BY "$timestamp","$political_party"' + ') "$dod" ' + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'AND "$base"."$political_party"="$dod"."$political_party") ' 'UNION ALL ' '(SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - 'NULL "political_party",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + 'NULL "$political_party",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM (' 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' - 'GROUP BY "timestamp"' - ') "base" ' + 'GROUP BY "$timestamp"' + ') "$base" ' 'FULL OUTER JOIN (' - 'SELECT TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'SELECT TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' - 'GROUP BY "timestamp"' - ') "dod" ' - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp")) ' - 'ORDER BY "timestamp","political_party"', str(query)) + 'GROUP BY "$timestamp"' + ') "$dod" ' + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) ' + 'ORDER BY "$timestamp","$political_party"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -555,7 +555,7 @@ def test_build_query_with_filter_isin_categorical_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "political_party" IN (\'d\')', str(query)) @@ -566,7 +566,7 @@ def test_build_query_with_filter_notin_categorical_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "political_party" NOT IN (\'d\')', str(query)) @@ -577,7 +577,7 @@ def test_build_query_with_filter_like_categorical_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "political_party" LIKE \'Rep%\'', str(query)) @@ -588,7 +588,7 @@ def test_build_query_with_filter_not_like_categorical_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "political_party" NOT LIKE \'Rep%\'', str(query)) @@ -599,7 +599,7 @@ def test_build_query_with_filter_isin_unique_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_id" IN (1)', str(query)) @@ -610,7 +610,7 @@ def test_build_query_with_filter_notin_unique_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_id" NOT IN (1)', str(query)) @@ -621,7 +621,7 @@ def test_build_query_with_filter_isin_unique_dim_display(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" IN (\'Donald Trump\')', str(query)) @@ -632,7 +632,7 @@ def test_build_query_with_filter_notin_unique_dim_display(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" NOT IN (\'Donald Trump\')', str(query)) @@ -643,7 +643,7 @@ def test_build_query_with_filter_like_unique_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) @@ -654,7 +654,7 @@ def test_build_query_with_filter_like_display_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) @@ -665,7 +665,7 @@ def test_build_query_with_filter_not_like_unique_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" NOT LIKE \'%Trump\'', str(query)) @@ -676,7 +676,7 @@ def test_build_query_with_filter_not_like_display_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" NOT LIKE \'%Trump\'', str(query)) @@ -687,7 +687,7 @@ def test_build_query_with_filter_like_categorical_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "political_party" LIKE \'Rep%\' ' 'OR "political_party" LIKE \'Dem%\'', str(query)) @@ -699,7 +699,7 @@ def test_build_query_with_filter_not_like_categorical_dim_multiple_patterns(self .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "political_party" NOT LIKE \'Rep%\' ' 'OR "political_party" NOT LIKE \'Dem%\'', str(query)) @@ -711,7 +711,7 @@ def test_build_query_with_filter_like_pattern_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "pattern" LIKE \'a%\' ' 'OR "pattern" LIKE \'b%\'', str(query)) @@ -723,7 +723,7 @@ def test_build_query_with_filter_not_like_pattern_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "pattern" NOT LIKE \'a%\' ' 'OR "pattern" NOT LIKE \'b%\'', str(query)) @@ -735,7 +735,7 @@ def test_build_query_with_filter_like_unique_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" LIKE \'%Trump\' ' 'OR "candidate_name" LIKE \'%Clinton\'', str(query)) @@ -747,7 +747,7 @@ def test_build_query_with_filter_like_display_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" LIKE \'%Trump\' ' 'OR "candidate_name" LIKE \'%Clinton\'', str(query)) @@ -759,7 +759,7 @@ def test_build_query_with_filter_not_like_unique_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" NOT LIKE \'%Trump\' ' 'OR "candidate_name" NOT LIKE \'%Clinton\'', str(query)) @@ -771,7 +771,7 @@ def test_build_query_with_filter_not_like_display_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" NOT LIKE \'%Trump\' ' 'OR "candidate_name" NOT LIKE \'%Clinton\'', str(query)) @@ -807,7 +807,7 @@ def test_build_query_with_filter_range_datetime_dimension(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2009-01-20\' AND \'2017-01-20\'', str(query)) @@ -818,7 +818,7 @@ def test_build_query_with_filter_boolean_true(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "is_winner"', str(query)) @@ -829,7 +829,7 @@ def test_build_query_with_filter_boolean_false(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE NOT "is_winner"', str(query)) @@ -845,7 +845,7 @@ def test_build_query_with_metric_filter_eq(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")=5', str(query)) @@ -856,7 +856,7 @@ def test_build_query_with_metric_filter_eq_left(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")=5', str(query)) @@ -867,7 +867,7 @@ def test_build_query_with_metric_filter_ne(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")<>5', str(query)) @@ -878,7 +878,7 @@ def test_build_query_with_metric_filter_ne_left(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")<>5', str(query)) @@ -889,7 +889,7 @@ def test_build_query_with_metric_filter_gt(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")>5', str(query)) @@ -900,7 +900,7 @@ def test_build_query_with_metric_filter_gt_left(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")>5', str(query)) @@ -911,7 +911,7 @@ def test_build_query_with_metric_filter_gte(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")>=5', str(query)) @@ -922,7 +922,7 @@ def test_build_query_with_metric_filter_gte_left(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")>=5', str(query)) @@ -933,7 +933,7 @@ def test_build_query_with_metric_filter_lt(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")<5', str(query)) @@ -944,7 +944,7 @@ def test_build_query_with_metric_filter_lt_left(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")<5', str(query)) @@ -955,7 +955,7 @@ def test_build_query_with_metric_filter_lte(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")<=5', str(query)) @@ -966,7 +966,7 @@ def test_build_query_with_metric_filter_lte_left(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")<=5', str(query)) @@ -982,11 +982,11 @@ def test_build_query_with_cumsum_operation(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp"', str(query)) def test_build_query_with_cummean_operation(self): query = slicer.data \ @@ -995,11 +995,11 @@ def test_build_query_with_cummean_operation(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -1014,18 +1014,18 @@ def test_single_reference_dod_with_no_dimension_uses_multiple_from_clauses_inste .query self.assertEqual('SELECT ' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM (' 'SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician"' - ') "base",(' + ') "$base",(' 'SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician"' - ') "dod"', str(query)) + ') "$dod"', str(query)) def test_single_reference_dod_with_dimension_but_not_reference_dimension_in_query_using_filter(self): query = slicer.data \ @@ -1037,31 +1037,31 @@ def test_single_reference_dod_with_dimension_but_not_reference_dimension_in_quer .query self.assertEqual('SELECT ' - 'COALESCE("base"."political_party","dod"."political_party") "political_party",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$political_party","$dod"."$political_party") "$political_party",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - '"political_party" "political_party",' - 'SUM("votes") "votes" ' + '"political_party" "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "political_party"' - ') "base" ' # end-nested + 'GROUP BY "$political_party"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - '"political_party" "political_party",' - 'SUM("votes") "votes" ' + '"political_party" "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "political_party"' - ') "dod" ' # end-nested + 'GROUP BY "$political_party"' + ') "$dod" ' # end-nested - 'ON "base"."political_party"="dod"."political_party" ' - 'ORDER BY "political_party"', str(query)) + 'ON "$base"."$political_party"="$dod"."$political_party" ' + 'ORDER BY "$political_party"', str(query)) def test_dimension_with_single_reference_dod(self): query = slicer.data \ @@ -1072,29 +1072,29 @@ def test_dimension_with_single_reference_dod(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_dimension_with_single_reference_wow(self): query = slicer.data \ @@ -1105,29 +1105,29 @@ def test_dimension_with_single_reference_wow(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'week\',1,"wow"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"wow"."votes" "votes_wow" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'week\',1,"$wow"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$wow"."$votes" "$votes_wow" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "wow" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$wow" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'week\',1,"wow"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'week\',1,"$wow"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_dimension_with_single_reference_mom(self): query = slicer.data \ @@ -1138,29 +1138,29 @@ def test_dimension_with_single_reference_mom(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'month\',1,"mom"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"mom"."votes" "votes_mom" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'month\',1,"$mom"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$mom"."$votes" "$votes_mom" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "mom" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$mom" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'month\',1,"mom"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'month\',1,"$mom"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_dimension_with_single_reference_qoq(self): query = slicer.data \ @@ -1171,29 +1171,29 @@ def test_dimension_with_single_reference_qoq(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'quarter\',1,"qoq"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"qoq"."votes" "votes_qoq" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'quarter\',1,"$qoq"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$qoq"."$votes" "$votes_qoq" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "qoq" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$qoq" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'quarter\',1,"qoq"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'quarter\',1,"$qoq"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_dimension_with_single_reference_yoy(self): query = slicer.data \ @@ -1204,29 +1204,29 @@ def test_dimension_with_single_reference_yoy(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"yoy"."votes" "votes_yoy" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$yoy"."$votes" "$votes_yoy" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "yoy" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_dimension_with_single_reference_as_a_delta(self): query = slicer.data \ @@ -1237,29 +1237,29 @@ def test_dimension_with_single_reference_as_a_delta(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"base"."votes"-"dod"."votes" "votes_dod_delta" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$base"."$votes"-"$dod"."$votes" "$votes_dod_delta" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_dimension_with_single_reference_as_a_delta_percentage(self): query = slicer.data \ @@ -1270,29 +1270,29 @@ def test_dimension_with_single_reference_as_a_delta_percentage(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '("base"."votes"-"dod"."votes")*100/NULLIF("dod"."votes",0) "votes_dod_delta_percent" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '("$base"."$votes"-"$dod"."$votes")*100/NULLIF("$dod"."$votes",0) "$votes_dod_delta_percent" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_reference_on_dimension_with_weekly_interval(self): weekly_timestamp = slicer.dimensions.timestamp(f.weekly) @@ -1304,29 +1304,29 @@ def test_reference_on_dimension_with_weekly_interval(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'IW\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'IW\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_reference_on_dimension_with_weekly_interval_no_interval_on_reference(self): query = slicer.data \ @@ -1337,29 +1337,29 @@ def test_reference_on_dimension_with_weekly_interval_no_interval_on_reference(se .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'IW\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'IW\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_reference_on_dimension_with_monthly_interval(self): query = slicer.data \ @@ -1370,29 +1370,29 @@ def test_reference_on_dimension_with_monthly_interval(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'MM\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'MM\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'MM\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'MM\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_reference_on_dimension_with_quarterly_interval(self): query = slicer.data \ @@ -1403,29 +1403,29 @@ def test_reference_on_dimension_with_quarterly_interval(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'Q\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'Q\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'Q\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'Q\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_reference_on_dimension_with_annual_interval(self): query = slicer.data \ @@ -1436,29 +1436,29 @@ def test_reference_on_dimension_with_annual_interval(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'Y\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'Y\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'Y\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'Y\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_dimension_with_multiple_references(self): query = slicer.data \ @@ -1472,44 +1472,44 @@ def test_dimension_with_multiple_references(self): self.assertEqual('SELECT ' 'COALESCE(' - '"base"."timestamp",' - 'TIMESTAMPADD(\'day\',1,"dod"."timestamp"),' - 'TIMESTAMPADD(\'year\',1,"yoy"."timestamp")' - ') "timestamp",' + '"$base"."$timestamp",' + 'TIMESTAMPADD(\'day\',1,"$dod"."$timestamp"),' + 'TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")' + ') "$timestamp",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod",' - '("base"."votes"-"yoy"."votes")*100/NULLIF("yoy"."votes",0) "votes_yoy_delta_percent" ' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod",' + '("$base"."$votes"-"$yoy"."$votes")*100/NULLIF("$yoy"."$votes",0) "$votes_yoy_delta_percent" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "yoy" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_reference_joins_nested_query_on_dimensions(self): query = slicer.data \ @@ -1521,33 +1521,33 @@ def test_reference_joins_nested_query_on_dimensions(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' - 'COALESCE("base"."political_party","yoy"."political_party") "political_party",' - '"base"."votes" "votes",' - '"yoy"."votes" "votes_yoy" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' + 'COALESCE("$base"."$political_party","$yoy"."$political_party") "$political_party",' + '"$base"."$votes" "$votes",' + '"$yoy"."$votes" "$votes_yoy" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"political_party" "political_party",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"political_party" "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp","political_party"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp","$political_party"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"political_party" "political_party",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"political_party" "$political_party",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp","political_party"' - ') "yoy" ' # end-nested + 'GROUP BY "$timestamp","$political_party"' + ') "$yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' - 'AND "base"."political_party"="yoy"."political_party" ' - 'ORDER BY "timestamp","political_party"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' + 'AND "$base"."$political_party"="$yoy"."$political_party" ' + 'ORDER BY "$timestamp","$political_party"', str(query)) def test_reference_with_unique_dimension_includes_display_definition(self): query = slicer.data \ @@ -1559,36 +1559,36 @@ def test_reference_with_unique_dimension_includes_display_definition(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' - 'COALESCE("base"."candidate","yoy"."candidate") "candidate",' - 'COALESCE("base"."candidate_display","yoy"."candidate_display") "candidate_display",' - '"base"."votes" "votes",' - '"yoy"."votes" "votes_yoy" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' + 'COALESCE("$base"."$candidate","$yoy"."$candidate") "$candidate",' + 'COALESCE("$base"."$candidate_display","$yoy"."$candidate_display") "$candidate_display",' + '"$base"."$votes" "$votes",' + '"$yoy"."$votes" "$votes_yoy" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"candidate_id" "candidate",' - '"candidate_name" "candidate_display",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"candidate_id" "$candidate",' + '"candidate_name" "$candidate_display",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp","candidate","candidate_display"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp","$candidate","$candidate_display"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"candidate_id" "candidate",' - '"candidate_name" "candidate_display",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"candidate_id" "$candidate",' + '"candidate_name" "$candidate_display",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp","candidate","candidate_display"' - ') "yoy" ' # end-nested + 'GROUP BY "$timestamp","$candidate","$candidate_display"' + ') "$yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' - 'AND "base"."candidate"="yoy"."candidate" ' - 'ORDER BY "timestamp","candidate_display"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' + 'AND "$base"."$candidate"="$yoy"."$candidate" ' + 'ORDER BY "$timestamp","$candidate_display"', str(query)) def test_adjust_reference_dimension_filters_in_reference_query(self): query = slicer.data \ @@ -1601,31 +1601,31 @@ def test_adjust_reference_dimension_filters_in_reference_query(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_filters(self): query = slicer.data \ @@ -1640,33 +1640,33 @@ def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_fil .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' 'AND "political_party" IN (\'d\') ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' 'AND "political_party" IN (\'d\') ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_reference(self): query = slicer.data \ @@ -1677,29 +1677,29 @@ def test_adapt_dow_for_leap_year_for_yoy_reference(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"yoy"."votes" "votes_yoy" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$yoy"."$votes" "$votes_yoy" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'IW\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "timestamp",' - 'SUM("votes") "votes" ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "yoy" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): query = slicer.data \ @@ -1710,29 +1710,29 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"base"."votes"-"yoy"."votes" "votes_yoy_delta" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$base"."$votes"-"$yoy"."$votes" "$votes_yoy_delta" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'IW\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "timestamp",' - 'SUM("votes") "votes" ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "yoy" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): query = slicer.data \ @@ -1743,29 +1743,29 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '("base"."votes"-"yoy"."votes")*100/NULLIF("yoy"."votes",0) "votes_yoy_delta_percent" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '("$base"."$votes"-"$yoy"."$votes")*100/NULLIF("$yoy"."$votes",0) "$votes_yoy_delta_percent" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'IW\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "timestamp",' - 'SUM("votes") "votes" ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "yoy" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_reference_with_date_filter(self): query = slicer.data \ @@ -1777,31 +1777,31 @@ def test_adapt_dow_for_leap_year_for_yoy_reference_with_date_filter(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'year\',1,"yoy"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"yoy"."votes" "votes_yoy" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$yoy"."$votes" "$votes_yoy" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'IW\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "timestamp",' - 'SUM("votes") "votes" ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'year\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'GROUP BY "timestamp"' - ') "yoy" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_adding_duplicate_reference_does_not_join_more_queries(self): query = slicer.data \ @@ -1813,29 +1813,29 @@ def test_adding_duplicate_reference_does_not_join_more_queries(self): .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension(self): query = slicer.data \ @@ -1848,31 +1848,31 @@ def test_use_same_nested_query_for_joining_references_with_same_period_and_dimen .query self.assertEqual('SELECT ' - 'COALESCE("base"."timestamp",TIMESTAMPADD(\'day\',1,"dod"."timestamp")) "timestamp",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod",' - '"base"."votes"-"dod"."votes" "votes_dod_delta",' - '("base"."votes"-"dod"."votes")*100/NULLIF("dod"."votes",0) "votes_dod_delta_percent" ' + 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod",' + '"$base"."$votes"-"$dod"."$votes" "$votes_dod_delta",' + '("$base"."$votes"-"$dod"."$votes")*100/NULLIF("$dod"."$votes",0) "$votes_dod_delta_percent" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension_with_different_periods(self): query = slicer.data \ @@ -1887,45 +1887,45 @@ def test_use_same_nested_query_for_joining_references_with_same_period_and_dimen self.assertEqual('SELECT ' 'COALESCE(' - '"base"."timestamp",' - 'TIMESTAMPADD(\'day\',1,"dod"."timestamp"),' - 'TIMESTAMPADD(\'year\',1,"yoy"."timestamp")' - ') "timestamp",' - '"base"."votes" "votes",' - '"dod"."votes" "votes_dod",' - '"base"."votes"-"dod"."votes" "votes_dod_delta",' - '"yoy"."votes" "votes_yoy",' - '"base"."votes"-"yoy"."votes" "votes_yoy_delta" ' + '"$base"."$timestamp",' + 'TIMESTAMPADD(\'day\',1,"$dod"."$timestamp"),' + 'TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")' + ') "$timestamp",' + '"$base"."$votes" "$votes",' + '"$dod"."$votes" "$votes_dod",' + '"$base"."$votes"-"$dod"."$votes" "$votes_dod_delta",' + '"$yoy"."$votes" "$votes_yoy",' + '"$base"."$votes"-"$yoy"."$votes" "$votes_yoy_delta" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "base" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "dod" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$dod" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'day\',1,"dod"."timestamp") ' + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp"' - ') "yoy" ' # end-nested + 'GROUP BY "$timestamp"' + ') "$yoy" ' # end-nested - 'ON "base"."timestamp"=TIMESTAMPADD(\'year\',1,"yoy"."timestamp") ' - 'ORDER BY "timestamp"', str(query)) + 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' + 'ORDER BY "$timestamp"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -1940,15 +1940,15 @@ def test_dimension_with_join_includes_join_in_query(self): .query self.assertEqual('SELECT ' - 'TRUNC("politician"."timestamp",\'DD\') "timestamp",' - '"politician"."district_id" "district",' - '"district"."district_name" "district_display",' - 'SUM("politician"."votes") "votes" ' + 'TRUNC("politician"."timestamp",\'DD\') "$timestamp",' + '"politician"."district_id" "$district",' + '"district"."district_name" "$district_display",' + 'SUM("politician"."votes") "$votes" ' 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' - 'GROUP BY "timestamp","district","district_display" ' - 'ORDER BY "timestamp","district_display"', str(query)) + 'GROUP BY "$timestamp","$district","$district_display" ' + 'ORDER BY "$timestamp","$district_display"', str(query)) def test_dimension_with_multiple_joins_includes_joins_ordered__in_query(self): query = slicer.data \ @@ -1959,18 +1959,18 @@ def test_dimension_with_multiple_joins_includes_joins_ordered__in_query(self): .query self.assertEqual('SELECT ' - 'TRUNC("politician"."timestamp",\'DD\') "timestamp",' - '"politician"."district_id" "district",' - '"district"."district_name" "district_display",' - 'SUM("politician"."votes") "votes",' - 'COUNT("voter"."id") "voters" ' + 'TRUNC("politician"."timestamp",\'DD\') "$timestamp",' + '"politician"."district_id" "$district",' + '"district"."district_name" "$district_display",' + 'SUM("politician"."votes") "$votes",' + 'COUNT("voter"."id") "$voters" ' 'FROM "politics"."politician" ' 'JOIN "politics"."voter" ' 'ON "politician"."id"="voter"."politician_id" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' - 'GROUP BY "timestamp","district","district_display" ' - 'ORDER BY "timestamp","district_display"', str(query)) + 'GROUP BY "$timestamp","$district","$district_display" ' + 'ORDER BY "$timestamp","$district_display"', str(query)) def test_dimension_with_recursive_join_joins_all_join_tables(self): query = slicer.data \ @@ -1980,17 +1980,17 @@ def test_dimension_with_recursive_join_joins_all_join_tables(self): .query self.assertEqual('SELECT ' - 'TRUNC("politician"."timestamp",\'DD\') "timestamp",' - '"district"."state_id" "state",' - '"state"."state_name" "state_display",' - 'SUM("politician"."votes") "votes" ' + 'TRUNC("politician"."timestamp",\'DD\') "$timestamp",' + '"district"."state_id" "$state",' + '"state"."state_name" "$state_display",' + 'SUM("politician"."votes") "$votes" ' 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' 'JOIN "locations"."state" ' 'ON "district"."state_id"="state"."id" ' - 'GROUP BY "timestamp","state","state_display" ' - 'ORDER BY "timestamp","state_display"', str(query)) + 'GROUP BY "$timestamp","$state","$state_display" ' + 'ORDER BY "$timestamp","$state_display"', str(query)) def test_metric_with_join_includes_join_in_query(self): query = slicer.data \ @@ -1999,13 +1999,13 @@ def test_metric_with_join_includes_join_in_query(self): .query self.assertEqual('SELECT ' - '"politician"."political_party" "political_party",' - 'COUNT("voter"."id") "voters" ' + '"politician"."political_party" "$political_party",' + 'COUNT("voter"."id") "$voters" ' 'FROM "politics"."politician" ' 'JOIN "politics"."voter" ' 'ON "politician"."id"="voter"."politician_id" ' - 'GROUP BY "political_party" ' - 'ORDER BY "political_party"', str(query)) + 'GROUP BY "$political_party" ' + 'ORDER BY "$political_party"', str(query)) def test_dimension_filter_with_join_on_display_definition_does_not_include_join_in_query(self): query = slicer.data \ @@ -2014,7 +2014,7 @@ def test_dimension_filter_with_join_on_display_definition_does_not_include_join_ .query self.assertEqual('SELECT ' - 'SUM("votes") "votes" ' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' 'WHERE "district_id" IN (1)', str(query)) @@ -2025,7 +2025,7 @@ def test_dimension_filter_display_field_with_join_includes_join_in_query(self): .query self.assertEqual('SELECT ' - 'SUM("politician"."votes") "votes" ' + 'SUM("politician"."votes") "$votes" ' 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' @@ -2038,7 +2038,7 @@ def test_dimension_filter_with_recursive_join_includes_join_in_query(self): .query self.assertEqual('SELECT ' - 'SUM("politician"."votes") "votes" ' + 'SUM("politician"."votes") "$votes" ' 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' @@ -2051,7 +2051,7 @@ def test_dimension_filter_with_deep_recursive_join_includes_joins_in_query(self) .query self.assertEqual('SELECT ' - 'SUM("politician"."votes") "votes" ' + 'SUM("politician"."votes") "$votes" ' 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' @@ -2073,11 +2073,11 @@ def test_build_query_order_by_dimension(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp"', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp"', str(query)) def test_build_query_order_by_dimension_display(self): query = slicer.data \ @@ -2087,12 +2087,12 @@ def test_build_query_order_by_dimension_display(self): .query self.assertEqual('SELECT ' - '"candidate_id" "candidate",' - '"candidate_name" "candidate_display",' - 'SUM("votes") "votes" ' + '"candidate_id" "$candidate",' + '"candidate_name" "$candidate_display",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "candidate","candidate_display" ' - 'ORDER BY "candidate_display"', str(query)) + 'GROUP BY "$candidate","$candidate_display" ' + 'ORDER BY "$candidate_display"', str(query)) def test_build_query_order_by_dimension_asc(self): query = slicer.data \ @@ -2102,11 +2102,11 @@ def test_build_query_order_by_dimension_asc(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp" ASC', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp" ASC', str(query)) def test_build_query_order_by_dimension_desc(self): query = slicer.data \ @@ -2116,11 +2116,11 @@ def test_build_query_order_by_dimension_desc(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp" DESC', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp" DESC', str(query)) def test_build_query_order_by_metric(self): query = slicer.data \ @@ -2130,11 +2130,11 @@ def test_build_query_order_by_metric(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "votes"', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$votes"', str(query)) def test_build_query_order_by_metric_asc(self): query = slicer.data \ @@ -2144,11 +2144,11 @@ def test_build_query_order_by_metric_asc(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "votes" ASC', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$votes" ASC', str(query)) def test_build_query_order_by_metric_desc(self): query = slicer.data \ @@ -2158,11 +2158,11 @@ def test_build_query_order_by_metric_desc(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "votes" DESC', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$votes" DESC', str(query)) def test_build_query_order_by_multiple_dimensions(self): query = slicer.data \ @@ -2173,13 +2173,13 @@ def test_build_query_order_by_multiple_dimensions(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"candidate_id" "candidate",' - '"candidate_name" "candidate_display",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"candidate_id" "$candidate",' + '"candidate_name" "$candidate_display",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp","candidate","candidate_display" ' - 'ORDER BY "timestamp","candidate"', str(query)) + 'GROUP BY "$timestamp","$candidate","$candidate_display" ' + 'ORDER BY "$timestamp","$candidate"', str(query)) def test_build_query_order_by_multiple_dimensions_with_different_orientations(self): query = slicer.data \ @@ -2190,13 +2190,13 @@ def test_build_query_order_by_multiple_dimensions_with_different_orientations(se .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - '"candidate_id" "candidate",' - '"candidate_name" "candidate_display",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + '"candidate_id" "$candidate",' + '"candidate_name" "$candidate_display",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp","candidate","candidate_display" ' - 'ORDER BY "timestamp" DESC,"candidate" ASC', str(query)) + 'GROUP BY "$timestamp","$candidate","$candidate_display" ' + 'ORDER BY "$timestamp" DESC,"$candidate" ASC', str(query)) def test_build_query_order_by_metrics_and_dimensions(self): query = slicer.data \ @@ -2207,11 +2207,11 @@ def test_build_query_order_by_metrics_and_dimensions(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp","votes"', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp","$votes"', str(query)) def test_build_query_order_by_metrics_and_dimensions_with_different_orientations(self): query = slicer.data \ @@ -2222,11 +2222,11 @@ def test_build_query_order_by_metrics_and_dimensions_with_different_orientations .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp" ASC,"votes" DESC', str(query)) + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp" ASC,"$votes" DESC', str(query)) @patch('fireant.slicer.queries.builder.fetch_data') @@ -2239,11 +2239,11 @@ def test_set_limit(self, mock_fetch_data: Mock): mock_fetch_data.assert_called_once_with(ANY, 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp" LIMIT 20', + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp" LIMIT 20', dimensions=DimensionMatcher(slicer.dimensions.timestamp)) def test_set_offset(self, mock_fetch_data: Mock): @@ -2254,11 +2254,11 @@ def test_set_offset(self, mock_fetch_data: Mock): mock_fetch_data.assert_called_once_with(ANY, 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp" ' + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp" ' 'OFFSET 20', dimensions=DimensionMatcher(slicer.dimensions.timestamp)) @@ -2270,11 +2270,11 @@ def test_set_limit_and_offset(self, mock_fetch_data: Mock): mock_fetch_data.assert_called_once_with(ANY, 'SELECT ' - 'TRUNC("timestamp",\'DD\') "timestamp",' - 'SUM("votes") "votes" ' + 'TRUNC("timestamp",\'DD\') "$timestamp",' + 'SUM("votes") "$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "timestamp" ' - 'ORDER BY "timestamp" ' + 'GROUP BY "$timestamp" ' + 'ORDER BY "$timestamp" ' 'LIMIT 20 ' 'OFFSET 20', dimensions=DimensionMatcher(slicer.dimensions.timestamp)) @@ -2321,7 +2321,7 @@ def test_pass_query_from_builder_as_arg(self, mock_fetch_data: Mock): .fetch() mock_fetch_data.assert_called_once_with(ANY, - 'SELECT SUM("votes") "votes" ' + 'SELECT SUM("votes") "$votes" ' 'FROM "politics"."politician"', dimensions=ANY) diff --git a/fireant/tests/slicer/queries/test_database.py b/fireant/tests/slicer/queries/test_database.py index fd8def54..befc0f2d 100644 --- a/fireant/tests/slicer/queries/test_database.py +++ b/fireant/tests/slicer/queries/test_database.py @@ -1,4 +1,3 @@ -import time from unittest import TestCase from unittest.mock import ( MagicMock, @@ -8,6 +7,7 @@ import numpy as np import pandas as pd +import time from fireant.slicer.queries.database import ( clean_and_apply_index, @@ -21,6 +21,10 @@ slicer, uni_dim_df, ) +from fireant.utils import ( + format_key, + format_key as f, +) class FetchDataTests(TestCase): @@ -42,7 +46,8 @@ def test_fetch_data_called_on_database(self): def test_index_set_on_data_frame_result(self): fetch_data(self.mock_database, self.mock_query, self.mock_dimensions) - self.mock_data_frame.set_index.assert_called_once_with([d.key for d in self.mock_dimensions]) + self.mock_data_frame.set_index.assert_called_once_with([f(d.key) + for d in self.mock_dimensions]) @patch('fireant.slicer.queries.database.query_logger.debug') def test_debug_query_log_called_with_query(self, mock_logger): @@ -97,7 +102,7 @@ def add_nans(df): cont_uni_dim_nans_df = cont_uni_dim_df \ - .append(cont_uni_dim_df.groupby(level='timestamp').apply(add_nans)) \ + .append(cont_uni_dim_df.groupby(level=format_key('timestamp')).apply(add_nans)) \ .sort_index() @@ -108,7 +113,7 @@ def totals(df): cont_uni_dim_nans_totals_df = cont_uni_dim_nans_df \ - .append(cont_uni_dim_nans_df.groupby(level='timestamp').apply(totals)) \ + .append(cont_uni_dim_nans_df.groupby(level=format_key('timestamp')).apply(totals)) \ .sort_index() \ .sort_index(level=[0, 1], ascending=False) # This sorts the DF so that the first instance of NaN is the totals diff --git a/fireant/tests/slicer/queries/test_dimension_choices.py b/fireant/tests/slicer/queries/test_dimension_choices.py index 168f36cb..b5a59d26 100644 --- a/fireant/tests/slicer/queries/test_dimension_choices.py +++ b/fireant/tests/slicer/queries/test_dimension_choices.py @@ -13,9 +13,9 @@ def test_query_choices_for_cat_dimension(self): .query self.assertEqual('SELECT ' - '"political_party" "political_party" ' + '"political_party" "$political_party" ' 'FROM "politics"."politician" ' - 'GROUP BY "political_party"', str(query)) + 'GROUP BY "$political_party"', str(query)) def test_query_choices_for_uni_dimension(self): query = slicer.dimensions.candidate \ @@ -23,10 +23,10 @@ def test_query_choices_for_uni_dimension(self): .query self.assertEqual('SELECT ' - '"candidate_id" "candidate",' - '"candidate_name" "candidate_display" ' + '"candidate_id" "$candidate",' + '"candidate_name" "$candidate_display" ' 'FROM "politics"."politician" ' - 'GROUP BY "candidate","candidate_display"', str(query)) + 'GROUP BY "$candidate","$candidate_display"', str(query)) def test_query_choices_for_uni_dimension_with_join(self): query = slicer.dimensions.district \ @@ -34,12 +34,12 @@ def test_query_choices_for_uni_dimension_with_join(self): .query self.assertEqual('SELECT ' - '"politician"."district_id" "district",' - '"district"."district_name" "district_display" ' + '"politician"."district_id" "$district",' + '"district"."district_name" "$district_display" ' 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' - 'GROUP BY "district","district_display"', str(query)) + 'GROUP BY "$district","$district_display"', str(query)) def test_no_choices_attr_for_datetime_dimension(self): with self.assertRaises(AttributeError): @@ -56,8 +56,8 @@ def test_filter_choices(self): .query self.assertEqual('SELECT ' - '"candidate_id" "candidate",' - '"candidate_name" "candidate_display" ' + '"candidate_id" "$candidate",' + '"candidate_name" "$candidate_display" ' 'FROM "politics"."politician" ' 'WHERE "political_party" IN (\'d\',\'r\') ' - 'GROUP BY "candidate","candidate_display"', str(query)) + 'GROUP BY "$candidate","$candidate_display"', str(query)) diff --git a/fireant/tests/slicer/test_operations.py b/fireant/tests/slicer/test_operations.py index 9ba4164e..b4748e55 100644 --- a/fireant/tests/slicer/test_operations.py +++ b/fireant/tests/slicer/test_operations.py @@ -24,7 +24,7 @@ def test_apply_to_timeseries(self): result = cumsum.apply(cont_dim_df) expected = pd.Series([2, 4, 6, 8, 10, 12], - name='wins', + name='$wins', index=cont_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -33,7 +33,7 @@ def test_apply_to_timeseries_with_uni_dim(self): result = cumsum.apply(cont_uni_dim_df) expected = pd.Series([1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6], - name='wins', + name='$wins', index=cont_uni_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -42,7 +42,7 @@ def test_apply_to_timeseries_with_uni_dim_and_ref(self): result = cumsum.apply(cont_uni_dim_ref_df) expected = pd.Series([1, 1, 2, 2, 3, 3, 4, 4, 5, 5], - name='wins', + name='$wins', index=cont_uni_dim_ref_df.index) pandas.testing.assert_series_equal(result, expected) @@ -53,7 +53,7 @@ def test_apply_to_timeseries(self): result = cumprod.apply(cont_dim_df) expected = pd.Series([2, 4, 8, 16, 32, 64], - name='wins', + name='$wins', index=cont_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -62,7 +62,7 @@ def test_apply_to_timeseries_with_uni_dim(self): result = cumprod.apply(cont_uni_dim_df) expected = pd.Series([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], - name='wins', + name='$wins', index=cont_uni_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -71,7 +71,7 @@ def test_apply_to_timeseries_with_uni_dim_and_ref(self): result = cumprod.apply(cont_uni_dim_ref_df) expected = pd.Series([1, 1, 1, 1, 1, 1, 1, 1, 1, 1], - name='wins', + name='$wins', index=cont_uni_dim_ref_df.index) pandas.testing.assert_series_equal(result, expected) @@ -82,7 +82,7 @@ def test_apply_to_timeseries(self): result = cummean.apply(cont_dim_df) expected = pd.Series([15220449.0, 15941233.0, 17165799.3, 18197903.25, 18672764.6, 18612389.3], - name='votes', + name='$votes', index=cont_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -92,7 +92,7 @@ def test_apply_to_timeseries_with_uni_dim(self): expected = pd.Series([5574387.0, 9646062.0, 5903886.0, 10037347.0, 6389131.0, 10776668.3, 6793838.5, 11404064.75, 7010664.2, 11662100.4, 6687706.0, 11924683.3], - name='votes', + name='$votes', index=cont_uni_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -102,7 +102,7 @@ def test_apply_to_timeseries_with_uni_dim_and_ref(self): expected = pd.Series([6233385.0, 10428632.0, 6796503.0, 11341971.5, 7200322.3, 11990065.6, 7369733.5, 12166110.0, 6910369.8, 12380407.6], - name='votes', + name='$votes', index=cont_uni_dim_ref_df.index) pandas.testing.assert_series_equal(result, expected) @@ -113,7 +113,7 @@ def test_apply_to_timeseries(self): result = rolling_mean.apply(cont_dim_df) expected = pd.Series([np.nan, np.nan, 2.0, 2.0, 2.0, 2.0], - name='wins', + name='$wins', index=cont_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -122,7 +122,7 @@ def test_apply_to_timeseries_with_uni_dim(self): result = rolling_mean.apply(cont_uni_dim_df) expected = pd.Series([np.nan, np.nan, np.nan, np.nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], - name='wins', + name='$wins', index=cont_uni_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -131,6 +131,6 @@ def test_apply_to_timeseries_with_uni_dim_and_ref(self): result = rolling_mean.apply(cont_uni_dim_ref_df) expected = pd.Series([np.nan, np.nan, np.nan, np.nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], - name='wins', + name='$wins', index=cont_uni_dim_ref_df.index) pandas.testing.assert_series_equal(result, expected) diff --git a/fireant/tests/slicer/widgets/test_csv.py b/fireant/tests/slicer/widgets/test_csv.py index 44faaf8b..37dcb6e6 100644 --- a/fireant/tests/slicer/widgets/test_csv.py +++ b/fireant/tests/slicer/widgets/test_csv.py @@ -17,6 +17,7 @@ slicer, uni_dim_df, ) +from fireant.utils import format_key as f class CSVWidgetTests(TestCase): @@ -26,7 +27,7 @@ def test_single_metric(self): result = CSV(slicer.metrics.votes) \ .transform(single_metric_df, slicer, [], []) - expected = single_metric_df.copy()[['votes']] + expected = single_metric_df.copy()[[f('votes')]] expected.columns = ['Votes'] self.assertEqual(result, expected.to_csv()) @@ -35,7 +36,7 @@ def test_multiple_metrics(self): result = CSV(slicer.metrics.votes, slicer.metrics.wins) \ .transform(multi_metric_df, slicer, [], []) - expected = multi_metric_df.copy()[['votes', 'wins']] + expected = multi_metric_df.copy()[[f('votes'), f('wins')]] expected.columns = ['Votes', 'Wins'] self.assertEqual(result, expected.to_csv()) @@ -44,7 +45,7 @@ def test_multiple_metrics_reversed(self): result = CSV(slicer.metrics.wins, slicer.metrics.votes) \ .transform(multi_metric_df, slicer, [], []) - expected = multi_metric_df.copy()[['wins', 'votes']] + expected = multi_metric_df.copy()[[f('wins'), f('votes')]] expected.columns = ['Wins', 'Votes'] self.assertEqual(result, expected.to_csv()) @@ -53,7 +54,7 @@ def test_time_series_dim(self): result = CSV(slicer.metrics.wins) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) - expected = cont_dim_df.copy()[['wins']] + expected = cont_dim_df.copy()[[f('wins')]] expected.index.names = ['Timestamp'] expected.columns = ['Wins'] @@ -63,7 +64,7 @@ def test_time_series_dim_with_operation(self): result = CSV(CumSum(slicer.metrics.votes)) \ .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp], []) - expected = cont_dim_operation_df.copy()[['cumsum(votes)']] + expected = cont_dim_operation_df.copy()[[f('cumsum(votes)')]] expected.index.names = ['Timestamp'] expected.columns = ['CumSum(Votes)'] @@ -73,7 +74,7 @@ def test_cat_dim(self): result = CSV(slicer.metrics.wins) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) - expected = cat_dim_df.copy()[['wins']] + expected = cat_dim_df.copy()[[f('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] @@ -84,9 +85,8 @@ def test_uni_dim(self): .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) expected = uni_dim_df.copy() \ - .set_index('candidate_display', append=True) \ - .reset_index('candidate', drop=True) \ - [['wins']] + .set_index(f('candidate_display'), append=True) \ + .reset_index(f('candidate'), drop=True)[[f('wins')]] expected.index.names = ['Candidate'] expected.columns = ['Wins'] @@ -99,12 +99,12 @@ def test_uni_dim_no_display_definition(self): candidate.display_definition = None uni_dim_df_copy = uni_dim_df.copy() - del uni_dim_df_copy[slicer.dimensions.candidate.display_key] + del uni_dim_df_copy[f(slicer.dimensions.candidate.display_key)] result = CSV(slicer.metrics.wins) \ .transform(uni_dim_df_copy, slicer, [candidate], []) - expected = uni_dim_df_copy.copy()[['wins']] + expected = uni_dim_df_copy.copy()[[f('wins')]] expected.index.names = ['Candidate'] expected.columns = ['Wins'] @@ -115,8 +115,8 @@ def test_multi_dims_time_series_and_uni(self): .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ - .set_index('state_display', append=True) \ - .reset_index('state', drop=False)[['wins']] + .set_index(f('state_display'), append=True) \ + .reset_index(f('state'), drop=False)[[f('wins')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Wins'] @@ -126,7 +126,7 @@ def test_pivoted_single_dimension_no_effect(self): result = CSV(slicer.metrics.wins, pivot=True) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) - expected = cat_dim_df.copy()[['wins']] + expected = cat_dim_df.copy()[[f('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] @@ -136,7 +136,7 @@ def test_pivoted_multi_dims_time_series_and_cat(self): result = CSV(slicer.metrics.wins, pivot=True) \ .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party], []) - expected = cont_cat_dim_df.copy()[['wins']] + expected = cont_cat_dim_df.copy()[[f('wins')]] expected.index.names = ['Timestamp', 'Party'] expected.columns = ['Wins'] expected = expected.unstack(level=[1]) @@ -148,8 +148,8 @@ def test_pivoted_multi_dims_time_series_and_uni(self): .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ - .set_index('state_display', append=True) \ - .reset_index('state', drop=True)[['votes']] + .set_index(f('state_display'), append=True) \ + .reset_index(f('state'), drop=True)[[f('votes')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Votes'] expected = expected.unstack(level=[1]) @@ -168,8 +168,8 @@ def test_time_series_ref(self): ]) expected = cont_uni_dim_ref_df.copy() \ - .set_index('state_display', append=True) \ - .reset_index('state', drop=True)[['votes', 'votes_eoe']] + .set_index(f('state_display'), append=True) \ + .reset_index(f('state'), drop=True)[[f('votes'), f('votes_eoe')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Votes', 'Votes (EoE)'] diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py index 194da18c..883aee9b 100644 --- a/fireant/tests/slicer/widgets/test_datatables.py +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -24,6 +24,7 @@ slicer, uni_dim_df, ) +from fireant.utils import format_key as f class DataTablesTransformerTests(TestCase): @@ -237,7 +238,7 @@ def test_uni_dim_no_display_definition(self): candidate.display_definition = None uni_dim_df_copy = uni_dim_df.copy() - del uni_dim_df_copy[slicer.dimensions.candidate.display_key] + del uni_dim_df_copy[f(slicer.dimensions.candidate.display_key)] result = DataTablesJS(slicer.metrics.wins) \ .transform(uni_dim_df_copy, slicer, [candidate], []) diff --git a/fireant/tests/slicer/widgets/test_pandas.py b/fireant/tests/slicer/widgets/test_pandas.py index 3bc9bb28..0af133c7 100644 --- a/fireant/tests/slicer/widgets/test_pandas.py +++ b/fireant/tests/slicer/widgets/test_pandas.py @@ -18,6 +18,7 @@ slicer, uni_dim_df, ) +from fireant.utils import format_key as f class DataTablesTransformerTests(TestCase): @@ -27,7 +28,7 @@ def test_single_metric(self): result = Pandas(slicer.metrics.votes) \ .transform(single_metric_df, slicer, [], []) - expected = single_metric_df.copy()[['votes']] + expected = single_metric_df.copy()[[f('votes')]] expected.columns = ['Votes'] pandas.testing.assert_frame_equal(result, expected) @@ -36,7 +37,7 @@ def test_multiple_metrics(self): result = Pandas(slicer.metrics.votes, slicer.metrics.wins) \ .transform(multi_metric_df, slicer, [], []) - expected = multi_metric_df.copy()[['votes', 'wins']] + expected = multi_metric_df.copy()[[f('votes'), f('wins')]] expected.columns = ['Votes', 'Wins'] pandas.testing.assert_frame_equal(result, expected) @@ -45,7 +46,7 @@ def test_multiple_metrics_reversed(self): result = Pandas(slicer.metrics.wins, slicer.metrics.votes) \ .transform(multi_metric_df, slicer, [], []) - expected = multi_metric_df.copy()[['wins', 'votes']] + expected = multi_metric_df.copy()[[f('wins'), f('votes')]] expected.columns = ['Wins', 'Votes'] pandas.testing.assert_frame_equal(result, expected) @@ -54,7 +55,7 @@ def test_time_series_dim(self): result = Pandas(slicer.metrics.wins) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) - expected = cont_dim_df.copy()[['wins']] + expected = cont_dim_df.copy()[[f('wins')]] expected.index.names = ['Timestamp'] expected.columns = ['Wins'] @@ -64,7 +65,7 @@ def test_time_series_dim_with_operation(self): result = Pandas(CumSum(slicer.metrics.votes)) \ .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp], []) - expected = cont_dim_operation_df.copy()[['cumsum(votes)']] + expected = cont_dim_operation_df.copy()[[f('cumsum(votes)')]] expected.index.names = ['Timestamp'] expected.columns = ['CumSum(Votes)'] @@ -74,7 +75,7 @@ def test_cat_dim(self): result = Pandas(slicer.metrics.wins) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) - expected = cat_dim_df.copy()[['wins']] + expected = cat_dim_df.copy()[[f('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] @@ -85,9 +86,9 @@ def test_uni_dim(self): .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) expected = uni_dim_df.copy() \ - .set_index('candidate_display', append=True) \ - .reset_index('candidate', drop=True) \ - [['wins']] + .set_index(f('candidate_display'), append=True) \ + .reset_index(f('candidate'), drop=True) \ + [[f('wins')]] expected.index.names = ['Candidate'] expected.columns = ['Wins'] @@ -100,12 +101,12 @@ def test_uni_dim_no_display_definition(self): candidate.display_definition = None uni_dim_df_copy = uni_dim_df.copy() - del uni_dim_df_copy[slicer.dimensions.candidate.display_key] + del uni_dim_df_copy[f(slicer.dimensions.candidate.display_key)] result = Pandas(slicer.metrics.wins) \ .transform(uni_dim_df_copy, slicer, [candidate], []) - expected = uni_dim_df_copy.copy()[['wins']] + expected = uni_dim_df_copy.copy()[[f('wins')]] expected.index.names = ['Candidate'] expected.columns = ['Wins'] @@ -116,8 +117,8 @@ def test_multi_dims_time_series_and_uni(self): .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ - .set_index('state_display', append=True) \ - .reset_index('state', drop=False)[['wins']] + .set_index(f('state_display'), append=True) \ + .reset_index(f('state'), drop=False)[[f('wins')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Wins'] @@ -127,7 +128,7 @@ def test_pivoted_single_dimension_no_effect(self): result = Pandas(slicer.metrics.wins, pivot=True) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) - expected = cat_dim_df.copy()[['wins']] + expected = cat_dim_df.copy()[[f('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] @@ -137,7 +138,7 @@ def test_pivoted_multi_dims_time_series_and_cat(self): result = Pandas(slicer.metrics.wins, pivot=True) \ .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party], []) - expected = cont_cat_dim_df.copy()[['wins']] + expected = cont_cat_dim_df.copy()[[f('wins')]] expected.index.names = ['Timestamp', 'Party'] expected.columns = ['Wins'] expected = expected.unstack(level=[1]) @@ -149,8 +150,8 @@ def test_pivoted_multi_dims_time_series_and_uni(self): .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ - .set_index('state_display', append=True) \ - .reset_index('state', drop=True)[['votes']] + .set_index(f('state_display'), append=True) \ + .reset_index(f('state'), drop=True)[[f('votes')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Votes'] expected = expected.unstack(level=[1]) @@ -168,8 +169,8 @@ def test_time_series_ref(self): ]) expected = cont_uni_dim_ref_df.copy() \ - .set_index('state_display', append=True) \ - .reset_index('state', drop=True)[['votes', 'votes_eoe']] + .set_index(f('state_display'), append=True) \ + .reset_index(f('state'), drop=True)[[f('votes'), f('votes_eoe')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Votes', 'Votes (EoE)'] @@ -186,9 +187,9 @@ def test_metric_format(self): result = Pandas(votes) \ .transform(cont_dim_df / 3, slicer, [slicer.dimensions.timestamp], []) - expected = cont_dim_df.copy()[['votes']] - expected['votes'] = ['${0:.2f}€'.format(x) - for x in expected['votes'] / 3] + expected = cont_dim_df.copy()[[f('votes')]] + expected[f('votes')] = ['${0:.2f}€'.format(x) + for x in expected[f('votes')] / 3] expected.index.names = ['Timestamp'] expected.columns = ['Votes'] diff --git a/fireant/utils.py b/fireant/utils.py index ecc662cb..319b60d8 100644 --- a/fireant/utils.py +++ b/fireant/utils.py @@ -115,3 +115,9 @@ def groupby_first_level(index): return [x[1:] for x in list(index) if x[1:] not in seen and not seen.add(x[1:])] + + +def format_key(key): + if key is None: + return key + return '${}'.format(key) diff --git a/requirements.txt b/requirements.txt index 4a9eb3d3..7c2e2c7b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ six pandas==0.22.0 -pypika==0.14.5 +pypika==0.14.6 pymysql==0.8.0 toposort==1.5 typing==3.6.2 diff --git a/setup.py b/setup.py index a6470b5b..46c2d4ab 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ def find_version(*file_paths): install_requires=[ 'six', 'pandas==0.22.0', - 'pypika==0.14.5', + 'pypika==0.14.6', 'toposort==1.5', 'typing==3.6.2', ], From 9aef44bc92b52f01fe8c199f6502c4c928b45e00 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 5 Jul 2018 12:50:13 +0200 Subject: [PATCH 076/123] Bumped version to 1.0.0.dev32 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index a2cc8978..2a27d546 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev31' +__version__ = '1.0.0.dev32' From 35e24d597bc05fd91dd6b0af761ae05b0bbfb74f Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 5 Jul 2018 16:33:48 +0200 Subject: [PATCH 077/123] Fixed dimension choices query for unique dimensions with display definition --- fireant/slicer/queries/builder.py | 6 +-- .../slicer/queries/test_dimension_choices.py | 37 +++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index eddec28a..8d397eb1 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -255,9 +255,9 @@ def fetch(self, limit=None, offset=None, hint=None, force_include=()) -> pd.Seri str(query), dimensions=self._dimensions) - display_key = getattr(dimension, 'display_key', None) - if display_key is not None: - return data[display_key] + df_key = format_key(getattr(dimension, 'display_key', None)) + if df_key is not None: + return data[df_key] display_key = 'display' if hasattr(dimension, 'display_values'): diff --git a/fireant/tests/slicer/queries/test_dimension_choices.py b/fireant/tests/slicer/queries/test_dimension_choices.py index b5a59d26..fb195c71 100644 --- a/fireant/tests/slicer/queries/test_dimension_choices.py +++ b/fireant/tests/slicer/queries/test_dimension_choices.py @@ -1,5 +1,11 @@ from unittest import TestCase +from unittest.mock import ( + ANY, + Mock, + patch, +) +from ..matchers import DimensionMatcher from ..mocks import slicer @@ -61,3 +67,34 @@ def test_filter_choices(self): 'FROM "politics"."politician" ' 'WHERE "political_party" IN (\'d\',\'r\') ' 'GROUP BY "$candidate","$candidate_display"', str(query)) + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +@patch('fireant.slicer.queries.builder.fetch_data') +class DimensionsChoicesFetchTests(TestCase): + def test_query_choices_for_cat_dimension(self, mock_fetch_data: Mock): + slicer.dimensions.political_party \ + .choices \ + .fetch() + + mock_fetch_data.assert_called_once_with(ANY, + 'SELECT ' + '"political_party" "$political_party" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$political_party" ' + 'ORDER BY "$political_party"', + dimensions=DimensionMatcher(slicer.dimensions.political_party)) + + def test_query_choices_for_uni_dimension(self, mock_fetch_data: Mock): + slicer.dimensions.candidate \ + .choices \ + .fetch() + + mock_fetch_data.assert_called_once_with(ANY, + 'SELECT ' + '"candidate_id" "$candidate",' + '"candidate_name" "$candidate_display" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$candidate","$candidate_display" ' + 'ORDER BY "$candidate_display"', + dimensions=DimensionMatcher(slicer.dimensions.candidate)) From 9f58004558a49cd2c194b95c518f731e7a3ab9a1 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 5 Jul 2018 16:54:10 +0200 Subject: [PATCH 078/123] Bumped version to 1.0.0.dev33 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 2a27d546..dbcb7577 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev32' +__version__ = '1.0.0.dev33' From b968a3b312e4a7c0cfcf7b027de91fdda8f67773 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 6 Jul 2018 10:10:01 +0200 Subject: [PATCH 079/123] Moved the limit/offset functions to the slicer query builder function from the fetch function --- fireant/slicer/queries/builder.py | 42 +++++++++++++------- fireant/tests/slicer/queries/test_builder.py | 12 ++++-- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 8d397eb1..b34f4b47 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -34,6 +34,8 @@ def __init__(self, slicer, table): self._dimensions = [] self._filters = [] self._references = [] + self._limit = None + self._offset = None @immutable def filter(self, *filters): @@ -43,6 +45,22 @@ def filter(self, *filters): """ self._filters += filters + @immutable + def limit(self, limit): + """ + :param limit: + A limit on the number of database rows returned. + """ + self._limit = limit + + @immutable + def offset(self, offset): + """ + :param offset: + A offset on the number of database rows returned. + """ + self._offset = offset + @property def query(self): """ @@ -143,22 +161,18 @@ def query(self): for (term, orientation) in orders: query = query.orderby(term, order=orientation) - return query + return query.limit(self._limit).offset(self._offset) - def fetch(self, limit=None, offset=None, hint=None) -> Iterable[Dict]: + def fetch(self, hint=None) -> Iterable[Dict]: """ Fetch the data for this query and transform it into the widgets. - :param limit: - A limit on the number of database rows returned. - :param offset: - A offset on the number of database rows returned. :param hint: A query hint label used with database vendors which support it. Adds a label comment to the query. :return: A list of dict (JSON) objects containing the widget configurations. """ - query = self.query.limit(limit).offset(offset) + query = self.query if hint and hasattr(query, 'hint') and callable(query.hint): query = query.hint(hint) @@ -216,16 +230,14 @@ def query(self): base_table=self.table, joins=self.slicer.joins, dimensions=self._dimensions, - filters=self._filters) + filters=self._filters) \ + .limit(self._limit) \ + .offset(self._offset) - def fetch(self, limit=None, offset=None, hint=None, force_include=()) -> pd.Series: + def fetch(self, hint=None, force_include=()) -> pd.Series: """ Fetch the data for this query and transform it into the widgets. - :param limit: - A limit on the number of database rows returned. - :param offset: - A offset on the number of database rows returned. :param force_include: A list of dimension values to include in the result set. This can be used to avoid having necessary results cut off due to the pagination. These results will be returned at the head of the results. @@ -248,8 +260,8 @@ def fetch(self, limit=None, offset=None, hint=None, force_include=()) -> pd.Seri # Ensure that these values are included query = query.orderby(include, order=Order.desc) - # Add ordering and pagination - query = query.orderby(definition).limit(limit).offset(offset) + # Order by the dimension definition that the choices are for + query = query.orderby(definition) data = fetch_data(self.slicer.database, str(query), diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 99e3618e..63a8fd46 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -2235,7 +2235,8 @@ def test_set_limit(self, mock_fetch_data: Mock): slicer.data \ .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ - .fetch(limit=20) + .limit(20) \ + .fetch() mock_fetch_data.assert_called_once_with(ANY, 'SELECT ' @@ -2250,7 +2251,8 @@ def test_set_offset(self, mock_fetch_data: Mock): slicer.data \ .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ - .fetch(offset=20) + .offset(20) \ + .fetch() mock_fetch_data.assert_called_once_with(ANY, 'SELECT ' @@ -2266,7 +2268,9 @@ def test_set_limit_and_offset(self, mock_fetch_data: Mock): slicer.data \ .widget(f.DataTablesJS(slicer.metrics.votes)) \ .dimension(slicer.dimensions.timestamp) \ - .fetch(limit=20, offset=20) + .limit(20) \ + .offset(30) \ + .fetch() mock_fetch_data.assert_called_once_with(ANY, 'SELECT ' @@ -2276,7 +2280,7 @@ def test_set_limit_and_offset(self, mock_fetch_data: Mock): 'GROUP BY "$timestamp" ' 'ORDER BY "$timestamp" ' 'LIMIT 20 ' - 'OFFSET 20', + 'OFFSET 30', dimensions=DimensionMatcher(slicer.dimensions.timestamp)) From a8ff2deb8c2a121a17e6b4a70bedfb5e71322551 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 6 Jul 2018 10:20:23 +0200 Subject: [PATCH 080/123] Bumped version to 1.0.0.dev34 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index dbcb7577..9aa63849 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev33' +__version__ = '1.0.0.dev34' From 2c8fe4e9a984fc91f0c18b0b4569e74b6e66bafa Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 6 Jul 2018 10:34:30 +0200 Subject: [PATCH 081/123] Fixed the key used to store operations in the data query result data frame --- fireant/slicer/queries/builder.py | 3 ++- fireant/tests/slicer/queries/test_builder.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index b34f4b47..1ce17f57 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -183,7 +183,8 @@ def fetch(self, hint=None) -> Iterable[Dict]: # Apply operations operations = find_operations_for_widgets(self._widgets) for operation in operations: - data_frame[operation.key] = operation.apply(data_frame) + df_key = format_key(operation.key) + data_frame[df_key] = operation.apply(data_frame) # Apply transformations return [widget.transform(data_frame, self.slicer, self._dimensions, self._references) diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 63a8fd46..ee57f4a8 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -9,6 +9,7 @@ from pypika import Order import fireant as f +from fireant.utils import format_key from fireant.slicer.exceptions import ( MetricRequiredException, ) @@ -2426,5 +2427,6 @@ def test_operations_results_stored_in_data_frame(self, mock_fetch_data: Mock): .widget(mock_widget) \ .fetch() - self.assertIn(mock_operation.key, mock_df) - self.assertEqual(mock_df[mock_operation.key], mock_operation.apply.return_value) + f_op_key = format_key(mock_operation.key) + self.assertIn(f_op_key, mock_df) + self.assertEqual(mock_df[f_op_key], mock_operation.apply.return_value) From f1209afe978ce6cb4bb5ffa1131063ea420fcda1 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 6 Jul 2018 10:34:47 +0200 Subject: [PATCH 082/123] Bumped version to 1.0.0.dev35 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 9aa63849..e75489f4 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev34' +__version__ = '1.0.0.dev35' From 49b0368dd94f9c04ec8c25ee963aadb44d5e78a8 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 6 Jul 2018 11:49:08 +0200 Subject: [PATCH 083/123] Upgraded pypika to 0.14.7 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7c2e2c7b..3498df31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ six pandas==0.22.0 -pypika==0.14.6 +pypika==0.14.7 pymysql==0.8.0 toposort==1.5 typing==3.6.2 diff --git a/setup.py b/setup.py index 46c2d4ab..d1789b5a 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ def find_version(*file_paths): install_requires=[ 'six', 'pandas==0.22.0', - 'pypika==0.14.6', + 'pypika==0.14.7', 'toposort==1.5', 'typing==3.6.2', ], From d661a26025458772184e10f1a08082efe4ee5b39 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 9 Jul 2018 11:15:54 +0200 Subject: [PATCH 084/123] Fixed the pivoted data tables widgets to render the data in the correct dimension order and removed column definitions for dimension value combinations that do not exist in the data frame. --- fireant/slicer/widgets/datatables.py | 15 +++++++-------- fireant/slicer/widgets/helpers.py | 12 +++++++++--- fireant/tests/slicer/widgets/test_datatables.py | 10 +++++----- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index 7e542920..abb4c02f 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -1,5 +1,3 @@ -import itertools - import pandas as pd from fireant import ( @@ -68,8 +66,8 @@ def _render_dimensional_metric_cell(row_data: pd.Series, metric: Metric): # Group by the last dimension, drop it, and fill the dict with either the raw metric values or the next level of # dicts. - for key, next_row in row_data.groupby(level=-1): - next_row.reset_index(level=-1, drop=True, inplace=True) + for key, next_row in row_data.groupby(level=1): + next_row.reset_index(level=1, drop=True, inplace=True) df_key = format_key(metric.key) level[key] = _render_dimensional_metric_cell(next_row, metric) \ @@ -214,14 +212,15 @@ def _metric_columns_pivoted(self, references, df_columns, render_column_label): :return: """ columns = [] + single_metric = 1 == len(self.items) for metric in self.items: - dimension_value_sets = [list(level) - for level in df_columns.levels[1:]] + dimension_value_sets = [row[1:] + for row in list(df_columns)] - for dimension_values in itertools.product(*dimension_value_sets): + for dimension_values in dimension_value_sets: for reference in [None] + references: key = reference_key(metric, reference) - title = render_column_label(dimension_values, metric, reference) + title = render_column_label(dimension_values, None if single_metric else metric, reference) data = '.'.join([key] + [str(x) for x in dimension_values]) columns.append(dict(title=title, diff --git a/fireant/slicer/widgets/helpers.py b/fireant/slicer/widgets/helpers.py index 26aabfb2..4cef2d77 100644 --- a/fireant/slicer/widgets/helpers.py +++ b/fireant/slicer/widgets/helpers.py @@ -61,7 +61,9 @@ def render_series_label(dimension_values, metric=None, reference=None): a tuple of dimension values. Can be zero-length or longer. :return: """ - used_dimensions = dimensions if metric is None else dimensions[1:] + num_used_dimensions = len(dimensions) - len(dimension_values) + used_dimensions = dimensions[num_used_dimensions:] + dimension_values = utils.wrap_list(dimension_values) dimension_labels = [utils.deep_get(dimension_display_values, [utils.format_key(dimension.key), dimension_value], @@ -70,12 +72,16 @@ def render_series_label(dimension_values, metric=None, reference=None): else 'Totals' for dimension, dimension_value in zip(used_dimensions, dimension_values)] + label = ", ".join(dimension_labels) + if metric is None: - return ", ".join(dimension_labels) + if reference is not None: + return '{} ({})'.format(label, reference.label) + return label if dimension_labels: return '{} ({})'.format(reference_label(metric, reference), - ', '.join(dimension_labels)) + label) return reference_label(metric, reference) diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py index 883aee9b..7591f36b 100644 --- a/fireant/tests/slicer/widgets/test_datatables.py +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -587,15 +587,15 @@ def test_pivoted_multi_dims_time_series_and_cat(self): 'render': {'_': 'value'}, }, { 'data': 'wins.d', - 'title': 'Wins (Democrat)', + 'title': 'Democrat', 'render': {'_': 'value', 'display': 'display'}, }, { 'data': 'wins.i', - 'title': 'Wins (Independent)', + 'title': 'Independent', 'render': {'_': 'value', 'display': 'display'}, }, { 'data': 'wins.r', - 'title': 'Wins (Republican)', + 'title': 'Republican', 'render': {'_': 'value', 'display': 'display'}, }], 'data': [{ @@ -654,11 +654,11 @@ def test_pivoted_multi_dims_time_series_and_uni(self): 'render': {'_': 'value'}, }, { 'data': 'votes.1', - 'title': 'Votes (Texas)', + 'title': 'Texas', 'render': {'_': 'value', 'display': 'display'}, }, { 'data': 'votes.2', - 'title': 'Votes (California)', + 'title': 'California', 'render': {'_': 'value', 'display': 'display'}, }], 'data': [{ From 966710f3b98195000998fce83df0db688062dedd Mon Sep 17 00:00:00 2001 From: Michael England Date: Mon, 9 Jul 2018 11:21:47 +0200 Subject: [PATCH 085/123] bumped v1.0 version to dev36 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index e75489f4..51337cca 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev35' +__version__ = '1.0.0.dev36' From ef084f886933ff80974d94f6adbdc90cc8530f3e Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 10 Jul 2018 10:42:02 +0200 Subject: [PATCH 086/123] Fixed the db query alias prefix to distinguish between metrics and dimensions to avoid collisions when an aliases is shared. --- fireant/slicer/operations.py | 10 +- fireant/slicer/queries/builder.py | 25 +- fireant/slicer/queries/database.py | 11 +- fireant/slicer/queries/makers.py | 12 +- fireant/slicer/queries/references.py | 34 +- fireant/slicer/references.py | 4 +- fireant/slicer/widgets/datatables.py | 13 +- fireant/slicer/widgets/helpers.py | 9 +- fireant/slicer/widgets/highcharts.py | 6 +- fireant/slicer/widgets/pandas.py | 22 +- fireant/tests/slicer/mocks.py | 69 +- fireant/tests/slicer/queries/test_builder.py | 1232 ++++++++--------- fireant/tests/slicer/queries/test_database.py | 13 +- .../slicer/queries/test_dimension_choices.py | 36 +- fireant/tests/slicer/test_operations.py | 24 +- fireant/tests/slicer/widgets/test_csv.py | 41 +- .../tests/slicer/widgets/test_datatables.py | 6 +- fireant/tests/slicer/widgets/test_pandas.py | 49 +- fireant/utils.py | 14 +- 19 files changed, 839 insertions(+), 791 deletions(-) diff --git a/fireant/slicer/operations.py b/fireant/slicer/operations.py index 07325705..54c26f0f 100644 --- a/fireant/slicer/operations.py +++ b/fireant/slicer/operations.py @@ -1,7 +1,7 @@ import numpy as np import pandas as pd -from fireant.utils import format_key +from fireant.utils import format_metric_key from .metrics import Metric @@ -79,7 +79,7 @@ def __repr__(self): class CumSum(_Cumulative): def apply(self, data_frame): - df_key = format_key(self.arg.key) + df_key = format_metric_key(self.arg.key) if isinstance(data_frame.index, pd.MultiIndex): levels = self._group_levels(data_frame.index) @@ -93,7 +93,7 @@ def apply(self, data_frame): class CumProd(_Cumulative): def apply(self, data_frame): - df_key = format_key(self.arg.key) + df_key = format_metric_key(self.arg.key) if isinstance(data_frame.index, pd.MultiIndex): levels = self._group_levels(data_frame.index) @@ -111,7 +111,7 @@ def cummean(x): return x.cumsum() / np.arange(1, len(x) + 1) def apply(self, data_frame): - df_key = format_key(self.arg.key) + df_key = format_metric_key(self.arg.key) if isinstance(data_frame.index, pd.MultiIndex): levels = self._group_levels(data_frame.index) @@ -148,7 +148,7 @@ def rolling_mean(self, x): return x.rolling(self.window, self.min_periods).mean() def apply(self, data_frame): - df_key = format_key(self.arg.key) + df_key = format_metric_key(self.arg.key) if isinstance(data_frame.index, pd.MultiIndex): levels = self._group_levels(data_frame.index) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 1ce17f57..5633d689 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -1,14 +1,16 @@ -import pandas as pd -from pypika import ( - Order, -) from typing import ( Dict, Iterable, ) +import pandas as pd +from pypika import ( + Order, +) + from fireant.utils import ( - format_key, + format_dimension_key, + format_metric_key, immutable, ) from .database import fetch_data @@ -25,6 +27,7 @@ make_slicer_query_with_references_and_totals, ) from ..base import SlicerElement +from ..dimensions import Dimension class QueryBuilder(object): @@ -128,6 +131,10 @@ def orderby(self, element: SlicerElement, orientation=None): The directionality to order by, either ascending or descending. :return: """ + format_key = format_dimension_key \ + if isinstance(element, Dimension) \ + else format_metric_key + self._orders += [(element.definition.as_(format_key(element.key)), orientation)] @property @@ -183,7 +190,7 @@ def fetch(self, hint=None) -> Iterable[Dict]: # Apply operations operations = find_operations_for_widgets(self._widgets) for operation in operations: - df_key = format_key(operation.key) + df_key = format_metric_key(operation.key) data_frame[df_key] = operation.apply(data_frame) # Apply transformations @@ -250,9 +257,9 @@ def fetch(self, hint=None, force_include=()) -> pd.Series: query = query.hint(hint) dimension = self._dimensions[0] - definition = dimension.display_definition.as_(format_key(dimension.display_key)) \ + definition = dimension.display_definition.as_(format_dimension_key(dimension.display_key)) \ if dimension.has_display_field \ - else dimension.definition.as_(format_key(dimension.key)) + else dimension.definition.as_(format_dimension_key(dimension.key)) if force_include: include = self.slicer.database.to_char(dimension.definition) \ @@ -268,7 +275,7 @@ def fetch(self, hint=None, force_include=()) -> pd.Series: str(query), dimensions=self._dimensions) - df_key = format_key(getattr(dimension, 'display_key', None)) + df_key = format_dimension_key(getattr(dimension, 'display_key', None)) if df_key is not None: return data[df_key] diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index ef0b4670..33cb1872 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -1,10 +1,11 @@ -import pandas as pd import time from typing import Iterable +import pandas as pd + from fireant.database.base import Database from fireant.formats import NULL_VALUE -from fireant.utils import format_key +from fireant.utils import format_dimension_key from .logger import ( query_logger, slow_query_logger, @@ -54,7 +55,7 @@ def clean_and_apply_index(data_frame: pd.DataFrame, dimensions: Iterable[Dimensi if not dimensions: return data_frame - dimension_keys = [format_key(d.key) + dimension_keys = [format_dimension_key(d.key) for d in dimensions] for i, dimension in enumerate(dimensions): @@ -63,7 +64,7 @@ def clean_and_apply_index(data_frame: pd.DataFrame, dimensions: Iterable[Dimensi # With that in mind, we leave the NaNs in them to represent Totals. continue - level = format_key(dimension.key) + level = format_dimension_key(dimension.key) data_frame[level] = fill_nans_in_level(data_frame, dimension, dimension_keys[:i]) \ .apply(lambda x: str(x) if not pd.isnull(x) else None) @@ -87,7 +88,7 @@ def fill_nans_in_level(data_frame, dimension, preceding_dimension_keys): :return: The level in the data_frame with the nulls replaced with empty string """ - level = format_key(dimension.key) + level = format_dimension_key(dimension.key) if dimension.is_rollup: if preceding_dimension_keys: diff --git a/fireant/slicer/queries/makers.py b/fireant/slicer/queries/makers.py index 990851f1..162c2879 100644 --- a/fireant/slicer/queries/makers.py +++ b/fireant/slicer/queries/makers.py @@ -4,6 +4,8 @@ from fireant.utils import ( flatten, format_key, + format_dimension_key, + format_metric_key, ) from .finders import ( find_joins_for_tables, @@ -199,9 +201,9 @@ def make_slicer_query_with_references(database, base_table, joins, dimensions, m else: container_query = container_query.from_(ref_query) - ref_dimension_definitions.append([offset_func(ref_query.field(format_key(dimension.key))) + ref_dimension_definitions.append([offset_func(ref_query.field(format_dimension_key(dimension.key))) if ref_dimension == dimension - else ref_query.field(format_key(dimension.key)) + else ref_query.field(format_dimension_key(dimension.key)) for dimension in dimensions]) ref_terms += make_terms_for_references(references, @@ -220,7 +222,7 @@ def make_slicer_query_with_references(database, base_table, joins, dimensions, m def make_terms_for_metrics(metrics): - return [metric.definition.as_(format_key(metric.key)) + return [metric.definition.as_(format_metric_key(metric.key)) for metric in metrics] @@ -243,12 +245,12 @@ def make_terms_for_dimension(dimension, window=None): window(dimension.definition, dimension.interval) if window and hasattr(dimension, 'interval') else dimension.definition - ).as_(format_key(dimension.key)) + ).as_(format_dimension_key(dimension.key)) # Include the display definition if there is one return [ dimension_definition, - dimension.display_definition.as_(format_key(dimension.display_key)) + dimension.display_definition.as_(format_dimension_key(dimension.display_key)) ] if dimension.has_display_field else [ dimension_definition ] diff --git a/fireant/slicer/queries/references.py b/fireant/slicer/queries/references.py index e0e990bf..5ef8d196 100644 --- a/fireant/slicer/queries/references.py +++ b/fireant/slicer/queries/references.py @@ -1,4 +1,8 @@ import copy +from typing import ( + Callable, + Iterable, +) from pypika import functions as fn from pypika.queries import QueryBuilder @@ -8,16 +12,15 @@ NullValue, Term, ) -from typing import ( - Callable, - Iterable, -) from fireant.slicer.references import ( reference_key, reference_term, ) -from fireant.utils import format_key +from fireant.utils import ( + format_dimension_key, + format_metric_key, +) from ..dimensions import Dimension from ..intervals import weekly @@ -45,7 +48,7 @@ def make_terms_for_references(references, original_query, ref_query, metrics): original_query, ref_query) - terms += [ref_metric(metric).as_(format_key(reference_key(metric, reference))) + terms += [ref_metric(metric).as_(format_metric_key(reference_key(metric, reference))) for metric in metrics] return terms @@ -74,7 +77,10 @@ def make_dimension_terms_for_reference_container_query(original_query, terms = [] for dimension, ref_dimension_definition in zip(dimensions, ref_dimension_definitions): - term = _select_for_reference_container_query(format_key(dimension.key), + f_key = format_dimension_key(dimension.key) + f_display_key = format_dimension_key(dimension.display_key) + + term = _select_for_reference_container_query(f_key, dimension.definition, original_query, ref_dimension_definition) @@ -84,9 +90,9 @@ def make_dimension_terms_for_reference_container_query(original_query, continue # Select the display definitions as a field from the ref query - ref_display_definition = [definition.table.field(format_key(dimension.display_key)) + ref_display_definition = [definition.table.field(f_display_key) for definition in ref_dimension_definition] - display_term = _select_for_reference_container_query(format_key(dimension.display_key), + display_term = _select_for_reference_container_query(f_display_key, dimension.display_definition, original_query, ref_display_definition) @@ -96,7 +102,9 @@ def make_dimension_terms_for_reference_container_query(original_query, def make_metric_terms_for_reference_container_query(original_query, metrics): - return [_select_for_reference_container_query(format_key(metric.key), metric.definition, original_query) + return [_select_for_reference_container_query(format_metric_key(metric.key), + metric.definition, + original_query) for metric in metrics] @@ -181,13 +189,15 @@ def make_reference_join_criterion(ref_dimension: Dimension, join_criterion = None for dimension in all_dimensions: - ref_query_field = ref_query.field(format_key(dimension.key)) + f_key = format_dimension_key(dimension.key) + + ref_query_field = ref_query.field(f_key) # If this is the reference dimension, it needs to be offset by the reference interval if ref_dimension == dimension: ref_query_field = offset_func(ref_query_field) - next_criterion = original_query.field(format_key(dimension.key)) == ref_query_field + next_criterion = original_query.field(f_key) == ref_query_field join_criterion = next_criterion \ if join_criterion is None \ diff --git a/fireant/slicer/references.py b/fireant/slicer/references.py index c06dbcc6..99960be3 100644 --- a/fireant/slicer/references.py +++ b/fireant/slicer/references.py @@ -109,10 +109,10 @@ def reference_term(reference: Reference, """ def original_field(metric): - return original_query.field(utils.format_key(metric.key)) + return original_query.field(utils.format_metric_key(metric.key)) def ref_field(metric): - return ref_query.field(utils.format_key(metric.key)) + return ref_query.field(utils.format_metric_key(metric.key)) if reference.delta: if reference.delta_percent: diff --git a/fireant/slicer/widgets/datatables.py b/fireant/slicer/widgets/datatables.py index abb4c02f..9a913733 100644 --- a/fireant/slicer/widgets/datatables.py +++ b/fireant/slicer/widgets/datatables.py @@ -7,7 +7,10 @@ formats, utils, ) -from fireant.utils import format_key +from fireant.utils import ( + format_dimension_key, + format_metric_key, +) from .base import ( TransformableWidget, ) @@ -69,7 +72,7 @@ def _render_dimensional_metric_cell(row_data: pd.Series, metric: Metric): for key, next_row in row_data.groupby(level=1): next_row.reset_index(level=1, drop=True, inplace=True) - df_key = format_key(metric.key) + df_key = format_metric_key(metric.key) level[key] = _render_dimensional_metric_cell(next_row, metric) \ if isinstance(next_row.index, pd.MultiIndex) \ else _format_metric_cell(next_row[df_key], metric) @@ -128,7 +131,7 @@ def transform(self, data_frame, slicer, dimensions, references): """ dimension_display_values = extract_display_values(dimensions, data_frame) - metric_keys = [format_key(reference_key(metric, reference)) + metric_keys = [format_metric_key(reference_key(metric, reference)) for metric in self.items for reference in [None] + references] data_frame = data_frame[metric_keys] @@ -242,13 +245,13 @@ def _data_row(self, dimensions, dimension_values, dimension_display_values, refe row = {} for dimension, dimension_value in zip(dimensions, utils.wrap_list(dimension_values)): - df_key = format_key(dimension.key) + df_key = format_dimension_key(dimension.key) row[dimension.key] = _render_dimension_cell(dimension_value, dimension_display_values.get(df_key)) for metric in self.items: for reference in [None] + references: key = reference_key(metric, reference) - df_key = format_key(key) + df_key = format_metric_key(key) row[key] = _render_dimensional_metric_cell(row_data, metric) \ if isinstance(row_data.index, pd.MultiIndex) \ diff --git a/fireant/slicer/widgets/helpers.py b/fireant/slicer/widgets/helpers.py index 4cef2d77..22d8d769 100644 --- a/fireant/slicer/widgets/helpers.py +++ b/fireant/slicer/widgets/helpers.py @@ -23,13 +23,15 @@ def extract_display_values(dimensions, data_frame): display_values = {} for dimension in dimensions: - key = utils.format_key(dimension.key) + key = utils.format_dimension_key(dimension.key) if hasattr(dimension, 'display_values'): display_values[key] = dimension.display_values elif getattr(dimension, 'display_key', None): - display_values[key] = data_frame[utils.format_key(dimension.display_key)] \ + f_display_key = utils.format_dimension_key(dimension.display_key) + + display_values[key] = data_frame[f_display_key] \ .groupby(level=key) \ .first() @@ -66,7 +68,8 @@ def render_series_label(dimension_values, metric=None, reference=None): dimension_values = utils.wrap_list(dimension_values) dimension_labels = [utils.deep_get(dimension_display_values, - [utils.format_key(dimension.key), dimension_value], + [utils.format_dimension_key(dimension.key), + dimension_value], dimension_value) if not pd.isnull(dimension_value) else 'Totals' diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 0a07e95f..90eb7dab 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -345,7 +345,7 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, series_color = next(colors) for reference, dash_style in zip([None] + references, itertools.cycle(DASH_STYLES)): - metric_key = utils.format_key(reference_key(series.metric, reference)) + metric_key = utils.format_metric_key(reference_key(series.metric, reference)) hc_series.append({ "type": series.type, @@ -374,6 +374,8 @@ def _render_series(self, axis, axis_idx, axis_color, colors, data_frame_groups, def _render_pie_series(self, series, reference, dimension_values, data_frame, render_series_label): metric = series.metric name = reference_label(metric, reference) + df_key = utils.format_metric_key(series.metric.key) + return { "name": name, "type": series.type, @@ -381,7 +383,7 @@ def _render_pie_series(self, series, reference, dimension_values, data_frame, re "data": [{ "name": render_series_label(dimension_values) if dimension_values else name, "y": formats.metric_value(y), - } for dimension_values, y in data_frame[utils.format_key(series.metric.key)].iteritems()], + } for dimension_values, y in data_frame[df_key].iteritems()], 'tooltip': { 'valueDecimals': metric.precision, 'valuePrefix': metric.prefix, diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index 45600366..613740a6 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -1,7 +1,10 @@ import pandas as pd from fireant import Metric -from fireant.utils import format_key +from fireant.utils import ( + format_dimension_key, + format_metric_key, +) from .base import ( TransformableWidget, ) @@ -38,28 +41,29 @@ def transform(self, data_frame, slicer, dimensions, references): if any([metric.precision is not None, metric.prefix is not None, metric.suffix is not None]): - df_key = format_key(metric.key) + df_key = format_metric_key(metric.key) result[df_key] = result[df_key] \ .apply(lambda x: formats.metric_display(x, metric.prefix, metric.suffix, metric.precision)) for dimension in dimensions: if dimension.has_display_field: - result = result.set_index(format_key(dimension.display_key), append=True) - result = result.reset_index(format_key(dimension.key), drop=True) + result = result.set_index(format_dimension_key(dimension.display_key), append=True) + result = result.reset_index(format_dimension_key(dimension.key), drop=True) if hasattr(dimension, 'display_values'): self._replace_display_values_in_index(dimension, result) if isinstance(data_frame.index, pd.MultiIndex): - index_levels = [format_key(dimension.display_key) + index_levels = [dimension.display_key if dimension.has_display_field - else format_key(dimension.key) + else dimension.key for dimension in dimensions] - result = result.reorder_levels(index_levels) + result = result.reorder_levels([format_dimension_key(level) + for level in index_levels]) - result = result[[format_key(reference_key(item, reference)) + result = result[[format_metric_key(reference_key(item, reference)) for reference in [None] + references for item in self.items]] @@ -82,7 +86,7 @@ def _replace_display_values_in_index(self, dimension, result): Replaces the raw values of a (categorical) dimension in the index with their corresponding display values. """ if isinstance(result.index, pd.MultiIndex): - df_key = format_key(dimension.key) + df_key = format_dimension_key(dimension.key) values = [dimension.display_values.get(x, x) for x in result.index.get_level_values(df_key)] result.index.set_levels(level=df_key, levels=values) diff --git a/fireant/tests/slicer/mocks.py b/fireant/tests/slicer/mocks.py index bbffd705..3719d3ab 100644 --- a/fireant/tests/slicer/mocks.py +++ b/fireant/tests/slicer/mocks.py @@ -1,12 +1,12 @@ from collections import ( OrderedDict, ) -from unittest.mock import Mock - -import pandas as pd from datetime import ( datetime, ) +from unittest.mock import Mock + +import pandas as pd from pypika import ( JoinType, Table, @@ -15,7 +15,10 @@ from fireant import * from fireant.slicer.references import ReferenceType -from fireant.utils import format_key as f +from fireant.utils import ( + format_dimension_key as fd, + format_metric_key as fm, +) class TestDatabase(VerticaDatabase): @@ -209,14 +212,14 @@ def __eq__(self, other): (6, 11): False, } -df_columns = [f('timestamp'), - f('candidate'), f('candidate_display'), - f('political_party'), - f('election'), f('election_display'), - f('state'), f('state_display'), - f('winner'), - f('votes'), - f('wins')] +df_columns = [fd('timestamp'), + fd('candidate'), fd('candidate_display'), + fd('political_party'), + fd('election'), fd('election_display'), + fd('state'), fd('state_display'), + fd('winner'), + fm('votes'), + fm('wins')] def PoliticsRow(timestamp, candidate, candidate_display, political_party, election, election_display, state, @@ -226,10 +229,12 @@ def PoliticsRow(timestamp, candidate, candidate_display, political_party, electi winner, votes, wins ) + records = [] for (election_id, candidate_id, state_id), votes in election_candidate_state_votes.items(): election_year = elections[election_id] winner = election_candidate_wins[(election_id, candidate_id)] + records.append(PoliticsRow( timestamp=datetime(int(election_year), 1, 1), candidate=candidate_id, candidate_display=candidates[candidate_id], @@ -243,38 +248,38 @@ def PoliticsRow(timestamp, candidate, candidate_display, political_party, electi mock_politics_database = pd.DataFrame.from_records(records, columns=df_columns) -single_metric_df = pd.DataFrame(mock_politics_database[[f('votes')]] +single_metric_df = pd.DataFrame(mock_politics_database[[fm('votes')]] .sum()).T -multi_metric_df = pd.DataFrame(mock_politics_database[[f('votes'), f('wins')]] +multi_metric_df = pd.DataFrame(mock_politics_database[[fm('votes'), fm('wins')]] .sum()).T -cont_dim_df = mock_politics_database[[f('timestamp'), f('votes'), f('wins')]] \ - .groupby(f('timestamp')) \ +cont_dim_df = mock_politics_database[[fd('timestamp'), fm('votes'), fm('wins')]] \ + .groupby(fd('timestamp')) \ .sum() -cat_dim_df = mock_politics_database[[f('political_party'), f('votes'), f('wins')]] \ - .groupby(f('political_party')) \ +cat_dim_df = mock_politics_database[[fd('political_party'), fm('votes'), fm('wins')]] \ + .groupby(fd('political_party')) \ .sum() -uni_dim_df = mock_politics_database[[f('candidate'), f('candidate_display'), f('votes'), f('wins')]] \ - .groupby([f('candidate'), f('candidate_display')]) \ +uni_dim_df = mock_politics_database[[fd('candidate'), fd('candidate_display'), fm('votes'), fm('wins')]] \ + .groupby([fd('candidate'), fd('candidate_display')]) \ .sum() \ - .reset_index(f('candidate_display')) + .reset_index(fd('candidate_display')) -cont_cat_dim_df = mock_politics_database[[f('timestamp'), f('political_party'), f('votes'), f('wins')]] \ - .groupby([f('timestamp'), f('political_party')]) \ +cont_cat_dim_df = mock_politics_database[[fd('timestamp'), fd('political_party'), fm('votes'), fm('wins')]] \ + .groupby([fd('timestamp'), fd('political_party')]) \ .sum() -cont_uni_dim_df = mock_politics_database[[f('timestamp'), f('state'), f('state_display'), f('votes'), f('wins')]] \ - .groupby([f('timestamp'), f('state'), f('state_display')]) \ +cont_uni_dim_df = mock_politics_database[[fd('timestamp'), fd('state'), fd('state_display'), fm('votes'), fm('wins')]] \ + .groupby([fd('timestamp'), fd('state'), fd('state_display')]) \ .sum() \ - .reset_index(f('state_display')) + .reset_index(fd('state_display')) cont_dim_operation_df = cont_dim_df.copy() -operation_key = f('cumsum(votes)') -cont_dim_operation_df[operation_key] = cont_dim_df[f('votes')].cumsum() +operation_key = fm('cumsum(votes)') +cont_dim_operation_df[operation_key] = cont_dim_df[fm('votes')].cumsum() def ref(data_frame, columns): @@ -302,7 +307,7 @@ def ref_delta(ref_data_frame, columns): return ref_data_frame.join(delta_data_frame) -_columns = [f('votes'), f('wins')] +_columns = [fm('votes'), fm('wins')] cont_uni_dim_ref_df = ref(cont_uni_dim_df, _columns) cont_uni_dim_ref_delta_df = ref_delta(cont_uni_dim_ref_df, _columns) @@ -355,8 +360,8 @@ def _totals(df): elif not isinstance(l.index, (pd.DatetimeIndex, pd.RangeIndex)): l.index = l.index.astype('str') -cont_cat_dim_totals_df = totals(cont_cat_dim_df, [f('political_party')], _columns) -cont_uni_dim_totals_df = totals(cont_uni_dim_df, [f('state')], _columns) -cont_uni_dim_all_totals_df = totals(cont_uni_dim_df, [f('timestamp'), f('state')], _columns) +cont_cat_dim_totals_df = totals(cont_cat_dim_df, [fd('political_party')], _columns) +cont_uni_dim_totals_df = totals(cont_uni_dim_df, [fd('state')], _columns) +cont_uni_dim_all_totals_df = totals(cont_uni_dim_df, [fd('timestamp'), fd('state')], _columns) ElectionOverElection = ReferenceType('eoe', 'EoE', 'year', 4) diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index ee57f4a8..769aa72d 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -1,3 +1,4 @@ +from datetime import date from unittest import TestCase from unittest.mock import ( ANY, @@ -5,17 +6,12 @@ patch, ) -from datetime import date from pypika import Order import fireant as f -from fireant.utils import format_key -from fireant.slicer.exceptions import ( - MetricRequiredException, -) -from ..matchers import ( - DimensionMatcher, -) +from fireant.slicer.exceptions import MetricRequiredException +from fireant.utils import format_metric_key +from ..matchers import DimensionMatcher from ..mocks import slicer @@ -55,7 +51,7 @@ def test_build_query_with_single_metric(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician"', str(query)) def test_build_query_with_multiple_metrics(self): @@ -64,8 +60,8 @@ def test_build_query_with_multiple_metrics(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes",' - 'SUM("is_winner") "$wins" ' + 'SUM("votes") "$m$votes",' + 'SUM("is_winner") "$m$wins" ' 'FROM "politics"."politician"', str(query)) def test_build_query_with_multiple_visualizations(self): @@ -75,8 +71,8 @@ def test_build_query_with_multiple_visualizations(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes",' - 'SUM("is_winner") "$wins" ' + 'SUM("votes") "$m$votes",' + 'SUM("is_winner") "$m$wins" ' 'FROM "politics"."politician"', str(query)) def test_build_query_for_chart_visualization_with_single_axis(self): @@ -86,7 +82,7 @@ def test_build_query_for_chart_visualization_with_single_axis(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician"', str(query)) def test_build_query_for_chart_visualization_with_multiple_axes(self): @@ -97,8 +93,8 @@ def test_build_query_for_chart_visualization_with_multiple_axes(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes",' - 'SUM("is_winner") "$wins" ' + 'SUM("votes") "$m$votes",' + 'SUM("is_winner") "$m$wins" ' 'FROM "politics"."politician"', str(query)) @@ -113,11 +109,11 @@ def test_build_query_with_datetime_dimension(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp"', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) def test_build_query_with_datetime_dimension_hourly(self): query = slicer.data \ @@ -126,11 +122,11 @@ def test_build_query_with_datetime_dimension_hourly(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'HH\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'HH\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp"', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) def test_build_query_with_datetime_dimension_daily(self): query = slicer.data \ @@ -139,11 +135,11 @@ def test_build_query_with_datetime_dimension_daily(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp"', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) def test_build_query_with_datetime_dimension_weekly(self): query = slicer.data \ @@ -152,11 +148,11 @@ def test_build_query_with_datetime_dimension_weekly(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'IW\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp"', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) def test_build_query_with_datetime_dimension_monthly(self): query = slicer.data \ @@ -165,11 +161,11 @@ def test_build_query_with_datetime_dimension_monthly(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'MM\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'MM\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp"', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) def test_build_query_with_datetime_dimension_quarterly(self): query = slicer.data \ @@ -178,11 +174,11 @@ def test_build_query_with_datetime_dimension_quarterly(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'Q\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'Q\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp"', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) def test_build_query_with_datetime_dimension_annually(self): query = slicer.data \ @@ -191,11 +187,11 @@ def test_build_query_with_datetime_dimension_annually(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'Y\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'Y\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp"', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) def test_build_query_with_boolean_dimension(self): query = slicer.data \ @@ -204,11 +200,11 @@ def test_build_query_with_boolean_dimension(self): .query self.assertEqual('SELECT ' - '"is_winner" "$winner",' - 'SUM("votes") "$votes" ' + '"is_winner" "$d$winner",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$winner" ' - 'ORDER BY "$winner"', str(query)) + 'GROUP BY "$d$winner" ' + 'ORDER BY "$d$winner"', str(query)) def test_build_query_with_categorical_dimension(self): query = slicer.data \ @@ -217,11 +213,11 @@ def test_build_query_with_categorical_dimension(self): .query self.assertEqual('SELECT ' - '"political_party" "$political_party",' - 'SUM("votes") "$votes" ' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$political_party" ' - 'ORDER BY "$political_party"', str(query)) + 'GROUP BY "$d$political_party" ' + 'ORDER BY "$d$political_party"', str(query)) def test_build_query_with_unique_dimension(self): query = slicer.data \ @@ -230,12 +226,12 @@ def test_build_query_with_unique_dimension(self): .query self.assertEqual('SELECT ' - '"election_id" "$election",' - '"election_year" "$election_display",' - 'SUM("votes") "$votes" ' + '"election_id" "$d$election",' + '"election_year" "$d$election_display",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$election","$election_display" ' - 'ORDER BY "$election_display"', str(query)) + 'GROUP BY "$d$election","$d$election_display" ' + 'ORDER BY "$d$election_display"', str(query)) def test_build_query_with_pattern_dimension(self): query = slicer.data \ @@ -248,11 +244,11 @@ def test_build_query_with_pattern_dimension(self): 'WHEN "pattern" LIKE \'groupA%\' THEN \'groupA%\' ' 'WHEN "pattern" LIKE \'groupB%\' THEN \'groupB%\' ' 'ELSE \'No Group\' ' - 'END "$pattern",' - 'SUM("votes") "$votes" ' + 'END "$d$pattern",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$pattern" ' - 'ORDER BY "$pattern"', str(query)) + 'GROUP BY "$d$pattern" ' + 'ORDER BY "$d$pattern"', str(query)) def test_build_query_with_pattern_no_values(self): query = slicer.data \ @@ -261,11 +257,11 @@ def test_build_query_with_pattern_no_values(self): .query self.assertEqual('SELECT ' - '\'No Group\' "$pattern",' - 'SUM("votes") "$votes" ' + '\'No Group\' "$d$pattern",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$pattern" ' - 'ORDER BY "$pattern"', str(query)) + 'GROUP BY "$d$pattern" ' + 'ORDER BY "$d$pattern"', str(query)) def test_build_query_with_multiple_dimensions(self): query = slicer.data \ @@ -275,13 +271,13 @@ def test_build_query_with_multiple_dimensions(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"candidate_id" "$candidate",' - '"candidate_name" "$candidate_display",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp","$candidate","$candidate_display" ' - 'ORDER BY "$timestamp","$candidate_display"', str(query)) + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display" ' + 'ORDER BY "$d$timestamp","$d$candidate_display"', str(query)) def test_build_query_with_multiple_dimensions_and_visualizations(self): query = slicer.data \ @@ -294,13 +290,13 @@ def test_build_query_with_multiple_dimensions_and_visualizations(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"political_party" "$political_party",' - 'SUM("votes") "$votes",' - 'SUM("is_winner") "$wins" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes",' + 'SUM("is_winner") "$m$wins" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp","$political_party" ' - 'ORDER BY "$timestamp","$political_party"', str(query)) + 'GROUP BY "$d$timestamp","$d$political_party" ' + 'ORDER BY "$d$timestamp","$d$political_party"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -314,19 +310,19 @@ def test_build_query_with_totals_cat_dimension(self): .query self.assertEqual('(SELECT ' - '"political_party" "$political_party",' - 'SUM("votes") "$votes" ' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$political_party") ' + 'GROUP BY "$d$political_party") ' 'UNION ALL ' '(SELECT ' - 'NULL "$political_party",' - 'SUM("votes") "$votes" ' + 'NULL "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician") ' - 'ORDER BY "$political_party"', str(query)) + 'ORDER BY "$d$political_party"', str(query)) def test_build_query_with_totals_uni_dimension(self): query = slicer.data \ @@ -335,21 +331,21 @@ def test_build_query_with_totals_uni_dimension(self): .query self.assertEqual('(SELECT ' - '"candidate_id" "$candidate",' - '"candidate_name" "$candidate_display",' - 'SUM("votes") "$votes" ' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$candidate","$candidate_display") ' + 'GROUP BY "$d$candidate","$d$candidate_display") ' 'UNION ALL ' '(SELECT ' - 'NULL "$candidate",' - 'NULL "$candidate_display",' - 'SUM("votes") "$votes" ' + 'NULL "$d$candidate",' + 'NULL "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician") ' - 'ORDER BY "$candidate_display"', str(query)) + 'ORDER BY "$d$candidate_display"', str(query)) def test_build_query_with_totals_on_dimension_and_subsequent_dimensions(self): query = slicer.data \ @@ -360,25 +356,25 @@ def test_build_query_with_totals_on_dimension_and_subsequent_dimensions(self): .query self.assertEqual('(SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"candidate_id" "$candidate",' - '"candidate_name" "$candidate_display",' - '"political_party" "$political_party",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp","$candidate","$candidate_display","$political_party") ' + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display","$d$political_party") ' 'UNION ALL ' '(SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'NULL "$candidate",' - 'NULL "$candidate_display",' - 'NULL "$political_party",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'NULL "$d$candidate",' + 'NULL "$d$candidate_display",' + 'NULL "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp") ' - 'ORDER BY "$timestamp","$candidate_display","$political_party"', str(query)) + 'GROUP BY "$d$timestamp") ' + 'ORDER BY "$d$timestamp","$d$candidate_display","$d$political_party"', str(query)) def test_build_query_with_totals_on_multiple_dimensions_dimension(self): query = slicer.data \ @@ -389,37 +385,37 @@ def test_build_query_with_totals_on_multiple_dimensions_dimension(self): .query self.assertEqual('(SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"candidate_id" "$candidate",' - '"candidate_name" "$candidate_display",' - '"political_party" "$political_party",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp","$candidate","$candidate_display","$political_party") ' + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display","$d$political_party") ' 'UNION ALL ' '(SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'NULL "$candidate",' - 'NULL "$candidate_display",' - 'NULL "$political_party",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'NULL "$d$candidate",' + 'NULL "$d$candidate_display",' + 'NULL "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp") ' + 'GROUP BY "$d$timestamp") ' 'UNION ALL ' '(SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"candidate_id" "$candidate",' - '"candidate_name" "$candidate_display",' - 'NULL "$political_party",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'NULL "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp","$candidate","$candidate_display") ' + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display") ' - 'ORDER BY "$timestamp","$candidate_display","$political_party"', str(query)) + 'ORDER BY "$d$timestamp","$d$candidate_display","$d$political_party"', str(query)) def test_build_query_with_totals_cat_dimension_with_references(self): query = slicer.data \ @@ -432,54 +428,54 @@ def test_build_query_with_totals_cat_dimension_with_references(self): # Important that in reference queries when using totals that the null dimensions are omitted from the nested # queries and selected in the container query self.assertEqual('(SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - 'COALESCE("$base"."$political_party","$dod"."$political_party") "$political_party",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + 'COALESCE("$base"."$d$political_party","$dod"."$d$political_party") "$d$political_party",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM (' 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"political_party" "$political_party",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp","$political_party"' + 'GROUP BY "$d$timestamp","$d$political_party"' ') "$base" ' 'FULL OUTER JOIN (' 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"political_party" "$political_party",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp","$political_party"' + 'GROUP BY "$d$timestamp","$d$political_party"' ') "$dod" ' - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'AND "$base"."$political_party"="$dod"."$political_party") ' + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'AND "$base"."$d$political_party"="$dod"."$d$political_party") ' 'UNION ALL ' '(SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - 'NULL "$political_party",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + 'NULL "$d$political_party",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM (' 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' 'FULL OUTER JOIN (' - 'SELECT TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'SELECT TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) ' - 'ORDER BY "$timestamp","$political_party"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + 'ORDER BY "$d$timestamp","$d$political_party"', str(query)) def test_build_query_with_totals_cat_dimension_with_references_and_date_filters(self): query = slicer.data \ @@ -491,58 +487,58 @@ def test_build_query_with_totals_cat_dimension_with_references_and_date_filters( .query self.assertEqual('(SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - 'COALESCE("$base"."$political_party","$dod"."$political_party") "$political_party",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + 'COALESCE("$base"."$d$political_party","$dod"."$d$political_party") "$d$political_party",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM (' 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"political_party" "$political_party",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' - 'GROUP BY "$timestamp","$political_party"' + 'GROUP BY "$d$timestamp","$d$political_party"' ') "$base" ' 'FULL OUTER JOIN (' 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"political_party" "$political_party",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' - 'GROUP BY "$timestamp","$political_party"' + 'GROUP BY "$d$timestamp","$d$political_party"' ') "$dod" ' - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'AND "$base"."$political_party"="$dod"."$political_party") ' + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'AND "$base"."$d$political_party"="$dod"."$d$political_party") ' 'UNION ALL ' '(SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - 'NULL "$political_party",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + 'NULL "$d$political_party",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM (' 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' 'FULL OUTER JOIN (' - 'SELECT TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'SELECT TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) ' - 'ORDER BY "$timestamp","$political_party"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + 'ORDER BY "$d$timestamp","$d$political_party"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -556,7 +552,7 @@ def test_build_query_with_filter_isin_categorical_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "political_party" IN (\'d\')', str(query)) @@ -567,7 +563,7 @@ def test_build_query_with_filter_notin_categorical_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "political_party" NOT IN (\'d\')', str(query)) @@ -578,7 +574,7 @@ def test_build_query_with_filter_like_categorical_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "political_party" LIKE \'Rep%\'', str(query)) @@ -589,7 +585,7 @@ def test_build_query_with_filter_not_like_categorical_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "political_party" NOT LIKE \'Rep%\'', str(query)) @@ -600,7 +596,7 @@ def test_build_query_with_filter_isin_unique_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_id" IN (1)', str(query)) @@ -611,7 +607,7 @@ def test_build_query_with_filter_notin_unique_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_id" NOT IN (1)', str(query)) @@ -622,7 +618,7 @@ def test_build_query_with_filter_isin_unique_dim_display(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" IN (\'Donald Trump\')', str(query)) @@ -633,7 +629,7 @@ def test_build_query_with_filter_notin_unique_dim_display(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" NOT IN (\'Donald Trump\')', str(query)) @@ -644,7 +640,7 @@ def test_build_query_with_filter_like_unique_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) @@ -655,7 +651,7 @@ def test_build_query_with_filter_like_display_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) @@ -666,7 +662,7 @@ def test_build_query_with_filter_not_like_unique_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" NOT LIKE \'%Trump\'', str(query)) @@ -677,7 +673,7 @@ def test_build_query_with_filter_not_like_display_dim(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" NOT LIKE \'%Trump\'', str(query)) @@ -688,7 +684,7 @@ def test_build_query_with_filter_like_categorical_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "political_party" LIKE \'Rep%\' ' 'OR "political_party" LIKE \'Dem%\'', str(query)) @@ -700,7 +696,7 @@ def test_build_query_with_filter_not_like_categorical_dim_multiple_patterns(self .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "political_party" NOT LIKE \'Rep%\' ' 'OR "political_party" NOT LIKE \'Dem%\'', str(query)) @@ -712,7 +708,7 @@ def test_build_query_with_filter_like_pattern_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "pattern" LIKE \'a%\' ' 'OR "pattern" LIKE \'b%\'', str(query)) @@ -724,7 +720,7 @@ def test_build_query_with_filter_not_like_pattern_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "pattern" NOT LIKE \'a%\' ' 'OR "pattern" NOT LIKE \'b%\'', str(query)) @@ -736,7 +732,7 @@ def test_build_query_with_filter_like_unique_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" LIKE \'%Trump\' ' 'OR "candidate_name" LIKE \'%Clinton\'', str(query)) @@ -748,7 +744,7 @@ def test_build_query_with_filter_like_display_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" LIKE \'%Trump\' ' 'OR "candidate_name" LIKE \'%Clinton\'', str(query)) @@ -760,7 +756,7 @@ def test_build_query_with_filter_not_like_unique_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" NOT LIKE \'%Trump\' ' 'OR "candidate_name" NOT LIKE \'%Clinton\'', str(query)) @@ -772,7 +768,7 @@ def test_build_query_with_filter_not_like_display_dim_multiple_patterns(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "candidate_name" NOT LIKE \'%Trump\' ' 'OR "candidate_name" NOT LIKE \'%Clinton\'', str(query)) @@ -808,7 +804,7 @@ def test_build_query_with_filter_range_datetime_dimension(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2009-01-20\' AND \'2017-01-20\'', str(query)) @@ -819,7 +815,7 @@ def test_build_query_with_filter_boolean_true(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "is_winner"', str(query)) @@ -830,7 +826,7 @@ def test_build_query_with_filter_boolean_false(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE NOT "is_winner"', str(query)) @@ -846,7 +842,7 @@ def test_build_query_with_metric_filter_eq(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")=5', str(query)) @@ -857,7 +853,7 @@ def test_build_query_with_metric_filter_eq_left(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")=5', str(query)) @@ -868,7 +864,7 @@ def test_build_query_with_metric_filter_ne(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")<>5', str(query)) @@ -879,7 +875,7 @@ def test_build_query_with_metric_filter_ne_left(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")<>5', str(query)) @@ -890,7 +886,7 @@ def test_build_query_with_metric_filter_gt(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")>5', str(query)) @@ -901,7 +897,7 @@ def test_build_query_with_metric_filter_gt_left(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")>5', str(query)) @@ -912,7 +908,7 @@ def test_build_query_with_metric_filter_gte(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")>=5', str(query)) @@ -923,7 +919,7 @@ def test_build_query_with_metric_filter_gte_left(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")>=5', str(query)) @@ -934,7 +930,7 @@ def test_build_query_with_metric_filter_lt(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")<5', str(query)) @@ -945,7 +941,7 @@ def test_build_query_with_metric_filter_lt_left(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")<5', str(query)) @@ -956,7 +952,7 @@ def test_build_query_with_metric_filter_lte(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")<=5', str(query)) @@ -967,7 +963,7 @@ def test_build_query_with_metric_filter_lte_left(self): .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'HAVING SUM("votes")<=5', str(query)) @@ -983,11 +979,11 @@ def test_build_query_with_cumsum_operation(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp"', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) def test_build_query_with_cummean_operation(self): query = slicer.data \ @@ -996,11 +992,11 @@ def test_build_query_with_cummean_operation(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp"', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -1015,16 +1011,16 @@ def test_single_reference_dod_with_no_dimension_uses_multiple_from_clauses_inste .query self.assertEqual('SELECT ' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM (' 'SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician"' ') "$base",(' 'SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician"' ') "$dod"', str(query)) @@ -1038,31 +1034,31 @@ def test_single_reference_dod_with_dimension_but_not_reference_dimension_in_quer .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$political_party","$dod"."$political_party") "$political_party",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$political_party","$dod"."$d$political_party") "$d$political_party",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - '"political_party" "$political_party",' - 'SUM("votes") "$votes" ' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "$political_party"' + 'GROUP BY "$d$political_party"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - '"political_party" "$political_party",' - 'SUM("votes") "$votes" ' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "$political_party"' + 'GROUP BY "$d$political_party"' ') "$dod" ' # end-nested - 'ON "$base"."$political_party"="$dod"."$political_party" ' - 'ORDER BY "$political_party"', str(query)) + 'ON "$base"."$d$political_party"="$dod"."$d$political_party" ' + 'ORDER BY "$d$political_party"', str(query)) def test_dimension_with_single_reference_dod(self): query = slicer.data \ @@ -1073,29 +1069,29 @@ def test_dimension_with_single_reference_dod(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_dimension_with_single_reference_wow(self): query = slicer.data \ @@ -1106,29 +1102,29 @@ def test_dimension_with_single_reference_wow(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'week\',1,"$wow"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$wow"."$votes" "$votes_wow" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'week\',1,"$wow"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$wow"."$m$votes" "$m$votes_wow" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$wow" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'week\',1,"$wow"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'week\',1,"$wow"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_dimension_with_single_reference_mom(self): query = slicer.data \ @@ -1139,29 +1135,29 @@ def test_dimension_with_single_reference_mom(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'month\',1,"$mom"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$mom"."$votes" "$votes_mom" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'month\',1,"$mom"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$mom"."$m$votes" "$m$votes_mom" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$mom" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'month\',1,"$mom"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'month\',1,"$mom"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_dimension_with_single_reference_qoq(self): query = slicer.data \ @@ -1172,29 +1168,29 @@ def test_dimension_with_single_reference_qoq(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'quarter\',1,"$qoq"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$qoq"."$votes" "$votes_qoq" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'quarter\',1,"$qoq"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$qoq"."$m$votes" "$m$votes_qoq" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$qoq" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'quarter\',1,"$qoq"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'quarter\',1,"$qoq"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_dimension_with_single_reference_yoy(self): query = slicer.data \ @@ -1205,29 +1201,29 @@ def test_dimension_with_single_reference_yoy(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$yoy"."$votes" "$votes_yoy" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$yoy"."$m$votes" "$m$votes_yoy" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$yoy" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_dimension_with_single_reference_as_a_delta(self): query = slicer.data \ @@ -1238,29 +1234,29 @@ def test_dimension_with_single_reference_as_a_delta(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$base"."$votes"-"$dod"."$votes" "$votes_dod_delta" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$base"."$m$votes"-"$dod"."$m$votes" "$m$votes_dod_delta" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_dimension_with_single_reference_as_a_delta_percentage(self): query = slicer.data \ @@ -1271,29 +1267,29 @@ def test_dimension_with_single_reference_as_a_delta_percentage(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '("$base"."$votes"-"$dod"."$votes")*100/NULLIF("$dod"."$votes",0) "$votes_dod_delta_percent" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '("$base"."$m$votes"-"$dod"."$m$votes")*100/NULLIF("$dod"."$m$votes",0) "$m$votes_dod_delta_percent" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_reference_on_dimension_with_weekly_interval(self): weekly_timestamp = slicer.dimensions.timestamp(f.weekly) @@ -1305,29 +1301,29 @@ def test_reference_on_dimension_with_weekly_interval(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_reference_on_dimension_with_weekly_interval_no_interval_on_reference(self): query = slicer.data \ @@ -1338,29 +1334,29 @@ def test_reference_on_dimension_with_weekly_interval_no_interval_on_reference(se .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_reference_on_dimension_with_monthly_interval(self): query = slicer.data \ @@ -1371,29 +1367,29 @@ def test_reference_on_dimension_with_monthly_interval(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'MM\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'MM\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'MM\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'MM\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_reference_on_dimension_with_quarterly_interval(self): query = slicer.data \ @@ -1404,29 +1400,29 @@ def test_reference_on_dimension_with_quarterly_interval(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'Q\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'Q\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'Q\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'Q\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_reference_on_dimension_with_annual_interval(self): query = slicer.data \ @@ -1437,29 +1433,29 @@ def test_reference_on_dimension_with_annual_interval(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'Y\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'Y\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'Y\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'Y\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_dimension_with_multiple_references(self): query = slicer.data \ @@ -1473,44 +1469,44 @@ def test_dimension_with_multiple_references(self): self.assertEqual('SELECT ' 'COALESCE(' - '"$base"."$timestamp",' - 'TIMESTAMPADD(\'day\',1,"$dod"."$timestamp"),' - 'TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")' - ') "$timestamp",' + '"$base"."$d$timestamp",' + 'TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp"),' + 'TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")' + ') "$d$timestamp",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod",' - '("$base"."$votes"-"$yoy"."$votes")*100/NULLIF("$yoy"."$votes",0) "$votes_yoy_delta_percent" ' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod",' + '("$base"."$m$votes"-"$yoy"."$m$votes")*100/NULLIF("$yoy"."$m$votes",0) "$m$votes_yoy_delta_percent" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$yoy" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_reference_joins_nested_query_on_dimensions(self): query = slicer.data \ @@ -1522,33 +1518,33 @@ def test_reference_joins_nested_query_on_dimensions(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' - 'COALESCE("$base"."$political_party","$yoy"."$political_party") "$political_party",' - '"$base"."$votes" "$votes",' - '"$yoy"."$votes" "$votes_yoy" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' + 'COALESCE("$base"."$d$political_party","$yoy"."$d$political_party") "$d$political_party",' + '"$base"."$m$votes" "$m$votes",' + '"$yoy"."$m$votes" "$m$votes_yoy" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"political_party" "$political_party",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp","$political_party"' + 'GROUP BY "$d$timestamp","$d$political_party"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"political_party" "$political_party",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp","$political_party"' + 'GROUP BY "$d$timestamp","$d$political_party"' ') "$yoy" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' - 'AND "$base"."$political_party"="$yoy"."$political_party" ' - 'ORDER BY "$timestamp","$political_party"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'AND "$base"."$d$political_party"="$yoy"."$d$political_party" ' + 'ORDER BY "$d$timestamp","$d$political_party"', str(query)) def test_reference_with_unique_dimension_includes_display_definition(self): query = slicer.data \ @@ -1560,36 +1556,36 @@ def test_reference_with_unique_dimension_includes_display_definition(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' - 'COALESCE("$base"."$candidate","$yoy"."$candidate") "$candidate",' - 'COALESCE("$base"."$candidate_display","$yoy"."$candidate_display") "$candidate_display",' - '"$base"."$votes" "$votes",' - '"$yoy"."$votes" "$votes_yoy" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' + 'COALESCE("$base"."$d$candidate","$yoy"."$d$candidate") "$d$candidate",' + 'COALESCE("$base"."$d$candidate_display","$yoy"."$d$candidate_display") "$d$candidate_display",' + '"$base"."$m$votes" "$m$votes",' + '"$yoy"."$m$votes" "$m$votes_yoy" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"candidate_id" "$candidate",' - '"candidate_name" "$candidate_display",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp","$candidate","$candidate_display"' + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"candidate_id" "$candidate",' - '"candidate_name" "$candidate_display",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp","$candidate","$candidate_display"' + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display"' ') "$yoy" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' - 'AND "$base"."$candidate"="$yoy"."$candidate" ' - 'ORDER BY "$timestamp","$candidate_display"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'AND "$base"."$d$candidate"="$yoy"."$d$candidate" ' + 'ORDER BY "$d$timestamp","$d$candidate_display"', str(query)) def test_adjust_reference_dimension_filters_in_reference_query(self): query = slicer.data \ @@ -1602,31 +1598,31 @@ def test_adjust_reference_dimension_filters_in_reference_query(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_filters(self): query = slicer.data \ @@ -1641,33 +1637,33 @@ def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_fil .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' 'AND "political_party" IN (\'d\') ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' 'AND "political_party" IN (\'d\') ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_reference(self): query = slicer.data \ @@ -1678,29 +1674,29 @@ def test_adapt_dow_for_leap_year_for_yoy_reference(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$yoy"."$votes" "$votes_yoy" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$yoy"."$m$votes" "$m$votes_yoy" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$timestamp",' - 'SUM("votes") "$votes" ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$yoy" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): query = slicer.data \ @@ -1711,29 +1707,29 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$base"."$votes"-"$yoy"."$votes" "$votes_yoy_delta" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$base"."$m$votes"-"$yoy"."$m$votes" "$m$votes_yoy_delta" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$timestamp",' - 'SUM("votes") "$votes" ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$yoy" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): query = slicer.data \ @@ -1744,29 +1740,29 @@ def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '("$base"."$votes"-"$yoy"."$votes")*100/NULLIF("$yoy"."$votes",0) "$votes_yoy_delta_percent" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '("$base"."$m$votes"-"$yoy"."$m$votes")*100/NULLIF("$yoy"."$m$votes",0) "$m$votes_yoy_delta_percent" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$timestamp",' - 'SUM("votes") "$votes" ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$yoy" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_adapt_dow_for_leap_year_for_yoy_reference_with_date_filter(self): query = slicer.data \ @@ -1778,31 +1774,31 @@ def test_adapt_dow_for_leap_year_for_yoy_reference_with_date_filter(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$yoy"."$votes" "$votes_yoy" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$yoy"."$m$votes" "$m$votes_yoy" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$timestamp",' - 'SUM("votes") "$votes" ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE TIMESTAMPADD(\'year\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$yoy" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_adding_duplicate_reference_does_not_join_more_queries(self): query = slicer.data \ @@ -1814,29 +1810,29 @@ def test_adding_duplicate_reference_does_not_join_more_queries(self): .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension(self): query = slicer.data \ @@ -1849,31 +1845,31 @@ def test_use_same_nested_query_for_joining_references_with_same_period_and_dimen .query self.assertEqual('SELECT ' - 'COALESCE("$base"."$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$timestamp")) "$timestamp",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod",' - '"$base"."$votes"-"$dod"."$votes" "$votes_dod_delta",' - '("$base"."$votes"-"$dod"."$votes")*100/NULLIF("$dod"."$votes",0) "$votes_dod_delta_percent" ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod",' + '"$base"."$m$votes"-"$dod"."$m$votes" "$m$votes_dod_delta",' + '("$base"."$m$votes"-"$dod"."$m$votes")*100/NULLIF("$dod"."$m$votes",0) "$m$votes_dod_delta_percent" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension_with_different_periods(self): query = slicer.data \ @@ -1888,45 +1884,45 @@ def test_use_same_nested_query_for_joining_references_with_same_period_and_dimen self.assertEqual('SELECT ' 'COALESCE(' - '"$base"."$timestamp",' - 'TIMESTAMPADD(\'day\',1,"$dod"."$timestamp"),' - 'TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp")' - ') "$timestamp",' - '"$base"."$votes" "$votes",' - '"$dod"."$votes" "$votes_dod",' - '"$base"."$votes"-"$dod"."$votes" "$votes_dod_delta",' - '"$yoy"."$votes" "$votes_yoy",' - '"$base"."$votes"-"$yoy"."$votes" "$votes_yoy_delta" ' + '"$base"."$d$timestamp",' + 'TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp"),' + 'TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")' + ') "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod",' + '"$base"."$m$votes"-"$dod"."$m$votes" "$m$votes_dod_delta",' + '"$yoy"."$m$votes" "$m$votes_yoy",' + '"$base"."$m$votes"-"$yoy"."$m$votes" "$m$votes_yoy_delta" ' 'FROM ' '(' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$base" ' # end-nested 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$dod" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$timestamp") ' + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' 'FULL OUTER JOIN (' # nested 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp"' + 'GROUP BY "$d$timestamp"' ') "$yoy" ' # end-nested - 'ON "$base"."$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$timestamp") ' - 'ORDER BY "$timestamp"', str(query)) + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -1941,15 +1937,15 @@ def test_dimension_with_join_includes_join_in_query(self): .query self.assertEqual('SELECT ' - 'TRUNC("politician"."timestamp",\'DD\') "$timestamp",' - '"politician"."district_id" "$district",' - '"district"."district_name" "$district_display",' - 'SUM("politician"."votes") "$votes" ' + 'TRUNC("politician"."timestamp",\'DD\') "$d$timestamp",' + '"politician"."district_id" "$d$district",' + '"district"."district_name" "$d$district_display",' + 'SUM("politician"."votes") "$m$votes" ' 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' - 'GROUP BY "$timestamp","$district","$district_display" ' - 'ORDER BY "$timestamp","$district_display"', str(query)) + 'GROUP BY "$d$timestamp","$d$district","$d$district_display" ' + 'ORDER BY "$d$timestamp","$d$district_display"', str(query)) def test_dimension_with_multiple_joins_includes_joins_ordered__in_query(self): query = slicer.data \ @@ -1960,18 +1956,18 @@ def test_dimension_with_multiple_joins_includes_joins_ordered__in_query(self): .query self.assertEqual('SELECT ' - 'TRUNC("politician"."timestamp",\'DD\') "$timestamp",' - '"politician"."district_id" "$district",' - '"district"."district_name" "$district_display",' - 'SUM("politician"."votes") "$votes",' - 'COUNT("voter"."id") "$voters" ' + 'TRUNC("politician"."timestamp",\'DD\') "$d$timestamp",' + '"politician"."district_id" "$d$district",' + '"district"."district_name" "$d$district_display",' + 'SUM("politician"."votes") "$m$votes",' + 'COUNT("voter"."id") "$m$voters" ' 'FROM "politics"."politician" ' 'JOIN "politics"."voter" ' 'ON "politician"."id"="voter"."politician_id" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' - 'GROUP BY "$timestamp","$district","$district_display" ' - 'ORDER BY "$timestamp","$district_display"', str(query)) + 'GROUP BY "$d$timestamp","$d$district","$d$district_display" ' + 'ORDER BY "$d$timestamp","$d$district_display"', str(query)) def test_dimension_with_recursive_join_joins_all_join_tables(self): query = slicer.data \ @@ -1981,17 +1977,17 @@ def test_dimension_with_recursive_join_joins_all_join_tables(self): .query self.assertEqual('SELECT ' - 'TRUNC("politician"."timestamp",\'DD\') "$timestamp",' - '"district"."state_id" "$state",' - '"state"."state_name" "$state_display",' - 'SUM("politician"."votes") "$votes" ' + 'TRUNC("politician"."timestamp",\'DD\') "$d$timestamp",' + '"district"."state_id" "$d$state",' + '"state"."state_name" "$d$state_display",' + 'SUM("politician"."votes") "$m$votes" ' 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' 'JOIN "locations"."state" ' 'ON "district"."state_id"="state"."id" ' - 'GROUP BY "$timestamp","$state","$state_display" ' - 'ORDER BY "$timestamp","$state_display"', str(query)) + 'GROUP BY "$d$timestamp","$d$state","$d$state_display" ' + 'ORDER BY "$d$timestamp","$d$state_display"', str(query)) def test_metric_with_join_includes_join_in_query(self): query = slicer.data \ @@ -2000,13 +1996,13 @@ def test_metric_with_join_includes_join_in_query(self): .query self.assertEqual('SELECT ' - '"politician"."political_party" "$political_party",' - 'COUNT("voter"."id") "$voters" ' + '"politician"."political_party" "$d$political_party",' + 'COUNT("voter"."id") "$m$voters" ' 'FROM "politics"."politician" ' 'JOIN "politics"."voter" ' 'ON "politician"."id"="voter"."politician_id" ' - 'GROUP BY "$political_party" ' - 'ORDER BY "$political_party"', str(query)) + 'GROUP BY "$d$political_party" ' + 'ORDER BY "$d$political_party"', str(query)) def test_dimension_filter_with_join_on_display_definition_does_not_include_join_in_query(self): query = slicer.data \ @@ -2015,7 +2011,7 @@ def test_dimension_filter_with_join_on_display_definition_does_not_include_join_ .query self.assertEqual('SELECT ' - 'SUM("votes") "$votes" ' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' 'WHERE "district_id" IN (1)', str(query)) @@ -2026,7 +2022,7 @@ def test_dimension_filter_display_field_with_join_includes_join_in_query(self): .query self.assertEqual('SELECT ' - 'SUM("politician"."votes") "$votes" ' + 'SUM("politician"."votes") "$m$votes" ' 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' @@ -2039,7 +2035,7 @@ def test_dimension_filter_with_recursive_join_includes_join_in_query(self): .query self.assertEqual('SELECT ' - 'SUM("politician"."votes") "$votes" ' + 'SUM("politician"."votes") "$m$votes" ' 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' @@ -2052,7 +2048,7 @@ def test_dimension_filter_with_deep_recursive_join_includes_joins_in_query(self) .query self.assertEqual('SELECT ' - 'SUM("politician"."votes") "$votes" ' + 'SUM("politician"."votes") "$m$votes" ' 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' @@ -2074,11 +2070,11 @@ def test_build_query_order_by_dimension(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp"', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) def test_build_query_order_by_dimension_display(self): query = slicer.data \ @@ -2088,12 +2084,12 @@ def test_build_query_order_by_dimension_display(self): .query self.assertEqual('SELECT ' - '"candidate_id" "$candidate",' - '"candidate_name" "$candidate_display",' - 'SUM("votes") "$votes" ' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$candidate","$candidate_display" ' - 'ORDER BY "$candidate_display"', str(query)) + 'GROUP BY "$d$candidate","$d$candidate_display" ' + 'ORDER BY "$d$candidate_display"', str(query)) def test_build_query_order_by_dimension_asc(self): query = slicer.data \ @@ -2103,11 +2099,11 @@ def test_build_query_order_by_dimension_asc(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp" ASC', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp" ASC', str(query)) def test_build_query_order_by_dimension_desc(self): query = slicer.data \ @@ -2117,11 +2113,11 @@ def test_build_query_order_by_dimension_desc(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp" DESC', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp" DESC', str(query)) def test_build_query_order_by_metric(self): query = slicer.data \ @@ -2131,11 +2127,11 @@ def test_build_query_order_by_metric(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$votes"', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$m$votes"', str(query)) def test_build_query_order_by_metric_asc(self): query = slicer.data \ @@ -2145,11 +2141,11 @@ def test_build_query_order_by_metric_asc(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$votes" ASC', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$m$votes" ASC', str(query)) def test_build_query_order_by_metric_desc(self): query = slicer.data \ @@ -2159,11 +2155,11 @@ def test_build_query_order_by_metric_desc(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$votes" DESC', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$m$votes" DESC', str(query)) def test_build_query_order_by_multiple_dimensions(self): query = slicer.data \ @@ -2174,13 +2170,13 @@ def test_build_query_order_by_multiple_dimensions(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"candidate_id" "$candidate",' - '"candidate_name" "$candidate_display",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp","$candidate","$candidate_display" ' - 'ORDER BY "$timestamp","$candidate"', str(query)) + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display" ' + 'ORDER BY "$d$timestamp","$d$candidate"', str(query)) def test_build_query_order_by_multiple_dimensions_with_different_orientations(self): query = slicer.data \ @@ -2191,13 +2187,13 @@ def test_build_query_order_by_multiple_dimensions_with_different_orientations(se .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - '"candidate_id" "$candidate",' - '"candidate_name" "$candidate_display",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp","$candidate","$candidate_display" ' - 'ORDER BY "$timestamp" DESC,"$candidate" ASC', str(query)) + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display" ' + 'ORDER BY "$d$timestamp" DESC,"$d$candidate" ASC', str(query)) def test_build_query_order_by_metrics_and_dimensions(self): query = slicer.data \ @@ -2208,11 +2204,11 @@ def test_build_query_order_by_metrics_and_dimensions(self): .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp","$votes"', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp","$m$votes"', str(query)) def test_build_query_order_by_metrics_and_dimensions_with_different_orientations(self): query = slicer.data \ @@ -2223,11 +2219,11 @@ def test_build_query_order_by_metrics_and_dimensions_with_different_orientations .query self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp" ASC,"$votes" DESC', str(query)) + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp" ASC,"$m$votes" DESC', str(query)) @patch('fireant.slicer.queries.builder.fetch_data') @@ -2241,11 +2237,11 @@ def test_set_limit(self, mock_fetch_data: Mock): mock_fetch_data.assert_called_once_with(ANY, 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp" LIMIT 20', + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp" LIMIT 20', dimensions=DimensionMatcher(slicer.dimensions.timestamp)) def test_set_offset(self, mock_fetch_data: Mock): @@ -2257,11 +2253,11 @@ def test_set_offset(self, mock_fetch_data: Mock): mock_fetch_data.assert_called_once_with(ANY, 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp" ' 'OFFSET 20', dimensions=DimensionMatcher(slicer.dimensions.timestamp)) @@ -2275,11 +2271,11 @@ def test_set_limit_and_offset(self, mock_fetch_data: Mock): mock_fetch_data.assert_called_once_with(ANY, 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$timestamp",' - 'SUM("votes") "$votes" ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'GROUP BY "$timestamp" ' - 'ORDER BY "$timestamp" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp" ' 'LIMIT 20 ' 'OFFSET 30', dimensions=DimensionMatcher(slicer.dimensions.timestamp)) @@ -2326,7 +2322,7 @@ def test_pass_query_from_builder_as_arg(self, mock_fetch_data: Mock): .fetch() mock_fetch_data.assert_called_once_with(ANY, - 'SELECT SUM("votes") "$votes" ' + 'SELECT SUM("votes") "$m$votes" ' 'FROM "politics"."politician"', dimensions=ANY) @@ -2427,6 +2423,6 @@ def test_operations_results_stored_in_data_frame(self, mock_fetch_data: Mock): .widget(mock_widget) \ .fetch() - f_op_key = format_key(mock_operation.key) + f_op_key = format_metric_key(mock_operation.key) self.assertIn(f_op_key, mock_df) self.assertEqual(mock_df[f_op_key], mock_operation.apply.return_value) diff --git a/fireant/tests/slicer/queries/test_database.py b/fireant/tests/slicer/queries/test_database.py index befc0f2d..cab6a20f 100644 --- a/fireant/tests/slicer/queries/test_database.py +++ b/fireant/tests/slicer/queries/test_database.py @@ -1,3 +1,4 @@ +import time from unittest import TestCase from unittest.mock import ( MagicMock, @@ -7,7 +8,6 @@ import numpy as np import pandas as pd -import time from fireant.slicer.queries.database import ( clean_and_apply_index, @@ -21,10 +21,7 @@ slicer, uni_dim_df, ) -from fireant.utils import ( - format_key, - format_key as f, -) +from fireant.utils import format_dimension_key as fd class FetchDataTests(TestCase): @@ -46,7 +43,7 @@ def test_fetch_data_called_on_database(self): def test_index_set_on_data_frame_result(self): fetch_data(self.mock_database, self.mock_query, self.mock_dimensions) - self.mock_data_frame.set_index.assert_called_once_with([f(d.key) + self.mock_data_frame.set_index.assert_called_once_with([fd(d.key) for d in self.mock_dimensions]) @patch('fireant.slicer.queries.database.query_logger.debug') @@ -102,7 +99,7 @@ def add_nans(df): cont_uni_dim_nans_df = cont_uni_dim_df \ - .append(cont_uni_dim_df.groupby(level=format_key('timestamp')).apply(add_nans)) \ + .append(cont_uni_dim_df.groupby(level=fd('timestamp')).apply(add_nans)) \ .sort_index() @@ -113,7 +110,7 @@ def totals(df): cont_uni_dim_nans_totals_df = cont_uni_dim_nans_df \ - .append(cont_uni_dim_nans_df.groupby(level=format_key('timestamp')).apply(totals)) \ + .append(cont_uni_dim_nans_df.groupby(level=fd('timestamp')).apply(totals)) \ .sort_index() \ .sort_index(level=[0, 1], ascending=False) # This sorts the DF so that the first instance of NaN is the totals diff --git a/fireant/tests/slicer/queries/test_dimension_choices.py b/fireant/tests/slicer/queries/test_dimension_choices.py index fb195c71..63cb6a2b 100644 --- a/fireant/tests/slicer/queries/test_dimension_choices.py +++ b/fireant/tests/slicer/queries/test_dimension_choices.py @@ -19,9 +19,9 @@ def test_query_choices_for_cat_dimension(self): .query self.assertEqual('SELECT ' - '"political_party" "$political_party" ' + '"political_party" "$d$political_party" ' 'FROM "politics"."politician" ' - 'GROUP BY "$political_party"', str(query)) + 'GROUP BY "$d$political_party"', str(query)) def test_query_choices_for_uni_dimension(self): query = slicer.dimensions.candidate \ @@ -29,10 +29,10 @@ def test_query_choices_for_uni_dimension(self): .query self.assertEqual('SELECT ' - '"candidate_id" "$candidate",' - '"candidate_name" "$candidate_display" ' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display" ' 'FROM "politics"."politician" ' - 'GROUP BY "$candidate","$candidate_display"', str(query)) + 'GROUP BY "$d$candidate","$d$candidate_display"', str(query)) def test_query_choices_for_uni_dimension_with_join(self): query = slicer.dimensions.district \ @@ -40,12 +40,12 @@ def test_query_choices_for_uni_dimension_with_join(self): .query self.assertEqual('SELECT ' - '"politician"."district_id" "$district",' - '"district"."district_name" "$district_display" ' + '"politician"."district_id" "$d$district",' + '"district"."district_name" "$d$district_display" ' 'FROM "politics"."politician" ' 'OUTER JOIN "locations"."district" ' 'ON "politician"."district_id"="district"."id" ' - 'GROUP BY "$district","$district_display"', str(query)) + 'GROUP BY "$d$district","$d$district_display"', str(query)) def test_no_choices_attr_for_datetime_dimension(self): with self.assertRaises(AttributeError): @@ -62,11 +62,11 @@ def test_filter_choices(self): .query self.assertEqual('SELECT ' - '"candidate_id" "$candidate",' - '"candidate_name" "$candidate_display" ' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display" ' 'FROM "politics"."politician" ' 'WHERE "political_party" IN (\'d\',\'r\') ' - 'GROUP BY "$candidate","$candidate_display"', str(query)) + 'GROUP BY "$d$candidate","$d$candidate_display"', str(query)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -79,10 +79,10 @@ def test_query_choices_for_cat_dimension(self, mock_fetch_data: Mock): mock_fetch_data.assert_called_once_with(ANY, 'SELECT ' - '"political_party" "$political_party" ' + '"political_party" "$d$political_party" ' 'FROM "politics"."politician" ' - 'GROUP BY "$political_party" ' - 'ORDER BY "$political_party"', + 'GROUP BY "$d$political_party" ' + 'ORDER BY "$d$political_party"', dimensions=DimensionMatcher(slicer.dimensions.political_party)) def test_query_choices_for_uni_dimension(self, mock_fetch_data: Mock): @@ -92,9 +92,9 @@ def test_query_choices_for_uni_dimension(self, mock_fetch_data: Mock): mock_fetch_data.assert_called_once_with(ANY, 'SELECT ' - '"candidate_id" "$candidate",' - '"candidate_name" "$candidate_display" ' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display" ' 'FROM "politics"."politician" ' - 'GROUP BY "$candidate","$candidate_display" ' - 'ORDER BY "$candidate_display"', + 'GROUP BY "$d$candidate","$d$candidate_display" ' + 'ORDER BY "$d$candidate_display"', dimensions=DimensionMatcher(slicer.dimensions.candidate)) diff --git a/fireant/tests/slicer/test_operations.py b/fireant/tests/slicer/test_operations.py index b4748e55..d86daa07 100644 --- a/fireant/tests/slicer/test_operations.py +++ b/fireant/tests/slicer/test_operations.py @@ -24,7 +24,7 @@ def test_apply_to_timeseries(self): result = cumsum.apply(cont_dim_df) expected = pd.Series([2, 4, 6, 8, 10, 12], - name='$wins', + name='$m$wins', index=cont_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -33,7 +33,7 @@ def test_apply_to_timeseries_with_uni_dim(self): result = cumsum.apply(cont_uni_dim_df) expected = pd.Series([1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6], - name='$wins', + name='$m$wins', index=cont_uni_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -42,7 +42,7 @@ def test_apply_to_timeseries_with_uni_dim_and_ref(self): result = cumsum.apply(cont_uni_dim_ref_df) expected = pd.Series([1, 1, 2, 2, 3, 3, 4, 4, 5, 5], - name='$wins', + name='$m$wins', index=cont_uni_dim_ref_df.index) pandas.testing.assert_series_equal(result, expected) @@ -53,7 +53,7 @@ def test_apply_to_timeseries(self): result = cumprod.apply(cont_dim_df) expected = pd.Series([2, 4, 8, 16, 32, 64], - name='$wins', + name='$m$wins', index=cont_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -62,7 +62,7 @@ def test_apply_to_timeseries_with_uni_dim(self): result = cumprod.apply(cont_uni_dim_df) expected = pd.Series([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], - name='$wins', + name='$m$wins', index=cont_uni_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -71,7 +71,7 @@ def test_apply_to_timeseries_with_uni_dim_and_ref(self): result = cumprod.apply(cont_uni_dim_ref_df) expected = pd.Series([1, 1, 1, 1, 1, 1, 1, 1, 1, 1], - name='$wins', + name='$m$wins', index=cont_uni_dim_ref_df.index) pandas.testing.assert_series_equal(result, expected) @@ -82,7 +82,7 @@ def test_apply_to_timeseries(self): result = cummean.apply(cont_dim_df) expected = pd.Series([15220449.0, 15941233.0, 17165799.3, 18197903.25, 18672764.6, 18612389.3], - name='$votes', + name='$m$votes', index=cont_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -92,7 +92,7 @@ def test_apply_to_timeseries_with_uni_dim(self): expected = pd.Series([5574387.0, 9646062.0, 5903886.0, 10037347.0, 6389131.0, 10776668.3, 6793838.5, 11404064.75, 7010664.2, 11662100.4, 6687706.0, 11924683.3], - name='$votes', + name='$m$votes', index=cont_uni_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -102,7 +102,7 @@ def test_apply_to_timeseries_with_uni_dim_and_ref(self): expected = pd.Series([6233385.0, 10428632.0, 6796503.0, 11341971.5, 7200322.3, 11990065.6, 7369733.5, 12166110.0, 6910369.8, 12380407.6], - name='$votes', + name='$m$votes', index=cont_uni_dim_ref_df.index) pandas.testing.assert_series_equal(result, expected) @@ -113,7 +113,7 @@ def test_apply_to_timeseries(self): result = rolling_mean.apply(cont_dim_df) expected = pd.Series([np.nan, np.nan, 2.0, 2.0, 2.0, 2.0], - name='$wins', + name='$m$wins', index=cont_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -122,7 +122,7 @@ def test_apply_to_timeseries_with_uni_dim(self): result = rolling_mean.apply(cont_uni_dim_df) expected = pd.Series([np.nan, np.nan, np.nan, np.nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], - name='$wins', + name='$m$wins', index=cont_uni_dim_df.index) pandas.testing.assert_series_equal(result, expected) @@ -131,6 +131,6 @@ def test_apply_to_timeseries_with_uni_dim_and_ref(self): result = rolling_mean.apply(cont_uni_dim_ref_df) expected = pd.Series([np.nan, np.nan, np.nan, np.nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], - name='$wins', + name='$m$wins', index=cont_uni_dim_ref_df.index) pandas.testing.assert_series_equal(result, expected) diff --git a/fireant/tests/slicer/widgets/test_csv.py b/fireant/tests/slicer/widgets/test_csv.py index 37dcb6e6..52703447 100644 --- a/fireant/tests/slicer/widgets/test_csv.py +++ b/fireant/tests/slicer/widgets/test_csv.py @@ -17,7 +17,10 @@ slicer, uni_dim_df, ) -from fireant.utils import format_key as f +from fireant.utils import ( + format_dimension_key as fd, + format_metric_key as fm, +) class CSVWidgetTests(TestCase): @@ -27,7 +30,7 @@ def test_single_metric(self): result = CSV(slicer.metrics.votes) \ .transform(single_metric_df, slicer, [], []) - expected = single_metric_df.copy()[[f('votes')]] + expected = single_metric_df.copy()[[fm('votes')]] expected.columns = ['Votes'] self.assertEqual(result, expected.to_csv()) @@ -36,7 +39,7 @@ def test_multiple_metrics(self): result = CSV(slicer.metrics.votes, slicer.metrics.wins) \ .transform(multi_metric_df, slicer, [], []) - expected = multi_metric_df.copy()[[f('votes'), f('wins')]] + expected = multi_metric_df.copy()[[fm('votes'), fm('wins')]] expected.columns = ['Votes', 'Wins'] self.assertEqual(result, expected.to_csv()) @@ -45,7 +48,7 @@ def test_multiple_metrics_reversed(self): result = CSV(slicer.metrics.wins, slicer.metrics.votes) \ .transform(multi_metric_df, slicer, [], []) - expected = multi_metric_df.copy()[[f('wins'), f('votes')]] + expected = multi_metric_df.copy()[[fm('wins'), fm('votes')]] expected.columns = ['Wins', 'Votes'] self.assertEqual(result, expected.to_csv()) @@ -54,7 +57,7 @@ def test_time_series_dim(self): result = CSV(slicer.metrics.wins) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) - expected = cont_dim_df.copy()[[f('wins')]] + expected = cont_dim_df.copy()[[fm('wins')]] expected.index.names = ['Timestamp'] expected.columns = ['Wins'] @@ -64,7 +67,7 @@ def test_time_series_dim_with_operation(self): result = CSV(CumSum(slicer.metrics.votes)) \ .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp], []) - expected = cont_dim_operation_df.copy()[[f('cumsum(votes)')]] + expected = cont_dim_operation_df.copy()[[fm('cumsum(votes)')]] expected.index.names = ['Timestamp'] expected.columns = ['CumSum(Votes)'] @@ -74,7 +77,7 @@ def test_cat_dim(self): result = CSV(slicer.metrics.wins) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) - expected = cat_dim_df.copy()[[f('wins')]] + expected = cat_dim_df.copy()[[fm('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] @@ -85,8 +88,8 @@ def test_uni_dim(self): .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) expected = uni_dim_df.copy() \ - .set_index(f('candidate_display'), append=True) \ - .reset_index(f('candidate'), drop=True)[[f('wins')]] + .set_index(fd('candidate_display'), append=True) \ + .reset_index(fd('candidate'), drop=True)[[fm('wins')]] expected.index.names = ['Candidate'] expected.columns = ['Wins'] @@ -99,12 +102,12 @@ def test_uni_dim_no_display_definition(self): candidate.display_definition = None uni_dim_df_copy = uni_dim_df.copy() - del uni_dim_df_copy[f(slicer.dimensions.candidate.display_key)] + del uni_dim_df_copy[fd(slicer.dimensions.candidate.display_key)] result = CSV(slicer.metrics.wins) \ .transform(uni_dim_df_copy, slicer, [candidate], []) - expected = uni_dim_df_copy.copy()[[f('wins')]] + expected = uni_dim_df_copy.copy()[[fm('wins')]] expected.index.names = ['Candidate'] expected.columns = ['Wins'] @@ -115,8 +118,8 @@ def test_multi_dims_time_series_and_uni(self): .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ - .set_index(f('state_display'), append=True) \ - .reset_index(f('state'), drop=False)[[f('wins')]] + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=False)[[fm('wins')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Wins'] @@ -126,7 +129,7 @@ def test_pivoted_single_dimension_no_effect(self): result = CSV(slicer.metrics.wins, pivot=True) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) - expected = cat_dim_df.copy()[[f('wins')]] + expected = cat_dim_df.copy()[[fm('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] @@ -136,7 +139,7 @@ def test_pivoted_multi_dims_time_series_and_cat(self): result = CSV(slicer.metrics.wins, pivot=True) \ .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party], []) - expected = cont_cat_dim_df.copy()[[f('wins')]] + expected = cont_cat_dim_df.copy()[[fm('wins')]] expected.index.names = ['Timestamp', 'Party'] expected.columns = ['Wins'] expected = expected.unstack(level=[1]) @@ -148,8 +151,8 @@ def test_pivoted_multi_dims_time_series_and_uni(self): .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ - .set_index(f('state_display'), append=True) \ - .reset_index(f('state'), drop=True)[[f('votes')]] + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=True)[[fm('votes')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Votes'] expected = expected.unstack(level=[1]) @@ -168,8 +171,8 @@ def test_time_series_ref(self): ]) expected = cont_uni_dim_ref_df.copy() \ - .set_index(f('state_display'), append=True) \ - .reset_index(f('state'), drop=True)[[f('votes'), f('votes_eoe')]] + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=True)[[fm('votes'), fm('votes_eoe')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Votes', 'Votes (EoE)'] diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py index 7591f36b..7b86fd8b 100644 --- a/fireant/tests/slicer/widgets/test_datatables.py +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -1,8 +1,8 @@ +from datetime import date from unittest import TestCase from unittest.mock import Mock import pandas as pd -from datetime import date from fireant.slicer.widgets.datatables import ( DataTablesJS, @@ -24,7 +24,7 @@ slicer, uni_dim_df, ) -from fireant.utils import format_key as f +from fireant.utils import format_dimension_key as fd class DataTablesTransformerTests(TestCase): @@ -238,7 +238,7 @@ def test_uni_dim_no_display_definition(self): candidate.display_definition = None uni_dim_df_copy = uni_dim_df.copy() - del uni_dim_df_copy[f(slicer.dimensions.candidate.display_key)] + del uni_dim_df_copy[fd(slicer.dimensions.candidate.display_key)] result = DataTablesJS(slicer.metrics.wins) \ .transform(uni_dim_df_copy, slicer, [candidate], []) diff --git a/fireant/tests/slicer/widgets/test_pandas.py b/fireant/tests/slicer/widgets/test_pandas.py index 0af133c7..dbd4d015 100644 --- a/fireant/tests/slicer/widgets/test_pandas.py +++ b/fireant/tests/slicer/widgets/test_pandas.py @@ -18,7 +18,10 @@ slicer, uni_dim_df, ) -from fireant.utils import format_key as f +from fireant.utils import ( + format_dimension_key as fd, + format_metric_key as fm, +) class DataTablesTransformerTests(TestCase): @@ -28,7 +31,7 @@ def test_single_metric(self): result = Pandas(slicer.metrics.votes) \ .transform(single_metric_df, slicer, [], []) - expected = single_metric_df.copy()[[f('votes')]] + expected = single_metric_df.copy()[[fm('votes')]] expected.columns = ['Votes'] pandas.testing.assert_frame_equal(result, expected) @@ -37,7 +40,7 @@ def test_multiple_metrics(self): result = Pandas(slicer.metrics.votes, slicer.metrics.wins) \ .transform(multi_metric_df, slicer, [], []) - expected = multi_metric_df.copy()[[f('votes'), f('wins')]] + expected = multi_metric_df.copy()[[fm('votes'), fm('wins')]] expected.columns = ['Votes', 'Wins'] pandas.testing.assert_frame_equal(result, expected) @@ -46,7 +49,7 @@ def test_multiple_metrics_reversed(self): result = Pandas(slicer.metrics.wins, slicer.metrics.votes) \ .transform(multi_metric_df, slicer, [], []) - expected = multi_metric_df.copy()[[f('wins'), f('votes')]] + expected = multi_metric_df.copy()[[fm('wins'), fm('votes')]] expected.columns = ['Wins', 'Votes'] pandas.testing.assert_frame_equal(result, expected) @@ -55,7 +58,7 @@ def test_time_series_dim(self): result = Pandas(slicer.metrics.wins) \ .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) - expected = cont_dim_df.copy()[[f('wins')]] + expected = cont_dim_df.copy()[[fm('wins')]] expected.index.names = ['Timestamp'] expected.columns = ['Wins'] @@ -65,7 +68,7 @@ def test_time_series_dim_with_operation(self): result = Pandas(CumSum(slicer.metrics.votes)) \ .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp], []) - expected = cont_dim_operation_df.copy()[[f('cumsum(votes)')]] + expected = cont_dim_operation_df.copy()[[fm('cumsum(votes)')]] expected.index.names = ['Timestamp'] expected.columns = ['CumSum(Votes)'] @@ -75,7 +78,7 @@ def test_cat_dim(self): result = Pandas(slicer.metrics.wins) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) - expected = cat_dim_df.copy()[[f('wins')]] + expected = cat_dim_df.copy()[[fm('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] @@ -86,9 +89,9 @@ def test_uni_dim(self): .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) expected = uni_dim_df.copy() \ - .set_index(f('candidate_display'), append=True) \ - .reset_index(f('candidate'), drop=True) \ - [[f('wins')]] + .set_index(fd('candidate_display'), append=True) \ + .reset_index(fd('candidate'), drop=True) \ + [[fm('wins')]] expected.index.names = ['Candidate'] expected.columns = ['Wins'] @@ -101,12 +104,12 @@ def test_uni_dim_no_display_definition(self): candidate.display_definition = None uni_dim_df_copy = uni_dim_df.copy() - del uni_dim_df_copy[f(slicer.dimensions.candidate.display_key)] + del uni_dim_df_copy[fd(slicer.dimensions.candidate.display_key)] result = Pandas(slicer.metrics.wins) \ .transform(uni_dim_df_copy, slicer, [candidate], []) - expected = uni_dim_df_copy.copy()[[f('wins')]] + expected = uni_dim_df_copy.copy()[[fm('wins')]] expected.index.names = ['Candidate'] expected.columns = ['Wins'] @@ -117,8 +120,8 @@ def test_multi_dims_time_series_and_uni(self): .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ - .set_index(f('state_display'), append=True) \ - .reset_index(f('state'), drop=False)[[f('wins')]] + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=False)[[fm('wins')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Wins'] @@ -128,7 +131,7 @@ def test_pivoted_single_dimension_no_effect(self): result = Pandas(slicer.metrics.wins, pivot=True) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) - expected = cat_dim_df.copy()[[f('wins')]] + expected = cat_dim_df.copy()[[fm('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] @@ -138,7 +141,7 @@ def test_pivoted_multi_dims_time_series_and_cat(self): result = Pandas(slicer.metrics.wins, pivot=True) \ .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party], []) - expected = cont_cat_dim_df.copy()[[f('wins')]] + expected = cont_cat_dim_df.copy()[[fm('wins')]] expected.index.names = ['Timestamp', 'Party'] expected.columns = ['Wins'] expected = expected.unstack(level=[1]) @@ -150,8 +153,8 @@ def test_pivoted_multi_dims_time_series_and_uni(self): .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ - .set_index(f('state_display'), append=True) \ - .reset_index(f('state'), drop=True)[[f('votes')]] + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=True)[[fm('votes')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Votes'] expected = expected.unstack(level=[1]) @@ -169,8 +172,8 @@ def test_time_series_ref(self): ]) expected = cont_uni_dim_ref_df.copy() \ - .set_index(f('state_display'), append=True) \ - .reset_index(f('state'), drop=True)[[f('votes'), f('votes_eoe')]] + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=True)[[fm('votes'), fm('votes_eoe')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Votes', 'Votes (EoE)'] @@ -187,9 +190,9 @@ def test_metric_format(self): result = Pandas(votes) \ .transform(cont_dim_df / 3, slicer, [slicer.dimensions.timestamp], []) - expected = cont_dim_df.copy()[[f('votes')]] - expected[f('votes')] = ['${0:.2f}€'.format(x) - for x in expected[f('votes')] / 3] + expected = cont_dim_df.copy()[[fm('votes')]] + expected[fm('votes')] = ['${0:.2f}€'.format(x) + for x in expected[fm('votes')] / 3] expected.index.names = ['Timestamp'] expected.columns = ['Votes'] diff --git a/fireant/utils.py b/fireant/utils.py index 319b60d8..9512de97 100644 --- a/fireant/utils.py +++ b/fireant/utils.py @@ -117,7 +117,19 @@ def groupby_first_level(index): if x[1:] not in seen and not seen.add(x[1:])] -def format_key(key): +def format_key(key, prefix=None): if key is None: return key + + if prefix is not None: + return '${}${}'.format(prefix, key) + return '${}'.format(key) + + +def format_dimension_key(key): + return format_key(key, 'd') + + +def format_metric_key(key): + return format_key(key, 'm') From 1582491971906c2368e7ffc16a8b0a158cd5a9d4 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 10 Jul 2018 11:05:11 +0200 Subject: [PATCH 087/123] Bumped version to 1.0.0.dev37 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 51337cca..ba7d6db7 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev36' +__version__ = '1.0.0.dev37' From 9807d074d0ca3784685b673a80b26fe3e7804987 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 10 Jul 2018 14:59:03 +0200 Subject: [PATCH 088/123] Added special case for using rolling operation with date dimension and date filter in order to adjust the filter then truncate the resulting data frame --- fireant/slicer/operations.py | 47 +- fireant/slicer/queries/builder.py | 23 +- fireant/slicer/queries/special_cases.py | 103 + .../queries/test_build_dimension_filters.py | 295 ++ .../slicer/queries/test_build_dimensions.py | 452 ++++ .../tests/slicer/queries/test_build_joins.py | 138 + .../queries/test_build_metric_filters.py | 141 + .../slicer/queries/test_build_metrics.py | 61 + .../slicer/queries/test_build_operations.py | 61 + .../slicer/queries/test_build_orderbys.py | 173 ++ .../slicer/queries/test_build_pagination.py | 67 + .../slicer/queries/test_build_references.py | 958 +++++++ .../tests/slicer/queries/test_build_render.py | 147 + fireant/tests/slicer/queries/test_builder.py | 2377 ----------------- 14 files changed, 2652 insertions(+), 2391 deletions(-) create mode 100644 fireant/slicer/queries/special_cases.py create mode 100644 fireant/tests/slicer/queries/test_build_dimension_filters.py create mode 100644 fireant/tests/slicer/queries/test_build_dimensions.py create mode 100644 fireant/tests/slicer/queries/test_build_joins.py create mode 100644 fireant/tests/slicer/queries/test_build_metric_filters.py create mode 100644 fireant/tests/slicer/queries/test_build_metrics.py create mode 100644 fireant/tests/slicer/queries/test_build_operations.py create mode 100644 fireant/tests/slicer/queries/test_build_orderbys.py create mode 100644 fireant/tests/slicer/queries/test_build_pagination.py create mode 100644 fireant/tests/slicer/queries/test_build_references.py create mode 100644 fireant/tests/slicer/queries/test_build_render.py diff --git a/fireant/slicer/operations.py b/fireant/slicer/operations.py index 54c26f0f..1af7a41a 100644 --- a/fireant/slicer/operations.py +++ b/fireant/slicer/operations.py @@ -19,6 +19,10 @@ class Operation(object): def apply(self, data_frame): raise NotImplementedError() + @property + def metrics(self): + raise NotImplementedError() + @property def operations(self): return [] @@ -32,6 +36,17 @@ def __init__(self, key, label, prefix=None, suffix=None, precision=None): self.suffix = suffix self.precision = precision + def apply(self, data_frame): + raise NotImplementedError() + + @property + def metrics(self): + raise NotImplementedError() + + @property + def operations(self): + raise NotImplementedError() + def _group_levels(self, index): """ Get the index levels that need to be grouped. This is to avoid apply the cumulative function across separate @@ -57,6 +72,9 @@ def __init__(self, arg): self.arg = arg + def apply(self, data_frame): + raise NotImplementedError() + @property def metrics(self): return [metric @@ -70,9 +88,6 @@ def operations(self): if isinstance(operation, Operation) for op_and_children in [operation] + operation.operations] - def apply(self, data_frame): - raise NotImplementedError() - def __repr__(self): return self.key @@ -123,9 +138,9 @@ def apply(self, data_frame): return self.cummean(data_frame[df_key]) -class _Rolling(_BaseOperation): +class RollingOperation(_BaseOperation): def __init__(self, arg, window, min_periods=None): - super(_Rolling, self).__init__( + super(RollingOperation, self).__init__( key='{}({})'.format(self.__class__.__name__.lower(), getattr(arg, 'key', arg)), label='{}({})'.format(self.__class__.__name__, @@ -139,11 +154,31 @@ def __init__(self, arg, window, min_periods=None): self.window = window self.min_periods = min_periods + def _should_adjust(self, other_operations): + # Need to figure out if this rolling operation is has the largest window, and if it's the first of multiple + # rolling operations if there are more than one operation sharing the largest window. + first_max_rolling = list(sorted(other_operations, key=lambda operation: operation.window))[0] + + return first_max_rolling is self + def apply(self, data_frame): raise NotImplementedError() + @property + def metrics(self): + return [metric + for metric in [self.arg] + if isinstance(metric, Metric)] + + @property + def operations(self): + return [op_and_children + for operation in [self.arg] + if isinstance(operation, Operation) + for op_and_children in [operation] + operation.operations] + -class RollingMean(_Rolling): +class RollingMean(RollingOperation): def rolling_mean(self, x): return x.rolling(self.window, self.min_periods).mean() diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 5633d689..2457cfd3 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -13,6 +13,7 @@ format_metric_key, immutable, ) +from . import special_cases from .database import fetch_data from .finders import ( find_and_group_references_for_dimensions, @@ -154,14 +155,18 @@ def query(self): reference_groups = find_and_group_references_for_dimensions(self._references) totals_dimensions = find_dimensions_with_totals(self._dimensions) - query = make_slicer_query_with_references_and_totals(self.slicer.database, - self.table, - self.slicer.joins, - self._dimensions, - find_metrics_for_widgets(self._widgets), - self._filters, - reference_groups, - totals_dimensions) + operations = find_operations_for_widgets(self._widgets) + args = special_cases.apply_to_query_args(self.slicer.database, + self.table, + self.slicer.joins, + self._dimensions, + find_metrics_for_widgets(self._widgets), + self._filters, + reference_groups, + totals_dimensions, + operations) + + query = make_slicer_query_with_references_and_totals(*args) # Add ordering orders = (self._orders or make_orders_for_dimensions(self._dimensions)) @@ -193,6 +198,8 @@ def fetch(self, hint=None) -> Iterable[Dict]: df_key = format_metric_key(operation.key) data_frame[df_key] = operation.apply(data_frame) + data_frame = special_cases.apply_operations_to_data_frame(operations, data_frame) + # Apply transformations return [widget.transform(data_frame, self.slicer, self._dimensions, self._references) for widget in self._widgets] diff --git a/fireant/slicer/queries/special_cases.py b/fireant/slicer/queries/special_cases.py new file mode 100644 index 00000000..e2e6efc9 --- /dev/null +++ b/fireant/slicer/queries/special_cases.py @@ -0,0 +1,103 @@ +from datetime import timedelta + +import pandas as pd + +from fireant.slicer.dimensions import DatetimeDimension +from fireant.slicer.filters import RangeFilter +from fireant.slicer.operations import RollingOperation + + +def adjust_daterange_filter_for_rolling_window(dimensions, operations, filters): + """ + This function adjusts date filters for a rolling operation in order to select enough date to compute the values for + within the original range. + + It only applies when using a date dimension in the first position and a RangeFilter is used on that dimension. It + is meant to be applied to a slicer query. + + :param dimensions: + The dimensions applied to a slicer query + :param operations: + The dimensions used in widgets in a slicer query + :param filters: + The filters applied to a slicer query + :return: + """ + has_datetime_dimension_in_first_dimension_pos = not len(dimensions) \ + or not isinstance(dimensions[0], DatetimeDimension) + if has_datetime_dimension_in_first_dimension_pos: + return filters + + has_rolling = any([isinstance(operation, RollingOperation) + for operation in operations]) + if not has_rolling: + return filters + + dim0 = dimensions[0] + filters_on_dim0 = [filter_ + for filter_ in filters + if isinstance(filter_, RangeFilter) + and str(filter_.definition.term) == str(dim0.definition)] + if not 0 < len(filters_on_dim0): + return filters + + max_rolling_period = max(operation.window + for operation in operations + if isinstance(operation, RollingOperation)) + + for filter_ in filters_on_dim0: + # Monkey patch the update start date on the date filter + args = {dim0.interval + 's': max_rolling_period} \ + if 'quarter' != dim0.interval \ + else {'months': max_rolling_period * 3} + filter_.definition.start.value -= timedelta(**args) + + return filters + + +def adjust_dataframe_for_rolling_window(operations, data_frame): + """ + This function adjusts the resulting data frame after executing a slicer query with a rolling operation. If there is + a date dimension in the first level of the data frame's index and a rolling operation is applied, it will slice the + dates following the max window to remove it. This way, the adjustment of date filters applied in + #adjust_daterange_filter_for_rolling_window are removed from the data frame but also in case there are no filters, + the first few date data points will be removed where the rolling window cannot be calculated. + + :param operations: + :param data_frame: + :return: + """ + has_rolling = any([isinstance(operation, RollingOperation) + for operation in operations]) + if not has_rolling: + return data_frame + + max_rolling_period = max(operation.window + for operation in operations + if isinstance(operation, RollingOperation)) + + if isinstance(data_frame.index, pd.DatetimeIndex): + return data_frame.iloc[max_rolling_period - 1:] + + if isinstance(data_frame.index, pd.MultiIndex) \ + and isinstance(data_frame.index.levels[0], pd.DatetimeIndex): + num_levels = len(data_frame.index.levels) + + return data_frame.groupby(level=list(range(1, num_levels))) \ + .apply(lambda df: df.iloc[max_rolling_period - 1:]) \ + .reset_index(level=list(range(num_levels - 1)), drop=True) + + return data_frame + + +def apply_to_query_args(database, table, joins, dimensions, metrics, filters, reference_groups, totals_dimensions, + operations): + filters = adjust_daterange_filter_for_rolling_window(dimensions, operations, filters) + + return database, table, joins, dimensions, metrics, filters, reference_groups, totals_dimensions + + +def apply_operations_to_data_frame(operations, data_frame): + data_frame = adjust_dataframe_for_rolling_window(operations, data_frame) + + return data_frame diff --git a/fireant/tests/slicer/queries/test_build_dimension_filters.py b/fireant/tests/slicer/queries/test_build_dimension_filters.py new file mode 100644 index 00000000..4e1741e6 --- /dev/null +++ b/fireant/tests/slicer/queries/test_build_dimension_filters.py @@ -0,0 +1,295 @@ +from datetime import date +from unittest import TestCase + +import fireant as f +from ..mocks import slicer + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderDimensionFilterTests(TestCase): + maxDiff = None + + def test_build_query_with_filter_isin_categorical_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.political_party.isin(['d'])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "political_party" IN (\'d\')', str(query)) + + def test_build_query_with_filter_notin_categorical_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.political_party.notin(['d'])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "political_party" NOT IN (\'d\')', str(query)) + + def test_build_query_with_filter_like_categorical_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.political_party.like('Rep%')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "political_party" LIKE \'Rep%\'', str(query)) + + def test_build_query_with_filter_not_like_categorical_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.political_party.not_like('Rep%')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "political_party" NOT LIKE \'Rep%\'', str(query)) + + def test_build_query_with_filter_isin_unique_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate.isin([1])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_id" IN (1)', str(query)) + + def test_build_query_with_filter_notin_unique_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate.notin([1])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_id" NOT IN (1)', str(query)) + + def test_build_query_with_filter_isin_unique_dim_display(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate.isin(['Donald Trump'], use_display=True)) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" IN (\'Donald Trump\')', str(query)) + + def test_build_query_with_filter_notin_unique_dim_display(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate.notin(['Donald Trump'], use_display=True)) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" NOT IN (\'Donald Trump\')', str(query)) + + def test_build_query_with_filter_like_unique_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate.like('%Trump')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) + + def test_build_query_with_filter_like_display_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate_display.like('%Trump')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) + + def test_build_query_with_filter_not_like_unique_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate.not_like('%Trump')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" NOT LIKE \'%Trump\'', str(query)) + + def test_build_query_with_filter_not_like_display_dim(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate_display.not_like('%Trump')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" NOT LIKE \'%Trump\'', str(query)) + + def test_build_query_with_filter_like_categorical_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.political_party.like('Rep%', 'Dem%')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "political_party" LIKE \'Rep%\' ' + 'OR "political_party" LIKE \'Dem%\'', str(query)) + + def test_build_query_with_filter_not_like_categorical_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.political_party.not_like('Rep%', 'Dem%')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "political_party" NOT LIKE \'Rep%\' ' + 'OR "political_party" NOT LIKE \'Dem%\'', str(query)) + + def test_build_query_with_filter_like_pattern_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.pattern.like('a%', 'b%')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "pattern" LIKE \'a%\' ' + 'OR "pattern" LIKE \'b%\'', str(query)) + + def test_build_query_with_filter_not_like_pattern_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.pattern.not_like('a%', 'b%')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "pattern" NOT LIKE \'a%\' ' + 'OR "pattern" NOT LIKE \'b%\'', str(query)) + + def test_build_query_with_filter_like_unique_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate.like('%Trump', '%Clinton')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" LIKE \'%Trump\' ' + 'OR "candidate_name" LIKE \'%Clinton\'', str(query)) + + def test_build_query_with_filter_like_display_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate_display.like('%Trump', '%Clinton')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" LIKE \'%Trump\' ' + 'OR "candidate_name" LIKE \'%Clinton\'', str(query)) + + def test_build_query_with_filter_not_like_unique_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate.not_like('%Trump', '%Clinton')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" NOT LIKE \'%Trump\' ' + 'OR "candidate_name" NOT LIKE \'%Clinton\'', str(query)) + + def test_build_query_with_filter_not_like_display_dim_multiple_patterns(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.candidate_display.not_like('%Trump', '%Clinton')) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "candidate_name" NOT LIKE \'%Trump\' ' + 'OR "candidate_name" NOT LIKE \'%Clinton\'', str(query)) + + def test_build_query_with_filter_isin_raise_exception_when_display_definition_undefined(self): + with self.assertRaises(f.QueryException): + slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.deepjoin.isin([1], use_display=True)) + + def test_build_query_with_filter_notin_raise_exception_when_display_definition_undefined(self): + with self.assertRaises(f.QueryException): + slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.deepjoin.notin([1], use_display=True)) + + def test_build_query_with_filter_like_raise_exception_when_display_definition_undefined(self): + with self.assertRaises(f.QueryException): + slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.deepjoin.like('test')) + + def test_build_query_with_filter_not_like_raise_exception_when_display_definition_undefined(self): + with self.assertRaises(f.QueryException): + slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.deepjoin.not_like('test')) + + def test_build_query_with_filter_range_datetime_dimension(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.timestamp.between(date(2009, 1, 20), date(2017, 1, 20))) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2009-01-20\' AND \'2017-01-20\'', str(query)) + + def test_build_query_with_filter_boolean_true(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.winner.is_(True)) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "is_winner"', str(query)) + + def test_build_query_with_filter_boolean_false(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.winner.is_(False)) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE NOT "is_winner"', str(query)) diff --git a/fireant/tests/slicer/queries/test_build_dimensions.py b/fireant/tests/slicer/queries/test_build_dimensions.py new file mode 100644 index 00000000..ee3adb3a --- /dev/null +++ b/fireant/tests/slicer/queries/test_build_dimensions.py @@ -0,0 +1,452 @@ +from datetime import date +from unittest import TestCase + +import fireant as f +from ..mocks import slicer + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderDimensionTests(TestCase): + maxDiff = None + + def test_build_query_with_datetime_dimension(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_build_query_with_datetime_dimension_hourly(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp(f.hourly)) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'HH\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_build_query_with_datetime_dimension_daily(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp(f.daily)) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_build_query_with_datetime_dimension_weekly(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp(f.weekly)) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_build_query_with_datetime_dimension_monthly(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp(f.monthly)) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'MM\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_build_query_with_datetime_dimension_quarterly(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp(f.quarterly)) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'Q\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_build_query_with_datetime_dimension_annually(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp(f.annually)) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'Y\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_build_query_with_boolean_dimension(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.winner) \ + .query + + self.assertEqual('SELECT ' + '"is_winner" "$d$winner",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$winner" ' + 'ORDER BY "$d$winner"', str(query)) + + def test_build_query_with_categorical_dimension(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.political_party) \ + .query + + self.assertEqual('SELECT ' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$political_party" ' + 'ORDER BY "$d$political_party"', str(query)) + + def test_build_query_with_unique_dimension(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.election) \ + .query + + self.assertEqual('SELECT ' + '"election_id" "$d$election",' + '"election_year" "$d$election_display",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$election","$d$election_display" ' + 'ORDER BY "$d$election_display"', str(query)) + + def test_build_query_with_pattern_dimension(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.pattern(['groupA%', 'groupB%'])) \ + .query + + self.assertEqual('SELECT ' + 'CASE ' + 'WHEN "pattern" LIKE \'groupA%\' THEN \'groupA%\' ' + 'WHEN "pattern" LIKE \'groupB%\' THEN \'groupB%\' ' + 'ELSE \'No Group\' ' + 'END "$d$pattern",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$pattern" ' + 'ORDER BY "$d$pattern"', str(query)) + + def test_build_query_with_pattern_no_values(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.pattern) \ + .query + + self.assertEqual('SELECT ' + '\'No Group\' "$d$pattern",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$pattern" ' + 'ORDER BY "$d$pattern"', str(query)) + + def test_build_query_with_multiple_dimensions(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .dimension(slicer.dimensions.candidate) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display" ' + 'ORDER BY "$d$timestamp","$d$candidate_display"', str(query)) + + def test_build_query_with_multiple_dimensions_and_visualizations(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes, slicer.metrics.wins)) \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes)) + .axis(f.HighCharts.LineChart(slicer.metrics.wins))) \ + .dimension(slicer.dimensions.timestamp) \ + .dimension(slicer.dimensions.political_party) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes",' + 'SUM("is_winner") "$m$wins" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp","$d$political_party" ' + 'ORDER BY "$d$timestamp","$d$political_party"', str(query)) + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderDimensionTotalsTests(TestCase): + maxDiff = None + + def test_build_query_with_totals_cat_dimension(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.political_party.rollup()) \ + .query + + self.assertEqual('(SELECT ' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$political_party") ' + + 'UNION ALL ' + + '(SELECT ' + 'NULL "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician") ' + + 'ORDER BY "$d$political_party"', str(query)) + + def test_build_query_with_totals_uni_dimension(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.candidate.rollup()) \ + .query + + self.assertEqual('(SELECT ' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$candidate","$d$candidate_display") ' + + 'UNION ALL ' + + '(SELECT ' + 'NULL "$d$candidate",' + 'NULL "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician") ' + + 'ORDER BY "$d$candidate_display"', str(query)) + + def test_build_query_with_totals_on_dimension_and_subsequent_dimensions(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp, + slicer.dimensions.candidate.rollup(), + slicer.dimensions.political_party) \ + .query + + self.assertEqual('(SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display","$d$political_party") ' + + 'UNION ALL ' + + '(SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'NULL "$d$candidate",' + 'NULL "$d$candidate_display",' + 'NULL "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp") ' + 'ORDER BY "$d$timestamp","$d$candidate_display","$d$political_party"', str(query)) + + def test_build_query_with_totals_on_multiple_dimensions_dimension(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp, + slicer.dimensions.candidate.rollup(), + slicer.dimensions.political_party.rollup()) \ + .query + + self.assertEqual('(SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display","$d$political_party") ' + + 'UNION ALL ' + + '(SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'NULL "$d$candidate",' + 'NULL "$d$candidate_display",' + 'NULL "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp") ' + + 'UNION ALL ' + + '(SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'NULL "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display") ' + + 'ORDER BY "$d$timestamp","$d$candidate_display","$d$political_party"', str(query)) + + def test_build_query_with_totals_cat_dimension_with_references(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp, + slicer.dimensions.political_party.rollup()) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .query + + # Important that in reference queries when using totals that the null dimensions are omitted from the nested + # queries and selected in the container query + self.assertEqual('(SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + 'COALESCE("$base"."$d$political_party","$dod"."$d$political_party") "$d$political_party",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM (' + + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp","$d$political_party"' + ') "$base" ' + + 'FULL OUTER JOIN (' + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp","$d$political_party"' + ') "$dod" ' + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'AND "$base"."$d$political_party"="$dod"."$d$political_party") ' + + 'UNION ALL ' + + '(SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + 'NULL "$d$political_party",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM (' + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' + + 'FULL OUTER JOIN (' + 'SELECT TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + 'ORDER BY "$d$timestamp","$d$political_party"', str(query)) + + def test_build_query_with_totals_cat_dimension_with_references_and_date_filters(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .dimension(slicer.dimensions.political_party.rollup()) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .filter(slicer.dimensions.timestamp.between(date(2018, 1, 1), date(2019, 1, 1))) \ + .query + + self.assertEqual('(SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + 'COALESCE("$base"."$d$political_party","$dod"."$d$political_party") "$d$political_party",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM (' + + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' + 'GROUP BY "$d$timestamp","$d$political_party"' + ') "$base" ' + + 'FULL OUTER JOIN (' + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' + 'GROUP BY "$d$timestamp","$d$political_party"' + ') "$dod" ' + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'AND "$base"."$d$political_party"="$dod"."$d$political_party") ' + + 'UNION ALL ' + + '(SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + 'NULL "$d$political_party",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM (' + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' + + 'FULL OUTER JOIN (' + 'SELECT TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + 'ORDER BY "$d$timestamp","$d$political_party"', str(query)) diff --git a/fireant/tests/slicer/queries/test_build_joins.py b/fireant/tests/slicer/queries/test_build_joins.py new file mode 100644 index 00000000..d4d89326 --- /dev/null +++ b/fireant/tests/slicer/queries/test_build_joins.py @@ -0,0 +1,138 @@ +from unittest import TestCase + +import fireant as f +from ..mocks import slicer + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderJoinTests(TestCase): + maxDiff = None + + def test_dimension_with_join_includes_join_in_query(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .dimension(slicer.dimensions.district) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("politician"."timestamp",\'DD\') "$d$timestamp",' + '"politician"."district_id" "$d$district",' + '"district"."district_name" "$d$district_display",' + 'SUM("politician"."votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'GROUP BY "$d$timestamp","$d$district","$d$district_display" ' + 'ORDER BY "$d$timestamp","$d$district_display"', str(query)) + + def test_dimension_with_multiple_joins_includes_joins_ordered__in_query(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes, + slicer.metrics.voters)) \ + .dimension(slicer.dimensions.timestamp) \ + .dimension(slicer.dimensions.district) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("politician"."timestamp",\'DD\') "$d$timestamp",' + '"politician"."district_id" "$d$district",' + '"district"."district_name" "$d$district_display",' + 'SUM("politician"."votes") "$m$votes",' + 'COUNT("voter"."id") "$m$voters" ' + 'FROM "politics"."politician" ' + 'JOIN "politics"."voter" ' + 'ON "politician"."id"="voter"."politician_id" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'GROUP BY "$d$timestamp","$d$district","$d$district_display" ' + 'ORDER BY "$d$timestamp","$d$district_display"', str(query)) + + def test_dimension_with_recursive_join_joins_all_join_tables(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .dimension(slicer.dimensions.state) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("politician"."timestamp",\'DD\') "$d$timestamp",' + '"district"."state_id" "$d$state",' + '"state"."state_name" "$d$state_display",' + 'SUM("politician"."votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'JOIN "locations"."state" ' + 'ON "district"."state_id"="state"."id" ' + 'GROUP BY "$d$timestamp","$d$state","$d$state_display" ' + 'ORDER BY "$d$timestamp","$d$state_display"', str(query)) + + def test_metric_with_join_includes_join_in_query(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.voters)) \ + .dimension(slicer.dimensions.political_party) \ + .query + + self.assertEqual('SELECT ' + '"politician"."political_party" "$d$political_party",' + 'COUNT("voter"."id") "$m$voters" ' + 'FROM "politics"."politician" ' + 'JOIN "politics"."voter" ' + 'ON "politician"."id"="voter"."politician_id" ' + 'GROUP BY "$d$political_party" ' + 'ORDER BY "$d$political_party"', str(query)) + + def test_dimension_filter_with_join_on_display_definition_does_not_include_join_in_query(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.district.isin([1])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "district_id" IN (1)', str(query)) + + def test_dimension_filter_display_field_with_join_includes_join_in_query(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.district.isin(['District 4'], use_display=True)) \ + .query + + self.assertEqual('SELECT ' + 'SUM("politician"."votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'WHERE "district"."district_name" IN (\'District 4\')', str(query)) + + def test_dimension_filter_with_recursive_join_includes_join_in_query(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.state.isin([1])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("politician"."votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'WHERE "district"."state_id" IN (1)', str(query)) + + def test_dimension_filter_with_deep_recursive_join_includes_joins_in_query(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.dimensions.deepjoin.isin([1])) \ + .query + + self.assertEqual('SELECT ' + 'SUM("politician"."votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'OUTER JOIN "locations"."district" ' + 'ON "politician"."district_id"="district"."id" ' + 'JOIN "locations"."state" ' + 'ON "district"."state_id"="state"."id" ' + 'JOIN "test"."deep" ' + 'ON "deep"."id"="state"."ref_id" ' + 'WHERE "deep"."id" IN (1)', str(query)) diff --git a/fireant/tests/slicer/queries/test_build_metric_filters.py b/fireant/tests/slicer/queries/test_build_metric_filters.py new file mode 100644 index 00000000..25c4ed6b --- /dev/null +++ b/fireant/tests/slicer/queries/test_build_metric_filters.py @@ -0,0 +1,141 @@ +from unittest import TestCase + +import fireant as f +from ..mocks import slicer + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderMetricFilterTests(TestCase): + maxDiff = None + + def test_build_query_with_metric_filter_eq(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.metrics.votes == 5) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")=5', str(query)) + + def test_build_query_with_metric_filter_eq_left(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(5 == slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")=5', str(query)) + + def test_build_query_with_metric_filter_ne(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.metrics.votes != 5) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")<>5', str(query)) + + def test_build_query_with_metric_filter_ne_left(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(5 != slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")<>5', str(query)) + + def test_build_query_with_metric_filter_gt(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.metrics.votes > 5) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")>5', str(query)) + + def test_build_query_with_metric_filter_gt_left(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(5 < slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")>5', str(query)) + + def test_build_query_with_metric_filter_gte(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.metrics.votes >= 5) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")>=5', str(query)) + + def test_build_query_with_metric_filter_gte_left(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(5 <= slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")>=5', str(query)) + + def test_build_query_with_metric_filter_lt(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.metrics.votes < 5) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")<5', str(query)) + + def test_build_query_with_metric_filter_lt_left(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(5 > slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")<5', str(query)) + + def test_build_query_with_metric_filter_lte(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(slicer.metrics.votes <= 5) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")<=5', str(query)) + + def test_build_query_with_metric_filter_lte_left(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .filter(5 >= slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'HAVING SUM("votes")<=5', str(query)) diff --git a/fireant/tests/slicer/queries/test_build_metrics.py b/fireant/tests/slicer/queries/test_build_metrics.py new file mode 100644 index 00000000..71c94089 --- /dev/null +++ b/fireant/tests/slicer/queries/test_build_metrics.py @@ -0,0 +1,61 @@ +from unittest import TestCase + +import fireant as f +from ..mocks import slicer + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderMetricTests(TestCase): + maxDiff = None + + def test_build_query_with_single_metric(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician"', str(query)) + + def test_build_query_with_multiple_metrics(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes, slicer.metrics.wins)) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes",' + 'SUM("is_winner") "$m$wins" ' + 'FROM "politics"."politician"', str(query)) + + def test_build_query_with_multiple_visualizations(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .widget(f.DataTablesJS(slicer.metrics.wins)) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes",' + 'SUM("is_winner") "$m$wins" ' + 'FROM "politics"."politician"', str(query)) + + def test_build_query_for_chart_visualization_with_single_axis(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician"', str(query)) + + def test_build_query_for_chart_visualization_with_multiple_axes(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes)) + .axis(f.HighCharts.LineChart(slicer.metrics.wins))) \ + .query + + self.assertEqual('SELECT ' + 'SUM("votes") "$m$votes",' + 'SUM("is_winner") "$m$wins" ' + 'FROM "politics"."politician"', str(query)) diff --git a/fireant/tests/slicer/queries/test_build_operations.py b/fireant/tests/slicer/queries/test_build_operations.py new file mode 100644 index 00000000..16d15d48 --- /dev/null +++ b/fireant/tests/slicer/queries/test_build_operations.py @@ -0,0 +1,61 @@ +from unittest import TestCase + +import fireant as f +from ..mocks import slicer + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderOperationTests(TestCase): + maxDiff = None + + def test_build_query_with_cumsum_operation(self): + query = slicer.data \ + .widget(f.DataTablesJS(f.CumSum(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_build_query_with_cummean_operation(self): + query = slicer.data \ + .widget(f.DataTablesJS(f.CumMean(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_build_query_with_cumprod_operation(self): + query = slicer.data \ + .widget(f.DataTablesJS(f.CumProd(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_build_query_with_rollingmean_operation(self): + query = slicer.data \ + .widget(f.DataTablesJS(f.RollingMean(slicer.metrics.votes, 3, 3))) \ + .dimension(slicer.dimensions.timestamp) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) diff --git a/fireant/tests/slicer/queries/test_build_orderbys.py b/fireant/tests/slicer/queries/test_build_orderbys.py new file mode 100644 index 00000000..47ce1657 --- /dev/null +++ b/fireant/tests/slicer/queries/test_build_orderbys.py @@ -0,0 +1,173 @@ +from unittest import TestCase + +from pypika import Order + +import fireant as f +from ..mocks import slicer + + +class QueryBuilderOrderTests(TestCase): + maxDiff = None + + def test_build_query_order_by_dimension(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.dimensions.timestamp) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_build_query_order_by_dimension_display(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.candidate) \ + .orderby(slicer.dimensions.candidate_display) \ + .query + + self.assertEqual('SELECT ' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$candidate","$d$candidate_display" ' + 'ORDER BY "$d$candidate_display"', str(query)) + + def test_build_query_order_by_dimension_asc(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.dimensions.timestamp, orientation=Order.asc) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp" ASC', str(query)) + + def test_build_query_order_by_dimension_desc(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.dimensions.timestamp, orientation=Order.desc) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp" DESC', str(query)) + + def test_build_query_order_by_metric(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$m$votes"', str(query)) + + def test_build_query_order_by_metric_asc(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.metrics.votes, orientation=Order.asc) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$m$votes" ASC', str(query)) + + def test_build_query_order_by_metric_desc(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.metrics.votes, orientation=Order.desc) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$m$votes" DESC', str(query)) + + def test_build_query_order_by_multiple_dimensions(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp, slicer.dimensions.candidate) \ + .orderby(slicer.dimensions.timestamp) \ + .orderby(slicer.dimensions.candidate) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display" ' + 'ORDER BY "$d$timestamp","$d$candidate"', str(query)) + + def test_build_query_order_by_multiple_dimensions_with_different_orientations(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp, slicer.dimensions.candidate) \ + .orderby(slicer.dimensions.timestamp, orientation=Order.desc) \ + .orderby(slicer.dimensions.candidate, orientation=Order.asc) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display" ' + 'ORDER BY "$d$timestamp" DESC,"$d$candidate" ASC', str(query)) + + def test_build_query_order_by_metrics_and_dimensions(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.dimensions.timestamp) \ + .orderby(slicer.metrics.votes) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp","$m$votes"', str(query)) + + def test_build_query_order_by_metrics_and_dimensions_with_different_orientations(self): + query = slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .orderby(slicer.dimensions.timestamp, orientation=Order.asc) \ + .orderby(slicer.metrics.votes, orientation=Order.desc) \ + .query + + self.assertEqual('SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp" ASC,"$m$votes" DESC', str(query)) diff --git a/fireant/tests/slicer/queries/test_build_pagination.py b/fireant/tests/slicer/queries/test_build_pagination.py new file mode 100644 index 00000000..7ea916ec --- /dev/null +++ b/fireant/tests/slicer/queries/test_build_pagination.py @@ -0,0 +1,67 @@ +from unittest import TestCase +from unittest.mock import ( + ANY, + Mock, + patch, +) + +import fireant as f +from ..matchers import ( + DimensionMatcher, +) +from ..mocks import slicer + + +@patch('fireant.slicer.queries.builder.fetch_data') +class QueryBuildPaginationTests(TestCase): + def test_set_limit(self, mock_fetch_data: Mock): + slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .limit(20) \ + .fetch() + + mock_fetch_data.assert_called_once_with(ANY, + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp" LIMIT 20', + dimensions=DimensionMatcher(slicer.dimensions.timestamp)) + + def test_set_offset(self, mock_fetch_data: Mock): + slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .offset(20) \ + .fetch() + + mock_fetch_data.assert_called_once_with(ANY, + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp" ' + 'OFFSET 20', + dimensions=DimensionMatcher(slicer.dimensions.timestamp)) + + def test_set_limit_and_offset(self, mock_fetch_data: Mock): + slicer.data \ + .widget(f.DataTablesJS(slicer.metrics.votes)) \ + .dimension(slicer.dimensions.timestamp) \ + .limit(20) \ + .offset(30) \ + .fetch() + + mock_fetch_data.assert_called_once_with(ANY, + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp" ' + 'ORDER BY "$d$timestamp" ' + 'LIMIT 20 ' + 'OFFSET 30', + dimensions=DimensionMatcher(slicer.dimensions.timestamp)) diff --git a/fireant/tests/slicer/queries/test_build_references.py b/fireant/tests/slicer/queries/test_build_references.py new file mode 100644 index 00000000..d6ef2dd9 --- /dev/null +++ b/fireant/tests/slicer/queries/test_build_references.py @@ -0,0 +1,958 @@ +from datetime import date +from unittest import TestCase + +import fireant as f +from ..mocks import slicer + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +class QueryBuilderDatetimeReferenceTests(TestCase): + maxDiff = None + + def test_single_reference_dod_with_no_dimension_uses_multiple_from_clauses_instead_of_joins(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + + 'FROM (' + 'SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician"' + ') "$base",(' + 'SELECT ' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician"' + ') "$dod"', str(query)) + + def test_single_reference_dod_with_dimension_but_not_reference_dimension_in_query_using_filter(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.political_party) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .filter(slicer.dimensions.timestamp.between(date(2000, 1, 1), date(2000, 3, 1))) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$political_party","$dod"."$d$political_party") "$d$political_party",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' + 'GROUP BY "$d$political_party"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' + 'GROUP BY "$d$political_party"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$political_party"="$dod"."$d$political_party" ' + 'ORDER BY "$d$political_party"', str(query)) + + def test_dimension_with_single_reference_dod(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_dimension_with_single_reference_wow(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.WeekOverWeek(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'week\',1,"$wow"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$wow"."$m$votes" "$m$votes_wow" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$wow" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'week\',1,"$wow"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_dimension_with_single_reference_mom(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.MonthOverMonth(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'month\',1,"$mom"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$mom"."$m$votes" "$m$votes_mom" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$mom" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'month\',1,"$mom"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_dimension_with_single_reference_qoq(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.QuarterOverQuarter(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'quarter\',1,"$qoq"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$qoq"."$m$votes" "$m$votes_qoq" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$qoq" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'quarter\',1,"$qoq"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_dimension_with_single_reference_yoy(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$yoy"."$m$votes" "$m$votes_yoy" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$yoy" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_dimension_with_single_reference_as_a_delta(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp, delta=True)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$base"."$m$votes"-"$dod"."$m$votes" "$m$votes_dod_delta" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_dimension_with_single_reference_as_a_delta_percentage(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp, delta_percent=True)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '("$base"."$m$votes"-"$dod"."$m$votes")*100/NULLIF("$dod"."$m$votes",' + '0) "$m$votes_dod_delta_percent" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_reference_on_dimension_with_weekly_interval(self): + weekly_timestamp = slicer.dimensions.timestamp(f.weekly) + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(weekly_timestamp) \ + .reference(f.DayOverDay(weekly_timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_reference_on_dimension_with_weekly_interval_no_interval_on_reference(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp(f.weekly)) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_reference_on_dimension_with_monthly_interval(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp(f.monthly)) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'MM\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'MM\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_reference_on_dimension_with_quarterly_interval(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp(f.quarterly)) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'Q\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'Q\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_reference_on_dimension_with_annual_interval(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp(f.annually)) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'Y\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'Y\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_dimension_with_multiple_references(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp, delta_percent=True)) \ + .query + + self.assertEqual('SELECT ' + + 'COALESCE(' + '"$base"."$d$timestamp",' + 'TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp"),' + 'TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")' + ') "$d$timestamp",' + + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod",' + '("$base"."$m$votes"-"$yoy"."$m$votes")*100/NULLIF("$yoy"."$m$votes",' + '0) "$m$votes_yoy_delta_percent" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$yoy" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_reference_joins_nested_query_on_dimensions(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .dimension(slicer.dimensions.political_party) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) ' + '"$d$timestamp",' + 'COALESCE("$base"."$d$political_party","$yoy"."$d$political_party") "$d$political_party",' + '"$base"."$m$votes" "$m$votes",' + '"$yoy"."$m$votes" "$m$votes_yoy" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp","$d$political_party"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"political_party" "$d$political_party",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp","$d$political_party"' + ') "$yoy" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'AND "$base"."$d$political_party"="$yoy"."$d$political_party" ' + 'ORDER BY "$d$timestamp","$d$political_party"', str(query)) + + def test_reference_with_unique_dimension_includes_display_definition(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .dimension(slicer.dimensions.candidate) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) ' + '"$d$timestamp",' + 'COALESCE("$base"."$d$candidate","$yoy"."$d$candidate") "$d$candidate",' + 'COALESCE("$base"."$d$candidate_display","$yoy"."$d$candidate_display") ' + '"$d$candidate_display",' + '"$base"."$m$votes" "$m$votes",' + '"$yoy"."$m$votes" "$m$votes_yoy" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + '"candidate_id" "$d$candidate",' + '"candidate_name" "$d$candidate_display",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display"' + ') "$yoy" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'AND "$base"."$d$candidate"="$yoy"."$d$candidate" ' + 'ORDER BY "$d$timestamp","$d$candidate_display"', str(query)) + + def test_adjust_reference_dimension_filters_in_reference_query(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .filter(slicer.dimensions.timestamp + .between(date(2018, 1, 1), date(2018, 1, 31))) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_filters(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ + .filter(slicer.dimensions.timestamp + .between(date(2018, 1, 1), date(2018, 1, 31))) \ + .filter(slicer.dimensions.political_party + .isin(['d'])) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' + 'AND "political_party" IN (\'d\') ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' + 'AND "political_party" IN (\'d\') ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_adapt_dow_for_leap_year_for_yoy_reference(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp(f.weekly)) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$yoy"."$m$votes" "$m$votes_yoy" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$yoy" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp(f.weekly)) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp, delta=True)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$base"."$m$votes"-"$yoy"."$m$votes" "$m$votes_yoy_delta" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$yoy" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp(f.weekly)) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp, delta_percent=True)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '("$base"."$m$votes"-"$yoy"."$m$votes")*100/NULLIF("$yoy"."$m$votes",' + '0) "$m$votes_yoy_delta_percent" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$yoy" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_adapt_dow_for_leap_year_for_yoy_reference_with_date_filter(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp(f.weekly)) \ + .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ + .filter(slicer.dimensions.timestamp.between(date(2018, 1, 1), date(2018, 1, 31))) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$yoy"."$m$votes" "$m$votes_yoy" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'IW\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'WHERE TIMESTAMPADD(\'year\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' + 'GROUP BY "$d$timestamp"' + ') "$yoy" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_adding_duplicate_reference_does_not_join_more_queries(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp), + f.DayOverDay(slicer.dimensions.timestamp)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp), + f.DayOverDay(slicer.dimensions.timestamp, delta=True), + f.DayOverDay(slicer.dimensions.timestamp, delta_percent=True)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' + '"$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod",' + '"$base"."$m$votes"-"$dod"."$m$votes" "$m$votes_dod_delta",' + '("$base"."$m$votes"-"$dod"."$m$votes")*100/NULLIF("$dod"."$m$votes",' + '0) "$m$votes_dod_delta_percent" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) + + def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension_with_different_periods(self): + query = slicer.data \ + .widget(f.HighCharts() + .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ + .dimension(slicer.dimensions.timestamp) \ + .reference(f.DayOverDay(slicer.dimensions.timestamp), + f.DayOverDay(slicer.dimensions.timestamp, delta=True), + f.YearOverYear(slicer.dimensions.timestamp), + f.YearOverYear(slicer.dimensions.timestamp, delta=True)) \ + .query + + self.assertEqual('SELECT ' + 'COALESCE(' + '"$base"."$d$timestamp",' + 'TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp"),' + 'TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")' + ') "$d$timestamp",' + '"$base"."$m$votes" "$m$votes",' + '"$dod"."$m$votes" "$m$votes_dod",' + '"$base"."$m$votes"-"$dod"."$m$votes" "$m$votes_dod_delta",' + '"$yoy"."$m$votes" "$m$votes_yoy",' + '"$base"."$m$votes"-"$yoy"."$m$votes" "$m$votes_yoy_delta" ' + 'FROM ' + + '(' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$base" ' # end-nested + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$dod" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' + + 'FULL OUTER JOIN (' # nested + 'SELECT ' + 'TRUNC("timestamp",\'DD\') "$d$timestamp",' + 'SUM("votes") "$m$votes" ' + 'FROM "politics"."politician" ' + 'GROUP BY "$d$timestamp"' + ') "$yoy" ' # end-nested + + 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' + 'ORDER BY "$d$timestamp"', str(query)) diff --git a/fireant/tests/slicer/queries/test_build_render.py b/fireant/tests/slicer/queries/test_build_render.py new file mode 100644 index 00000000..93e8028b --- /dev/null +++ b/fireant/tests/slicer/queries/test_build_render.py @@ -0,0 +1,147 @@ +from unittest import TestCase +from unittest.mock import ( + ANY, + Mock, + patch, +) + +import fireant as f +from fireant.utils import ( + format_metric_key, +) +from ..matchers import ( + DimensionMatcher, +) +from ..mocks import slicer + + +# noinspection SqlDialectInspection,SqlNoDataSourceInspection +@patch('fireant.slicer.queries.builder.fetch_data') +class QueryBuilderRenderTests(TestCase): + def test_pass_slicer_database_as_arg(self, mock_fetch_data: Mock): + mock_widget = f.Widget(slicer.metrics.votes) + mock_widget.transform = Mock() + + slicer.data \ + .widget(mock_widget) \ + .fetch() + + mock_fetch_data.assert_called_once_with(slicer.database, + ANY, + dimensions=ANY) + + def test_pass_query_from_builder_as_arg(self, mock_fetch_data: Mock): + mock_widget = f.Widget(slicer.metrics.votes) + mock_widget.transform = Mock() + + slicer.data \ + .widget(mock_widget) \ + .fetch() + + mock_fetch_data.assert_called_once_with(ANY, + 'SELECT SUM("votes") "$m$votes" ' + 'FROM "politics"."politician"', + dimensions=ANY) + + def test_builder_dimensions_as_arg_with_zero_dimensions(self, mock_fetch_data: Mock): + mock_widget = f.Widget(slicer.metrics.votes) + mock_widget.transform = Mock() + + slicer.data \ + .widget(mock_widget) \ + .fetch() + + mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=[]) + + def test_builder_dimensions_as_arg_with_one_dimension(self, mock_fetch_data: Mock): + mock_widget = f.Widget(slicer.metrics.votes) + mock_widget.transform = Mock() + + dimensions = [slicer.dimensions.state] + + slicer.data \ + .widget(mock_widget) \ + .dimension(*dimensions) \ + .fetch() + + mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=DimensionMatcher(*dimensions)) + + def test_builder_dimensions_as_arg_with_multiple_dimensions(self, mock_fetch_data: Mock): + mock_widget = f.Widget(slicer.metrics.votes) + mock_widget.transform = Mock() + + dimensions = slicer.dimensions.timestamp, slicer.dimensions.state, slicer.dimensions.political_party + + slicer.data \ + .widget(mock_widget) \ + .dimension(*dimensions) \ + .fetch() + + mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=DimensionMatcher(*dimensions)) + + def test_call_transform_on_widget(self, mock_fetch_data: Mock): + mock_widget = f.Widget(slicer.metrics.votes) + mock_widget.transform = Mock() + + # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work + slicer.data \ + .dimension(slicer.dimensions.timestamp) \ + .widget(mock_widget) \ + .fetch() + + mock_widget.transform.assert_called_once_with(mock_fetch_data.return_value, + slicer, + DimensionMatcher(slicer.dimensions.timestamp), + []) + + def test_returns_results_from_widget_transform(self, mock_fetch_data: Mock): + mock_widget = f.Widget(slicer.metrics.votes) + mock_widget.transform = Mock() + + # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work + result = slicer.data \ + .dimension(slicer.dimensions.timestamp) \ + .widget(mock_widget) \ + .fetch() + + self.assertListEqual(result, [mock_widget.transform.return_value]) + + def test_operations_evaluated(self, mock_fetch_data: Mock): + mock_operation = Mock(name='mock_operation ', spec=f.Operation) + mock_operation.key, mock_operation.definition = 'mock_operation', slicer.table.abc + mock_operation.metrics = [] + + mock_widget = f.Widget(mock_operation) + mock_widget.transform = Mock() + + mock_df = {} + mock_fetch_data.return_value = mock_df + + # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work + slicer.data \ + .dimension(slicer.dimensions.timestamp) \ + .widget(mock_widget) \ + .fetch() + + mock_operation.apply.assert_called_once_with(mock_df) + + def test_operations_results_stored_in_data_frame(self, mock_fetch_data: Mock): + mock_operation = Mock(name='mock_operation ', spec=f.Operation) + mock_operation.key, mock_operation.definition = 'mock_operation', slicer.table.abc + mock_operation.metrics = [] + + mock_widget = f.Widget(mock_operation) + mock_widget.transform = Mock() + + mock_df = {} + mock_fetch_data.return_value = mock_df + + # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work + slicer.data \ + .dimension(slicer.dimensions.timestamp) \ + .widget(mock_widget) \ + .fetch() + + f_op_key = format_metric_key(mock_operation.key) + self.assertIn(f_op_key, mock_df) + self.assertEqual(mock_df[f_op_key], mock_operation.apply.return_value) diff --git a/fireant/tests/slicer/queries/test_builder.py b/fireant/tests/slicer/queries/test_builder.py index 769aa72d..e9ea9993 100644 --- a/fireant/tests/slicer/queries/test_builder.py +++ b/fireant/tests/slicer/queries/test_builder.py @@ -1,17 +1,7 @@ -from datetime import date from unittest import TestCase -from unittest.mock import ( - ANY, - Mock, - patch, -) - -from pypika import Order import fireant as f from fireant.slicer.exceptions import MetricRequiredException -from fireant.utils import format_metric_key -from ..matchers import DimensionMatcher from ..mocks import slicer @@ -41,2244 +31,6 @@ def test_orderby_is_immutable(self): self.assertIsNot(query1, query2) -# noinspection SqlDialectInspection,SqlNoDataSourceInspection -class QueryBuilderMetricTests(TestCase): - maxDiff = None - - def test_build_query_with_single_metric(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician"', str(query)) - - def test_build_query_with_multiple_metrics(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes, slicer.metrics.wins)) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes",' - 'SUM("is_winner") "$m$wins" ' - 'FROM "politics"."politician"', str(query)) - - def test_build_query_with_multiple_visualizations(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .widget(f.DataTablesJS(slicer.metrics.wins)) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes",' - 'SUM("is_winner") "$m$wins" ' - 'FROM "politics"."politician"', str(query)) - - def test_build_query_for_chart_visualization_with_single_axis(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician"', str(query)) - - def test_build_query_for_chart_visualization_with_multiple_axes(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes)) - .axis(f.HighCharts.LineChart(slicer.metrics.wins))) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes",' - 'SUM("is_winner") "$m$wins" ' - 'FROM "politics"."politician"', str(query)) - - -# noinspection SqlDialectInspection,SqlNoDataSourceInspection -class QueryBuilderDimensionTests(TestCase): - maxDiff = None - - def test_build_query_with_datetime_dimension(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_build_query_with_datetime_dimension_hourly(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp(f.hourly)) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'HH\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_build_query_with_datetime_dimension_daily(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp(f.daily)) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_build_query_with_datetime_dimension_weekly(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp(f.weekly)) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'IW\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_build_query_with_datetime_dimension_monthly(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp(f.monthly)) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'MM\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_build_query_with_datetime_dimension_quarterly(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp(f.quarterly)) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'Q\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_build_query_with_datetime_dimension_annually(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp(f.annually)) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'Y\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_build_query_with_boolean_dimension(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.winner) \ - .query - - self.assertEqual('SELECT ' - '"is_winner" "$d$winner",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$winner" ' - 'ORDER BY "$d$winner"', str(query)) - - def test_build_query_with_categorical_dimension(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.political_party) \ - .query - - self.assertEqual('SELECT ' - '"political_party" "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$political_party" ' - 'ORDER BY "$d$political_party"', str(query)) - - def test_build_query_with_unique_dimension(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.election) \ - .query - - self.assertEqual('SELECT ' - '"election_id" "$d$election",' - '"election_year" "$d$election_display",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$election","$d$election_display" ' - 'ORDER BY "$d$election_display"', str(query)) - - def test_build_query_with_pattern_dimension(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.pattern(['groupA%', 'groupB%'])) \ - .query - - self.assertEqual('SELECT ' - 'CASE ' - 'WHEN "pattern" LIKE \'groupA%\' THEN \'groupA%\' ' - 'WHEN "pattern" LIKE \'groupB%\' THEN \'groupB%\' ' - 'ELSE \'No Group\' ' - 'END "$d$pattern",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$pattern" ' - 'ORDER BY "$d$pattern"', str(query)) - - def test_build_query_with_pattern_no_values(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.pattern) \ - .query - - self.assertEqual('SELECT ' - '\'No Group\' "$d$pattern",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$pattern" ' - 'ORDER BY "$d$pattern"', str(query)) - - def test_build_query_with_multiple_dimensions(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .dimension(slicer.dimensions.candidate) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"candidate_id" "$d$candidate",' - '"candidate_name" "$d$candidate_display",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display" ' - 'ORDER BY "$d$timestamp","$d$candidate_display"', str(query)) - - def test_build_query_with_multiple_dimensions_and_visualizations(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes, slicer.metrics.wins)) \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes)) - .axis(f.HighCharts.LineChart(slicer.metrics.wins))) \ - .dimension(slicer.dimensions.timestamp) \ - .dimension(slicer.dimensions.political_party) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"political_party" "$d$political_party",' - 'SUM("votes") "$m$votes",' - 'SUM("is_winner") "$m$wins" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp","$d$political_party" ' - 'ORDER BY "$d$timestamp","$d$political_party"', str(query)) - - -# noinspection SqlDialectInspection,SqlNoDataSourceInspection -class QueryBuilderDimensionTotalsTests(TestCase): - maxDiff = None - - def test_build_query_with_totals_cat_dimension(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.political_party.rollup()) \ - .query - - self.assertEqual('(SELECT ' - '"political_party" "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$political_party") ' - - 'UNION ALL ' - - '(SELECT ' - 'NULL "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician") ' - - 'ORDER BY "$d$political_party"', str(query)) - - def test_build_query_with_totals_uni_dimension(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.candidate.rollup()) \ - .query - - self.assertEqual('(SELECT ' - '"candidate_id" "$d$candidate",' - '"candidate_name" "$d$candidate_display",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$candidate","$d$candidate_display") ' - - 'UNION ALL ' - - '(SELECT ' - 'NULL "$d$candidate",' - 'NULL "$d$candidate_display",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician") ' - - 'ORDER BY "$d$candidate_display"', str(query)) - - def test_build_query_with_totals_on_dimension_and_subsequent_dimensions(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp, - slicer.dimensions.candidate.rollup(), - slicer.dimensions.political_party) \ - .query - - self.assertEqual('(SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"candidate_id" "$d$candidate",' - '"candidate_name" "$d$candidate_display",' - '"political_party" "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display","$d$political_party") ' - - 'UNION ALL ' - - '(SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'NULL "$d$candidate",' - 'NULL "$d$candidate_display",' - 'NULL "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp") ' - 'ORDER BY "$d$timestamp","$d$candidate_display","$d$political_party"', str(query)) - - def test_build_query_with_totals_on_multiple_dimensions_dimension(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp, - slicer.dimensions.candidate.rollup(), - slicer.dimensions.political_party.rollup()) \ - .query - - self.assertEqual('(SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"candidate_id" "$d$candidate",' - '"candidate_name" "$d$candidate_display",' - '"political_party" "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display","$d$political_party") ' - - 'UNION ALL ' - - '(SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'NULL "$d$candidate",' - 'NULL "$d$candidate_display",' - 'NULL "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp") ' - - 'UNION ALL ' - - '(SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"candidate_id" "$d$candidate",' - '"candidate_name" "$d$candidate_display",' - 'NULL "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display") ' - - 'ORDER BY "$d$timestamp","$d$candidate_display","$d$political_party"', str(query)) - - def test_build_query_with_totals_cat_dimension_with_references(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp, - slicer.dimensions.political_party.rollup()) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ - .query - - # Important that in reference queries when using totals that the null dimensions are omitted from the nested - # queries and selected in the container query - self.assertEqual('(SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - 'COALESCE("$base"."$d$political_party","$dod"."$d$political_party") "$d$political_party",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM (' - - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"political_party" "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp","$d$political_party"' - ') "$base" ' - - 'FULL OUTER JOIN (' - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"political_party" "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp","$d$political_party"' - ') "$dod" ' - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'AND "$base"."$d$political_party"="$dod"."$d$political_party") ' - - 'UNION ALL ' - - '(SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - 'NULL "$d$political_party",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM (' - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' - - 'FULL OUTER JOIN (' - 'SELECT TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' - 'ORDER BY "$d$timestamp","$d$political_party"', str(query)) - - def test_build_query_with_totals_cat_dimension_with_references_and_date_filters(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .dimension(slicer.dimensions.political_party.rollup()) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ - .filter(slicer.dimensions.timestamp.between(date(2018, 1, 1), date(2019, 1, 1))) \ - .query - - self.assertEqual('(SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - 'COALESCE("$base"."$d$political_party","$dod"."$d$political_party") "$d$political_party",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM (' - - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"political_party" "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' - 'GROUP BY "$d$timestamp","$d$political_party"' - ') "$base" ' - - 'FULL OUTER JOIN (' - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"political_party" "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' - 'GROUP BY "$d$timestamp","$d$political_party"' - ') "$dod" ' - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'AND "$base"."$d$political_party"="$dod"."$d$political_party") ' - - 'UNION ALL ' - - '(SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - 'NULL "$d$political_party",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM (' - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' - - 'FULL OUTER JOIN (' - 'SELECT TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2019-01-01\' ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) ' - 'ORDER BY "$d$timestamp","$d$political_party"', str(query)) - - -# noinspection SqlDialectInspection,SqlNoDataSourceInspection -class QueryBuilderDimensionFilterTests(TestCase): - maxDiff = None - - def test_build_query_with_filter_isin_categorical_dim(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.political_party.isin(['d'])) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "political_party" IN (\'d\')', str(query)) - - def test_build_query_with_filter_notin_categorical_dim(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.political_party.notin(['d'])) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "political_party" NOT IN (\'d\')', str(query)) - - def test_build_query_with_filter_like_categorical_dim(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.political_party.like('Rep%')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "political_party" LIKE \'Rep%\'', str(query)) - - def test_build_query_with_filter_not_like_categorical_dim(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.political_party.not_like('Rep%')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "political_party" NOT LIKE \'Rep%\'', str(query)) - - def test_build_query_with_filter_isin_unique_dim(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.candidate.isin([1])) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "candidate_id" IN (1)', str(query)) - - def test_build_query_with_filter_notin_unique_dim(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.candidate.notin([1])) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "candidate_id" NOT IN (1)', str(query)) - - def test_build_query_with_filter_isin_unique_dim_display(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.candidate.isin(['Donald Trump'], use_display=True)) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "candidate_name" IN (\'Donald Trump\')', str(query)) - - def test_build_query_with_filter_notin_unique_dim_display(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.candidate.notin(['Donald Trump'], use_display=True)) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "candidate_name" NOT IN (\'Donald Trump\')', str(query)) - - def test_build_query_with_filter_like_unique_dim(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.candidate.like('%Trump')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) - - def test_build_query_with_filter_like_display_dim(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.candidate_display.like('%Trump')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) - - def test_build_query_with_filter_not_like_unique_dim(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.candidate.not_like('%Trump')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "candidate_name" NOT LIKE \'%Trump\'', str(query)) - - def test_build_query_with_filter_not_like_display_dim(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.candidate_display.not_like('%Trump')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "candidate_name" NOT LIKE \'%Trump\'', str(query)) - - def test_build_query_with_filter_like_categorical_dim_multiple_patterns(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.political_party.like('Rep%', 'Dem%')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "political_party" LIKE \'Rep%\' ' - 'OR "political_party" LIKE \'Dem%\'', str(query)) - - def test_build_query_with_filter_not_like_categorical_dim_multiple_patterns(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.political_party.not_like('Rep%', 'Dem%')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "political_party" NOT LIKE \'Rep%\' ' - 'OR "political_party" NOT LIKE \'Dem%\'', str(query)) - - def test_build_query_with_filter_like_pattern_dim_multiple_patterns(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.pattern.like('a%', 'b%')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "pattern" LIKE \'a%\' ' - 'OR "pattern" LIKE \'b%\'', str(query)) - - def test_build_query_with_filter_not_like_pattern_dim_multiple_patterns(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.pattern.not_like('a%', 'b%')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "pattern" NOT LIKE \'a%\' ' - 'OR "pattern" NOT LIKE \'b%\'', str(query)) - - def test_build_query_with_filter_like_unique_dim_multiple_patterns(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.candidate.like('%Trump', '%Clinton')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "candidate_name" LIKE \'%Trump\' ' - 'OR "candidate_name" LIKE \'%Clinton\'', str(query)) - - def test_build_query_with_filter_like_display_dim_multiple_patterns(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.candidate_display.like('%Trump', '%Clinton')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "candidate_name" LIKE \'%Trump\' ' - 'OR "candidate_name" LIKE \'%Clinton\'', str(query)) - - def test_build_query_with_filter_not_like_unique_dim_multiple_patterns(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.candidate.not_like('%Trump', '%Clinton')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "candidate_name" NOT LIKE \'%Trump\' ' - 'OR "candidate_name" NOT LIKE \'%Clinton\'', str(query)) - - def test_build_query_with_filter_not_like_display_dim_multiple_patterns(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.candidate_display.not_like('%Trump', '%Clinton')) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "candidate_name" NOT LIKE \'%Trump\' ' - 'OR "candidate_name" NOT LIKE \'%Clinton\'', str(query)) - - def test_build_query_with_filter_isin_raise_exception_when_display_definition_undefined(self): - with self.assertRaises(f.QueryException): - slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.deepjoin.isin([1], use_display=True)) - - def test_build_query_with_filter_notin_raise_exception_when_display_definition_undefined(self): - with self.assertRaises(f.QueryException): - slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.deepjoin.notin([1], use_display=True)) - - def test_build_query_with_filter_like_raise_exception_when_display_definition_undefined(self): - with self.assertRaises(f.QueryException): - slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.deepjoin.like('test')) - - def test_build_query_with_filter_not_like_raise_exception_when_display_definition_undefined(self): - with self.assertRaises(f.QueryException): - slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.deepjoin.not_like('test')) - - def test_build_query_with_filter_range_datetime_dimension(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.timestamp.between(date(2009, 1, 20), date(2017, 1, 20))) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "timestamp" BETWEEN \'2009-01-20\' AND \'2017-01-20\'', str(query)) - - def test_build_query_with_filter_boolean_true(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.winner.is_(True)) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "is_winner"', str(query)) - - def test_build_query_with_filter_boolean_false(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.winner.is_(False)) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE NOT "is_winner"', str(query)) - - -# noinspection SqlDialectInspection,SqlNoDataSourceInspection -class QueryBuilderMetricFilterTests(TestCase): - maxDiff = None - - def test_build_query_with_metric_filter_eq(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.metrics.votes == 5) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'HAVING SUM("votes")=5', str(query)) - - def test_build_query_with_metric_filter_eq_left(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(5 == slicer.metrics.votes) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'HAVING SUM("votes")=5', str(query)) - - def test_build_query_with_metric_filter_ne(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.metrics.votes != 5) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'HAVING SUM("votes")<>5', str(query)) - - def test_build_query_with_metric_filter_ne_left(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(5 != slicer.metrics.votes) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'HAVING SUM("votes")<>5', str(query)) - - def test_build_query_with_metric_filter_gt(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.metrics.votes > 5) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'HAVING SUM("votes")>5', str(query)) - - def test_build_query_with_metric_filter_gt_left(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(5 < slicer.metrics.votes) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'HAVING SUM("votes")>5', str(query)) - - def test_build_query_with_metric_filter_gte(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.metrics.votes >= 5) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'HAVING SUM("votes")>=5', str(query)) - - def test_build_query_with_metric_filter_gte_left(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(5 <= slicer.metrics.votes) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'HAVING SUM("votes")>=5', str(query)) - - def test_build_query_with_metric_filter_lt(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.metrics.votes < 5) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'HAVING SUM("votes")<5', str(query)) - - def test_build_query_with_metric_filter_lt_left(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(5 > slicer.metrics.votes) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'HAVING SUM("votes")<5', str(query)) - - def test_build_query_with_metric_filter_lte(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.metrics.votes <= 5) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'HAVING SUM("votes")<=5', str(query)) - - def test_build_query_with_metric_filter_lte_left(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(5 >= slicer.metrics.votes) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'HAVING SUM("votes")<=5', str(query)) - - -# noinspection SqlDialectInspection,SqlNoDataSourceInspection -class QueryBuilderOperationTests(TestCase): - maxDiff = None - - def test_build_query_with_cumsum_operation(self): - query = slicer.data \ - .widget(f.DataTablesJS(f.CumSum(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_build_query_with_cummean_operation(self): - query = slicer.data \ - .widget(f.DataTablesJS(f.CumMean(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp"', str(query)) - - -# noinspection SqlDialectInspection,SqlNoDataSourceInspection -class QueryBuilderDatetimeReferenceTests(TestCase): - maxDiff = None - - def test_single_reference_dod_with_no_dimension_uses_multiple_from_clauses_instead_of_joins(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - - 'FROM (' - 'SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician"' - ') "$base",(' - 'SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician"' - ') "$dod"', str(query)) - - def test_single_reference_dod_with_dimension_but_not_reference_dimension_in_query_using_filter(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.political_party) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ - .filter(slicer.dimensions.timestamp.between(date(2000, 1, 1), date(2000, 3, 1))) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$political_party","$dod"."$d$political_party") "$d$political_party",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM ' - - '(' # nested - 'SELECT ' - '"political_party" "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "timestamp" BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "$d$political_party"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - '"political_party" "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2000-01-01\' AND \'2000-03-01\' ' - 'GROUP BY "$d$political_party"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$political_party"="$dod"."$d$political_party" ' - 'ORDER BY "$d$political_party"', str(query)) - - def test_dimension_with_single_reference_dod(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_dimension_with_single_reference_wow(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .reference(f.WeekOverWeek(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'week\',1,"$wow"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$wow"."$m$votes" "$m$votes_wow" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$wow" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'week\',1,"$wow"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_dimension_with_single_reference_mom(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .reference(f.MonthOverMonth(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'month\',1,"$mom"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$mom"."$m$votes" "$m$votes_mom" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$mom" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'month\',1,"$mom"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_dimension_with_single_reference_qoq(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .reference(f.QuarterOverQuarter(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'quarter\',1,"$qoq"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$qoq"."$m$votes" "$m$votes_qoq" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$qoq" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'quarter\',1,"$qoq"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_dimension_with_single_reference_yoy(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$yoy"."$m$votes" "$m$votes_yoy" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$yoy" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_dimension_with_single_reference_as_a_delta(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp, delta=True)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$base"."$m$votes"-"$dod"."$m$votes" "$m$votes_dod_delta" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_dimension_with_single_reference_as_a_delta_percentage(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp, delta_percent=True)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '("$base"."$m$votes"-"$dod"."$m$votes")*100/NULLIF("$dod"."$m$votes",0) "$m$votes_dod_delta_percent" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_reference_on_dimension_with_weekly_interval(self): - weekly_timestamp = slicer.dimensions.timestamp(f.weekly) - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(weekly_timestamp) \ - .reference(f.DayOverDay(weekly_timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_reference_on_dimension_with_weekly_interval_no_interval_on_reference(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp(f.weekly)) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_reference_on_dimension_with_monthly_interval(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp(f.monthly)) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'MM\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'MM\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_reference_on_dimension_with_quarterly_interval(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp(f.quarterly)) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'Q\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'Q\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_reference_on_dimension_with_annual_interval(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp(f.annually)) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'Y\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'Y\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_dimension_with_multiple_references(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ - .reference(f.YearOverYear(slicer.dimensions.timestamp, delta_percent=True)) \ - .query - - self.assertEqual('SELECT ' - - 'COALESCE(' - '"$base"."$d$timestamp",' - 'TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp"),' - 'TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")' - ') "$d$timestamp",' - - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod",' - '("$base"."$m$votes"-"$yoy"."$m$votes")*100/NULLIF("$yoy"."$m$votes",0) "$m$votes_yoy_delta_percent" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$yoy" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_reference_joins_nested_query_on_dimensions(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .dimension(slicer.dimensions.political_party) \ - .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' - 'COALESCE("$base"."$d$political_party","$yoy"."$d$political_party") "$d$political_party",' - '"$base"."$m$votes" "$m$votes",' - '"$yoy"."$m$votes" "$m$votes_yoy" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"political_party" "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp","$d$political_party"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"political_party" "$d$political_party",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp","$d$political_party"' - ') "$yoy" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' - 'AND "$base"."$d$political_party"="$yoy"."$d$political_party" ' - 'ORDER BY "$d$timestamp","$d$political_party"', str(query)) - - def test_reference_with_unique_dimension_includes_display_definition(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .dimension(slicer.dimensions.candidate) \ - .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' - 'COALESCE("$base"."$d$candidate","$yoy"."$d$candidate") "$d$candidate",' - 'COALESCE("$base"."$d$candidate_display","$yoy"."$d$candidate_display") "$d$candidate_display",' - '"$base"."$m$votes" "$m$votes",' - '"$yoy"."$m$votes" "$m$votes_yoy" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"candidate_id" "$d$candidate",' - '"candidate_name" "$d$candidate_display",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"candidate_id" "$d$candidate",' - '"candidate_name" "$d$candidate_display",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display"' - ') "$yoy" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' - 'AND "$base"."$d$candidate"="$yoy"."$d$candidate" ' - 'ORDER BY "$d$timestamp","$d$candidate_display"', str(query)) - - def test_adjust_reference_dimension_filters_in_reference_query(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ - .filter(slicer.dimensions.timestamp - .between(date(2018, 1, 1), date(2018, 1, 31))) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_adjust_reference_dimension_filters_in_reference_query_with_multiple_filters(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp)) \ - .filter(slicer.dimensions.timestamp - .between(date(2018, 1, 1), date(2018, 1, 31))) \ - .filter(slicer.dimensions.political_party - .isin(['d'])) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'AND "political_party" IN (\'d\') ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE TIMESTAMPADD(\'day\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'AND "political_party" IN (\'d\') ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_adapt_dow_for_leap_year_for_yoy_reference(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp(f.weekly)) \ - .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$yoy"."$m$votes" "$m$votes_yoy" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$yoy" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_adapt_dow_for_leap_year_for_yoy_delta_reference(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp(f.weekly)) \ - .reference(f.YearOverYear(slicer.dimensions.timestamp, delta=True)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$base"."$m$votes"-"$yoy"."$m$votes" "$m$votes_yoy_delta" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$yoy" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_adapt_dow_for_leap_year_for_yoy_delta_percent_reference(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp(f.weekly)) \ - .reference(f.YearOverYear(slicer.dimensions.timestamp, delta_percent=True)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '("$base"."$m$votes"-"$yoy"."$m$votes")*100/NULLIF("$yoy"."$m$votes",0) "$m$votes_yoy_delta_percent" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$yoy" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_adapt_dow_for_leap_year_for_yoy_reference_with_date_filter(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp(f.weekly)) \ - .reference(f.YearOverYear(slicer.dimensions.timestamp)) \ - .filter(slicer.dimensions.timestamp.between(date(2018, 1, 1), date(2018, 1, 31))) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$yoy"."$m$votes" "$m$votes_yoy" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'IW\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "timestamp" BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TIMESTAMPADD(\'year\',-1,TRUNC(TIMESTAMPADD(\'year\',1,"timestamp"),\'IW\')) "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE TIMESTAMPADD(\'year\',1,"timestamp") BETWEEN \'2018-01-01\' AND \'2018-01-31\' ' - 'GROUP BY "$d$timestamp"' - ') "$yoy" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_adding_duplicate_reference_does_not_join_more_queries(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp), - f.DayOverDay(slicer.dimensions.timestamp)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp), - f.DayOverDay(slicer.dimensions.timestamp, delta=True), - f.DayOverDay(slicer.dimensions.timestamp, delta_percent=True)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE("$base"."$d$timestamp",TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp")) "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod",' - '"$base"."$m$votes"-"$dod"."$m$votes" "$m$votes_dod_delta",' - '("$base"."$m$votes"-"$dod"."$m$votes")*100/NULLIF("$dod"."$m$votes",0) "$m$votes_dod_delta_percent" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_use_same_nested_query_for_joining_references_with_same_period_and_dimension_with_different_periods(self): - query = slicer.data \ - .widget(f.HighCharts() - .axis(f.HighCharts.LineChart(slicer.metrics.votes))) \ - .dimension(slicer.dimensions.timestamp) \ - .reference(f.DayOverDay(slicer.dimensions.timestamp), - f.DayOverDay(slicer.dimensions.timestamp, delta=True), - f.YearOverYear(slicer.dimensions.timestamp), - f.YearOverYear(slicer.dimensions.timestamp, delta=True)) \ - .query - - self.assertEqual('SELECT ' - 'COALESCE(' - '"$base"."$d$timestamp",' - 'TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp"),' - 'TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp")' - ') "$d$timestamp",' - '"$base"."$m$votes" "$m$votes",' - '"$dod"."$m$votes" "$m$votes_dod",' - '"$base"."$m$votes"-"$dod"."$m$votes" "$m$votes_dod_delta",' - '"$yoy"."$m$votes" "$m$votes_yoy",' - '"$base"."$m$votes"-"$yoy"."$m$votes" "$m$votes_yoy_delta" ' - 'FROM ' - - '(' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$base" ' # end-nested - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$dod" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'day\',1,"$dod"."$d$timestamp") ' - - 'FULL OUTER JOIN (' # nested - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp"' - ') "$yoy" ' # end-nested - - 'ON "$base"."$d$timestamp"=TIMESTAMPADD(\'year\',1,"$yoy"."$d$timestamp") ' - 'ORDER BY "$d$timestamp"', str(query)) - - -# noinspection SqlDialectInspection,SqlNoDataSourceInspection -class QueryBuilderJoinTests(TestCase): - maxDiff = None - - def test_dimension_with_join_includes_join_in_query(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .dimension(slicer.dimensions.district) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("politician"."timestamp",\'DD\') "$d$timestamp",' - '"politician"."district_id" "$d$district",' - '"district"."district_name" "$d$district_display",' - 'SUM("politician"."votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'OUTER JOIN "locations"."district" ' - 'ON "politician"."district_id"="district"."id" ' - 'GROUP BY "$d$timestamp","$d$district","$d$district_display" ' - 'ORDER BY "$d$timestamp","$d$district_display"', str(query)) - - def test_dimension_with_multiple_joins_includes_joins_ordered__in_query(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes, - slicer.metrics.voters)) \ - .dimension(slicer.dimensions.timestamp) \ - .dimension(slicer.dimensions.district) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("politician"."timestamp",\'DD\') "$d$timestamp",' - '"politician"."district_id" "$d$district",' - '"district"."district_name" "$d$district_display",' - 'SUM("politician"."votes") "$m$votes",' - 'COUNT("voter"."id") "$m$voters" ' - 'FROM "politics"."politician" ' - 'JOIN "politics"."voter" ' - 'ON "politician"."id"="voter"."politician_id" ' - 'OUTER JOIN "locations"."district" ' - 'ON "politician"."district_id"="district"."id" ' - 'GROUP BY "$d$timestamp","$d$district","$d$district_display" ' - 'ORDER BY "$d$timestamp","$d$district_display"', str(query)) - - def test_dimension_with_recursive_join_joins_all_join_tables(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .dimension(slicer.dimensions.state) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("politician"."timestamp",\'DD\') "$d$timestamp",' - '"district"."state_id" "$d$state",' - '"state"."state_name" "$d$state_display",' - 'SUM("politician"."votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'OUTER JOIN "locations"."district" ' - 'ON "politician"."district_id"="district"."id" ' - 'JOIN "locations"."state" ' - 'ON "district"."state_id"="state"."id" ' - 'GROUP BY "$d$timestamp","$d$state","$d$state_display" ' - 'ORDER BY "$d$timestamp","$d$state_display"', str(query)) - - def test_metric_with_join_includes_join_in_query(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.voters)) \ - .dimension(slicer.dimensions.political_party) \ - .query - - self.assertEqual('SELECT ' - '"politician"."political_party" "$d$political_party",' - 'COUNT("voter"."id") "$m$voters" ' - 'FROM "politics"."politician" ' - 'JOIN "politics"."voter" ' - 'ON "politician"."id"="voter"."politician_id" ' - 'GROUP BY "$d$political_party" ' - 'ORDER BY "$d$political_party"', str(query)) - - def test_dimension_filter_with_join_on_display_definition_does_not_include_join_in_query(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.district.isin([1])) \ - .query - - self.assertEqual('SELECT ' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'WHERE "district_id" IN (1)', str(query)) - - def test_dimension_filter_display_field_with_join_includes_join_in_query(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.district.isin(['District 4'], use_display=True)) \ - .query - - self.assertEqual('SELECT ' - 'SUM("politician"."votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'OUTER JOIN "locations"."district" ' - 'ON "politician"."district_id"="district"."id" ' - 'WHERE "district"."district_name" IN (\'District 4\')', str(query)) - - def test_dimension_filter_with_recursive_join_includes_join_in_query(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.state.isin([1])) \ - .query - - self.assertEqual('SELECT ' - 'SUM("politician"."votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'OUTER JOIN "locations"."district" ' - 'ON "politician"."district_id"="district"."id" ' - 'WHERE "district"."state_id" IN (1)', str(query)) - - def test_dimension_filter_with_deep_recursive_join_includes_joins_in_query(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .filter(slicer.dimensions.deepjoin.isin([1])) \ - .query - - self.assertEqual('SELECT ' - 'SUM("politician"."votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'OUTER JOIN "locations"."district" ' - 'ON "politician"."district_id"="district"."id" ' - 'JOIN "locations"."state" ' - 'ON "district"."state_id"="state"."id" ' - 'JOIN "test"."deep" ' - 'ON "deep"."id"="state"."ref_id" ' - 'WHERE "deep"."id" IN (1)', str(query)) - - -class QueryBuilderOrderTests(TestCase): - maxDiff = None - - def test_build_query_order_by_dimension(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .orderby(slicer.dimensions.timestamp) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp"', str(query)) - - def test_build_query_order_by_dimension_display(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.candidate) \ - .orderby(slicer.dimensions.candidate_display) \ - .query - - self.assertEqual('SELECT ' - '"candidate_id" "$d$candidate",' - '"candidate_name" "$d$candidate_display",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$candidate","$d$candidate_display" ' - 'ORDER BY "$d$candidate_display"', str(query)) - - def test_build_query_order_by_dimension_asc(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .orderby(slicer.dimensions.timestamp, orientation=Order.asc) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp" ASC', str(query)) - - def test_build_query_order_by_dimension_desc(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .orderby(slicer.dimensions.timestamp, orientation=Order.desc) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp" DESC', str(query)) - - def test_build_query_order_by_metric(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .orderby(slicer.metrics.votes) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$m$votes"', str(query)) - - def test_build_query_order_by_metric_asc(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .orderby(slicer.metrics.votes, orientation=Order.asc) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$m$votes" ASC', str(query)) - - def test_build_query_order_by_metric_desc(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .orderby(slicer.metrics.votes, orientation=Order.desc) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$m$votes" DESC', str(query)) - - def test_build_query_order_by_multiple_dimensions(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp, slicer.dimensions.candidate) \ - .orderby(slicer.dimensions.timestamp) \ - .orderby(slicer.dimensions.candidate) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"candidate_id" "$d$candidate",' - '"candidate_name" "$d$candidate_display",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display" ' - 'ORDER BY "$d$timestamp","$d$candidate"', str(query)) - - def test_build_query_order_by_multiple_dimensions_with_different_orientations(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp, slicer.dimensions.candidate) \ - .orderby(slicer.dimensions.timestamp, orientation=Order.desc) \ - .orderby(slicer.dimensions.candidate, orientation=Order.asc) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - '"candidate_id" "$d$candidate",' - '"candidate_name" "$d$candidate_display",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp","$d$candidate","$d$candidate_display" ' - 'ORDER BY "$d$timestamp" DESC,"$d$candidate" ASC', str(query)) - - def test_build_query_order_by_metrics_and_dimensions(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .orderby(slicer.dimensions.timestamp) \ - .orderby(slicer.metrics.votes) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp","$m$votes"', str(query)) - - def test_build_query_order_by_metrics_and_dimensions_with_different_orientations(self): - query = slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .orderby(slicer.dimensions.timestamp, orientation=Order.asc) \ - .orderby(slicer.metrics.votes, orientation=Order.desc) \ - .query - - self.assertEqual('SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp" ASC,"$m$votes" DESC', str(query)) - - -@patch('fireant.slicer.queries.builder.fetch_data') -class QueryBuildPaginationTests(TestCase): - def test_set_limit(self, mock_fetch_data: Mock): - slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .limit(20) \ - .fetch() - - mock_fetch_data.assert_called_once_with(ANY, - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp" LIMIT 20', - dimensions=DimensionMatcher(slicer.dimensions.timestamp)) - - def test_set_offset(self, mock_fetch_data: Mock): - slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .offset(20) \ - .fetch() - - mock_fetch_data.assert_called_once_with(ANY, - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp" ' - 'OFFSET 20', - dimensions=DimensionMatcher(slicer.dimensions.timestamp)) - - def test_set_limit_and_offset(self, mock_fetch_data: Mock): - slicer.data \ - .widget(f.DataTablesJS(slicer.metrics.votes)) \ - .dimension(slicer.dimensions.timestamp) \ - .limit(20) \ - .offset(30) \ - .fetch() - - mock_fetch_data.assert_called_once_with(ANY, - 'SELECT ' - 'TRUNC("timestamp",\'DD\') "$d$timestamp",' - 'SUM("votes") "$m$votes" ' - 'FROM "politics"."politician" ' - 'GROUP BY "$d$timestamp" ' - 'ORDER BY "$d$timestamp" ' - 'LIMIT 20 ' - 'OFFSET 30', - dimensions=DimensionMatcher(slicer.dimensions.timestamp)) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -2297,132 +49,3 @@ def test_datatablesjs_requires_at_least_one_metric(self): slicer.data \ .widget(f.DataTablesJS()) - -# noinspection SqlDialectInspection,SqlNoDataSourceInspection -@patch('fireant.slicer.queries.builder.fetch_data') -class QueryBuilderRenderTests(TestCase): - def test_pass_slicer_database_as_arg(self, mock_fetch_data: Mock): - mock_widget = f.Widget(slicer.metrics.votes) - mock_widget.transform = Mock() - - slicer.data \ - .widget(mock_widget) \ - .fetch() - - mock_fetch_data.assert_called_once_with(slicer.database, - ANY, - dimensions=ANY) - - def test_pass_query_from_builder_as_arg(self, mock_fetch_data: Mock): - mock_widget = f.Widget(slicer.metrics.votes) - mock_widget.transform = Mock() - - slicer.data \ - .widget(mock_widget) \ - .fetch() - - mock_fetch_data.assert_called_once_with(ANY, - 'SELECT SUM("votes") "$m$votes" ' - 'FROM "politics"."politician"', - dimensions=ANY) - - def test_builder_dimensions_as_arg_with_zero_dimensions(self, mock_fetch_data: Mock): - mock_widget = f.Widget(slicer.metrics.votes) - mock_widget.transform = Mock() - - slicer.data \ - .widget(mock_widget) \ - .fetch() - - mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=[]) - - def test_builder_dimensions_as_arg_with_one_dimension(self, mock_fetch_data: Mock): - mock_widget = f.Widget(slicer.metrics.votes) - mock_widget.transform = Mock() - - dimensions = [slicer.dimensions.state] - - slicer.data \ - .widget(mock_widget) \ - .dimension(*dimensions) \ - .fetch() - - mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=DimensionMatcher(*dimensions)) - - def test_builder_dimensions_as_arg_with_multiple_dimensions(self, mock_fetch_data: Mock): - mock_widget = f.Widget(slicer.metrics.votes) - mock_widget.transform = Mock() - - dimensions = slicer.dimensions.timestamp, slicer.dimensions.state, slicer.dimensions.political_party - - slicer.data \ - .widget(mock_widget) \ - .dimension(*dimensions) \ - .fetch() - - mock_fetch_data.assert_called_once_with(ANY, ANY, dimensions=DimensionMatcher(*dimensions)) - - def test_call_transform_on_widget(self, mock_fetch_data: Mock): - mock_widget = f.Widget(slicer.metrics.votes) - mock_widget.transform = Mock() - - # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work - slicer.data \ - .dimension(slicer.dimensions.timestamp) \ - .widget(mock_widget) \ - .fetch() - - mock_widget.transform.assert_called_once_with(mock_fetch_data.return_value, - slicer, - DimensionMatcher(slicer.dimensions.timestamp), - []) - - def test_returns_results_from_widget_transform(self, mock_fetch_data: Mock): - mock_widget = f.Widget(slicer.metrics.votes) - mock_widget.transform = Mock() - - # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work - result = slicer.data \ - .dimension(slicer.dimensions.timestamp) \ - .widget(mock_widget) \ - .fetch() - - self.assertListEqual(result, [mock_widget.transform.return_value]) - - def test_operations_evaluated(self, mock_fetch_data: Mock): - mock_operation = Mock(name='mock_operation ', spec=f.Operation) - mock_operation.key, mock_operation.definition = 'mock_operation', slicer.table.abc - - mock_widget = f.Widget(mock_operation) - mock_widget.transform = Mock() - - mock_df = {} - mock_fetch_data.return_value = mock_df - - # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work - slicer.data \ - .dimension(slicer.dimensions.timestamp) \ - .widget(mock_widget) \ - .fetch() - - mock_operation.apply.assert_called_once_with(mock_df) - - def test_operations_results_stored_in_data_frame(self, mock_fetch_data: Mock): - mock_operation = Mock(name='mock_operation ', spec=f.Operation) - mock_operation.key, mock_operation.definition = 'mock_operation', slicer.table.abc - - mock_widget = f.Widget(mock_operation) - mock_widget.transform = Mock() - - mock_df = {} - mock_fetch_data.return_value = mock_df - - # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work - slicer.data \ - .dimension(slicer.dimensions.timestamp) \ - .widget(mock_widget) \ - .fetch() - - f_op_key = format_metric_key(mock_operation.key) - self.assertIn(f_op_key, mock_df) - self.assertEqual(mock_df[f_op_key], mock_operation.apply.return_value) From 174779cf12c0577bee5671c01428158bfc11157e Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 10 Jul 2018 16:16:30 +0200 Subject: [PATCH 089/123] Bumped version to 1.0.0.dev38 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index ba7d6db7..c60d495c 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev37' +__version__ = '1.0.0.dev38' From 9cdce0572de14488fd0f5959e2e67b2146d3a46b Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 19 Jul 2018 13:19:17 +0200 Subject: [PATCH 090/123] Fixed the metric format function to correctly handle for inf and nan --- fireant/formats.py | 6 +- fireant/slicer/dimensions.py | 10 ++++ fireant/tests/slicer/widgets/test_pandas.py | 62 +++++++++++++++++++++ 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/fireant/formats.py b/fireant/formats.py index 58cada1c..5368ecd6 100644 --- a/fireant/formats.py +++ b/fireant/formats.py @@ -111,6 +111,9 @@ def metric_display(value, prefix=None, suffix=None, precision=None): :return: A formatted string containing the display value for the metric. """ + if pd.isnull(value) or value in {np.inf, -np.inf}: + return '' + if isinstance(value, float): if precision is not None: float_format = '%d' if precision == 0 else '%.{}f'.format(precision) @@ -129,9 +132,6 @@ def metric_display(value, prefix=None, suffix=None, precision=None): float_format = '%d' value = locale.format(float_format, value, grouping=True) - if value is INFINITY: - return "∞" - return '{prefix}{value}{suffix}'.format( prefix=prefix or '', value=str(value), diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 1fdd407a..be0c3bf8 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -316,12 +316,22 @@ def __call__(self, groups): :return: A copy of the dimension with the interval set. """ + self.groups = groups + cases = Case() for group in groups: cases = cases.when(self.field.like(group), group) self.definition = cases.else_(self._DEFAULT) + def __repr__(self): + dimension = super().__repr__() + + if self.groups is not None: + return '{}({})'.format(dimension, self.groups) + + return dimension + class TotalsDimension(Dimension): def __init__(self, dimension): diff --git a/fireant/tests/slicer/widgets/test_pandas.py b/fireant/tests/slicer/widgets/test_pandas.py index dbd4d015..76a767ea 100644 --- a/fireant/tests/slicer/widgets/test_pandas.py +++ b/fireant/tests/slicer/widgets/test_pandas.py @@ -1,5 +1,7 @@ +import copy from unittest import TestCase +import numpy as np import pandas as pd import pandas.testing @@ -197,3 +199,63 @@ def test_metric_format(self): expected.columns = ['Votes'] pandas.testing.assert_frame_equal(result, expected) + + def test_nan_in_metrics(self): + cat_dim_df_with_nan = cat_dim_df.copy() + cat_dim_df_with_nan['$m$wins'] = cat_dim_df_with_nan['$m$wins'].apply(float) + cat_dim_df_with_nan.iloc[2, 1] = np.nan + + result = Pandas(slicer.metrics.wins) \ + .transform(cat_dim_df_with_nan, slicer, [slicer.dimensions.political_party], []) + + expected = cat_dim_df_with_nan.copy()[[fm('wins')]] + expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') + expected.columns = ['Wins'] + + pandas.testing.assert_frame_equal(result, expected) + + def test_inf_in_metrics(self): + cat_dim_df_with_nan = cat_dim_df.copy() + cat_dim_df_with_nan['$m$wins'] = cat_dim_df_with_nan['$m$wins'].apply(float) + cat_dim_df_with_nan.iloc[2, 1] = np.inf + + result = Pandas(slicer.metrics.wins) \ + .transform(cat_dim_df_with_nan, slicer, [slicer.dimensions.political_party], []) + + expected = cat_dim_df_with_nan.copy()[[fm('wins')]] + expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') + expected.columns = ['Wins'] + + pandas.testing.assert_frame_equal(result, expected) + + def test_neginf_in_metrics(self): + cat_dim_df_with_nan = cat_dim_df.copy() + cat_dim_df_with_nan['$m$wins'] = cat_dim_df_with_nan['$m$wins'].apply(float) + cat_dim_df_with_nan.iloc[2, 1] = np.inf + + result = Pandas(slicer.metrics.wins) \ + .transform(cat_dim_df_with_nan, slicer, [slicer.dimensions.political_party], []) + + expected = cat_dim_df_with_nan.copy()[[fm('wins')]] + expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') + expected.columns = ['Wins'] + + pandas.testing.assert_frame_equal(result, expected) + + def test_inf_in_metrics_with_precision_zero(self): + cat_dim_df_with_nan = cat_dim_df.copy() + cat_dim_df_with_nan['$m$wins'] = cat_dim_df_with_nan['$m$wins'].apply(float) + cat_dim_df_with_nan.iloc[2, 1] = np.inf + + slicer_modified = copy.deepcopy(slicer) + slicer_modified.metrics.wins.precision = 0 + + result = Pandas(slicer_modified.metrics.wins) \ + .transform(cat_dim_df_with_nan, slicer_modified, [slicer_modified.dimensions.political_party], []) + + expected = cat_dim_df_with_nan.copy()[[fm('wins')]] + expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') + expected['$m$wins'] = ['6', '0', ''] + expected.columns = ['Wins'] + + pandas.testing.assert_frame_equal(result, expected) \ No newline at end of file From 3e98dee2f655ee2facfff4a9168918ef9d240ea7 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 19 Jul 2018 18:08:36 +0200 Subject: [PATCH 091/123] Fixed an issue using int values in data tables with pivot=true when there are NaNs in the data frame, then int gets converted to a float and that breaks the column definitions --- fireant/slicer/queries/database.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index 33cb1872..62f90ab4 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -66,6 +66,10 @@ def clean_and_apply_index(data_frame: pd.DataFrame, dimensions: Iterable[Dimensi level = format_dimension_key(dimension.key) data_frame[level] = fill_nans_in_level(data_frame, dimension, dimension_keys[:i]) \ + .apply( + # Handles an annoying case of pandas in which the ENTIRE data frame gets converted from int to float if + # the are NaNs, even if there are no NaNs in the column :/ + lambda x: int(x) if isinstance(x, float) and float.is_integer(x) else x) \ .apply(lambda x: str(x) if not pd.isnull(x) else None) # Set index on dimension columns From 15e6e8f1067e48f35f4eda28702b3cc686bfc0e5 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 19 Jul 2018 18:43:24 +0200 Subject: [PATCH 092/123] Fixed the behavior of antipattern to negate the whole group of patterns together instead of individually --- fireant/slicer/dimensions.py | 4 +-- fireant/slicer/filters.py | 17 +++++++------ .../queries/test_build_dimension_filters.py | 25 ++++++++++--------- requirements.txt | 2 +- setup.py | 2 +- 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/fireant/slicer/dimensions.py b/fireant/slicer/dimensions.py index 1fdd407a..3820dcf7 100644 --- a/fireant/slicer/dimensions.py +++ b/fireant/slicer/dimensions.py @@ -12,7 +12,7 @@ BooleanFilter, ContainsFilter, ExcludesFilter, - NotLikeFilter, + AntiPatternFilter, PatternFilter, RangeFilter, ) @@ -101,7 +101,7 @@ def not_like(self, pattern, *patterns): A slicer query filter used to filter a slicer query to results where this dimension's display definition matches the pattern. """ - return NotLikeFilter(getattr(self, self.pattern_definition_attribute), pattern, *patterns) + return AntiPatternFilter(getattr(self, self.pattern_definition_attribute), pattern, *patterns) class CategoricalDimension(PatternFilterableMixin, Dimension): diff --git a/fireant/slicer/filters.py b/fireant/slicer/filters.py index 1c2d04b0..ef29235d 100644 --- a/fireant/slicer/filters.py +++ b/fireant/slicer/filters.py @@ -60,16 +60,19 @@ def __init__(self, dimension_definition, start, stop): class PatternFilter(DimensionFilter): - def _apply(self, dimension_definition, pattern): - return dimension_definition.like(pattern) + def _apply(self, dimension_definition, patterns): + definition = dimension_definition.like(patterns[0]) + + for pattern in patterns[1:]: + definition |= dimension_definition.like(pattern) + + return definition def __init__(self, dimension_definition, pattern, *patterns): - definition = self._apply(dimension_definition, pattern) - for extra_pattern in patterns: - definition |= self._apply(dimension_definition, extra_pattern) + definition = self._apply(dimension_definition, [pattern, *patterns]) super(PatternFilter, self).__init__(definition) -class NotLikeFilter(PatternFilter): +class AntiPatternFilter(PatternFilter): def _apply(self, dimension_definition, pattern): - return dimension_definition.not_like(pattern) + return super(AntiPatternFilter, self)._apply(dimension_definition, pattern).negate() diff --git a/fireant/tests/slicer/queries/test_build_dimension_filters.py b/fireant/tests/slicer/queries/test_build_dimension_filters.py index 4e1741e6..c979052b 100644 --- a/fireant/tests/slicer/queries/test_build_dimension_filters.py +++ b/fireant/tests/slicer/queries/test_build_dimension_filters.py @@ -1,6 +1,7 @@ -from datetime import date from unittest import TestCase +from datetime import date + import fireant as f from ..mocks import slicer @@ -51,7 +52,7 @@ def test_build_query_with_filter_not_like_categorical_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "political_party" NOT LIKE \'Rep%\'', str(query)) + 'WHERE NOT "political_party" LIKE \'Rep%\'', str(query)) def test_build_query_with_filter_isin_unique_dim(self): query = slicer.data \ @@ -128,7 +129,7 @@ def test_build_query_with_filter_not_like_unique_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "candidate_name" NOT LIKE \'%Trump\'', str(query)) + 'WHERE NOT "candidate_name" LIKE \'%Trump\'', str(query)) def test_build_query_with_filter_not_like_display_dim(self): query = slicer.data \ @@ -139,7 +140,7 @@ def test_build_query_with_filter_not_like_display_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "candidate_name" NOT LIKE \'%Trump\'', str(query)) + 'WHERE NOT "candidate_name" LIKE \'%Trump\'', str(query)) def test_build_query_with_filter_like_categorical_dim_multiple_patterns(self): query = slicer.data \ @@ -162,8 +163,8 @@ def test_build_query_with_filter_not_like_categorical_dim_multiple_patterns(self self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "political_party" NOT LIKE \'Rep%\' ' - 'OR "political_party" NOT LIKE \'Dem%\'', str(query)) + 'WHERE NOT ("political_party" LIKE \'Rep%\' ' + 'OR "political_party" LIKE \'Dem%\')', str(query)) def test_build_query_with_filter_like_pattern_dim_multiple_patterns(self): query = slicer.data \ @@ -186,8 +187,8 @@ def test_build_query_with_filter_not_like_pattern_dim_multiple_patterns(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "pattern" NOT LIKE \'a%\' ' - 'OR "pattern" NOT LIKE \'b%\'', str(query)) + 'WHERE NOT ("pattern" LIKE \'a%\' ' + 'OR "pattern" LIKE \'b%\')', str(query)) def test_build_query_with_filter_like_unique_dim_multiple_patterns(self): query = slicer.data \ @@ -222,8 +223,8 @@ def test_build_query_with_filter_not_like_unique_dim_multiple_patterns(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "candidate_name" NOT LIKE \'%Trump\' ' - 'OR "candidate_name" NOT LIKE \'%Clinton\'', str(query)) + 'WHERE NOT ("candidate_name" LIKE \'%Trump\' ' + 'OR "candidate_name" LIKE \'%Clinton\')', str(query)) def test_build_query_with_filter_not_like_display_dim_multiple_patterns(self): query = slicer.data \ @@ -234,8 +235,8 @@ def test_build_query_with_filter_not_like_display_dim_multiple_patterns(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "candidate_name" NOT LIKE \'%Trump\' ' - 'OR "candidate_name" NOT LIKE \'%Clinton\'', str(query)) + 'WHERE NOT ("candidate_name" LIKE \'%Trump\' ' + 'OR "candidate_name" LIKE \'%Clinton\')', str(query)) def test_build_query_with_filter_isin_raise_exception_when_display_definition_undefined(self): with self.assertRaises(f.QueryException): diff --git a/requirements.txt b/requirements.txt index 3498df31..bc723c45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ six pandas==0.22.0 -pypika==0.14.7 +pypika==0.14.9 pymysql==0.8.0 toposort==1.5 typing==3.6.2 diff --git a/setup.py b/setup.py index d1789b5a..d34e5084 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ def find_version(*file_paths): install_requires=[ 'six', 'pandas==0.22.0', - 'pypika==0.14.7', + 'pypika==0.14.9', 'toposort==1.5', 'typing==3.6.2', ], From a0ee1b31855b05032e458d2ca5cdb39220925ce8 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 19 Jul 2018 18:48:39 +0200 Subject: [PATCH 093/123] upgraded to dev39 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index c60d495c..59b5a612 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev38' +__version__ = '1.0.0.dev39' From 4cc80bef2cf94e0ba8b8622138f7ddc6d68cbaa3 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 6 Aug 2018 17:36:47 +0200 Subject: [PATCH 094/123] Added react-table transformer and changed pivot behavior of Pandas and CSV transformers to pivot any dimension and to transpose the data frame --- fireant/formats.py | 2 + fireant/slicer/base.py | 2 +- fireant/slicer/queries/database.py | 42 +- fireant/slicer/widgets/__init__.py | 1 + fireant/slicer/widgets/base.py | 10 +- fireant/slicer/widgets/csv.py | 1 + fireant/slicer/widgets/pandas.py | 52 +- fireant/slicer/widgets/reacttable.py | 324 +++++ fireant/tests/slicer/mocks.py | 20 +- fireant/tests/slicer/queries/test_database.py | 8 +- fireant/tests/slicer/widgets/test_csv.py | 44 +- fireant/tests/slicer/widgets/test_pandas.py | 87 +- .../tests/slicer/widgets/test_reacttable.py | 1145 +++++++++++++++++ fireant/utils.py | 34 + 14 files changed, 1683 insertions(+), 89 deletions(-) create mode 100644 fireant/slicer/widgets/reacttable.py create mode 100644 fireant/tests/slicer/widgets/test_reacttable.py diff --git a/fireant/formats.py b/fireant/formats.py index 5368ecd6..611b5e06 100644 --- a/fireant/formats.py +++ b/fireant/formats.py @@ -10,6 +10,8 @@ INFINITY = "Infinity" NULL_VALUE = 'null' +TOTALS_VALUE = 'totals' +RAW_VALUE = 'raw' NO_TIME = time(0) diff --git a/fireant/slicer/base.py b/fireant/slicer/base.py index c32665e9..3ad8ce0c 100644 --- a/fireant/slicer/base.py +++ b/fireant/slicer/base.py @@ -19,7 +19,7 @@ def __init__(self, key, label=None, definition=None, display_definition=None): used for querying labels. """ self.key = key - self.label = label or key + self.label = label if label is not None else key self.definition = definition self.display_definition = display_definition diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index 62f90ab4..867c6369 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -1,10 +1,14 @@ import time +from functools import partial from typing import Iterable import pandas as pd from fireant.database.base import Database -from fireant.formats import NULL_VALUE +from fireant.formats import ( + NULL_VALUE, + TOTALS_VALUE, +) from fireant.utils import format_dimension_key from .logger import ( query_logger, @@ -65,7 +69,7 @@ def clean_and_apply_index(data_frame: pd.DataFrame, dimensions: Iterable[Dimensi continue level = format_dimension_key(dimension.key) - data_frame[level] = fill_nans_in_level(data_frame, dimension, dimension_keys[:i]) \ + data_frame[level] = fill_nans_in_level(data_frame, dimensions[:i + 1]) \ .apply( # Handles an annoying case of pandas in which the ENTIRE data frame gets converted from int to float if # the are NaNs, even if there are no NaNs in the column :/ @@ -76,7 +80,7 @@ def clean_and_apply_index(data_frame: pd.DataFrame, dimensions: Iterable[Dimensi return data_frame.set_index(dimension_keys) -def fill_nans_in_level(data_frame, dimension, preceding_dimension_keys): +def fill_nans_in_level(data_frame, dimensions): """ In case there are NaN values representing both totals (from ROLLUP) and database nulls, we need to replace the real nulls with an empty string in order to distinguish between them. We choose to replace the actual database nulls @@ -85,33 +89,39 @@ def fill_nans_in_level(data_frame, dimension, preceding_dimension_keys): :param data_frame: The data_frame we are replacing values in. - :param dimension: - The level of the data frame to replace nulls in. This function should be called once per non-conitnuous - dimension, in the order of the dimensions. - :param preceding_dimension_keys: + :param dimensions: + A list of dimensions with the last item in the list being the dimension to fill nans for. This function requires + the dimension being processed as well as the preceding dimensions since a roll up in a higher level dimension + results in nulls for lower level dimension. :return: The level in the data_frame with the nulls replaced with empty string """ - level = format_dimension_key(dimension.key) + level = format_dimension_key(dimensions[-1].key) + + number_rollup_dimensions = sum(dimension.is_rollup for dimension in dimensions) + if 0 < number_rollup_dimensions: + fill_nan_for_nulls = partial(_fill_nan_for_nulls, offset=number_rollup_dimensions) + + if 1 < len(dimensions): + preceding_dimension_keys = [format_dimension_key(d.key) + for d in dimensions[:-1]] - if dimension.is_rollup: - if preceding_dimension_keys: return (data_frame .groupby(preceding_dimension_keys)[level] - .apply(_fill_nan_for_nulls)) + .apply(fill_nan_for_nulls)) - return _fill_nan_for_nulls(data_frame[level]) + return fill_nan_for_nulls(data_frame[level]) return data_frame[level].fillna(NULL_VALUE) -def _fill_nan_for_nulls(df): +def _fill_nan_for_nulls(df, offset=1): """ Fills the first NaN with a literal string "null" if there are two NaN values, otherwise nothing is filled. :param df: :return: """ - if 1 < pd.isnull(df).sum(): - return df.fillna(NULL_VALUE, limit=1) - return df + if offset < pd.isnull(df).sum(): + return df.fillna(NULL_VALUE, limit=1).fillna(TOTALS_VALUE) + return df.fillna(TOTALS_VALUE) diff --git a/fireant/slicer/widgets/__init__.py b/fireant/slicer/widgets/__init__.py index 87a36f36..7eae2bf5 100644 --- a/fireant/slicer/widgets/__init__.py +++ b/fireant/slicer/widgets/__init__.py @@ -1,6 +1,7 @@ from .base import Widget from .csv import CSV from .datatables import DataTablesJS +from .reacttable import ReactTable from .highcharts import HighCharts from .matplotlib import Matplotlib from .pandas import Pandas diff --git a/fireant/slicer/widgets/base.py b/fireant/slicer/widgets/base.py index 3e3bff9f..71b5c0cc 100644 --- a/fireant/slicer/widgets/base.py +++ b/fireant/slicer/widgets/base.py @@ -1,11 +1,15 @@ -from fireant import Metric +from typing import Union + +from fireant import ( + Metric, + Operation, +) from fireant.slicer.exceptions import MetricRequiredException from fireant.utils import immutable -from ..operations import Operation class Widget: - def __init__(self, *items: Metric): + def __init__(self, *items: Union[Metric, Operation]): self.items = list(items) @immutable diff --git a/fireant/slicer/widgets/csv.py b/fireant/slicer/widgets/csv.py index c8f62109..a22aa741 100644 --- a/fireant/slicer/widgets/csv.py +++ b/fireant/slicer/widgets/csv.py @@ -4,4 +4,5 @@ class CSV(Pandas): def transform(self, data_frame, slicer, dimensions, references): result_df = super(CSV, self).transform(data_frame, slicer, dimensions, references) + result_df.columns.names = [None] return result_df.to_csv() diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index 613740a6..18b784c6 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -18,9 +18,10 @@ class Pandas(TransformableWidget): - def __init__(self, metric, *metrics: Metric, pivot=False, max_columns=None): + def __init__(self, metric, *metrics: Metric, pivot=(), transpose=False, max_columns=None): super(Pandas, self).__init__(metric, *metrics) self.pivot = pivot + self.transpose = transpose self.max_columns = min(max_columns, HARD_MAX_COLUMNS) \ if max_columns is not None \ else HARD_MAX_COLUMNS @@ -71,15 +72,46 @@ def transform(self, data_frame, slicer, dimensions, references): result.index.names = [dimension.label or dimension.key for dimension in dimensions] - result.columns = [reference_label(item, reference) - for reference in [None] + references - for item in self.items] + result.columns = pd.Index([reference_label(item, reference) + for item in self.items + for reference in [None] + references], + name='Metrics') - if not self.pivot: - return result + return self.pivot_data_frame(result, [d.label or d.key for d in self.pivot], self.transpose) + + @staticmethod + def pivot_data_frame(data_frame, pivot=(), transpose=False): + """ + Pivot and transpose the data frame. Dimensions including in the `pivot` arg will be unshifted to columns. If + `transpose` is True the data frame will be transposed. If there is only index level in the data frame (ie. one + dimension), and that dimension is pivoted, then the data frame will just be transposed. If there is a single + metric in the data frame and at least one dimension pivoted, the metrics column level will be dropped for + simplicity. + + :param data_frame: + :param pivot: + :param transpose: + :return: + """ + if not (pivot or transpose): + return data_frame + + # NOTE: Don't pivot a single dimension data frame. This turns the data frame into a series and pivots the + # metrics anyway. Instead, transpose the data frame. + should_transpose_instead_of_pivot = len(pivot) == len(data_frame.index.names) + + if pivot and not should_transpose_instead_of_pivot: + data_frame = data_frame.unstack(level=pivot) + + if transpose or should_transpose_instead_of_pivot: + data_frame = data_frame.transpose() + + # If there are more than one column levels and the last level is a single metric, drop the level + if isinstance(data_frame.columns, pd.MultiIndex) and 1 == len(data_frame.columns.levels[0]): + data_frame.name = data_frame.columns.levels[0][0] # capture the name of the metrics column + data_frame.columns = data_frame.columns.droplevel(0) # drop the metrics level - pivot_levels = result.index.names[1:] - return result.unstack(level=pivot_levels) + return data_frame.fillna('') def _replace_display_values_in_index(self, dimension, result): """ @@ -89,7 +121,9 @@ def _replace_display_values_in_index(self, dimension, result): df_key = format_dimension_key(dimension.key) values = [dimension.display_values.get(x, x) for x in result.index.get_level_values(df_key)] - result.index.set_levels(level=df_key, levels=values) + result.index.set_levels(level=df_key, + levels=values, + inplace=True) return result values = [dimension.display_values.get(x, x) diff --git a/fireant/slicer/widgets/reacttable.py b/fireant/slicer/widgets/reacttable.py new file mode 100644 index 00000000..3fcde13f --- /dev/null +++ b/fireant/slicer/widgets/reacttable.py @@ -0,0 +1,324 @@ +from collections import OrderedDict + +import pandas as pd + +from fireant.formats import ( + RAW_VALUE, + TOTALS_VALUE, +) +from fireant.utils import ( + format_dimension_key, + format_metric_key, + getdeepattr, + setdeepattr, +) +from .pandas import Pandas +from ..dimensions import ( + DatetimeDimension, + Dimension, +) +from ..intervals import ( + annually, + daily, + hourly, + monthly, + quarterly, + weekly, +) +from ..metrics import Metric +from ..references import ( + reference_key, + reference_label, +) + +DATE_FORMATS = { + hourly: '%Y-%m-%d %h', + daily: '%Y-%m-%d', + weekly: '%Y-%m', + monthly: '%Y', + quarterly: '%Y-%q', + annually: '%Y', +} + +metrics = Dimension('metrics', '') +metrics_dimension_key = format_dimension_key(metrics.key) + + +def map_index_level(index, level, func): + if isinstance(index, pd.MultiIndex): + values = index.levels[level] + return index.set_levels(values.map(func), level) + + assert level == 0 + + return index.map(func) + + +def fillna_index(index, value): + if isinstance(index, pd.MultiIndex): + return pd.MultiIndex.from_tuples([[value if pd.isnull(x) else x + for x in row] + for row in index], + names=index.names) + + return index.fillna(value) + + +def _get_item_display(item, value): + if item is None or all([item.prefix is None, item.suffix is None, item.precision is None]): + return + + return '{prefix}{value}{suffix}'.format( + prefix=(item.prefix or ""), + suffix=(item.suffix or ""), + value=value if item.precision is None else '{:.{precision}f}'.format(0.1234567890, precision=2) + ) + + +def make_column_format(item): + if item is None or all([item.prefix is None, item.suffix is None, item.precision is None]): + return + + return '{prefix}{format}{suffix}'.format( + prefix=(item.prefix or ""), + suffix=(item.suffix or "").replace('%', '%%'), + format='%s' if item.precision is None else '%.{}f'.format(item.precision) + ) + + +class ReferenceItem: + def __init__(self, item, reference): + if reference is None: + self.key = item.key + self.label = item.label + else: + self.key = reference_key(item, reference) + self.label = reference_label(item, reference) + + self.prefix = item.prefix + self.suffix = item.suffix + self.precision = item.precision + + +class TotalsItem: + key = TOTALS_VALUE + label = 'Totals' + prefix = suffix = precision = None + + +class ReactTable(Pandas): + def __init__(self, metric, *metrics: Metric, pivot=(), transpose=False, max_columns=None): + super(ReactTable, self).__init__(metric, *metrics, + pivot=pivot, + transpose=transpose, + max_columns=max_columns) + + def __repr__(self): + return '{}({})'.format(self.__class__.__name__, + ','.join(str(m) for m in self.items)) + + def map_display_values(self, df, dimensions): + """ + WRITEME + + :param df: + :param dimensions: + :return: + """ + dimension_display_values = {} + + for dimension in dimensions: + f_dimension_key = format_dimension_key(dimension.key) + + if dimension.has_display_field: + f_display_key = format_dimension_key(dimension.display_key) + + dimension_display_values[f_dimension_key] = \ + df[f_display_key].groupby(f_dimension_key).first().fillna('null').to_dict() + + del df[f_display_key] + + if hasattr(dimension, 'display_values'): + dimension_display_values[f_dimension_key] = dimension.display_values + + return dimension_display_values + + @staticmethod + def format_data_frame(df, dimensions): + """ + This function prepares the raw data frame for transformation by formatting dates in the index and removing any + remaining NaN/NaT values. It also names the column as metrics so that it can be treated like a dimension level. + + :param df: + :param dimensions: + :return: + """ + for i, dimension in enumerate(dimensions): + if isinstance(dimension, DatetimeDimension): + date_format = DATE_FORMATS.get(dimension.interval, DATE_FORMATS[daily]) + df.index = map_index_level(df.index, i, lambda dt: dt.strftime(date_format)) + + df.index = fillna_index(df.index, TOTALS_VALUE) + df.columns.name = metrics_dimension_key + + @staticmethod + def transform_dimension_column_headers(data_frame, dimensions): + """ + Convert the unpivoted dimensions into ReactTable column header definitions. + + :param data_frame: + :param dimensions: + :return: + """ + dimension_map = {format_dimension_key(d.key): d + for d in dimensions + [metrics]} + + columns = [] + if isinstance(data_frame.index, pd.RangeIndex): + return columns + + for f_dimension_key in data_frame.index.names: + dimension = dimension_map[f_dimension_key] + + columns.append({ + 'Header': getattr(dimension, 'label', dimension.key), + 'accessor': f_dimension_key, + }) + + return columns + + @staticmethod + def transform_metric_column_headers(data_frame, item_map, dimension_display_values): + """ + Convert the metrics into ReactTable column header definitions. This includes any pivoted dimensions, which will + result in multiple rows of headers. + + :param data_frame: + :param item_map: + :param dimension_display_values: + :return: + """ + + def get_header(column_value, f_dimension_key, is_totals): + if f_dimension_key == metrics_dimension_key or is_totals: + item = item_map[column_value] + return getattr(item, 'label', item.key) + + else: + return getdeepattr(dimension_display_values, (f_dimension_key, column_value), column_value) + + def _make_columns(df, previous_levels=()): + f_dimension_key = df.index.names[0] + groups = df.groupby(level=0) \ + if isinstance(df.index, pd.MultiIndex) else \ + [(level, None) for level in df.index] + + columns = [] + for column_value, group in groups: + is_totals = TOTALS_VALUE == column_value + + column = {'Header': get_header(column_value, f_dimension_key, is_totals)} + + levels = previous_levels + (column_value,) + if group is not None: + next_level_df = group.reset_index(level=0, drop=True) + column['columns'] = _make_columns(next_level_df, levels) + + else: + if hasattr(data_frame, 'name'): + levels += (data_frame.name,) + column['accessor'] = '.'.join(levels) + + if is_totals: + column['className'] = 'fireant-totals' + + columns.append(column) + + return columns + + column_frame = data_frame.columns.to_frame() + return _make_columns(column_frame) + + @staticmethod + def transform_data(data_frame, item_map, dimension_display_values): + """ + WRITEME + + :param data_frame: + :param item_map: + :param dimension_display_values: + :return: + """ + result = [] + + for index, series in data_frame.iterrows(): + if not isinstance(index, tuple): + index = (index,) + + index = [x + if x not in item_map + else getattr(item_map[x], 'label', item_map[x].key) + for x in index] + + row = {} + for key, value in zip(data_frame.index.names, index): + if key is None: + continue + + data = {RAW_VALUE: value} + display = getdeepattr(dimension_display_values, (key, value)) + if display is not None: + data['display'] = display + + row[key] = data + + for key, value in series.iteritems(): + data = {RAW_VALUE: value} + item = item_map.get(key[0] if isinstance(key, tuple) else key) + display = _get_item_display(item, value) + if display is not None: + data['display'] = display + + setdeepattr(row, key, data) + + result.append(row) + + return result + + def transform(self, data_frame, slicer, dimensions, references): + """ + WRITEME + + :param data_frame: + :param slicer: + :param dimensions: + :param references: + :return: + """ + df_dimension_columns = [format_dimension_key(d.display_key) + for d in dimensions + if d.has_display_field] + item_map = OrderedDict([(format_metric_key(reference_key(i, reference)), ReferenceItem(i, reference)) + for i in self.items + for reference in [None] + references]) + df_metric_columns = list(item_map.keys()) + + # Add an extra item to map the totals key to it's label + item_map[TOTALS_VALUE] = TotalsItem + + df = data_frame[df_dimension_columns + df_metric_columns].copy() + + dimension_display_values = self.map_display_values(df, dimensions) + self.format_data_frame(df, dimensions) + + dimension_keys = [format_dimension_key(dimension.key) for dimension in self.pivot] + df = self.pivot_data_frame(df, dimension_keys, self.transpose) + + dimension_columns = self.transform_dimension_column_headers(df, dimensions) + metric_columns = self.transform_metric_column_headers(df, item_map, dimension_display_values) + data = self.transform_data(df, item_map, dimension_display_values) + + return { + 'columns': dimension_columns + metric_columns, + 'data': data, + } diff --git a/fireant/tests/slicer/mocks.py b/fireant/tests/slicer/mocks.py index 3719d3ab..08d5b7ea 100644 --- a/fireant/tests/slicer/mocks.py +++ b/fireant/tests/slicer/mocks.py @@ -1,12 +1,12 @@ from collections import ( OrderedDict, ) -from datetime import ( - datetime, -) from unittest.mock import Mock import pandas as pd +from datetime import ( + datetime, +) from pypika import ( JoinType, Table, @@ -108,7 +108,9 @@ def __eq__(self, other): definition=fn.Count(voters_table.id)), Metric('turnout', label='Turnout', - definition=fn.Sum(politicians_table.votes) / fn.Count(voters_table.id)), + definition=fn.Sum(politicians_table.votes) / fn.Count(voters_table.id), + suffix='%', + precision=2), ), ) @@ -282,6 +284,10 @@ def PoliticsRow(timestamp, candidate, candidate_display, political_party, electi cont_dim_operation_df[operation_key] = cont_dim_df[fm('votes')].cumsum() +def split(list, i): + return list[:i], list[i:] + + def ref(data_frame, columns): ref_cols = {column: '%s_eoe' % column for column in columns} @@ -291,9 +297,9 @@ def ref(data_frame, columns): .rename(columns=ref_cols)[list(ref_cols.values())] return (cont_uni_dim_df - .copy() - .join(ref_df) - .iloc[2:]) + .copy() + .join(ref_df) + .iloc[2:]) def ref_delta(ref_data_frame, columns): diff --git a/fireant/tests/slicer/queries/test_database.py b/fireant/tests/slicer/queries/test_database.py index cab6a20f..4eb10b04 100644 --- a/fireant/tests/slicer/queries/test_database.py +++ b/fireant/tests/slicer/queries/test_database.py @@ -141,11 +141,11 @@ def test_set_cat_dim_index_with_nan_converted_to_empty_str(self): self.assertListEqual(result.index.tolist(), ['d', 'i', 'r', 'null']) - def test_convert_cat_totals_converted_to_none(self): + def test_convert_cat_totals_converted_to_totals(self): result = clean_and_apply_index(cat_dim_nans_df.reset_index(), [slicer.dimensions.political_party.rollup()]) - self.assertListEqual(result.index.tolist(), ['d', 'i', 'r', None]) + self.assertListEqual(result.index.tolist(), ['d', 'i', 'r', 'totals']) def test_convert_numeric_values_to_string(self): result = clean_and_apply_index(uni_dim_df.reset_index(), [slicer.dimensions.candidate]) @@ -167,13 +167,13 @@ def test_convert_uni_totals(self): result = clean_and_apply_index(uni_dim_nans_df.reset_index(), [slicer.dimensions.candidate.rollup()]) - self.assertListEqual(result.index.tolist(), [str(x + 1) for x in range(11)] + [None]) + self.assertListEqual(result.index.tolist(), [str(x + 1) for x in range(11)] + ['totals']) def test_set_index_for_multiindex_with_nans_and_totals(self): result = clean_and_apply_index(cont_uni_dim_nans_totals_df.reset_index(), [slicer.dimensions.timestamp, slicer.dimensions.state.rollup()]) - self.assertListEqual(result.index.get_level_values(1).unique().tolist(), ['2', '1', 'null', np.nan]) + self.assertListEqual(result.index.get_level_values(1).unique().tolist(), ['2', '1', 'null', 'totals']) class FetchDimensionOptionsTests(TestCase): diff --git a/fireant/tests/slicer/widgets/test_csv.py b/fireant/tests/slicer/widgets/test_csv.py index 52703447..569e0e38 100644 --- a/fireant/tests/slicer/widgets/test_csv.py +++ b/fireant/tests/slicer/widgets/test_csv.py @@ -33,7 +33,7 @@ def test_single_metric(self): expected = single_metric_df.copy()[[fm('votes')]] expected.columns = ['Votes'] - self.assertEqual(result, expected.to_csv()) + self.assertEqual(expected.to_csv(), result) def test_multiple_metrics(self): result = CSV(slicer.metrics.votes, slicer.metrics.wins) \ @@ -42,7 +42,7 @@ def test_multiple_metrics(self): expected = multi_metric_df.copy()[[fm('votes'), fm('wins')]] expected.columns = ['Votes', 'Wins'] - self.assertEqual(result, expected.to_csv()) + self.assertEqual(expected.to_csv(), result) def test_multiple_metrics_reversed(self): result = CSV(slicer.metrics.wins, slicer.metrics.votes) \ @@ -51,7 +51,7 @@ def test_multiple_metrics_reversed(self): expected = multi_metric_df.copy()[[fm('wins'), fm('votes')]] expected.columns = ['Wins', 'Votes'] - self.assertEqual(result, expected.to_csv()) + self.assertEqual(expected.to_csv(), result) def test_time_series_dim(self): result = CSV(slicer.metrics.wins) \ @@ -61,7 +61,7 @@ def test_time_series_dim(self): expected.index.names = ['Timestamp'] expected.columns = ['Wins'] - self.assertEqual(result, expected.to_csv()) + self.assertEqual(expected.to_csv(), result) def test_time_series_dim_with_operation(self): result = CSV(CumSum(slicer.metrics.votes)) \ @@ -71,7 +71,7 @@ def test_time_series_dim_with_operation(self): expected.index.names = ['Timestamp'] expected.columns = ['CumSum(Votes)'] - self.assertEqual(result, expected.to_csv()) + self.assertEqual(expected.to_csv(), result) def test_cat_dim(self): result = CSV(slicer.metrics.wins) \ @@ -81,7 +81,7 @@ def test_cat_dim(self): expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] - self.assertEqual(result, expected.to_csv()) + self.assertEqual(expected.to_csv(), result) def test_uni_dim(self): result = CSV(slicer.metrics.wins) \ @@ -93,7 +93,7 @@ def test_uni_dim(self): expected.index.names = ['Candidate'] expected.columns = ['Wins'] - self.assertEqual(result, expected.to_csv()) + self.assertEqual(expected.to_csv(), result) def test_uni_dim_no_display_definition(self): import copy @@ -111,7 +111,7 @@ def test_uni_dim_no_display_definition(self): expected.index.names = ['Candidate'] expected.columns = ['Wins'] - self.assertEqual(result, expected.to_csv()) + self.assertEqual(expected.to_csv(), result) def test_multi_dims_time_series_and_uni(self): result = CSV(slicer.metrics.wins) \ @@ -123,41 +123,43 @@ def test_multi_dims_time_series_and_uni(self): expected.index.names = ['Timestamp', 'State'] expected.columns = ['Wins'] - self.assertEqual(result, expected.to_csv()) + self.assertEqual(expected.to_csv(), result) - def test_pivoted_single_dimension_no_effect(self): - result = CSV(slicer.metrics.wins, pivot=True) \ + def test_pivoted_single_dimension_transposes_data_frame(self): + result = CSV(slicer.metrics.wins, pivot=[slicer.dimensions.political_party]) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) expected = cat_dim_df.copy()[[fm('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] + expected.columns.names = ['Metrics'] + expected = expected.transpose() - self.assertEqual(result, expected.to_csv()) + self.assertEqual(expected.to_csv(), result) def test_pivoted_multi_dims_time_series_and_cat(self): - result = CSV(slicer.metrics.wins, pivot=True) \ + result = CSV(slicer.metrics.wins, pivot=[slicer.dimensions.political_party]) \ .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party], []) expected = cont_cat_dim_df.copy()[[fm('wins')]] - expected.index.names = ['Timestamp', 'Party'] - expected.columns = ['Wins'] expected = expected.unstack(level=[1]) + expected.index.names = ['Timestamp'] + expected.columns = ['Democrat', 'Independent', 'Republican'] - self.assertEqual(result, expected.to_csv()) + self.assertEqual(expected.to_csv(), result) def test_pivoted_multi_dims_time_series_and_uni(self): - result = CSV(slicer.metrics.votes, pivot=True) \ + result = CSV(slicer.metrics.votes, pivot=[slicer.dimensions.state]) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ .set_index(fd('state_display'), append=True) \ .reset_index(fd('state'), drop=True)[[fm('votes')]] - expected.index.names = ['Timestamp', 'State'] - expected.columns = ['Votes'] expected = expected.unstack(level=[1]) + expected.index.names = ['Timestamp'] + expected.columns = ['California', 'Texas'] - self.assertEqual(result, expected.to_csv()) + self.assertEqual(expected.to_csv(), result) def test_time_series_ref(self): result = CSV(slicer.metrics.votes) \ @@ -176,4 +178,4 @@ def test_time_series_ref(self): expected.index.names = ['Timestamp', 'State'] expected.columns = ['Votes', 'Votes (EoE)'] - self.assertEqual(result, expected.to_csv()) + self.assertEqual(expected.to_csv(), result) diff --git a/fireant/tests/slicer/widgets/test_pandas.py b/fireant/tests/slicer/widgets/test_pandas.py index 76a767ea..27a77d42 100644 --- a/fireant/tests/slicer/widgets/test_pandas.py +++ b/fireant/tests/slicer/widgets/test_pandas.py @@ -26,7 +26,7 @@ ) -class DataTablesTransformerTests(TestCase): +class PandasTransformerTests(TestCase): maxDiff = None def test_single_metric(self): @@ -35,8 +35,9 @@ def test_single_metric(self): expected = single_metric_df.copy()[[fm('votes')]] expected.columns = ['Votes'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_multiple_metrics(self): result = Pandas(slicer.metrics.votes, slicer.metrics.wins) \ @@ -44,8 +45,9 @@ def test_multiple_metrics(self): expected = multi_metric_df.copy()[[fm('votes'), fm('wins')]] expected.columns = ['Votes', 'Wins'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_multiple_metrics_reversed(self): result = Pandas(slicer.metrics.wins, slicer.metrics.votes) \ @@ -53,8 +55,9 @@ def test_multiple_metrics_reversed(self): expected = multi_metric_df.copy()[[fm('wins'), fm('votes')]] expected.columns = ['Wins', 'Votes'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_time_series_dim(self): result = Pandas(slicer.metrics.wins) \ @@ -63,8 +66,9 @@ def test_time_series_dim(self): expected = cont_dim_df.copy()[[fm('wins')]] expected.index.names = ['Timestamp'] expected.columns = ['Wins'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_time_series_dim_with_operation(self): result = Pandas(CumSum(slicer.metrics.votes)) \ @@ -73,8 +77,9 @@ def test_time_series_dim_with_operation(self): expected = cont_dim_operation_df.copy()[[fm('cumsum(votes)')]] expected.index.names = ['Timestamp'] expected.columns = ['CumSum(Votes)'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_cat_dim(self): result = Pandas(slicer.metrics.wins) \ @@ -83,8 +88,9 @@ def test_cat_dim(self): expected = cat_dim_df.copy()[[fm('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_uni_dim(self): result = Pandas(slicer.metrics.wins) \ @@ -96,8 +102,9 @@ def test_uni_dim(self): [[fm('wins')]] expected.index.names = ['Candidate'] expected.columns = ['Wins'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_uni_dim_no_display_definition(self): import copy @@ -114,8 +121,9 @@ def test_uni_dim_no_display_definition(self): expected = uni_dim_df_copy.copy()[[fm('wins')]] expected.index.names = ['Candidate'] expected.columns = ['Wins'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_multi_dims_time_series_and_uni(self): result = Pandas(slicer.metrics.wins) \ @@ -126,42 +134,59 @@ def test_multi_dims_time_series_and_uni(self): .reset_index(fd('state'), drop=False)[[fm('wins')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Wins'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) - def test_pivoted_single_dimension_no_effect(self): - result = Pandas(slicer.metrics.wins, pivot=True) \ + def test_transpose_single_dimension(self): + result = Pandas(slicer.metrics.wins, transpose=True) \ .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) expected = cat_dim_df.copy()[[fm('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] + expected.columns.name = 'Metrics' + expected = expected.transpose() - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) + + def test_pivoted_single_dimension_transposes_data_frame(self): + result = Pandas(slicer.metrics.wins, pivot=[slicer.dimensions.political_party]) \ + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) + + expected = cat_dim_df.copy()[[fm('wins')]] + expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') + expected.columns = ['Wins'] + expected.columns.name = 'Metrics' + expected = expected.transpose() + + pandas.testing.assert_frame_equal(expected, result) def test_pivoted_multi_dims_time_series_and_cat(self): - result = Pandas(slicer.metrics.wins, pivot=True) \ + result = Pandas(slicer.metrics.wins, pivot=[slicer.dimensions.political_party]) \ .transform(cont_cat_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.political_party], []) expected = cont_cat_dim_df.copy()[[fm('wins')]] - expected.index.names = ['Timestamp', 'Party'] - expected.columns = ['Wins'] - expected = expected.unstack(level=[1]) + expected = expected.unstack(level=[1]).fillna(value='') + expected.index.names = ['Timestamp'] + expected.columns = ['Democrat', 'Independent', 'Republican'] + expected.columns.names = ['Party'] - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_pivoted_multi_dims_time_series_and_uni(self): - result = Pandas(slicer.metrics.votes, pivot=True) \ + result = Pandas(slicer.metrics.votes, pivot=[slicer.dimensions.state]) \ .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) expected = cont_uni_dim_df.copy() \ .set_index(fd('state_display'), append=True) \ .reset_index(fd('state'), drop=True)[[fm('votes')]] - expected.index.names = ['Timestamp', 'State'] - expected.columns = ['Votes'] expected = expected.unstack(level=[1]) + expected.index.names = ['Timestamp'] + expected.columns = ['California', 'Texas'] + expected.columns.names = ['State'] - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_time_series_ref(self): result = Pandas(slicer.metrics.votes) \ @@ -178,8 +203,9 @@ def test_time_series_ref(self): .reset_index(fd('state'), drop=True)[[fm('votes'), fm('votes_eoe')]] expected.index.names = ['Timestamp', 'State'] expected.columns = ['Votes', 'Votes (EoE)'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_metric_format(self): import copy @@ -197,8 +223,9 @@ def test_metric_format(self): for x in expected[fm('votes')] / 3] expected.index.names = ['Timestamp'] expected.columns = ['Votes'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_nan_in_metrics(self): cat_dim_df_with_nan = cat_dim_df.copy() @@ -211,8 +238,9 @@ def test_nan_in_metrics(self): expected = cat_dim_df_with_nan.copy()[[fm('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_inf_in_metrics(self): cat_dim_df_with_nan = cat_dim_df.copy() @@ -225,8 +253,9 @@ def test_inf_in_metrics(self): expected = cat_dim_df_with_nan.copy()[[fm('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_neginf_in_metrics(self): cat_dim_df_with_nan = cat_dim_df.copy() @@ -239,8 +268,9 @@ def test_neginf_in_metrics(self): expected = cat_dim_df_with_nan.copy()[[fm('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected.columns = ['Wins'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) + pandas.testing.assert_frame_equal(expected, result) def test_inf_in_metrics_with_precision_zero(self): cat_dim_df_with_nan = cat_dim_df.copy() @@ -257,5 +287,6 @@ def test_inf_in_metrics_with_precision_zero(self): expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') expected['$m$wins'] = ['6', '0', ''] expected.columns = ['Wins'] + expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(result, expected) \ No newline at end of file + pandas.testing.assert_frame_equal(expected, result) \ No newline at end of file diff --git a/fireant/tests/slicer/widgets/test_reacttable.py b/fireant/tests/slicer/widgets/test_reacttable.py new file mode 100644 index 00000000..c754bb3a --- /dev/null +++ b/fireant/tests/slicer/widgets/test_reacttable.py @@ -0,0 +1,1145 @@ +from unittest import TestCase + +from fireant.slicer.widgets.reacttable import ReactTable +from fireant.tests.slicer.mocks import ( + CumSum, + ElectionOverElection, + cat_dim_df, + cont_dim_df, + cont_dim_operation_df, + cont_uni_dim_all_totals_df, + cont_uni_dim_df, + cont_uni_dim_ref_df, + cont_uni_dim_totals_df, + multi_metric_df, + single_metric_df, + slicer, + uni_dim_df, +) +from fireant.utils import format_dimension_key as fd + + +class ReactTableTransformerTests(TestCase): + maxDiff = None + + def test_single_metric(self): + result = ReactTable(slicer.metrics.votes) \ + .transform(single_metric_df, slicer, [], []) + + self.assertEqual({ + 'columns': [{'Header': 'Votes', 'accessor': '$m$votes'}], + 'data': [{'$m$votes': {'raw': 111674336}}] + }, result) + + def test_multiple_metrics(self): + result = ReactTable(slicer.metrics.votes, slicer.metrics.wins) \ + .transform(multi_metric_df, slicer, [], []) + + self.assertEqual({ + 'columns': [{'Header': 'Votes', 'accessor': '$m$votes'}, + {'Header': 'Wins', 'accessor': '$m$wins'}], + 'data': [{'$m$votes': {'raw': 111674336}, '$m$wins': {'raw': 12}}] + }, result) + + def test_multiple_metrics_reversed(self): + result = ReactTable(slicer.metrics.wins, slicer.metrics.votes) \ + .transform(multi_metric_df, slicer, [], []) + + self.assertEqual({ + 'columns': [{'Header': 'Wins', 'accessor': '$m$wins'}, + {'Header': 'Votes', 'accessor': '$m$votes'}], + 'data': [{'$m$votes': {'raw': 111674336}, '$m$wins': {'raw': 12}}] + }, result) + + def test_time_series_dim(self): + result = ReactTable(slicer.metrics.wins) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) + + self.assertEqual({ + 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, + {'Header': 'Wins', 'accessor': '$m$wins'}], + 'data': [{'$d$timestamp': {'raw': '1996-01-01'}, '$m$wins': {'raw': 2}}, + {'$d$timestamp': {'raw': '2000-01-01'}, '$m$wins': {'raw': 2}}, + {'$d$timestamp': {'raw': '2004-01-01'}, '$m$wins': {'raw': 2}}, + {'$d$timestamp': {'raw': '2008-01-01'}, '$m$wins': {'raw': 2}}, + {'$d$timestamp': {'raw': '2012-01-01'}, '$m$wins': {'raw': 2}}, + {'$d$timestamp': {'raw': '2016-01-01'}, '$m$wins': {'raw': 2}}] + }, result) + + def test_time_series_dim_with_operation(self): + result = ReactTable(CumSum(slicer.metrics.votes)) \ + .transform(cont_dim_operation_df, slicer, [slicer.dimensions.timestamp], []) + + self.assertEqual({ + 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, + {'Header': 'CumSum(Votes)', 'accessor': '$m$cumsum(votes)'}], + 'data': [{ + '$d$timestamp': {'raw': '1996-01-01'}, + '$m$cumsum(votes)': {'raw': 15220449} + }, { + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$cumsum(votes)': {'raw': 31882466} + }, { + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$cumsum(votes)': {'raw': 51497398} + }, { + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$cumsum(votes)': {'raw': 72791613} + }, { + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$cumsum(votes)': {'raw': 93363823} + }, { + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$cumsum(votes)': {'raw': 111674336} + }] + }, result) + + def test_cat_dim(self): + result = ReactTable(slicer.metrics.wins) \ + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) + + self.assertEqual({ + 'columns': [{'Header': 'Party', 'accessor': '$d$political_party'}, + {'Header': 'Wins', 'accessor': '$m$wins'}], + 'data': [{ + '$d$political_party': {'display': 'Democrat', 'raw': 'd'}, + '$m$wins': {'raw': 6} + }, { + '$d$political_party': {'display': 'Independent', 'raw': 'i'}, + '$m$wins': {'raw': 0} + }, { + '$d$political_party': {'display': 'Republican', 'raw': 'r'}, + '$m$wins': {'raw': 6} + }] + }, result) + + def test_uni_dim(self): + result = ReactTable(slicer.metrics.wins) \ + .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) + + self.assertEqual({ + 'columns': [{'Header': 'Candidate', 'accessor': '$d$candidate'}, + {'Header': 'Wins', 'accessor': '$m$wins'}], + 'data': [{ + '$d$candidate': {'display': 'Bill Clinton', 'raw': '1'}, + '$m$wins': {'raw': 2} + }, { + '$d$candidate': {'display': 'Bob Dole', 'raw': '2'}, + '$m$wins': {'raw': 0} + }, { + '$d$candidate': {'display': 'Ross Perot', 'raw': '3'}, + '$m$wins': {'raw': 0} + }, { + '$d$candidate': {'display': 'George Bush', 'raw': '4'}, + '$m$wins': {'raw': 4} + }, { + '$d$candidate': {'display': 'Al Gore', 'raw': '5'}, + '$m$wins': {'raw': 0} + }, { + '$d$candidate': {'display': 'John Kerry', 'raw': '6'}, + '$m$wins': {'raw': 0} + }, { + '$d$candidate': {'display': 'Barrack Obama', 'raw': '7'}, + '$m$wins': {'raw': 4} + }, { + '$d$candidate': {'display': 'John McCain', 'raw': '8'}, + '$m$wins': {'raw': 0} + }, { + '$d$candidate': {'display': 'Mitt Romney', 'raw': '9'}, + '$m$wins': {'raw': 0} + }, { + '$d$candidate': {'display': 'Donald Trump', 'raw': '10'}, + '$m$wins': {'raw': 2} + }, { + '$d$candidate': {'display': 'Hillary Clinton', 'raw': '11'}, + '$m$wins': {'raw': 0} + }] + }, result) + + def test_uni_dim_no_display_definition(self): + import copy + candidate = copy.copy(slicer.dimensions.candidate) + candidate.display_key = None + candidate.display_definition = None + + uni_dim_df_copy = uni_dim_df.copy() + del uni_dim_df_copy[fd(slicer.dimensions.candidate.display_key)] + + result = ReactTable(slicer.metrics.wins) \ + .transform(uni_dim_df_copy, slicer, [candidate], []) + + self.assertEqual({ + 'columns': [{'Header': 'Candidate', 'accessor': '$d$candidate'}, + {'Header': 'Wins', 'accessor': '$m$wins'}], + 'data': [{'$d$candidate': {'raw': '1'}, '$m$wins': {'raw': 2}}, + {'$d$candidate': {'raw': '2'}, '$m$wins': {'raw': 0}}, + {'$d$candidate': {'raw': '3'}, '$m$wins': {'raw': 0}}, + {'$d$candidate': {'raw': '4'}, '$m$wins': {'raw': 4}}, + {'$d$candidate': {'raw': '5'}, '$m$wins': {'raw': 0}}, + {'$d$candidate': {'raw': '6'}, '$m$wins': {'raw': 0}}, + {'$d$candidate': {'raw': '7'}, '$m$wins': {'raw': 4}}, + {'$d$candidate': {'raw': '8'}, '$m$wins': {'raw': 0}}, + {'$d$candidate': {'raw': '9'}, '$m$wins': {'raw': 0}}, + {'$d$candidate': {'raw': '10'}, '$m$wins': {'raw': 2}}, + {'$d$candidate': {'raw': '11'}, '$m$wins': {'raw': 0}}] + }, result) + + def test_multi_dims_time_series_and_uni(self): + result = ReactTable(slicer.metrics.wins) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) + + self.assertEqual({ + 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, + {'Header': 'State', 'accessor': '$d$state'}, + {'Header': 'Wins', 'accessor': '$m$wins'}], + 'data': [{ + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '1996-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '1996-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$wins': {'raw': 1} + }] + }, result) + + def test_multi_dims_with_one_level_totals(self): + result = ReactTable(slicer.metrics.wins) \ + .transform(cont_uni_dim_totals_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state.rollup()], + []) + + self.assertEqual({ + 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, + {'Header': 'State', 'accessor': '$d$state'}, + {'Header': 'Wins', 'accessor': '$m$wins'}], + 'data': [{ + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '1996-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '1996-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'raw': 'Totals'}, + '$d$timestamp': {'raw': '1996-01-01'}, + '$m$wins': {'raw': 2} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'raw': 'Totals'}, + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$wins': {'raw': 2} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'raw': 'Totals'}, + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$wins': {'raw': 2} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'raw': 'Totals'}, + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$wins': {'raw': 2} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'raw': 'Totals'}, + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$wins': {'raw': 2} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'raw': 'Totals'}, + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$wins': {'raw': 2} + }] + }, result) + + def test_multi_dims_with_all_levels_totals(self): + result = ReactTable(slicer.metrics.wins) \ + .transform(cont_uni_dim_all_totals_df, slicer, [slicer.dimensions.timestamp.rollup(), + slicer.dimensions.state.rollup()], []) + + self.assertEqual({ + 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, + {'Header': 'State', 'accessor': '$d$state'}, + {'Header': 'Wins', 'accessor': '$m$wins'}], + 'data': [{ + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '1996-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '1996-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'raw': 'Totals'}, + '$d$timestamp': {'raw': '1996-01-01'}, + '$m$wins': {'raw': 2} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'raw': 'Totals'}, + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$wins': {'raw': 2} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'raw': 'Totals'}, + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$wins': {'raw': 2} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'raw': 'Totals'}, + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$wins': {'raw': 2} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'raw': 'Totals'}, + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$wins': {'raw': 2} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$wins': {'raw': 1} + }, { + '$d$state': {'raw': 'Totals'}, + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$wins': {'raw': 2} + }, { + '$d$state': {'raw': 'Totals'}, + '$d$timestamp': {'raw': 'Totals'}, + '$m$wins': {'raw': 12} + }] + }, result) + + def test_time_series_ref(self): + result = ReactTable(slicer.metrics.votes) \ + .transform(cont_uni_dim_ref_df, + slicer, + [ + slicer.dimensions.timestamp, + slicer.dimensions.state + ], [ + ElectionOverElection(slicer.dimensions.timestamp) + ]) + + self.assertEqual({ + 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, + {'Header': 'State', 'accessor': '$d$state'}, + {'Header': 'Votes', 'accessor': '$m$votes'}, + {'Header': 'Votes (EoE)', 'accessor': '$m$votes_eoe'}], + 'data': [{ + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$votes': {'raw': 6233385.0}, + '$m$votes_eoe': {'raw': 5574387.0} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$votes': {'raw': 10428632.0}, + '$m$votes_eoe': {'raw': 9646062.0} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$votes': {'raw': 7359621.0}, + '$m$votes_eoe': {'raw': 6233385.0} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$votes': {'raw': 12255311.0}, + '$m$votes_eoe': {'raw': 10428632.0} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$votes': {'raw': 8007961.0}, + '$m$votes_eoe': {'raw': 7359621.0} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$votes': {'raw': 13286254.0}, + '$m$votes_eoe': {'raw': 12255311.0} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$votes': {'raw': 7877967.0}, + '$m$votes_eoe': {'raw': 8007961.0} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$votes': {'raw': 12694243.0}, + '$m$votes_eoe': {'raw': 13286254.0} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$votes': {'raw': 5072915.0}, + '$m$votes_eoe': {'raw': 7877967.0} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$votes': {'raw': 13237598.0}, + '$m$votes_eoe': {'raw': 12694243.0} + }] + }, result) + + def test_time_series_ref_multiple_metrics(self): + result = ReactTable(slicer.metrics.votes, slicer.metrics.wins) \ + .transform(cont_uni_dim_ref_df, + slicer, + [ + slicer.dimensions.timestamp, + slicer.dimensions.state + ], [ + ElectionOverElection(slicer.dimensions.timestamp) + ]) + + self.assertEqual({ + 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, + {'Header': 'State', 'accessor': '$d$state'}, + {'Header': 'Votes', 'accessor': '$m$votes'}, + {'Header': 'Votes (EoE)', 'accessor': '$m$votes_eoe'}, + {'Header': 'Wins', 'accessor': '$m$wins'}, + {'Header': 'Wins (EoE)', 'accessor': '$m$wins_eoe'}], + 'data': [{ + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$votes': {'raw': 6233385.0}, + '$m$votes_eoe': {'raw': 5574387.0}, + '$m$wins': {'raw': 1.0}, + '$m$wins_eoe': {'raw': 1.0} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$votes': {'raw': 10428632.0}, + '$m$votes_eoe': {'raw': 9646062.0}, + '$m$wins': {'raw': 1.0}, + '$m$wins_eoe': {'raw': 1.0} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$votes': {'raw': 7359621.0}, + '$m$votes_eoe': {'raw': 6233385.0}, + '$m$wins': {'raw': 1.0}, + '$m$wins_eoe': {'raw': 1.0} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$votes': {'raw': 12255311.0}, + '$m$votes_eoe': {'raw': 10428632.0}, + '$m$wins': {'raw': 1.0}, + '$m$wins_eoe': {'raw': 1.0} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$votes': {'raw': 8007961.0}, + '$m$votes_eoe': {'raw': 7359621.0}, + '$m$wins': {'raw': 1.0}, + '$m$wins_eoe': {'raw': 1.0} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$votes': {'raw': 13286254.0}, + '$m$votes_eoe': {'raw': 12255311.0}, + '$m$wins': {'raw': 1.0}, + '$m$wins_eoe': {'raw': 1.0} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$votes': {'raw': 7877967.0}, + '$m$votes_eoe': {'raw': 8007961.0}, + '$m$wins': {'raw': 1.0}, + '$m$wins_eoe': {'raw': 1.0} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$votes': {'raw': 12694243.0}, + '$m$votes_eoe': {'raw': 13286254.0}, + '$m$wins': {'raw': 1.0}, + '$m$wins_eoe': {'raw': 1.0} + }, { + '$d$state': {'display': 'Texas', 'raw': '1'}, + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$votes': {'raw': 5072915.0}, + '$m$votes_eoe': {'raw': 7877967.0}, + '$m$wins': {'raw': 1.0}, + '$m$wins_eoe': {'raw': 1.0} + }, { + '$d$state': {'display': 'California', 'raw': '2'}, + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$votes': {'raw': 13237598.0}, + '$m$votes_eoe': {'raw': 12694243.0}, + '$m$wins': {'raw': 1.0}, + '$m$wins_eoe': {'raw': 1.0} + }] + }, result) + + def test_transpose(self): + result = ReactTable(slicer.metrics.wins, transpose=True) \ + .transform(cat_dim_df, slicer, [slicer.dimensions.political_party], []) + + self.assertEqual({ + 'columns': [{'Header': '', 'accessor': '$d$metrics'}, + {'Header': 'Democrat', 'accessor': 'd'}, + {'Header': 'Independent', 'accessor': 'i'}, + {'Header': 'Republican', 'accessor': 'r'}], + 'data': [{ + '$d$metrics': {'raw': 'Wins'}, + 'd': {'raw': 6}, + 'i': {'raw': 0}, + 'r': {'raw': 6} + }] + }, result) + + def test_pivot_second_dimension_with_one_metric(self): + result = ReactTable(slicer.metrics.wins, pivot=[slicer.dimensions.state]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) + + self.assertEqual({ + 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, + {'Header': 'Texas', 'accessor': '1'}, + {'Header': 'California', 'accessor': '2'}], + 'data': [{ + '$d$timestamp': {'raw': '1996-01-01'}, + '1': {'raw': 1}, + '2': {'raw': 1} + }, { + '$d$timestamp': {'raw': '2000-01-01'}, + '1': {'raw': 1}, + '2': {'raw': 1} + }, { + '$d$timestamp': {'raw': '2004-01-01'}, + '1': {'raw': 1}, + '2': {'raw': 1} + }, { + '$d$timestamp': {'raw': '2008-01-01'}, + '1': {'raw': 1}, + '2': {'raw': 1} + }, { + '$d$timestamp': {'raw': '2012-01-01'}, + '1': {'raw': 1}, + '2': {'raw': 1} + }, { + '$d$timestamp': {'raw': '2016-01-01'}, + '1': {'raw': 1}, + '2': {'raw': 1} + }] + }, result) + + def test_pivot_second_dimension_with_multiple_metrics(self): + result = ReactTable(slicer.metrics.wins, slicer.metrics.votes, pivot=[slicer.dimensions.state]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) + + self.assertEqual({ + 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, + { + 'Header': 'Votes', + 'columns': [{'Header': 'Texas', 'accessor': '$m$votes.1'}, + {'Header': 'California', 'accessor': '$m$votes.2'}] + }, { + 'Header': 'Wins', + 'columns': [{'Header': 'Texas', 'accessor': '$m$wins.1'}, + {'Header': 'California', 'accessor': '$m$wins.2'}] + }], + 'data': [{ + '$d$timestamp': {'raw': '1996-01-01'}, + '$m$votes': {'1': {'raw': 5574387}, '2': {'raw': 9646062}}, + '$m$wins': {'1': {'raw': 1}, '2': {'raw': 1}} + }, { + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$votes': {'1': {'raw': 6233385}, '2': {'raw': 10428632}}, + '$m$wins': {'1': {'raw': 1}, '2': {'raw': 1}} + }, { + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$votes': {'1': {'raw': 7359621}, '2': {'raw': 12255311}}, + '$m$wins': {'1': {'raw': 1}, '2': {'raw': 1}} + }, { + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$votes': {'1': {'raw': 8007961}, '2': {'raw': 13286254}}, + '$m$wins': {'1': {'raw': 1}, '2': {'raw': 1}} + }, { + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$votes': {'1': {'raw': 7877967}, '2': {'raw': 12694243}}, + '$m$wins': {'1': {'raw': 1}, '2': {'raw': 1}} + }, { + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$votes': {'1': {'raw': 5072915}, '2': {'raw': 13237598}}, + '$m$wins': {'1': {'raw': 1}, '2': {'raw': 1}} + }] + }, result) + + def test_pivot_second_dimension_with_multiple_metrics_and_references(self): + result = ReactTable(slicer.metrics.votes, slicer.metrics.wins, pivot=[slicer.dimensions.state]) \ + .transform(cont_uni_dim_ref_df, + slicer, [ + slicer.dimensions.timestamp, slicer.dimensions.state + ], [ + ElectionOverElection(slicer.dimensions.timestamp) + ]) + + self.assertEqual({ + 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, + { + 'Header': 'Votes', + 'columns': [{'Header': 'Texas', 'accessor': '$m$votes.1'}, + {'Header': 'California', 'accessor': '$m$votes.2'}] + }, { + 'Header': 'Votes (EoE)', + 'columns': [{'Header': 'Texas', 'accessor': '$m$votes_eoe.1'}, + { + 'Header': 'California', + 'accessor': '$m$votes_eoe.2' + }] + }, { + 'Header': 'Wins', + 'columns': [{'Header': 'Texas', 'accessor': '$m$wins.1'}, + {'Header': 'California', 'accessor': '$m$wins.2'}] + }, { + 'Header': 'Wins (EoE)', + 'columns': [{'Header': 'Texas', 'accessor': '$m$wins_eoe.1'}, + { + 'Header': 'California', + 'accessor': '$m$wins_eoe.2' + }] + }], + 'data': [{ + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$votes': {'1': {'raw': 6233385.0}, '2': {'raw': 10428632.0}}, + '$m$votes_eoe': {'1': {'raw': 5574387.0}, '2': {'raw': 9646062.0}}, + '$m$wins': {'1': {'raw': 1.0}, '2': {'raw': 1.0}}, + '$m$wins_eoe': {'1': {'raw': 1.0}, '2': {'raw': 1.0}} + }, { + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$votes': {'1': {'raw': 7359621.0}, '2': {'raw': 12255311.0}}, + '$m$votes_eoe': {'1': {'raw': 6233385.0}, '2': {'raw': 10428632.0}}, + '$m$wins': {'1': {'raw': 1.0}, '2': {'raw': 1.0}}, + '$m$wins_eoe': {'1': {'raw': 1.0}, '2': {'raw': 1.0}} + }, { + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$votes': {'1': {'raw': 8007961.0}, '2': {'raw': 13286254.0}}, + '$m$votes_eoe': {'1': {'raw': 7359621.0}, '2': {'raw': 12255311.0}}, + '$m$wins': {'1': {'raw': 1.0}, '2': {'raw': 1.0}}, + '$m$wins_eoe': {'1': {'raw': 1.0}, '2': {'raw': 1.0}} + }, { + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$votes': {'1': {'raw': 7877967.0}, '2': {'raw': 12694243.0}}, + '$m$votes_eoe': {'1': {'raw': 8007961.0}, '2': {'raw': 13286254.0}}, + '$m$wins': {'1': {'raw': 1.0}, '2': {'raw': 1.0}}, + '$m$wins_eoe': {'1': {'raw': 1.0}, '2': {'raw': 1.0}} + }, { + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$votes': {'1': {'raw': 5072915.0}, '2': {'raw': 13237598.0}}, + '$m$votes_eoe': {'1': {'raw': 7877967.0}, '2': {'raw': 12694243.0}}, + '$m$wins': {'1': {'raw': 1.0}, '2': {'raw': 1.0}}, + '$m$wins_eoe': {'1': {'raw': 1.0}, '2': {'raw': 1.0}} + }] + }, result) + + def test_pivot_single_dimension_as_rows_single_metric_metrics_automatically_pivoted(self): + result = ReactTable(slicer.metrics.wins, pivot=[slicer.dimensions.candidate]) \ + .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) + + self.assertEqual({ + 'columns': [{'Header': '', 'accessor': '$d$metrics'}, + {'Header': 'Bill Clinton', 'accessor': '1'}, + {'Header': 'Bob Dole', 'accessor': '2'}, + {'Header': 'Ross Perot', 'accessor': '3'}, + {'Header': 'George Bush', 'accessor': '4'}, + {'Header': 'Al Gore', 'accessor': '5'}, + {'Header': 'John Kerry', 'accessor': '6'}, + {'Header': 'Barrack Obama', 'accessor': '7'}, + {'Header': 'John McCain', 'accessor': '8'}, + {'Header': 'Mitt Romney', 'accessor': '9'}, + {'Header': 'Donald Trump', 'accessor': '10'}, + {'Header': 'Hillary Clinton', 'accessor': '11'}], + 'data': [{ + '$d$metrics': {'raw': 'Wins'}, + '1': {'raw': 2}, + '10': {'raw': 2}, + '11': {'raw': 0}, + '2': {'raw': 0}, + '3': {'raw': 0}, + '4': {'raw': 4}, + '5': {'raw': 0}, + '6': {'raw': 0}, + '7': {'raw': 4}, + '8': {'raw': 0}, + '9': {'raw': 0} + }] + }, result) + + def test_pivot_single_dimension_as_rows_single_metric_and_transpose_set_to_true(self): + result = ReactTable(slicer.metrics.wins, pivot=[slicer.dimensions.candidate], transpose=True) \ + .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) + + self.assertEqual({ + 'columns': [{'Header': '', 'accessor': '$d$metrics'}, + {'Header': 'Bill Clinton', 'accessor': '1'}, + {'Header': 'Bob Dole', 'accessor': '2'}, + {'Header': 'Ross Perot', 'accessor': '3'}, + {'Header': 'George Bush', 'accessor': '4'}, + {'Header': 'Al Gore', 'accessor': '5'}, + {'Header': 'John Kerry', 'accessor': '6'}, + {'Header': 'Barrack Obama', 'accessor': '7'}, + {'Header': 'John McCain', 'accessor': '8'}, + {'Header': 'Mitt Romney', 'accessor': '9'}, + {'Header': 'Donald Trump', 'accessor': '10'}, + {'Header': 'Hillary Clinton', 'accessor': '11'}], + 'data': [{ + '$d$metrics': {'raw': 'Wins'}, + '1': {'raw': 2}, + '10': {'raw': 2}, + '11': {'raw': 0}, + '2': {'raw': 0}, + '3': {'raw': 0}, + '4': {'raw': 4}, + '5': {'raw': 0}, + '6': {'raw': 0}, + '7': {'raw': 4}, + '8': {'raw': 0}, + '9': {'raw': 0} + }] + }, result) + + def test_pivot_single_dimension_as_rows_multiple_metrics(self): + result = ReactTable(slicer.metrics.wins, slicer.metrics.votes, + pivot=[slicer.dimensions.candidate]) \ + .transform(uni_dim_df, slicer, [slicer.dimensions.candidate], []) + + self.assertEqual({ + 'columns': [{'Header': '', 'accessor': '$d$metrics'}, + {'Header': 'Bill Clinton', 'accessor': '1'}, + {'Header': 'Bob Dole', 'accessor': '2'}, + {'Header': 'Ross Perot', 'accessor': '3'}, + {'Header': 'George Bush', 'accessor': '4'}, + {'Header': 'Al Gore', 'accessor': '5'}, + {'Header': 'John Kerry', 'accessor': '6'}, + {'Header': 'Barrack Obama', 'accessor': '7'}, + {'Header': 'John McCain', 'accessor': '8'}, + {'Header': 'Mitt Romney', 'accessor': '9'}, + {'Header': 'Donald Trump', 'accessor': '10'}, + {'Header': 'Hillary Clinton', 'accessor': '11'}], + 'data': [{ + '$d$metrics': {'raw': 'Wins'}, + '1': {'raw': 2}, + '10': {'raw': 2}, + '11': {'raw': 0}, + '2': {'raw': 0}, + '3': {'raw': 0}, + '4': {'raw': 4}, + '5': {'raw': 0}, + '6': {'raw': 0}, + '7': {'raw': 4}, + '8': {'raw': 0}, + '9': {'raw': 0} + }, { + '$d$metrics': {'raw': 'Votes'}, + '1': {'raw': 7579518}, + '10': {'raw': 13438835}, + '11': {'raw': 4871678}, + '2': {'raw': 6564547}, + '3': {'raw': 1076384}, + '4': {'raw': 18403811}, + '5': {'raw': 8294949}, + '6': {'raw': 9578189}, + '7': {'raw': 24227234}, + '8': {'raw': 9491109}, + '9': {'raw': 8148082} + }] + }, result) + + def test_pivot_single_metric_time_series_dim(self): + result = ReactTable(slicer.metrics.wins) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) + + self.assertEqual({ + 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, + {'Header': 'Wins', 'accessor': '$m$wins'}], + 'data': [{'$d$timestamp': {'raw': '1996-01-01'}, '$m$wins': {'raw': 2}}, + {'$d$timestamp': {'raw': '2000-01-01'}, '$m$wins': {'raw': 2}}, + {'$d$timestamp': {'raw': '2004-01-01'}, '$m$wins': {'raw': 2}}, + {'$d$timestamp': {'raw': '2008-01-01'}, '$m$wins': {'raw': 2}}, + {'$d$timestamp': {'raw': '2012-01-01'}, '$m$wins': {'raw': 2}}, + {'$d$timestamp': {'raw': '2016-01-01'}, '$m$wins': {'raw': 2}}] + }, result) + + def test_pivot_multi_dims_with_all_levels_totals(self): + state = slicer.dimensions.state.rollup() + result = ReactTable(slicer.metrics.wins, slicer.metrics.votes, pivot=[state]) \ + .transform(cont_uni_dim_all_totals_df, slicer, [slicer.dimensions.timestamp.rollup(), + state], []) + + self.assertEqual({ + 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, + { + 'Header': 'Votes', + 'columns': [{'Header': 'Texas', 'accessor': '$m$votes.1'}, + {'Header': 'California', 'accessor': '$m$votes.2'}, + { + 'Header': 'Totals', + 'accessor': '$m$votes.totals', + 'className': 'fireant-totals' + }] + }, { + 'Header': 'Wins', + 'columns': [{'Header': 'Texas', 'accessor': '$m$wins.1'}, + {'Header': 'California', 'accessor': '$m$wins.2'}, + { + 'Header': 'Totals', + 'accessor': '$m$wins.totals', + 'className': 'fireant-totals' + }] + }], + 'data': [{ + '$d$timestamp': {'raw': '1996-01-01'}, + '$m$votes': { + '1': {'raw': 5574387.0}, + '2': {'raw': 9646062.0}, + 'totals': {'raw': 15220449.0} + }, + '$m$wins': { + '1': {'raw': 1.0}, + '2': {'raw': 1.0}, + 'totals': {'raw': 2.0} + } + }, { + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$votes': { + '1': {'raw': 6233385.0}, + '2': {'raw': 10428632.0}, + 'totals': {'raw': 16662017.0} + }, + '$m$wins': { + '1': {'raw': 1.0}, + '2': {'raw': 1.0}, + 'totals': {'raw': 2.0} + } + }, { + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$votes': { + '1': {'raw': 7359621.0}, + '2': {'raw': 12255311.0}, + 'totals': {'raw': 19614932.0} + }, + '$m$wins': { + '1': {'raw': 1.0}, + '2': {'raw': 1.0}, + 'totals': {'raw': 2.0} + } + }, { + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$votes': { + '1': {'raw': 8007961.0}, + '2': {'raw': 13286254.0}, + 'totals': {'raw': 21294215.0} + }, + '$m$wins': { + '1': {'raw': 1.0}, + '2': {'raw': 1.0}, + 'totals': {'raw': 2.0} + } + }, { + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$votes': { + '1': {'raw': 7877967.0}, + '2': {'raw': 12694243.0}, + 'totals': {'raw': 20572210.0} + }, + '$m$wins': { + '1': {'raw': 1.0}, + '2': {'raw': 1.0}, + 'totals': {'raw': 2.0} + } + }, { + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$votes': { + '1': {'raw': 5072915.0}, + '2': {'raw': 13237598.0}, + 'totals': {'raw': 18310513.0} + }, + '$m$wins': { + '1': {'raw': 1.0}, + '2': {'raw': 1.0}, + 'totals': {'raw': 2.0} + } + }, { + '$d$timestamp': {'raw': 'Totals'}, + '$m$votes': { + '1': {'raw': ''}, + '2': {'raw': ''}, + 'totals': {'raw': 111674336.0} + }, + '$m$wins': { + '1': {'raw': ''}, + '2': {'raw': ''}, + 'totals': {'raw': 12.0} + } + }] + }, result) + + def test_pivot_first_dimension_and_transpose_with_all_levels_totals(self): + state = slicer.dimensions.state.rollup() + result = ReactTable(slicer.metrics.wins, slicer.metrics.votes, pivot=[state], transpose=True) \ + .transform(cont_uni_dim_all_totals_df, slicer, [slicer.dimensions.timestamp.rollup(), + state], []) + + self.assertEqual({ + 'columns': [{'Header': '', 'accessor': '$d$metrics'}, + {'Header': 'State', 'accessor': '$d$state'}, + {'Header': '1996-01-01', 'accessor': '1996-01-01'}, + {'Header': '2000-01-01', 'accessor': '2000-01-01'}, + {'Header': '2004-01-01', 'accessor': '2004-01-01'}, + {'Header': '2008-01-01', 'accessor': '2008-01-01'}, + {'Header': '2012-01-01', 'accessor': '2012-01-01'}, + {'Header': '2016-01-01', 'accessor': '2016-01-01'}, + { + 'Header': 'Totals', + 'accessor': 'totals', + 'className': 'fireant-totals' + }], + 'data': [{ + '$d$metrics': {'raw': 'Wins'}, + '$d$state': {'display': 'Texas', 'raw': '1'}, + '1996-01-01': {'raw': 1.0}, + '2000-01-01': {'raw': 1.0}, + '2004-01-01': {'raw': 1.0}, + '2008-01-01': {'raw': 1.0}, + '2012-01-01': {'raw': 1.0}, + '2016-01-01': {'raw': 1.0}, + 'totals': {'raw': ''} + }, { + '$d$metrics': {'raw': 'Wins'}, + '$d$state': {'display': 'California', 'raw': '2'}, + '1996-01-01': {'raw': 1.0}, + '2000-01-01': {'raw': 1.0}, + '2004-01-01': {'raw': 1.0}, + '2008-01-01': {'raw': 1.0}, + '2012-01-01': {'raw': 1.0}, + '2016-01-01': {'raw': 1.0}, + 'totals': {'raw': ''} + }, { + '$d$metrics': {'raw': 'Wins'}, + '$d$state': {'raw': 'Totals'}, + '1996-01-01': {'raw': 2.0}, + '2000-01-01': {'raw': 2.0}, + '2004-01-01': {'raw': 2.0}, + '2008-01-01': {'raw': 2.0}, + '2012-01-01': {'raw': 2.0}, + '2016-01-01': {'raw': 2.0}, + 'totals': {'raw': 12.0} + }, { + '$d$metrics': {'raw': 'Votes'}, + '$d$state': {'display': 'Texas', 'raw': '1'}, + '1996-01-01': {'raw': 5574387.0}, + '2000-01-01': {'raw': 6233385.0}, + '2004-01-01': {'raw': 7359621.0}, + '2008-01-01': {'raw': 8007961.0}, + '2012-01-01': {'raw': 7877967.0}, + '2016-01-01': {'raw': 5072915.0}, + 'totals': {'raw': ''} + }, { + '$d$metrics': {'raw': 'Votes'}, + '$d$state': {'display': 'California', 'raw': '2'}, + '1996-01-01': {'raw': 9646062.0}, + '2000-01-01': {'raw': 10428632.0}, + '2004-01-01': {'raw': 12255311.0}, + '2008-01-01': {'raw': 13286254.0}, + '2012-01-01': {'raw': 12694243.0}, + '2016-01-01': {'raw': 13237598.0}, + 'totals': {'raw': ''} + }, { + '$d$metrics': {'raw': 'Votes'}, + '$d$state': {'raw': 'Totals'}, + '1996-01-01': {'raw': 15220449.0}, + '2000-01-01': {'raw': 16662017.0}, + '2004-01-01': {'raw': 19614932.0}, + '2008-01-01': {'raw': 21294215.0}, + '2012-01-01': {'raw': 20572210.0}, + '2016-01-01': {'raw': 18310513.0}, + 'totals': {'raw': 111674336.0} + }] + }, result) + + def test_pivot_second_dimension_and_transpose_with_all_levels_totals(self): + state = slicer.dimensions.state.rollup() + result = ReactTable(slicer.metrics.wins, slicer.metrics.votes, pivot=[state], transpose=True) \ + .transform(cont_uni_dim_all_totals_df, slicer, [slicer.dimensions.timestamp.rollup(), + state], []) + + self.assertEqual({ + 'columns': [{'Header': '', 'accessor': '$d$metrics'}, + {'Header': 'State', 'accessor': '$d$state'}, + {'Header': '1996-01-01', 'accessor': '1996-01-01'}, + {'Header': '2000-01-01', 'accessor': '2000-01-01'}, + {'Header': '2004-01-01', 'accessor': '2004-01-01'}, + {'Header': '2008-01-01', 'accessor': '2008-01-01'}, + {'Header': '2012-01-01', 'accessor': '2012-01-01'}, + {'Header': '2016-01-01', 'accessor': '2016-01-01'}, + { + 'Header': 'Totals', + 'accessor': 'totals', + 'className': 'fireant-totals' + }], + 'data': [{ + '$d$metrics': {'raw': 'Wins'}, + '$d$state': {'display': 'Texas', 'raw': '1'}, + '1996-01-01': {'raw': 1.0}, + '2000-01-01': {'raw': 1.0}, + '2004-01-01': {'raw': 1.0}, + '2008-01-01': {'raw': 1.0}, + '2012-01-01': {'raw': 1.0}, + '2016-01-01': {'raw': 1.0}, + 'totals': {'raw': ''} + }, { + '$d$metrics': {'raw': 'Wins'}, + '$d$state': {'display': 'California', 'raw': '2'}, + '1996-01-01': {'raw': 1.0}, + '2000-01-01': {'raw': 1.0}, + '2004-01-01': {'raw': 1.0}, + '2008-01-01': {'raw': 1.0}, + '2012-01-01': {'raw': 1.0}, + '2016-01-01': {'raw': 1.0}, + 'totals': {'raw': ''} + }, { + '$d$metrics': {'raw': 'Wins'}, + '$d$state': {'raw': 'Totals'}, + '1996-01-01': {'raw': 2.0}, + '2000-01-01': {'raw': 2.0}, + '2004-01-01': {'raw': 2.0}, + '2008-01-01': {'raw': 2.0}, + '2012-01-01': {'raw': 2.0}, + '2016-01-01': {'raw': 2.0}, + 'totals': {'raw': 12.0} + }, { + '$d$metrics': {'raw': 'Votes'}, + '$d$state': {'display': 'Texas', 'raw': '1'}, + '1996-01-01': {'raw': 5574387.0}, + '2000-01-01': {'raw': 6233385.0}, + '2004-01-01': {'raw': 7359621.0}, + '2008-01-01': {'raw': 8007961.0}, + '2012-01-01': {'raw': 7877967.0}, + '2016-01-01': {'raw': 5072915.0}, + 'totals': {'raw': ''} + }, { + '$d$metrics': {'raw': 'Votes'}, + '$d$state': {'display': 'California', 'raw': '2'}, + '1996-01-01': {'raw': 9646062.0}, + '2000-01-01': {'raw': 10428632.0}, + '2004-01-01': {'raw': 12255311.0}, + '2008-01-01': {'raw': 13286254.0}, + '2012-01-01': {'raw': 12694243.0}, + '2016-01-01': {'raw': 13237598.0}, + 'totals': {'raw': ''} + }, { + '$d$metrics': {'raw': 'Votes'}, + '$d$state': {'raw': 'Totals'}, + '1996-01-01': {'raw': 15220449.0}, + '2000-01-01': {'raw': 16662017.0}, + '2004-01-01': {'raw': 19614932.0}, + '2008-01-01': {'raw': 21294215.0}, + '2012-01-01': {'raw': 20572210.0}, + '2016-01-01': {'raw': 18310513.0}, + 'totals': {'raw': 111674336.0} + }] + }, result) diff --git a/fireant/utils.py b/fireant/utils.py index 9512de97..84349f9c 100644 --- a/fireant/utils.py +++ b/fireant/utils.py @@ -13,6 +13,34 @@ def deep_get(d, keys, default=None): d_level = d_level[key] return d_level + +def setdeepattr(d, key, value): + if not isinstance(key, (list, tuple)): + key = (key,) + + top, *rest = key + + if rest: + if top not in d: + d[top] = {} + + setdeepattr(d[top], rest, value) + + else: + d[top] = value + + +def getdeepattr(d, keys, default_value=None): + d_level = d + + for key in keys: + if key not in d_level: + return default_value + + d_level = d_level[key] + + return d_level + def flatten(items): return [item for level in items for item in wrap_list(level)] @@ -23,6 +51,12 @@ def slice_first(item): return item +a = 'a' \ + if True \ + else 'b' if True \ + else 'c' + + def filter_duplicates(iterable): filtered_list, seen = [], set() for item in iterable: From bdcad9668f741eb0ebf1d43133f09b216393cac1 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 6 Aug 2018 17:59:07 +0200 Subject: [PATCH 095/123] Fixed an issue with older python syntax --- fireant/slicer/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/slicer/filters.py b/fireant/slicer/filters.py index ef29235d..b3f5e315 100644 --- a/fireant/slicer/filters.py +++ b/fireant/slicer/filters.py @@ -69,7 +69,7 @@ def _apply(self, dimension_definition, patterns): return definition def __init__(self, dimension_definition, pattern, *patterns): - definition = self._apply(dimension_definition, [pattern, *patterns]) + definition = self._apply(dimension_definition, (pattern,) + patterns) super(PatternFilter, self).__init__(definition) From 46a35b182e1887a1a6a20cc74bf761086e9569d2 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 7 Aug 2018 09:36:11 +0200 Subject: [PATCH 096/123] Added a comment to the fireant transformer explaining how to use it. --- fireant/slicer/widgets/reacttable.py | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/fireant/slicer/widgets/reacttable.py b/fireant/slicer/widgets/reacttable.py index 3fcde13f..9ad1e542 100644 --- a/fireant/slicer/widgets/reacttable.py +++ b/fireant/slicer/widgets/reacttable.py @@ -107,6 +107,35 @@ class TotalsItem: class ReactTable(Pandas): + """ + This component does not work with react-table out of the box, some customization is needed in order to work with + the transformed data. + + ``` + // A Custom TdComponent implementation is required by Fireant in order to render display values + const TdComponent = ({ + toggleSort, + className, + children, + ...rest + }) => +
+ {_.get(children, 'display', children.raw) ||  } +
; + + const FireantReactTable = ({ + config, // The payload from fireant + }) => + ReactTableDefaults.defaultSortMethod(a.raw, b.raw, desc)}> + ; + ``` + """ + def __init__(self, metric, *metrics: Metric, pivot=(), transpose=False, max_columns=None): super(ReactTable, self).__init__(metric, *metrics, pivot=pivot, From 6fdd7a82069c4fc0eb79d0331e7ef2056ffc2f1b Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 7 Aug 2018 10:18:18 +0200 Subject: [PATCH 097/123] Fixed an issue with react-table when the result set is empty --- fireant/slicer/widgets/reacttable.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fireant/slicer/widgets/reacttable.py b/fireant/slicer/widgets/reacttable.py index 9ad1e542..a2278923 100644 --- a/fireant/slicer/widgets/reacttable.py +++ b/fireant/slicer/widgets/reacttable.py @@ -45,6 +45,10 @@ def map_index_level(index, level, func): + # If the index is empty, do not do anything + if 0 == index.size: + return index + if isinstance(index, pd.MultiIndex): values = index.levels[level] return index.set_levels(values.map(func), level) From 79996ef9d752f751dee0a7d9d833bc53b1c71067 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 7 Aug 2018 11:08:03 +0200 Subject: [PATCH 098/123] Added docstrings to react table --- fireant/slicer/queries/database.py | 19 +++-- fireant/slicer/widgets/helpers.py | 7 +- fireant/slicer/widgets/highcharts.py | 10 +-- fireant/slicer/widgets/pandas.py | 4 + fireant/slicer/widgets/reacttable.py | 115 ++++++++++++++++++++++----- fireant/utils.py | 95 +++++++++++++++++----- 6 files changed, 197 insertions(+), 53 deletions(-) diff --git a/fireant/slicer/queries/database.py b/fireant/slicer/queries/database.py index 867c6369..7d448f68 100644 --- a/fireant/slicer/queries/database.py +++ b/fireant/slicer/queries/database.py @@ -1,9 +1,8 @@ +import pandas as pd import time from functools import partial from typing import Iterable -import pandas as pd - from fireant.database.base import Database from fireant.formats import ( NULL_VALUE, @@ -100,7 +99,7 @@ def fill_nans_in_level(data_frame, dimensions): number_rollup_dimensions = sum(dimension.is_rollup for dimension in dimensions) if 0 < number_rollup_dimensions: - fill_nan_for_nulls = partial(_fill_nan_for_nulls, offset=number_rollup_dimensions) + fill_nan_for_nulls = partial(_fill_nan_for_nulls, n_rolled_up_dimensions=number_rollup_dimensions) if 1 < len(dimensions): preceding_dimension_keys = [format_dimension_key(d.key) @@ -115,13 +114,23 @@ def fill_nans_in_level(data_frame, dimensions): return data_frame[level].fillna(NULL_VALUE) -def _fill_nan_for_nulls(df, offset=1): +def _fill_nan_for_nulls(df, n_rolled_up_dimensions=1): """ Fills the first NaN with a literal string "null" if there are two NaN values, otherwise nothing is filled. :param df: + :param n_rolled_up_dimensions: + The number of rolled up dimensions preceding and including the dimension :return: """ - if offset < pd.isnull(df).sum(): + + # If there are rolled up dimensions, then fill only the first instance of NULL with a literal string "null" and + # the rest of the nulls are totals. This check compares the number of nulls to the number of rolled up dimensions, + # or expected nulls which are totals rows. If there are more nulls, there should be exactly + # `n_rolled_up_dimensions+1` nulls which means one is a true `null` value. + number_of_nulls_for_dimension = pd.isnull(df).sum() + if n_rolled_up_dimensions < number_of_nulls_for_dimension: + assert n_rolled_up_dimensions + 1 == number_of_nulls_for_dimension return df.fillna(NULL_VALUE, limit=1).fillna(TOTALS_VALUE) + return df.fillna(TOTALS_VALUE) diff --git a/fireant/slicer/widgets/helpers.py b/fireant/slicer/widgets/helpers.py index 22d8d769..49bb89f3 100644 --- a/fireant/slicer/widgets/helpers.py +++ b/fireant/slicer/widgets/helpers.py @@ -67,10 +67,9 @@ def render_series_label(dimension_values, metric=None, reference=None): used_dimensions = dimensions[num_used_dimensions:] dimension_values = utils.wrap_list(dimension_values) - dimension_labels = [utils.deep_get(dimension_display_values, - [utils.format_dimension_key(dimension.key), - dimension_value], - dimension_value) + dimension_labels = [utils.getdeepattr(dimension_display_values, + (utils.format_dimension_key(dimension.key), dimension_value), + dimension_value) if not pd.isnull(dimension_value) else 'Totals' for dimension, dimension_value in zip(used_dimensions, dimension_values)] diff --git a/fireant/slicer/widgets/highcharts.py b/fireant/slicer/widgets/highcharts.py index 90eb7dab..365dc0fe 100644 --- a/fireant/slicer/widgets/highcharts.py +++ b/fireant/slicer/widgets/highcharts.py @@ -1,4 +1,6 @@ import itertools + +import pandas as pd from datetime import ( datetime, ) @@ -7,8 +9,6 @@ Union, ) -import pandas as pd - from fireant import ( DatetimeDimension, Metric, @@ -261,9 +261,9 @@ def _render_x_axis(self, data_frame, dimensions, dimension_display_values): categories = ["All"] \ if isinstance(first_level, pd.RangeIndex) \ - else [utils.deep_get(dimension_display_values, - [first_level.name, dimension_value], - dimension_value or 'Totals') + else [utils.getdeepattr(dimension_display_values, + (first_level.name, dimension_value), + dimension_value or 'Totals') for dimension_value in first_level] return { diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index 18b784c6..26a74296 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -89,9 +89,13 @@ def pivot_data_frame(data_frame, pivot=(), transpose=False): simplicity. :param data_frame: + The result set data frame :param pivot: + A list of index keys for `data_frame` of levels to shift :param transpose: + A boolean true or false whether to transpose the data frame. :return: + The shifted/transposed data frame """ if not (pivot or transpose): return data_frame diff --git a/fireant/slicer/widgets/reacttable.py b/fireant/slicer/widgets/reacttable.py index a2278923..9a8d94cc 100644 --- a/fireant/slicer/widgets/reacttable.py +++ b/fireant/slicer/widgets/reacttable.py @@ -177,31 +177,45 @@ def map_display_values(self, df, dimensions): return dimension_display_values @staticmethod - def format_data_frame(df, dimensions): + def format_data_frame(data_frame, dimensions): """ This function prepares the raw data frame for transformation by formatting dates in the index and removing any remaining NaN/NaT values. It also names the column as metrics so that it can be treated like a dimension level. - :param df: + :param data_frame: + The result set data frame :param dimensions: :return: """ for i, dimension in enumerate(dimensions): if isinstance(dimension, DatetimeDimension): date_format = DATE_FORMATS.get(dimension.interval, DATE_FORMATS[daily]) - df.index = map_index_level(df.index, i, lambda dt: dt.strftime(date_format)) + data_frame.index = map_index_level(data_frame.index, i, lambda dt: dt.strftime(date_format)) - df.index = fillna_index(df.index, TOTALS_VALUE) - df.columns.name = metrics_dimension_key + data_frame.index = fillna_index(data_frame.index, TOTALS_VALUE) + data_frame.columns.name = metrics_dimension_key @staticmethod def transform_dimension_column_headers(data_frame, dimensions): """ - Convert the unpivoted dimensions into ReactTable column header definitions. + Convert the un-pivoted dimensions into ReactTable column header definitions. :param data_frame: + The result set data frame :param dimensions: + A list of dimensions in the data frame that are part of the index :return: + A list of column header definitions with the following structure. + + ``` + columns = [{ + Header: 'Column A', + accessor: 'a', + }, { + Header: 'Column B', + accessor: 'b', + }] + ``` """ dimension_map = {format_dimension_key(d.key): d for d in dimensions + [metrics]} @@ -227,9 +241,31 @@ def transform_metric_column_headers(data_frame, item_map, dimension_display_valu result in multiple rows of headers. :param data_frame: + The result set data frame :param item_map: + A map to find metrics/operations based on their keys found in the data frame. :param dimension_display_values: + A map for finding display values for dimensions based on their key and value. :return: + A list of column header definitions with the following structure. + + ``` + columns = [{ + Header: 'Column A', + columns: [{ + Header: 'SubColumn A.0', + accessor: 'a.0', + }, { + Header: 'SubColumn A.1', + accessor: 'a.1', + }] + }, { + Header: 'Column B', + columns: [ + ... + ] + }] + ``` """ def get_header(column_value, f_dimension_key, is_totals): @@ -237,28 +273,47 @@ def get_header(column_value, f_dimension_key, is_totals): item = item_map[column_value] return getattr(item, 'label', item.key) - else: - return getdeepattr(dimension_display_values, (f_dimension_key, column_value), column_value) + return getdeepattr(dimension_display_values, (f_dimension_key, column_value), column_value) + + def _make_columns(columns_frame, previous_levels=()): + """ + This function recursively creates the individual column definitions for React Table with the above tree + structure depending on how many index levels there are in the columns. - def _make_columns(df, previous_levels=()): - f_dimension_key = df.index.names[0] - groups = df.groupby(level=0) \ - if isinstance(df.index, pd.MultiIndex) else \ - [(level, None) for level in df.index] + :param columns_frame: + A data frame representing the columns of the result set data frame. + :param previous_levels: + A tuple containing the higher level index level values used for building the data accessor path + """ + f_dimension_key = columns_frame.index.names[0] + + # Group the columns if they are multi-index so we can get the proper sub-column values. This will yield + # one group per dimension value with the group data frame containing only the relevant sub-columns + groups = columns_frame.groupby(level=0) \ + if isinstance(columns_frame.index, pd.MultiIndex) else \ + [(level, None) for level in columns_frame.index] columns = [] for column_value, group in groups: is_totals = TOTALS_VALUE == column_value + # All column definitions have a header column = {'Header': get_header(column_value, f_dimension_key, is_totals)} levels = previous_levels + (column_value,) if group is not None: + # If there is a group, then drop this index level from the group data frame and recurse to build + # sub column definitions next_level_df = group.reset_index(level=0, drop=True) column['columns'] = _make_columns(next_level_df, levels) else: + # If there is no group, then this is a leaf, or a column header on the bottom row of the table + # head. These are effectively the actual columns in the table. All leaf column header definitions + # require an accessor for how to acccess data the for that column if hasattr(data_frame, 'name'): + # If the metrics column index level was dropped (due to there being a single metric), then the + # index level name will be set as the data frame's name. levels += (data_frame.name,) column['accessor'] = '.'.join(levels) @@ -275,12 +330,15 @@ def _make_columns(df, previous_levels=()): @staticmethod def transform_data(data_frame, item_map, dimension_display_values): """ - WRITEME + Builds a list of dicts containing the data for ReactTable. This aligns with the accessors set by + #transform_dimension_column_headers and #transform_metric_column_headers :param data_frame: + The result set data frame :param item_map: + A map to find metrics/operations based on their keys found in the data frame. :param dimension_display_values: - :return: + A map for finding display values for dimensions based on their key and value. """ result = [] @@ -288,25 +346,35 @@ def transform_data(data_frame, item_map, dimension_display_values): if not isinstance(index, tuple): index = (index,) - index = [x - if x not in item_map - else getattr(item_map[x], 'label', item_map[x].key) - for x in index] + # Get a list of values from the index. These can be metrics or dimensions so it checks in the item map if + # there is a display value for the value + index = [item + if item not in item_map + else getattr(item_map[item], 'label', item_map[item].key) + for item in index] row = {} + + # Add the index to the row for key, value in zip(data_frame.index.names, index): if key is None: continue data = {RAW_VALUE: value} + + # Try to find a display value for the item. If this is a metric the raw value is replaced with the + # display value because there is no raw value for a metric label display = getdeepattr(dimension_display_values, (key, value)) if display is not None: data['display'] = display row[key] = data + # Add the values to the row for key, value in series.iteritems(): data = {RAW_VALUE: value} + + # Try to find a display value for the item item = item_map.get(key[0] if isinstance(key, tuple) else key) display = _get_item_display(item, value) if display is not None: @@ -320,13 +388,20 @@ def transform_data(data_frame, item_map, dimension_display_values): def transform(self, data_frame, slicer, dimensions, references): """ - WRITEME + Transforms a data frame into a format for ReactTable. This is an object containing attributes `columns` and + `data` which align with the props in ReactTable with the same name. :param data_frame: + The result set data frame :param slicer: + The slicer that generated the data query :param dimensions: + A list of dimensions that were selected in the data query :param references: + A list of references that were selected in the data query :return: + An dict containing attributes `columns` and `data` which align with the props in ReactTable with the same + names. """ df_dimension_columns = [format_dimension_key(d.display_key) for d in dimensions diff --git a/fireant/utils.py b/fireant/utils.py index 84349f9c..ec1d383d 100644 --- a/fireant/utils.py +++ b/fireant/utils.py @@ -5,20 +5,49 @@ def wrap_list(value): return value if isinstance(value, (tuple, list)) else [value] -def deep_get(d, keys, default=None): - d_level = d - for key in keys: - if key not in d_level: - return default - d_level = d_level[key] - return d_level - - -def setdeepattr(d, key, value): - if not isinstance(key, (list, tuple)): - key = (key,) +def setdeepattr(d, keys, value): + """ + Similar to the built-in `setattr`, this function accepts a list/tuple of keys to set a value deep in a `dict` + + Given the following dict structure + ``` + d = { + 'A': { + '0': { + 'a': 1, + 'b': 2, + } + }, + } + ``` + + Calling `setdeepattr` with a key path to a value deep in the structure will set that value. If the value or any + of the objects in the key path do not exist, then a dict will be created. + + ``` + # Overwrites the value in `d` at A.0.a, which was 1, to 3 + setdeepattr(d, ('A', '0', 'a'), 3) + + # Adds an entry in `d` to A.0 with the key 'c' and the value 3 + setdeepattr(d, ('A', '0', 'c'), 3) + + # Adds an entry in `d` with the key 'X' and the value a new dict + # Adds an entry in `d` to `X` with the key '0' and the value a new dict + # Adds an entry in `d` to `X.0` with the key 'a' and the value 0 + setdeepattr(d, ('X', '0', 'a'), 0) + ``` + + :param d: + A dict value with nested dict attributes. + :param keys: + A list/tuple path of keys in `d` to the desired value + :param value: + The value to set at the given path `keys`. + """ + if not isinstance(keys, (list, tuple)): + keys = (keys,) - top, *rest = key + top, *rest = keys if rest: if top not in d: @@ -31,6 +60,40 @@ def setdeepattr(d, key, value): def getdeepattr(d, keys, default_value=None): + """ + Similar to the built-in `getattr`, this function accepts a list/tuple of keys to get a value deep in a `dict` + + Given the following dict structure + ``` + d = { + 'A': { + '0': { + 'a': 1, + 'b': 2, + } + }, + } + ``` + + Calling `getdeepattr` with a key path to a value deep in the structure will return that value. If the value or any + of the objects in the key path do not exist, then the default value is returned. + + ``` + assert 1 == getdeepattr(d, ('A', '0', 'a')) + assert 2 == getdeepattr(d, ('A', '0', 'b')) + assert 0 == getdeepattr(d, ('A', '0', 'c'), default_value=0) + assert 0 == getdeepattr(d, ('X', '0', 'a'), default_value=0) + ``` + + :param d: + A dict value with nested dict attributes. + :param keys: + A list/tuple path of keys in `d` to the desired value + :param default_value: + A default value that will be returned if the path `keys` does not yield a value. + :return: + The value following the path `keys` or `default_value` + """ d_level = d for key in keys: @@ -51,12 +114,6 @@ def slice_first(item): return item -a = 'a' \ - if True \ - else 'b' if True \ - else 'c' - - def filter_duplicates(iterable): filtered_list, seen = [], set() for item in iterable: From 7682e538205bf020f5695c6e1a5daec046cb746f Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 7 Aug 2018 12:02:49 +0200 Subject: [PATCH 099/123] Bumped version to dev40 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 59b5a612..54f4a483 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev39' +__version__ = '1.0.0.dev40' From 72d1f86c94bab70b1387f4214c3a4cf9f6f9b68e Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 7 Aug 2018 13:53:16 +0200 Subject: [PATCH 100/123] Set vertica_python to replace unicode errors --- fireant/database/vertica.py | 3 ++- fireant/tests/database/test_vertica.py | 18 +++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/fireant/database/vertica.py b/fireant/database/vertica.py index b376094a..2dd8a15f 100644 --- a/fireant/database/vertica.py +++ b/fireant/database/vertica.py @@ -51,7 +51,8 @@ def connect(self): return vertica_python.connect(host=self.host, port=self.port, database=self.database, user=self.user, password=self.password, - read_timeout=self.read_timeout) + read_timeout=self.read_timeout, + unicode_error='replace') def fetch(self, query): return super(VerticaDatabase, self).fetch(query) diff --git a/fireant/tests/database/test_vertica.py b/fireant/tests/database/test_vertica.py index 8400e368..6328e09e 100644 --- a/fireant/tests/database/test_vertica.py +++ b/fireant/tests/database/test_vertica.py @@ -1,16 +1,19 @@ from unittest import TestCase +from unittest.mock import ( + Mock, + patch, +) -from unittest.mock import patch, Mock +from pypika import Field from fireant import ( - hourly, + annually, daily, - weekly, + hourly, quarterly, - annually, + weekly, ) from fireant.database import VerticaDatabase -from pypika import Field class TestVertica(TestCase): @@ -36,8 +39,9 @@ def test_connect(self): self.assertEqual('OK', result) mock_vertica.connect.assert_called_once_with( - host='test_host', port=1234, database='test_database', - user='test_user', password='password', read_timeout=None, + host='test_host', port=1234, database='test_database', + user='test_user', password='password', + read_timeout=None, unicode_error='replace' ) def test_trunc_hour(self): From bc11af31e3ec033a87127582c529cda6a3058952 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 7 Aug 2018 14:14:35 +0200 Subject: [PATCH 101/123] Bumped version to dev41 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 54f4a483..3855146b 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev40' +__version__ = '1.0.0.dev41' From 82b1881ca4cdfec55977a38a1120cf58534e5a42 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 7 Aug 2018 14:53:39 +0200 Subject: [PATCH 102/123] Changed the pattern filters to be case insensitive --- fireant/slicer/filters.py | 5 ++- .../queries/test_build_dimension_filters.py | 44 +++++++++---------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/fireant/slicer/filters.py b/fireant/slicer/filters.py index b3f5e315..2199e679 100644 --- a/fireant/slicer/filters.py +++ b/fireant/slicer/filters.py @@ -1,4 +1,5 @@ from pypika import Not +from pypika.functions import Lower class Filter(object): @@ -61,10 +62,10 @@ def __init__(self, dimension_definition, start, stop): class PatternFilter(DimensionFilter): def _apply(self, dimension_definition, patterns): - definition = dimension_definition.like(patterns[0]) + definition = Lower(dimension_definition).like(Lower(patterns[0])) for pattern in patterns[1:]: - definition |= dimension_definition.like(pattern) + definition |= Lower(dimension_definition).like(Lower(pattern)) return definition diff --git a/fireant/tests/slicer/queries/test_build_dimension_filters.py b/fireant/tests/slicer/queries/test_build_dimension_filters.py index c979052b..1f23a0cf 100644 --- a/fireant/tests/slicer/queries/test_build_dimension_filters.py +++ b/fireant/tests/slicer/queries/test_build_dimension_filters.py @@ -41,7 +41,7 @@ def test_build_query_with_filter_like_categorical_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "political_party" LIKE \'Rep%\'', str(query)) + 'WHERE LOWER("political_party") LIKE LOWER(\'Rep%\')', str(query)) def test_build_query_with_filter_not_like_categorical_dim(self): query = slicer.data \ @@ -52,7 +52,7 @@ def test_build_query_with_filter_not_like_categorical_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE NOT "political_party" LIKE \'Rep%\'', str(query)) + 'WHERE NOT LOWER("political_party") LIKE LOWER(\'Rep%\')', str(query)) def test_build_query_with_filter_isin_unique_dim(self): query = slicer.data \ @@ -107,7 +107,7 @@ def test_build_query_with_filter_like_unique_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) + 'WHERE LOWER("candidate_name") LIKE LOWER(\'%Trump\')', str(query)) def test_build_query_with_filter_like_display_dim(self): query = slicer.data \ @@ -118,7 +118,7 @@ def test_build_query_with_filter_like_display_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "candidate_name" LIKE \'%Trump\'', str(query)) + 'WHERE LOWER("candidate_name") LIKE LOWER(\'%Trump\')', str(query)) def test_build_query_with_filter_not_like_unique_dim(self): query = slicer.data \ @@ -129,7 +129,7 @@ def test_build_query_with_filter_not_like_unique_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE NOT "candidate_name" LIKE \'%Trump\'', str(query)) + 'WHERE NOT LOWER("candidate_name") LIKE LOWER(\'%Trump\')', str(query)) def test_build_query_with_filter_not_like_display_dim(self): query = slicer.data \ @@ -140,7 +140,7 @@ def test_build_query_with_filter_not_like_display_dim(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE NOT "candidate_name" LIKE \'%Trump\'', str(query)) + 'WHERE NOT LOWER("candidate_name") LIKE LOWER(\'%Trump\')', str(query)) def test_build_query_with_filter_like_categorical_dim_multiple_patterns(self): query = slicer.data \ @@ -151,8 +151,8 @@ def test_build_query_with_filter_like_categorical_dim_multiple_patterns(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "political_party" LIKE \'Rep%\' ' - 'OR "political_party" LIKE \'Dem%\'', str(query)) + 'WHERE LOWER("political_party") LIKE LOWER(\'Rep%\') ' + 'OR LOWER("political_party") LIKE LOWER(\'Dem%\')', str(query)) def test_build_query_with_filter_not_like_categorical_dim_multiple_patterns(self): query = slicer.data \ @@ -163,8 +163,8 @@ def test_build_query_with_filter_not_like_categorical_dim_multiple_patterns(self self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE NOT ("political_party" LIKE \'Rep%\' ' - 'OR "political_party" LIKE \'Dem%\')', str(query)) + 'WHERE NOT (LOWER("political_party") LIKE LOWER(\'Rep%\') ' + 'OR LOWER("political_party") LIKE LOWER(\'Dem%\'))', str(query)) def test_build_query_with_filter_like_pattern_dim_multiple_patterns(self): query = slicer.data \ @@ -175,8 +175,8 @@ def test_build_query_with_filter_like_pattern_dim_multiple_patterns(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "pattern" LIKE \'a%\' ' - 'OR "pattern" LIKE \'b%\'', str(query)) + 'WHERE LOWER("pattern") LIKE LOWER(\'a%\') ' + 'OR LOWER("pattern") LIKE LOWER(\'b%\')', str(query)) def test_build_query_with_filter_not_like_pattern_dim_multiple_patterns(self): query = slicer.data \ @@ -187,8 +187,8 @@ def test_build_query_with_filter_not_like_pattern_dim_multiple_patterns(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE NOT ("pattern" LIKE \'a%\' ' - 'OR "pattern" LIKE \'b%\')', str(query)) + 'WHERE NOT (LOWER("pattern") LIKE LOWER(\'a%\') ' + 'OR LOWER("pattern") LIKE LOWER(\'b%\'))', str(query)) def test_build_query_with_filter_like_unique_dim_multiple_patterns(self): query = slicer.data \ @@ -199,8 +199,8 @@ def test_build_query_with_filter_like_unique_dim_multiple_patterns(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "candidate_name" LIKE \'%Trump\' ' - 'OR "candidate_name" LIKE \'%Clinton\'', str(query)) + 'WHERE LOWER("candidate_name") LIKE LOWER(\'%Trump\') ' + 'OR LOWER("candidate_name") LIKE LOWER(\'%Clinton\')', str(query)) def test_build_query_with_filter_like_display_dim_multiple_patterns(self): query = slicer.data \ @@ -211,8 +211,8 @@ def test_build_query_with_filter_like_display_dim_multiple_patterns(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE "candidate_name" LIKE \'%Trump\' ' - 'OR "candidate_name" LIKE \'%Clinton\'', str(query)) + 'WHERE LOWER("candidate_name") LIKE LOWER(\'%Trump\') ' + 'OR LOWER("candidate_name") LIKE LOWER(\'%Clinton\')', str(query)) def test_build_query_with_filter_not_like_unique_dim_multiple_patterns(self): query = slicer.data \ @@ -223,8 +223,8 @@ def test_build_query_with_filter_not_like_unique_dim_multiple_patterns(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE NOT ("candidate_name" LIKE \'%Trump\' ' - 'OR "candidate_name" LIKE \'%Clinton\')', str(query)) + 'WHERE NOT (LOWER("candidate_name") LIKE LOWER(\'%Trump\') ' + 'OR LOWER("candidate_name") LIKE LOWER(\'%Clinton\'))', str(query)) def test_build_query_with_filter_not_like_display_dim_multiple_patterns(self): query = slicer.data \ @@ -235,8 +235,8 @@ def test_build_query_with_filter_not_like_display_dim_multiple_patterns(self): self.assertEqual('SELECT ' 'SUM("votes") "$m$votes" ' 'FROM "politics"."politician" ' - 'WHERE NOT ("candidate_name" LIKE \'%Trump\' ' - 'OR "candidate_name" LIKE \'%Clinton\')', str(query)) + 'WHERE NOT (LOWER("candidate_name") LIKE LOWER(\'%Trump\') ' + 'OR LOWER("candidate_name") LIKE LOWER(\'%Clinton\'))', str(query)) def test_build_query_with_filter_isin_raise_exception_when_display_definition_undefined(self): with self.assertRaises(f.QueryException): From aee5f2afaec8454138e5948c8b5cbb63ebf677eb Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 7 Aug 2018 15:49:47 +0200 Subject: [PATCH 103/123] bumped fireant to dev42 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 3855146b..089cd3e3 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev41' +__version__ = '1.0.0.dev42' From 8c8f557caac622bf3d257180708fdef6cd3bd812 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 7 Aug 2018 17:27:49 +0200 Subject: [PATCH 104/123] Fixed operations to calculate for references --- fireant/slicer/operations.py | 25 +++---- fireant/slicer/queries/builder.py | 15 +++-- .../tests/slicer/queries/test_build_render.py | 33 +++++++++- fireant/tests/slicer/test_operations.py | 65 ++++++++++--------- 4 files changed, 85 insertions(+), 53 deletions(-) diff --git a/fireant/slicer/operations.py b/fireant/slicer/operations.py index 1af7a41a..cdae5a6b 100644 --- a/fireant/slicer/operations.py +++ b/fireant/slicer/operations.py @@ -2,6 +2,7 @@ import pandas as pd from fireant.utils import format_metric_key +from fireant.slicer.references import reference_key from .metrics import Metric @@ -16,7 +17,7 @@ class Operation(object): The `Operation` class represents an operation in the `Slicer` API. """ - def apply(self, data_frame): + def apply(self, data_frame, reference): raise NotImplementedError() @property @@ -36,7 +37,7 @@ def __init__(self, key, label, prefix=None, suffix=None, precision=None): self.suffix = suffix self.precision = precision - def apply(self, data_frame): + def apply(self, data_frame, reference): raise NotImplementedError() @property @@ -72,7 +73,7 @@ def __init__(self, arg): self.arg = arg - def apply(self, data_frame): + def apply(self, data_frame, reference): raise NotImplementedError() @property @@ -93,8 +94,8 @@ def __repr__(self): class CumSum(_Cumulative): - def apply(self, data_frame): - df_key = format_metric_key(self.arg.key) + def apply(self, data_frame, reference): + df_key = format_metric_key(reference_key(self.arg, reference)) if isinstance(data_frame.index, pd.MultiIndex): levels = self._group_levels(data_frame.index) @@ -107,8 +108,8 @@ def apply(self, data_frame): class CumProd(_Cumulative): - def apply(self, data_frame): - df_key = format_metric_key(self.arg.key) + def apply(self, data_frame, reference): + df_key = format_metric_key(reference_key(self.arg, reference)) if isinstance(data_frame.index, pd.MultiIndex): levels = self._group_levels(data_frame.index) @@ -125,8 +126,8 @@ class CumMean(_Cumulative): def cummean(x): return x.cumsum() / np.arange(1, len(x) + 1) - def apply(self, data_frame): - df_key = format_metric_key(self.arg.key) + def apply(self, data_frame, reference): + df_key = format_metric_key(reference_key(self.arg, reference)) if isinstance(data_frame.index, pd.MultiIndex): levels = self._group_levels(data_frame.index) @@ -161,7 +162,7 @@ def _should_adjust(self, other_operations): return first_max_rolling is self - def apply(self, data_frame): + def apply(self, data_frame, reference): raise NotImplementedError() @property @@ -182,8 +183,8 @@ class RollingMean(RollingOperation): def rolling_mean(self, x): return x.rolling(self.window, self.min_periods).mean() - def apply(self, data_frame): - df_key = format_metric_key(self.arg.key) + def apply(self, data_frame, reference): + df_key = format_metric_key(reference_key(self.arg, reference)) if isinstance(data_frame.index, pd.MultiIndex): levels = self._group_levels(data_frame.index) diff --git a/fireant/slicer/queries/builder.py b/fireant/slicer/queries/builder.py index 2457cfd3..022e8d74 100644 --- a/fireant/slicer/queries/builder.py +++ b/fireant/slicer/queries/builder.py @@ -1,12 +1,11 @@ -from typing import ( - Dict, - Iterable, -) - import pandas as pd from pypika import ( Order, ) +from typing import ( + Dict, + Iterable, +) from fireant.utils import ( format_dimension_key, @@ -27,6 +26,7 @@ make_slicer_query, make_slicer_query_with_references_and_totals, ) +from .references import reference_key from ..base import SlicerElement from ..dimensions import Dimension @@ -195,8 +195,9 @@ def fetch(self, hint=None) -> Iterable[Dict]: # Apply operations operations = find_operations_for_widgets(self._widgets) for operation in operations: - df_key = format_metric_key(operation.key) - data_frame[df_key] = operation.apply(data_frame) + for reference in [None] + self._references: + df_key = format_metric_key(reference_key(operation, reference)) + data_frame[df_key] = operation.apply(data_frame, reference) data_frame = special_cases.apply_operations_to_data_frame(operations, data_frame) diff --git a/fireant/tests/slicer/queries/test_build_render.py b/fireant/tests/slicer/queries/test_build_render.py index 93e8028b..0a8e5eae 100644 --- a/fireant/tests/slicer/queries/test_build_render.py +++ b/fireant/tests/slicer/queries/test_build_render.py @@ -2,6 +2,7 @@ from unittest.mock import ( ANY, Mock, + call, patch, ) @@ -12,7 +13,10 @@ from ..matchers import ( DimensionMatcher, ) -from ..mocks import slicer +from ..mocks import ( + ElectionOverElection, + slicer, +) # noinspection SqlDialectInspection,SqlNoDataSourceInspection @@ -123,7 +127,32 @@ def test_operations_evaluated(self, mock_fetch_data: Mock): .widget(mock_widget) \ .fetch() - mock_operation.apply.assert_called_once_with(mock_df) + mock_operation.apply.assert_called_once_with(mock_df, None) + + def test_operations_evaluated_for_each_reference(self, mock_fetch_data: Mock): + eoe = ElectionOverElection(slicer.dimensions.timestamp) + + mock_operation = Mock(name='mock_operation ', spec=f.Operation) + mock_operation.key, mock_operation.definition = 'mock_operation', slicer.table.abc + mock_operation.metrics = [] + + mock_widget = f.Widget(mock_operation) + mock_widget.transform = Mock() + + mock_df = {} + mock_fetch_data.return_value = mock_df + + # Need to keep widget the last call in the chain otherwise the object gets cloned and the assertion won't work + slicer.data \ + .dimension(slicer.dimensions.timestamp) \ + .reference(eoe) \ + .widget(mock_widget) \ + .fetch() + + mock_operation.apply.assert_has_calls([ + call(mock_df, None), + call(mock_df, eoe), + ]) def test_operations_results_stored_in_data_frame(self, mock_fetch_data: Mock): mock_operation = Mock(name='mock_operation ', spec=f.Operation) diff --git a/fireant/tests/slicer/test_operations.py b/fireant/tests/slicer/test_operations.py index d86daa07..a480cfa8 100644 --- a/fireant/tests/slicer/test_operations.py +++ b/fireant/tests/slicer/test_operations.py @@ -11,6 +11,7 @@ RollingMean, ) from fireant.tests.slicer.mocks import ( + ElectionOverElection, cont_dim_df, cont_uni_dim_df, cont_uni_dim_ref_df, @@ -21,116 +22,116 @@ class CumSumTests(TestCase): def test_apply_to_timeseries(self): cumsum = CumSum(slicer.metrics.wins) - result = cumsum.apply(cont_dim_df) + result = cumsum.apply(cont_dim_df, None) expected = pd.Series([2, 4, 6, 8, 10, 12], name='$m$wins', index=cont_dim_df.index) - pandas.testing.assert_series_equal(result, expected) + pandas.testing.assert_series_equal(expected, result) def test_apply_to_timeseries_with_uni_dim(self): cumsum = CumSum(slicer.metrics.wins) - result = cumsum.apply(cont_uni_dim_df) + result = cumsum.apply(cont_uni_dim_df, None) expected = pd.Series([1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6], name='$m$wins', index=cont_uni_dim_df.index) - pandas.testing.assert_series_equal(result, expected) + pandas.testing.assert_series_equal(expected, result) def test_apply_to_timeseries_with_uni_dim_and_ref(self): cumsum = CumSum(slicer.metrics.wins) - result = cumsum.apply(cont_uni_dim_ref_df) + result = cumsum.apply(cont_uni_dim_ref_df, ElectionOverElection(slicer.dimensions.timestamp)) - expected = pd.Series([1, 1, 2, 2, 3, 3, 4, 4, 5, 5], - name='$m$wins', + expected = pd.Series([1., 1., 2., 2., 3., 3., 4, 4, 5, 5], + name='$m$wins_eoe', index=cont_uni_dim_ref_df.index) - pandas.testing.assert_series_equal(result, expected) + pandas.testing.assert_series_equal(expected, result) class CumProdTests(TestCase): def test_apply_to_timeseries(self): cumprod = CumProd(slicer.metrics.wins) - result = cumprod.apply(cont_dim_df) + result = cumprod.apply(cont_dim_df, None) expected = pd.Series([2, 4, 8, 16, 32, 64], name='$m$wins', index=cont_dim_df.index) - pandas.testing.assert_series_equal(result, expected) + pandas.testing.assert_series_equal(expected, result) def test_apply_to_timeseries_with_uni_dim(self): cumprod = CumProd(slicer.metrics.wins) - result = cumprod.apply(cont_uni_dim_df) + result = cumprod.apply(cont_uni_dim_df, None) expected = pd.Series([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], name='$m$wins', index=cont_uni_dim_df.index) - pandas.testing.assert_series_equal(result, expected) + pandas.testing.assert_series_equal(expected, result) def test_apply_to_timeseries_with_uni_dim_and_ref(self): cumprod = CumProd(slicer.metrics.wins) - result = cumprod.apply(cont_uni_dim_ref_df) + result = cumprod.apply(cont_uni_dim_ref_df, ElectionOverElection(slicer.dimensions.timestamp)) - expected = pd.Series([1, 1, 1, 1, 1, 1, 1, 1, 1, 1], - name='$m$wins', + expected = pd.Series([1., 1, 1, 1, 1, 1, 1, 1, 1, 1], + name='$m$wins_eoe', index=cont_uni_dim_ref_df.index) - pandas.testing.assert_series_equal(result, expected) + pandas.testing.assert_series_equal(expected, result) class CumMeanTests(TestCase): def test_apply_to_timeseries(self): cummean = CumMean(slicer.metrics.votes) - result = cummean.apply(cont_dim_df) + result = cummean.apply(cont_dim_df, None) expected = pd.Series([15220449.0, 15941233.0, 17165799.3, 18197903.25, 18672764.6, 18612389.3], name='$m$votes', index=cont_dim_df.index) - pandas.testing.assert_series_equal(result, expected) + pandas.testing.assert_series_equal(expected, result) def test_apply_to_timeseries_with_uni_dim(self): cummean = CumMean(slicer.metrics.votes) - result = cummean.apply(cont_uni_dim_df) + result = cummean.apply(cont_uni_dim_df, None) expected = pd.Series([5574387.0, 9646062.0, 5903886.0, 10037347.0, 6389131.0, 10776668.3, 6793838.5, 11404064.75, 7010664.2, 11662100.4, 6687706.0, 11924683.3], name='$m$votes', index=cont_uni_dim_df.index) - pandas.testing.assert_series_equal(result, expected) + pandas.testing.assert_series_equal(expected, result) def test_apply_to_timeseries_with_uni_dim_and_ref(self): cummean = CumMean(slicer.metrics.votes) - result = cummean.apply(cont_uni_dim_ref_df) + result = cummean.apply(cont_uni_dim_ref_df, ElectionOverElection(slicer.dimensions.timestamp)) - expected = pd.Series([6233385.0, 10428632.0, 6796503.0, 11341971.5, 7200322.3, - 11990065.6, 7369733.5, 12166110.0, 6910369.8, 12380407.6], - name='$m$votes', + expected = pd.Series([5574387.0, 9646062.0, 5903886.0, 10037347.0, 6389131.0, + 10776668.333333334, 6793838.5, 11404064.75, 7010664.2, 11662100.4], + name='$m$votes_eoe', index=cont_uni_dim_ref_df.index) - pandas.testing.assert_series_equal(result, expected) + pandas.testing.assert_series_equal(expected, result) class RollingMeanTests(TestCase): def test_apply_to_timeseries(self): rolling_mean = RollingMean(slicer.metrics.wins, 3) - result = rolling_mean.apply(cont_dim_df) + result = rolling_mean.apply(cont_dim_df, None) expected = pd.Series([np.nan, np.nan, 2.0, 2.0, 2.0, 2.0], name='$m$wins', index=cont_dim_df.index) - pandas.testing.assert_series_equal(result, expected) + pandas.testing.assert_series_equal(expected, result) def test_apply_to_timeseries_with_uni_dim(self): rolling_mean = RollingMean(slicer.metrics.wins, 3) - result = rolling_mean.apply(cont_uni_dim_df) + result = rolling_mean.apply(cont_uni_dim_df, None) expected = pd.Series([np.nan, np.nan, np.nan, np.nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], name='$m$wins', index=cont_uni_dim_df.index) - pandas.testing.assert_series_equal(result, expected) + pandas.testing.assert_series_equal(expected, result) def test_apply_to_timeseries_with_uni_dim_and_ref(self): rolling_mean = RollingMean(slicer.metrics.wins, 3) - result = rolling_mean.apply(cont_uni_dim_ref_df) + result = rolling_mean.apply(cont_uni_dim_ref_df, ElectionOverElection(slicer.dimensions.timestamp)) expected = pd.Series([np.nan, np.nan, np.nan, np.nan, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], - name='$m$wins', + name='$m$wins_eoe', index=cont_uni_dim_ref_df.index) - pandas.testing.assert_series_equal(result, expected) + pandas.testing.assert_series_equal(expected, result) From 22d40301d5e10ea4c177e6fe094ea65f954d3250 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 7 Aug 2018 17:44:37 +0200 Subject: [PATCH 105/123] Added a fillna to reacttable to handle NaNs and Inf --- fireant/slicer/widgets/reacttable.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fireant/slicer/widgets/reacttable.py b/fireant/slicer/widgets/reacttable.py index 9a8d94cc..fcdda0c7 100644 --- a/fireant/slicer/widgets/reacttable.py +++ b/fireant/slicer/widgets/reacttable.py @@ -1,5 +1,6 @@ from collections import OrderedDict +import numpy as np import pandas as pd from fireant.formats import ( @@ -414,7 +415,9 @@ def transform(self, data_frame, slicer, dimensions, references): # Add an extra item to map the totals key to it's label item_map[TOTALS_VALUE] = TotalsItem - df = data_frame[df_dimension_columns + df_metric_columns].copy() + df = data_frame[df_dimension_columns + df_metric_columns].copy() \ + .fillna(value='NaN') \ + .replace([np.inf, -np.inf], 'Inf') dimension_display_values = self.map_display_values(df, dimensions) self.format_data_frame(df, dimensions) From db5e1a121055ada2d3485ed217ae2136c1b51952 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 7 Aug 2018 17:58:08 +0200 Subject: [PATCH 106/123] bumped fireant to dev43 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 089cd3e3..523085f5 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev42' +__version__ = '1.0.0.dev43' From ea00f8149267dacb4cb5ee7cabeb5a9bf0863deb Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 8 Aug 2018 14:12:14 +0200 Subject: [PATCH 107/123] Fixed react table's display value formatting and added test cases --- fireant/slicer/widgets/reacttable.py | 33 +++---- .../tests/slicer/widgets/test_reacttable.py | 91 ++++++++++++++++++- 2 files changed, 107 insertions(+), 17 deletions(-) diff --git a/fireant/slicer/widgets/reacttable.py b/fireant/slicer/widgets/reacttable.py index fcdda0c7..5054d3d4 100644 --- a/fireant/slicer/widgets/reacttable.py +++ b/fireant/slicer/widgets/reacttable.py @@ -69,25 +69,23 @@ def fillna_index(index, value): return index.fillna(value) -def _get_item_display(item, value): - if item is None or all([item.prefix is None, item.suffix is None, item.precision is None]): - return +def display_value(value, prefix=None, suffix=None, precision=None): + if isinstance(value, bool): + value = str(value).lower() - return '{prefix}{value}{suffix}'.format( - prefix=(item.prefix or ""), - suffix=(item.suffix or ""), - value=value if item.precision is None else '{:.{precision}f}'.format(0.1234567890, precision=2) - ) + def format_numeric_value(value): + if precision is not None and isinstance(value, float): + return '{:.{precision}f}'.format(value, precision=precision) + if isinstance(value, int): + return '{:,}'.format(value) -def make_column_format(item): - if item is None or all([item.prefix is None, item.suffix is None, item.precision is None]): - return + return '{}'.format(value) - return '{prefix}{format}{suffix}'.format( - prefix=(item.prefix or ""), - suffix=(item.suffix or "").replace('%', '%%'), - format='%s' if item.precision is None else '%.{}f'.format(item.precision) + return '{prefix}{value}{suffix}'.format( + prefix=(prefix or ""), + suffix=(suffix or ""), + value=format_numeric_value(value), ) @@ -377,7 +375,10 @@ def transform_data(data_frame, item_map, dimension_display_values): # Try to find a display value for the item item = item_map.get(key[0] if isinstance(key, tuple) else key) - display = _get_item_display(item, value) + display = display_value(value, item.prefix, item.suffix, item.precision) \ + if item is not None else \ + value + if display is not None: data['display'] = display diff --git a/fireant/tests/slicer/widgets/test_reacttable.py b/fireant/tests/slicer/widgets/test_reacttable.py index c754bb3a..29286d95 100644 --- a/fireant/tests/slicer/widgets/test_reacttable.py +++ b/fireant/tests/slicer/widgets/test_reacttable.py @@ -1,6 +1,9 @@ from unittest import TestCase -from fireant.slicer.widgets.reacttable import ReactTable +from fireant.slicer.widgets.reacttable import ( + ReactTable, + display_value, +) from fireant.tests.slicer.mocks import ( CumSum, ElectionOverElection, @@ -1143,3 +1146,89 @@ def test_pivot_second_dimension_and_transpose_with_all_levels_totals(self): 'totals': {'raw': 111674336.0} }] }, result) + + +class ReactTableDisplayValueFormat(TestCase): + def test_str_value_no_formats(self): + display = display_value('abcdef') + self.assertEqual('abcdef', display) + + def test_bool_true_value_no_formats(self): + display = display_value(True) + self.assertEqual('true', display) + + def test_bool_false_value_no_formats(self): + display = display_value(False) + self.assertEqual('false', display) + + def test_int_value_no_formats(self): + display = display_value(12345) + self.assertEqual('12,345', display) + + def test_decimal_value_no_formats(self): + display = display_value(12345.123456789) + self.assertEqual('12345.123456789', display) + + def test_str_value_with_prefix(self): + display = display_value('abcdef', prefix='$') + self.assertEqual('$abcdef', display) + + def test_bool_true_value_with_prefix(self): + display = display_value(True, prefix='$') + self.assertEqual('$true', display) + + def test_bool_false_value_with_prefix(self): + display = display_value(False, prefix='$') + self.assertEqual('$false', display) + + def test_int_value_with_prefix(self): + display = display_value(12345, prefix='$') + self.assertEqual('$12,345', display) + + def test_decimal_value_with_prefix(self): + display = display_value(12345.123456789, prefix='$') + self.assertEqual('$12345.123456789', display) + + def test_str_value_with_suffix(self): + display = display_value('abcdef', suffix='€') + self.assertEqual('abcdef€', display) + + def test_bool_true_value_with_suffix(self): + display = display_value(True, suffix='€') + self.assertEqual('true€', display) + + def test_bool_false_value_with_suffix(self): + display = display_value(False, suffix='€') + self.assertEqual('false€', display) + + def test_int_value_with_suffix(self): + display = display_value(12345, suffix='€') + self.assertEqual('12,345€', display) + + def test_decimal_value_with_suffix(self): + display = display_value(12345.123456789, suffix='€') + self.assertEqual('12345.123456789€', display) + + def test_str_value_with_precision(self): + display = display_value('abcdef', precision=2) + self.assertEqual('abcdef', display) + + def test_bool_true_value_with_precision(self): + display = display_value(True, precision=2) + self.assertEqual('true', display) + + def test_bool_false_value_with_precision(self): + display = display_value(False, precision=2) + self.assertEqual('false', display) + + def test_int_value_with_precision(self): + display = display_value(12345, precision=2) + self.assertEqual('12,345', display) + + def test_decimal_value_with_precision_0(self): + display = display_value(12345.123456789, precision=0) + self.assertEqual('12345', display) + + def test_decimal_value_with_precision_2(self): + display = display_value(12345.123456789, precision=2) + self.assertEqual('12345.12', display) From 9e134c0c0e5d25027d438c395b7961e9ad7f0c3f Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 8 Aug 2018 15:09:48 +0200 Subject: [PATCH 108/123] fixed react tables tests --- fireant/slicer/widgets/reacttable.py | 4 + .../tests/slicer/widgets/test_reacttable.py | 885 ++++++++++-------- 2 files changed, 517 insertions(+), 372 deletions(-) diff --git a/fireant/slicer/widgets/reacttable.py b/fireant/slicer/widgets/reacttable.py index 5054d3d4..28a6f7fb 100644 --- a/fireant/slicer/widgets/reacttable.py +++ b/fireant/slicer/widgets/reacttable.py @@ -371,6 +371,10 @@ def transform_data(data_frame, item_map, dimension_display_values): # Add the values to the row for key, value in series.iteritems(): + # pd.Series casts everything to float, cast it back to int if it's an int + if np.int64 == data_frame[key].dtype: + value = int(value) + data = {RAW_VALUE: value} # Try to find a display value for the item diff --git a/fireant/tests/slicer/widgets/test_reacttable.py b/fireant/tests/slicer/widgets/test_reacttable.py index 29286d95..ddd2b7c4 100644 --- a/fireant/tests/slicer/widgets/test_reacttable.py +++ b/fireant/tests/slicer/widgets/test_reacttable.py @@ -31,7 +31,7 @@ def test_single_metric(self): self.assertEqual({ 'columns': [{'Header': 'Votes', 'accessor': '$m$votes'}], - 'data': [{'$m$votes': {'raw': 111674336}}] + 'data': [{'$m$votes': {'display': '111,674,336', 'raw': 111674336}}] }, result) def test_multiple_metrics(self): @@ -41,7 +41,10 @@ def test_multiple_metrics(self): self.assertEqual({ 'columns': [{'Header': 'Votes', 'accessor': '$m$votes'}, {'Header': 'Wins', 'accessor': '$m$wins'}], - 'data': [{'$m$votes': {'raw': 111674336}, '$m$wins': {'raw': 12}}] + 'data': [{ + '$m$votes': {'display': '111,674,336', 'raw': 111674336}, + '$m$wins': {'display': '12', 'raw': 12} + }] }, result) def test_multiple_metrics_reversed(self): @@ -51,7 +54,10 @@ def test_multiple_metrics_reversed(self): self.assertEqual({ 'columns': [{'Header': 'Wins', 'accessor': '$m$wins'}, {'Header': 'Votes', 'accessor': '$m$votes'}], - 'data': [{'$m$votes': {'raw': 111674336}, '$m$wins': {'raw': 12}}] + 'data': [{ + '$m$votes': {'display': '111,674,336', 'raw': 111674336}, + '$m$wins': {'display': '12', 'raw': 12} + }] }, result) def test_time_series_dim(self): @@ -61,12 +67,30 @@ def test_time_series_dim(self): self.assertEqual({ 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, {'Header': 'Wins', 'accessor': '$m$wins'}], - 'data': [{'$d$timestamp': {'raw': '1996-01-01'}, '$m$wins': {'raw': 2}}, - {'$d$timestamp': {'raw': '2000-01-01'}, '$m$wins': {'raw': 2}}, - {'$d$timestamp': {'raw': '2004-01-01'}, '$m$wins': {'raw': 2}}, - {'$d$timestamp': {'raw': '2008-01-01'}, '$m$wins': {'raw': 2}}, - {'$d$timestamp': {'raw': '2012-01-01'}, '$m$wins': {'raw': 2}}, - {'$d$timestamp': {'raw': '2016-01-01'}, '$m$wins': {'raw': 2}}] + 'data': [{ + '$d$timestamp': {'raw': '1996-01-01'}, + '$m$wins': {'display': '2', 'raw': 2} + }, + { + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$wins': {'display': '2', 'raw': 2} + }, + { + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$wins': {'display': '2', 'raw': 2} + }, + { + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$wins': {'display': '2', 'raw': 2} + }, + { + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$wins': {'display': '2', 'raw': 2} + }, + { + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$wins': {'display': '2', 'raw': 2} + }] }, result) def test_time_series_dim_with_operation(self): @@ -78,22 +102,22 @@ def test_time_series_dim_with_operation(self): {'Header': 'CumSum(Votes)', 'accessor': '$m$cumsum(votes)'}], 'data': [{ '$d$timestamp': {'raw': '1996-01-01'}, - '$m$cumsum(votes)': {'raw': 15220449} + '$m$cumsum(votes)': {'display': '15,220,449', 'raw': 15220449} }, { '$d$timestamp': {'raw': '2000-01-01'}, - '$m$cumsum(votes)': {'raw': 31882466} + '$m$cumsum(votes)': {'display': '31,882,466', 'raw': 31882466} }, { '$d$timestamp': {'raw': '2004-01-01'}, - '$m$cumsum(votes)': {'raw': 51497398} + '$m$cumsum(votes)': {'display': '51,497,398', 'raw': 51497398} }, { '$d$timestamp': {'raw': '2008-01-01'}, - '$m$cumsum(votes)': {'raw': 72791613} + '$m$cumsum(votes)': {'display': '72,791,613', 'raw': 72791613} }, { '$d$timestamp': {'raw': '2012-01-01'}, - '$m$cumsum(votes)': {'raw': 93363823} + '$m$cumsum(votes)': {'display': '93,363,823', 'raw': 93363823} }, { '$d$timestamp': {'raw': '2016-01-01'}, - '$m$cumsum(votes)': {'raw': 111674336} + '$m$cumsum(votes)': {'display': '111,674,336', 'raw': 111674336} }] }, result) @@ -106,13 +130,13 @@ def test_cat_dim(self): {'Header': 'Wins', 'accessor': '$m$wins'}], 'data': [{ '$d$political_party': {'display': 'Democrat', 'raw': 'd'}, - '$m$wins': {'raw': 6} + '$m$wins': {'display': '6', 'raw': 6} }, { '$d$political_party': {'display': 'Independent', 'raw': 'i'}, - '$m$wins': {'raw': 0} + '$m$wins': {'display': '0', 'raw': 0} }, { '$d$political_party': {'display': 'Republican', 'raw': 'r'}, - '$m$wins': {'raw': 6} + '$m$wins': {'display': '6', 'raw': 6} }] }, result) @@ -125,37 +149,37 @@ def test_uni_dim(self): {'Header': 'Wins', 'accessor': '$m$wins'}], 'data': [{ '$d$candidate': {'display': 'Bill Clinton', 'raw': '1'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }, { '$d$candidate': {'display': 'Bob Dole', 'raw': '2'}, - '$m$wins': {'raw': 0} + '$m$wins': {'display': '0', 'raw': 0} }, { '$d$candidate': {'display': 'Ross Perot', 'raw': '3'}, - '$m$wins': {'raw': 0} + '$m$wins': {'display': '0', 'raw': 0} }, { '$d$candidate': {'display': 'George Bush', 'raw': '4'}, - '$m$wins': {'raw': 4} + '$m$wins': {'display': '4', 'raw': 4} }, { '$d$candidate': {'display': 'Al Gore', 'raw': '5'}, - '$m$wins': {'raw': 0} + '$m$wins': {'display': '0', 'raw': 0} }, { '$d$candidate': {'display': 'John Kerry', 'raw': '6'}, - '$m$wins': {'raw': 0} + '$m$wins': {'display': '0', 'raw': 0} }, { '$d$candidate': {'display': 'Barrack Obama', 'raw': '7'}, - '$m$wins': {'raw': 4} + '$m$wins': {'display': '4', 'raw': 4} }, { '$d$candidate': {'display': 'John McCain', 'raw': '8'}, - '$m$wins': {'raw': 0} + '$m$wins': {'display': '0', 'raw': 0} }, { '$d$candidate': {'display': 'Mitt Romney', 'raw': '9'}, - '$m$wins': {'raw': 0} + '$m$wins': {'display': '0', 'raw': 0} }, { '$d$candidate': {'display': 'Donald Trump', 'raw': '10'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }, { '$d$candidate': {'display': 'Hillary Clinton', 'raw': '11'}, - '$m$wins': {'raw': 0} + '$m$wins': {'display': '0', 'raw': 0} }] }, result) @@ -174,17 +198,22 @@ def test_uni_dim_no_display_definition(self): self.assertEqual({ 'columns': [{'Header': 'Candidate', 'accessor': '$d$candidate'}, {'Header': 'Wins', 'accessor': '$m$wins'}], - 'data': [{'$d$candidate': {'raw': '1'}, '$m$wins': {'raw': 2}}, - {'$d$candidate': {'raw': '2'}, '$m$wins': {'raw': 0}}, - {'$d$candidate': {'raw': '3'}, '$m$wins': {'raw': 0}}, - {'$d$candidate': {'raw': '4'}, '$m$wins': {'raw': 4}}, - {'$d$candidate': {'raw': '5'}, '$m$wins': {'raw': 0}}, - {'$d$candidate': {'raw': '6'}, '$m$wins': {'raw': 0}}, - {'$d$candidate': {'raw': '7'}, '$m$wins': {'raw': 4}}, - {'$d$candidate': {'raw': '8'}, '$m$wins': {'raw': 0}}, - {'$d$candidate': {'raw': '9'}, '$m$wins': {'raw': 0}}, - {'$d$candidate': {'raw': '10'}, '$m$wins': {'raw': 2}}, - {'$d$candidate': {'raw': '11'}, '$m$wins': {'raw': 0}}] + 'data': [{'$d$candidate': {'raw': '1'}, '$m$wins': {'display': '2', 'raw': 2}}, + {'$d$candidate': {'raw': '2'}, '$m$wins': {'display': '0', 'raw': 0}}, + {'$d$candidate': {'raw': '3'}, '$m$wins': {'display': '0', 'raw': 0}}, + {'$d$candidate': {'raw': '4'}, '$m$wins': {'display': '4', 'raw': 4}}, + {'$d$candidate': {'raw': '5'}, '$m$wins': {'display': '0', 'raw': 0}}, + {'$d$candidate': {'raw': '6'}, '$m$wins': {'display': '0', 'raw': 0}}, + {'$d$candidate': {'raw': '7'}, '$m$wins': {'display': '4', 'raw': 4}}, + {'$d$candidate': {'raw': '8'}, '$m$wins': {'display': '0', 'raw': 0}}, + {'$d$candidate': {'raw': '9'}, '$m$wins': {'display': '0', 'raw': 0}}, + { + '$d$candidate': {'raw': '10'}, + '$m$wins': {'display': '2', 'raw': 2} + }, { + '$d$candidate': {'raw': '11'}, + '$m$wins': {'display': '0', 'raw': 0} + }] }, result) def test_multi_dims_time_series_and_uni(self): @@ -198,51 +227,51 @@ def test_multi_dims_time_series_and_uni(self): 'data': [{ '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '1996-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '1996-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2000-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2000-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2004-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2004-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2008-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2008-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2012-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2012-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2016-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2016-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }] }, result) @@ -258,75 +287,75 @@ def test_multi_dims_with_one_level_totals(self): 'data': [{ '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '1996-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '1996-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'raw': 'Totals'}, '$d$timestamp': {'raw': '1996-01-01'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2000-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2000-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'raw': 'Totals'}, '$d$timestamp': {'raw': '2000-01-01'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2004-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2004-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'raw': 'Totals'}, '$d$timestamp': {'raw': '2004-01-01'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2008-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2008-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'raw': 'Totals'}, '$d$timestamp': {'raw': '2008-01-01'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2012-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2012-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'raw': 'Totals'}, '$d$timestamp': {'raw': '2012-01-01'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2016-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2016-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'raw': 'Totals'}, '$d$timestamp': {'raw': '2016-01-01'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }] }, result) @@ -342,79 +371,79 @@ def test_multi_dims_with_all_levels_totals(self): 'data': [{ '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '1996-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '1996-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'raw': 'Totals'}, '$d$timestamp': {'raw': '1996-01-01'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2000-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2000-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'raw': 'Totals'}, '$d$timestamp': {'raw': '2000-01-01'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2004-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2004-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'raw': 'Totals'}, '$d$timestamp': {'raw': '2004-01-01'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2008-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2008-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'raw': 'Totals'}, '$d$timestamp': {'raw': '2008-01-01'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2012-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2012-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'raw': 'Totals'}, '$d$timestamp': {'raw': '2012-01-01'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2016-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2016-01-01'}, - '$m$wins': {'raw': 1} + '$m$wins': {'display': '1', 'raw': 1} }, { '$d$state': {'raw': 'Totals'}, '$d$timestamp': {'raw': '2016-01-01'}, - '$m$wins': {'raw': 2} + '$m$wins': {'display': '2', 'raw': 2} }, { '$d$state': {'raw': 'Totals'}, '$d$timestamp': {'raw': 'Totals'}, - '$m$wins': {'raw': 12} + '$m$wins': {'display': '12', 'raw': 12} }] }, result) @@ -437,53 +466,53 @@ def test_time_series_ref(self): 'data': [{ '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2000-01-01'}, - '$m$votes': {'raw': 6233385.0}, - '$m$votes_eoe': {'raw': 5574387.0} + '$m$votes': {'display': '6,233,385', 'raw': 6233385}, + '$m$votes_eoe': {'display': '5574387.0', 'raw': 5574387.0} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2000-01-01'}, - '$m$votes': {'raw': 10428632.0}, - '$m$votes_eoe': {'raw': 9646062.0} + '$m$votes': {'display': '10,428,632', 'raw': 10428632}, + '$m$votes_eoe': {'display': '9646062.0', 'raw': 9646062.0} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2004-01-01'}, - '$m$votes': {'raw': 7359621.0}, - '$m$votes_eoe': {'raw': 6233385.0} + '$m$votes': {'display': '7,359,621', 'raw': 7359621}, + '$m$votes_eoe': {'display': '6233385.0', 'raw': 6233385.0} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2004-01-01'}, - '$m$votes': {'raw': 12255311.0}, - '$m$votes_eoe': {'raw': 10428632.0} + '$m$votes': {'display': '12,255,311', 'raw': 12255311}, + '$m$votes_eoe': {'display': '10428632.0', 'raw': 10428632.0} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2008-01-01'}, - '$m$votes': {'raw': 8007961.0}, - '$m$votes_eoe': {'raw': 7359621.0} + '$m$votes': {'display': '8,007,961', 'raw': 8007961}, + '$m$votes_eoe': {'display': '7359621.0', 'raw': 7359621.0} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2008-01-01'}, - '$m$votes': {'raw': 13286254.0}, - '$m$votes_eoe': {'raw': 12255311.0} + '$m$votes': {'display': '13,286,254', 'raw': 13286254}, + '$m$votes_eoe': {'display': '12255311.0', 'raw': 12255311.0} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2012-01-01'}, - '$m$votes': {'raw': 7877967.0}, - '$m$votes_eoe': {'raw': 8007961.0} + '$m$votes': {'display': '7,877,967', 'raw': 7877967}, + '$m$votes_eoe': {'display': '8007961.0', 'raw': 8007961.0} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2012-01-01'}, - '$m$votes': {'raw': 12694243.0}, - '$m$votes_eoe': {'raw': 13286254.0} + '$m$votes': {'display': '12,694,243', 'raw': 12694243}, + '$m$votes_eoe': {'display': '13286254.0', 'raw': 13286254.0} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2016-01-01'}, - '$m$votes': {'raw': 5072915.0}, - '$m$votes_eoe': {'raw': 7877967.0} + '$m$votes': {'display': '5,072,915', 'raw': 5072915}, + '$m$votes_eoe': {'display': '7877967.0', 'raw': 7877967.0} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2016-01-01'}, - '$m$votes': {'raw': 13237598.0}, - '$m$votes_eoe': {'raw': 12694243.0} + '$m$votes': {'display': '13,237,598', 'raw': 13237598}, + '$m$votes_eoe': {'display': '12694243.0', 'raw': 12694243.0} }] }, result) @@ -508,73 +537,73 @@ def test_time_series_ref_multiple_metrics(self): 'data': [{ '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2000-01-01'}, - '$m$votes': {'raw': 6233385.0}, - '$m$votes_eoe': {'raw': 5574387.0}, - '$m$wins': {'raw': 1.0}, - '$m$wins_eoe': {'raw': 1.0} + '$m$votes': {'display': '6,233,385', 'raw': 6233385}, + '$m$votes_eoe': {'display': '5574387.0', 'raw': 5574387.0}, + '$m$wins': {'display': '1', 'raw': 1}, + '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2000-01-01'}, - '$m$votes': {'raw': 10428632.0}, - '$m$votes_eoe': {'raw': 9646062.0}, - '$m$wins': {'raw': 1.0}, - '$m$wins_eoe': {'raw': 1.0} + '$m$votes': {'display': '10,428,632', 'raw': 10428632}, + '$m$votes_eoe': {'display': '9646062.0', 'raw': 9646062.0}, + '$m$wins': {'display': '1', 'raw': 1}, + '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2004-01-01'}, - '$m$votes': {'raw': 7359621.0}, - '$m$votes_eoe': {'raw': 6233385.0}, - '$m$wins': {'raw': 1.0}, - '$m$wins_eoe': {'raw': 1.0} + '$m$votes': {'display': '7,359,621', 'raw': 7359621}, + '$m$votes_eoe': {'display': '6233385.0', 'raw': 6233385.0}, + '$m$wins': {'display': '1', 'raw': 1}, + '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2004-01-01'}, - '$m$votes': {'raw': 12255311.0}, - '$m$votes_eoe': {'raw': 10428632.0}, - '$m$wins': {'raw': 1.0}, - '$m$wins_eoe': {'raw': 1.0} + '$m$votes': {'display': '12,255,311', 'raw': 12255311}, + '$m$votes_eoe': {'display': '10428632.0', 'raw': 10428632.0}, + '$m$wins': {'display': '1', 'raw': 1}, + '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2008-01-01'}, - '$m$votes': {'raw': 8007961.0}, - '$m$votes_eoe': {'raw': 7359621.0}, - '$m$wins': {'raw': 1.0}, - '$m$wins_eoe': {'raw': 1.0} + '$m$votes': {'display': '8,007,961', 'raw': 8007961}, + '$m$votes_eoe': {'display': '7359621.0', 'raw': 7359621.0}, + '$m$wins': {'display': '1', 'raw': 1}, + '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2008-01-01'}, - '$m$votes': {'raw': 13286254.0}, - '$m$votes_eoe': {'raw': 12255311.0}, - '$m$wins': {'raw': 1.0}, - '$m$wins_eoe': {'raw': 1.0} + '$m$votes': {'display': '13,286,254', 'raw': 13286254}, + '$m$votes_eoe': {'display': '12255311.0', 'raw': 12255311.0}, + '$m$wins': {'display': '1', 'raw': 1}, + '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2012-01-01'}, - '$m$votes': {'raw': 7877967.0}, - '$m$votes_eoe': {'raw': 8007961.0}, - '$m$wins': {'raw': 1.0}, - '$m$wins_eoe': {'raw': 1.0} + '$m$votes': {'display': '7,877,967', 'raw': 7877967}, + '$m$votes_eoe': {'display': '8007961.0', 'raw': 8007961.0}, + '$m$wins': {'display': '1', 'raw': 1}, + '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2012-01-01'}, - '$m$votes': {'raw': 12694243.0}, - '$m$votes_eoe': {'raw': 13286254.0}, - '$m$wins': {'raw': 1.0}, - '$m$wins_eoe': {'raw': 1.0} + '$m$votes': {'display': '12,694,243', 'raw': 12694243}, + '$m$votes_eoe': {'display': '13286254.0', 'raw': 13286254.0}, + '$m$wins': {'display': '1', 'raw': 1}, + '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2016-01-01'}, - '$m$votes': {'raw': 5072915.0}, - '$m$votes_eoe': {'raw': 7877967.0}, - '$m$wins': {'raw': 1.0}, - '$m$wins_eoe': {'raw': 1.0} + '$m$votes': {'display': '5,072,915', 'raw': 5072915}, + '$m$votes_eoe': {'display': '7877967.0', 'raw': 7877967.0}, + '$m$wins': {'display': '1', 'raw': 1}, + '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2016-01-01'}, - '$m$votes': {'raw': 13237598.0}, - '$m$votes_eoe': {'raw': 12694243.0}, - '$m$wins': {'raw': 1.0}, - '$m$wins_eoe': {'raw': 1.0} + '$m$votes': {'display': '13,237,598', 'raw': 13237598}, + '$m$votes_eoe': {'display': '12694243.0', 'raw': 12694243.0}, + '$m$wins': {'display': '1', 'raw': 1}, + '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} }] }, result) @@ -589,9 +618,9 @@ def test_transpose(self): {'Header': 'Republican', 'accessor': 'r'}], 'data': [{ '$d$metrics': {'raw': 'Wins'}, - 'd': {'raw': 6}, - 'i': {'raw': 0}, - 'r': {'raw': 6} + 'd': {'display': 6, 'raw': 6}, + 'i': {'display': 0, 'raw': 0}, + 'r': {'display': 6, 'raw': 6} }] }, result) @@ -605,28 +634,28 @@ def test_pivot_second_dimension_with_one_metric(self): {'Header': 'California', 'accessor': '2'}], 'data': [{ '$d$timestamp': {'raw': '1996-01-01'}, - '1': {'raw': 1}, - '2': {'raw': 1} + '1': {'display': 1, 'raw': 1}, + '2': {'display': 1, 'raw': 1} }, { '$d$timestamp': {'raw': '2000-01-01'}, - '1': {'raw': 1}, - '2': {'raw': 1} + '1': {'display': 1, 'raw': 1}, + '2': {'display': 1, 'raw': 1} }, { '$d$timestamp': {'raw': '2004-01-01'}, - '1': {'raw': 1}, - '2': {'raw': 1} + '1': {'display': 1, 'raw': 1}, + '2': {'display': 1, 'raw': 1} }, { '$d$timestamp': {'raw': '2008-01-01'}, - '1': {'raw': 1}, - '2': {'raw': 1} + '1': {'display': 1, 'raw': 1}, + '2': {'display': 1, 'raw': 1} }, { '$d$timestamp': {'raw': '2012-01-01'}, - '1': {'raw': 1}, - '2': {'raw': 1} + '1': {'display': 1, 'raw': 1}, + '2': {'display': 1, 'raw': 1} }, { '$d$timestamp': {'raw': '2016-01-01'}, - '1': {'raw': 1}, - '2': {'raw': 1} + '1': {'display': 1, 'raw': 1}, + '2': {'display': 1, 'raw': 1} }] }, result) @@ -647,28 +676,64 @@ def test_pivot_second_dimension_with_multiple_metrics(self): }], 'data': [{ '$d$timestamp': {'raw': '1996-01-01'}, - '$m$votes': {'1': {'raw': 5574387}, '2': {'raw': 9646062}}, - '$m$wins': {'1': {'raw': 1}, '2': {'raw': 1}} + '$m$votes': { + '1': {'display': '5,574,387', 'raw': 5574387}, + '2': {'display': '9,646,062', 'raw': 9646062} + }, + '$m$wins': { + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} + } }, { '$d$timestamp': {'raw': '2000-01-01'}, - '$m$votes': {'1': {'raw': 6233385}, '2': {'raw': 10428632}}, - '$m$wins': {'1': {'raw': 1}, '2': {'raw': 1}} + '$m$votes': { + '1': {'display': '6,233,385', 'raw': 6233385}, + '2': {'display': '10,428,632', 'raw': 10428632} + }, + '$m$wins': { + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} + } }, { '$d$timestamp': {'raw': '2004-01-01'}, - '$m$votes': {'1': {'raw': 7359621}, '2': {'raw': 12255311}}, - '$m$wins': {'1': {'raw': 1}, '2': {'raw': 1}} + '$m$votes': { + '1': {'display': '7,359,621', 'raw': 7359621}, + '2': {'display': '12,255,311', 'raw': 12255311} + }, + '$m$wins': { + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} + } }, { '$d$timestamp': {'raw': '2008-01-01'}, - '$m$votes': {'1': {'raw': 8007961}, '2': {'raw': 13286254}}, - '$m$wins': {'1': {'raw': 1}, '2': {'raw': 1}} + '$m$votes': { + '1': {'display': '8,007,961', 'raw': 8007961}, + '2': {'display': '13,286,254', 'raw': 13286254} + }, + '$m$wins': { + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} + } }, { '$d$timestamp': {'raw': '2012-01-01'}, - '$m$votes': {'1': {'raw': 7877967}, '2': {'raw': 12694243}}, - '$m$wins': {'1': {'raw': 1}, '2': {'raw': 1}} + '$m$votes': { + '1': {'display': '7,877,967', 'raw': 7877967}, + '2': {'display': '12,694,243', 'raw': 12694243} + }, + '$m$wins': { + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} + } }, { '$d$timestamp': {'raw': '2016-01-01'}, - '$m$votes': {'1': {'raw': 5072915}, '2': {'raw': 13237598}}, - '$m$wins': {'1': {'raw': 1}, '2': {'raw': 1}} + '$m$votes': { + '1': {'display': '5,072,915', 'raw': 5072915}, + '2': {'display': '13,237,598', 'raw': 13237598} + }, + '$m$wins': { + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} + } }] }, result) @@ -708,34 +773,94 @@ def test_pivot_second_dimension_with_multiple_metrics_and_references(self): }], 'data': [{ '$d$timestamp': {'raw': '2000-01-01'}, - '$m$votes': {'1': {'raw': 6233385.0}, '2': {'raw': 10428632.0}}, - '$m$votes_eoe': {'1': {'raw': 5574387.0}, '2': {'raw': 9646062.0}}, - '$m$wins': {'1': {'raw': 1.0}, '2': {'raw': 1.0}}, - '$m$wins_eoe': {'1': {'raw': 1.0}, '2': {'raw': 1.0}} + '$m$votes': { + '1': {'display': '6,233,385', 'raw': 6233385}, + '2': {'display': '10,428,632', 'raw': 10428632} + }, + '$m$votes_eoe': { + '1': {'display': '5574387.0', 'raw': 5574387.0}, + '2': {'display': '9646062.0', 'raw': 9646062.0} + }, + '$m$wins': { + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} + }, + '$m$wins_eoe': { + '1': {'display': '1.0', 'raw': 1.0}, + '2': {'display': '1.0', 'raw': 1.0} + } }, { '$d$timestamp': {'raw': '2004-01-01'}, - '$m$votes': {'1': {'raw': 7359621.0}, '2': {'raw': 12255311.0}}, - '$m$votes_eoe': {'1': {'raw': 6233385.0}, '2': {'raw': 10428632.0}}, - '$m$wins': {'1': {'raw': 1.0}, '2': {'raw': 1.0}}, - '$m$wins_eoe': {'1': {'raw': 1.0}, '2': {'raw': 1.0}} + '$m$votes': { + '1': {'display': '7,359,621', 'raw': 7359621}, + '2': {'display': '12,255,311', 'raw': 12255311} + }, + '$m$votes_eoe': { + '1': {'display': '6233385.0', 'raw': 6233385.0}, + '2': {'display': '10428632.0', 'raw': 10428632.0} + }, + '$m$wins': { + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} + }, + '$m$wins_eoe': { + '1': {'display': '1.0', 'raw': 1.0}, + '2': {'display': '1.0', 'raw': 1.0} + } }, { '$d$timestamp': {'raw': '2008-01-01'}, - '$m$votes': {'1': {'raw': 8007961.0}, '2': {'raw': 13286254.0}}, - '$m$votes_eoe': {'1': {'raw': 7359621.0}, '2': {'raw': 12255311.0}}, - '$m$wins': {'1': {'raw': 1.0}, '2': {'raw': 1.0}}, - '$m$wins_eoe': {'1': {'raw': 1.0}, '2': {'raw': 1.0}} + '$m$votes': { + '1': {'display': '8,007,961', 'raw': 8007961}, + '2': {'display': '13,286,254', 'raw': 13286254} + }, + '$m$votes_eoe': { + '1': {'display': '7359621.0', 'raw': 7359621.0}, + '2': {'display': '12255311.0', 'raw': 12255311.0} + }, + '$m$wins': { + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} + }, + '$m$wins_eoe': { + '1': {'display': '1.0', 'raw': 1.0}, + '2': {'display': '1.0', 'raw': 1.0} + } }, { '$d$timestamp': {'raw': '2012-01-01'}, - '$m$votes': {'1': {'raw': 7877967.0}, '2': {'raw': 12694243.0}}, - '$m$votes_eoe': {'1': {'raw': 8007961.0}, '2': {'raw': 13286254.0}}, - '$m$wins': {'1': {'raw': 1.0}, '2': {'raw': 1.0}}, - '$m$wins_eoe': {'1': {'raw': 1.0}, '2': {'raw': 1.0}} + '$m$votes': { + '1': {'display': '7,877,967', 'raw': 7877967}, + '2': {'display': '12,694,243', 'raw': 12694243} + }, + '$m$votes_eoe': { + '1': {'display': '8007961.0', 'raw': 8007961.0}, + '2': {'display': '13286254.0', 'raw': 13286254.0} + }, + '$m$wins': { + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} + }, + '$m$wins_eoe': { + '1': {'display': '1.0', 'raw': 1.0}, + '2': {'display': '1.0', 'raw': 1.0} + } }, { '$d$timestamp': {'raw': '2016-01-01'}, - '$m$votes': {'1': {'raw': 5072915.0}, '2': {'raw': 13237598.0}}, - '$m$votes_eoe': {'1': {'raw': 7877967.0}, '2': {'raw': 12694243.0}}, - '$m$wins': {'1': {'raw': 1.0}, '2': {'raw': 1.0}}, - '$m$wins_eoe': {'1': {'raw': 1.0}, '2': {'raw': 1.0}} + '$m$votes': { + '1': {'display': '5,072,915', 'raw': 5072915}, + '2': {'display': '13,237,598', 'raw': 13237598} + }, + '$m$votes_eoe': { + '1': {'display': '7877967.0', 'raw': 7877967.0}, + '2': {'display': '12694243.0', 'raw': 12694243.0} + }, + '$m$wins': { + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} + }, + '$m$wins_eoe': { + '1': {'display': '1.0', 'raw': 1.0}, + '2': {'display': '1.0', 'raw': 1.0} + } }] }, result) @@ -758,17 +883,17 @@ def test_pivot_single_dimension_as_rows_single_metric_metrics_automatically_pivo {'Header': 'Hillary Clinton', 'accessor': '11'}], 'data': [{ '$d$metrics': {'raw': 'Wins'}, - '1': {'raw': 2}, - '10': {'raw': 2}, - '11': {'raw': 0}, - '2': {'raw': 0}, - '3': {'raw': 0}, - '4': {'raw': 4}, - '5': {'raw': 0}, - '6': {'raw': 0}, - '7': {'raw': 4}, - '8': {'raw': 0}, - '9': {'raw': 0} + '1': {'display': 2, 'raw': 2}, + '10': {'display': 2, 'raw': 2}, + '11': {'display': 0, 'raw': 0}, + '2': {'display': 0, 'raw': 0}, + '3': {'display': 0, 'raw': 0}, + '4': {'display': 4, 'raw': 4}, + '5': {'display': 0, 'raw': 0}, + '6': {'display': 0, 'raw': 0}, + '7': {'display': 4, 'raw': 4}, + '8': {'display': 0, 'raw': 0}, + '9': {'display': 0, 'raw': 0} }] }, result) @@ -791,17 +916,17 @@ def test_pivot_single_dimension_as_rows_single_metric_and_transpose_set_to_true( {'Header': 'Hillary Clinton', 'accessor': '11'}], 'data': [{ '$d$metrics': {'raw': 'Wins'}, - '1': {'raw': 2}, - '10': {'raw': 2}, - '11': {'raw': 0}, - '2': {'raw': 0}, - '3': {'raw': 0}, - '4': {'raw': 4}, - '5': {'raw': 0}, - '6': {'raw': 0}, - '7': {'raw': 4}, - '8': {'raw': 0}, - '9': {'raw': 0} + '1': {'display': 2, 'raw': 2}, + '10': {'display': 2, 'raw': 2}, + '11': {'display': 0, 'raw': 0}, + '2': {'display': 0, 'raw': 0}, + '3': {'display': 0, 'raw': 0}, + '4': {'display': 4, 'raw': 4}, + '5': {'display': 0, 'raw': 0}, + '6': {'display': 0, 'raw': 0}, + '7': {'display': 4, 'raw': 4}, + '8': {'display': 0, 'raw': 0}, + '9': {'display': 0, 'raw': 0} }] }, result) @@ -825,30 +950,30 @@ def test_pivot_single_dimension_as_rows_multiple_metrics(self): {'Header': 'Hillary Clinton', 'accessor': '11'}], 'data': [{ '$d$metrics': {'raw': 'Wins'}, - '1': {'raw': 2}, - '10': {'raw': 2}, - '11': {'raw': 0}, - '2': {'raw': 0}, - '3': {'raw': 0}, - '4': {'raw': 4}, - '5': {'raw': 0}, - '6': {'raw': 0}, - '7': {'raw': 4}, - '8': {'raw': 0}, - '9': {'raw': 0} + '1': {'display': 2, 'raw': 2}, + '10': {'display': 2, 'raw': 2}, + '11': {'display': 0, 'raw': 0}, + '2': {'display': 0, 'raw': 0}, + '3': {'display': 0, 'raw': 0}, + '4': {'display': 4, 'raw': 4}, + '5': {'display': 0, 'raw': 0}, + '6': {'display': 0, 'raw': 0}, + '7': {'display': 4, 'raw': 4}, + '8': {'display': 0, 'raw': 0}, + '9': {'display': 0, 'raw': 0} }, { '$d$metrics': {'raw': 'Votes'}, - '1': {'raw': 7579518}, - '10': {'raw': 13438835}, - '11': {'raw': 4871678}, - '2': {'raw': 6564547}, - '3': {'raw': 1076384}, - '4': {'raw': 18403811}, - '5': {'raw': 8294949}, - '6': {'raw': 9578189}, - '7': {'raw': 24227234}, - '8': {'raw': 9491109}, - '9': {'raw': 8148082} + '1': {'display': 7579518, 'raw': 7579518}, + '10': {'display': 13438835, 'raw': 13438835}, + '11': {'display': 4871678, 'raw': 4871678}, + '2': {'display': 6564547, 'raw': 6564547}, + '3': {'display': 1076384, 'raw': 1076384}, + '4': {'display': 18403811, 'raw': 18403811}, + '5': {'display': 8294949, 'raw': 8294949}, + '6': {'display': 9578189, 'raw': 9578189}, + '7': {'display': 24227234, 'raw': 24227234}, + '8': {'display': 9491109, 'raw': 9491109}, + '9': {'display': 8148082, 'raw': 8148082} }] }, result) @@ -859,12 +984,25 @@ def test_pivot_single_metric_time_series_dim(self): self.assertEqual({ 'columns': [{'Header': 'Timestamp', 'accessor': '$d$timestamp'}, {'Header': 'Wins', 'accessor': '$m$wins'}], - 'data': [{'$d$timestamp': {'raw': '1996-01-01'}, '$m$wins': {'raw': 2}}, - {'$d$timestamp': {'raw': '2000-01-01'}, '$m$wins': {'raw': 2}}, - {'$d$timestamp': {'raw': '2004-01-01'}, '$m$wins': {'raw': 2}}, - {'$d$timestamp': {'raw': '2008-01-01'}, '$m$wins': {'raw': 2}}, - {'$d$timestamp': {'raw': '2012-01-01'}, '$m$wins': {'raw': 2}}, - {'$d$timestamp': {'raw': '2016-01-01'}, '$m$wins': {'raw': 2}}] + 'data': [{ + '$d$timestamp': {'raw': '1996-01-01'}, + '$m$wins': {'display': '2', 'raw': 2} + }, { + '$d$timestamp': {'raw': '2000-01-01'}, + '$m$wins': {'display': '2', 'raw': 2} + }, { + '$d$timestamp': {'raw': '2004-01-01'}, + '$m$wins': {'display': '2', 'raw': 2} + }, { + '$d$timestamp': {'raw': '2008-01-01'}, + '$m$wins': {'display': '2', 'raw': 2} + }, { + '$d$timestamp': {'raw': '2012-01-01'}, + '$m$wins': {'display': '2', 'raw': 2} + }, { + '$d$timestamp': {'raw': '2016-01-01'}, + '$m$wins': {'display': '2', 'raw': 2} + }] }, result) def test_pivot_multi_dims_with_all_levels_totals(self): @@ -897,86 +1035,89 @@ def test_pivot_multi_dims_with_all_levels_totals(self): 'data': [{ '$d$timestamp': {'raw': '1996-01-01'}, '$m$votes': { - '1': {'raw': 5574387.0}, - '2': {'raw': 9646062.0}, - 'totals': {'raw': 15220449.0} + '1': {'display': '5574387.0', 'raw': 5574387.0}, + '2': {'display': '9646062.0', 'raw': 9646062.0}, + 'totals': {'display': '15220449.0', 'raw': 15220449.0} }, '$m$wins': { - '1': {'raw': 1.0}, - '2': {'raw': 1.0}, - 'totals': {'raw': 2.0} + '1': {'display': '1.0', 'raw': 1.0}, + '2': {'display': '1.0', 'raw': 1.0}, + 'totals': {'display': '2.0', 'raw': 2.0} } }, { '$d$timestamp': {'raw': '2000-01-01'}, '$m$votes': { - '1': {'raw': 6233385.0}, - '2': {'raw': 10428632.0}, - 'totals': {'raw': 16662017.0} + '1': {'display': '6233385.0', 'raw': 6233385.0}, + '2': {'display': '10428632.0', 'raw': 10428632.0}, + 'totals': {'display': '16662017.0', 'raw': 16662017.0} }, '$m$wins': { - '1': {'raw': 1.0}, - '2': {'raw': 1.0}, - 'totals': {'raw': 2.0} + '1': {'display': '1.0', 'raw': 1.0}, + '2': {'display': '1.0', 'raw': 1.0}, + 'totals': {'display': '2.0', 'raw': 2.0} } }, { '$d$timestamp': {'raw': '2004-01-01'}, '$m$votes': { - '1': {'raw': 7359621.0}, - '2': {'raw': 12255311.0}, - 'totals': {'raw': 19614932.0} + '1': {'display': '7359621.0', 'raw': 7359621.0}, + '2': {'display': '12255311.0', 'raw': 12255311.0}, + 'totals': {'display': '19614932.0', 'raw': 19614932.0} }, '$m$wins': { - '1': {'raw': 1.0}, - '2': {'raw': 1.0}, - 'totals': {'raw': 2.0} + '1': {'display': '1.0', 'raw': 1.0}, + '2': {'display': '1.0', 'raw': 1.0}, + 'totals': {'display': '2.0', 'raw': 2.0} } }, { '$d$timestamp': {'raw': '2008-01-01'}, '$m$votes': { - '1': {'raw': 8007961.0}, - '2': {'raw': 13286254.0}, - 'totals': {'raw': 21294215.0} + '1': {'display': '8007961.0', 'raw': 8007961.0}, + '2': {'display': '13286254.0', 'raw': 13286254.0}, + 'totals': {'display': '21294215.0', 'raw': 21294215.0} }, '$m$wins': { - '1': {'raw': 1.0}, - '2': {'raw': 1.0}, - 'totals': {'raw': 2.0} + '1': {'display': '1.0', 'raw': 1.0}, + '2': {'display': '1.0', 'raw': 1.0}, + 'totals': {'display': '2.0', 'raw': 2.0} } }, { '$d$timestamp': {'raw': '2012-01-01'}, '$m$votes': { - '1': {'raw': 7877967.0}, - '2': {'raw': 12694243.0}, - 'totals': {'raw': 20572210.0} + '1': {'display': '7877967.0', 'raw': 7877967.0}, + '2': {'display': '12694243.0', 'raw': 12694243.0}, + 'totals': {'display': '20572210.0', 'raw': 20572210.0} }, '$m$wins': { - '1': {'raw': 1.0}, - '2': {'raw': 1.0}, - 'totals': {'raw': 2.0} + '1': {'display': '1.0', 'raw': 1.0}, + '2': {'display': '1.0', 'raw': 1.0}, + 'totals': {'display': '2.0', 'raw': 2.0} } }, { '$d$timestamp': {'raw': '2016-01-01'}, '$m$votes': { - '1': {'raw': 5072915.0}, - '2': {'raw': 13237598.0}, - 'totals': {'raw': 18310513.0} + '1': {'display': '5072915.0', 'raw': 5072915.0}, + '2': {'display': '13237598.0', 'raw': 13237598.0}, + 'totals': {'display': '18310513.0', 'raw': 18310513.0} }, '$m$wins': { - '1': {'raw': 1.0}, - '2': {'raw': 1.0}, - 'totals': {'raw': 2.0} + '1': {'display': '1.0', 'raw': 1.0}, + '2': {'display': '1.0', 'raw': 1.0}, + 'totals': {'display': '2.0', 'raw': 2.0} } }, { '$d$timestamp': {'raw': 'Totals'}, '$m$votes': { - '1': {'raw': ''}, - '2': {'raw': ''}, - 'totals': {'raw': 111674336.0} + '1': {'display': '', 'raw': ''}, + '2': {'display': '', 'raw': ''}, + 'totals': { + 'display': '111674336.0', + 'raw': 111674336.0 + } }, '$m$wins': { - '1': {'raw': ''}, - '2': {'raw': ''}, - 'totals': {'raw': 12.0} + '1': {'display': '', 'raw': ''}, + '2': {'display': '', 'raw': ''}, + 'totals': {'display': '12.0', 'raw': 12.0} } }] }, result) @@ -1004,63 +1145,63 @@ def test_pivot_first_dimension_and_transpose_with_all_levels_totals(self): 'data': [{ '$d$metrics': {'raw': 'Wins'}, '$d$state': {'display': 'Texas', 'raw': '1'}, - '1996-01-01': {'raw': 1.0}, - '2000-01-01': {'raw': 1.0}, - '2004-01-01': {'raw': 1.0}, - '2008-01-01': {'raw': 1.0}, - '2012-01-01': {'raw': 1.0}, - '2016-01-01': {'raw': 1.0}, - 'totals': {'raw': ''} + '1996-01-01': {'display': 1.0, 'raw': 1.0}, + '2000-01-01': {'display': 1.0, 'raw': 1.0}, + '2004-01-01': {'display': 1.0, 'raw': 1.0}, + '2008-01-01': {'display': 1.0, 'raw': 1.0}, + '2012-01-01': {'display': 1.0, 'raw': 1.0}, + '2016-01-01': {'display': 1.0, 'raw': 1.0}, + 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Wins'}, '$d$state': {'display': 'California', 'raw': '2'}, - '1996-01-01': {'raw': 1.0}, - '2000-01-01': {'raw': 1.0}, - '2004-01-01': {'raw': 1.0}, - '2008-01-01': {'raw': 1.0}, - '2012-01-01': {'raw': 1.0}, - '2016-01-01': {'raw': 1.0}, - 'totals': {'raw': ''} + '1996-01-01': {'display': 1.0, 'raw': 1.0}, + '2000-01-01': {'display': 1.0, 'raw': 1.0}, + '2004-01-01': {'display': 1.0, 'raw': 1.0}, + '2008-01-01': {'display': 1.0, 'raw': 1.0}, + '2012-01-01': {'display': 1.0, 'raw': 1.0}, + '2016-01-01': {'display': 1.0, 'raw': 1.0}, + 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Wins'}, '$d$state': {'raw': 'Totals'}, - '1996-01-01': {'raw': 2.0}, - '2000-01-01': {'raw': 2.0}, - '2004-01-01': {'raw': 2.0}, - '2008-01-01': {'raw': 2.0}, - '2012-01-01': {'raw': 2.0}, - '2016-01-01': {'raw': 2.0}, - 'totals': {'raw': 12.0} + '1996-01-01': {'display': 2.0, 'raw': 2.0}, + '2000-01-01': {'display': 2.0, 'raw': 2.0}, + '2004-01-01': {'display': 2.0, 'raw': 2.0}, + '2008-01-01': {'display': 2.0, 'raw': 2.0}, + '2012-01-01': {'display': 2.0, 'raw': 2.0}, + '2016-01-01': {'display': 2.0, 'raw': 2.0}, + 'totals': {'display': '12.0', 'raw': 12.0} }, { '$d$metrics': {'raw': 'Votes'}, '$d$state': {'display': 'Texas', 'raw': '1'}, - '1996-01-01': {'raw': 5574387.0}, - '2000-01-01': {'raw': 6233385.0}, - '2004-01-01': {'raw': 7359621.0}, - '2008-01-01': {'raw': 8007961.0}, - '2012-01-01': {'raw': 7877967.0}, - '2016-01-01': {'raw': 5072915.0}, - 'totals': {'raw': ''} + '1996-01-01': {'display': 5574387.0, 'raw': 5574387.0}, + '2000-01-01': {'display': 6233385.0, 'raw': 6233385.0}, + '2004-01-01': {'display': 7359621.0, 'raw': 7359621.0}, + '2008-01-01': {'display': 8007961.0, 'raw': 8007961.0}, + '2012-01-01': {'display': 7877967.0, 'raw': 7877967.0}, + '2016-01-01': {'display': 5072915.0, 'raw': 5072915.0}, + 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Votes'}, '$d$state': {'display': 'California', 'raw': '2'}, - '1996-01-01': {'raw': 9646062.0}, - '2000-01-01': {'raw': 10428632.0}, - '2004-01-01': {'raw': 12255311.0}, - '2008-01-01': {'raw': 13286254.0}, - '2012-01-01': {'raw': 12694243.0}, - '2016-01-01': {'raw': 13237598.0}, - 'totals': {'raw': ''} + '1996-01-01': {'display': 9646062.0, 'raw': 9646062.0}, + '2000-01-01': {'display': 10428632.0, 'raw': 10428632.0}, + '2004-01-01': {'display': 12255311.0, 'raw': 12255311.0}, + '2008-01-01': {'display': 13286254.0, 'raw': 13286254.0}, + '2012-01-01': {'display': 12694243.0, 'raw': 12694243.0}, + '2016-01-01': {'display': 13237598.0, 'raw': 13237598.0}, + 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Votes'}, '$d$state': {'raw': 'Totals'}, - '1996-01-01': {'raw': 15220449.0}, - '2000-01-01': {'raw': 16662017.0}, - '2004-01-01': {'raw': 19614932.0}, - '2008-01-01': {'raw': 21294215.0}, - '2012-01-01': {'raw': 20572210.0}, - '2016-01-01': {'raw': 18310513.0}, - 'totals': {'raw': 111674336.0} + '1996-01-01': {'display': 15220449.0, 'raw': 15220449.0}, + '2000-01-01': {'display': 16662017.0, 'raw': 16662017.0}, + '2004-01-01': {'display': 19614932.0, 'raw': 19614932.0}, + '2008-01-01': {'display': 21294215.0, 'raw': 21294215.0}, + '2012-01-01': {'display': 20572210.0, 'raw': 20572210.0}, + '2016-01-01': {'display': 18310513.0, 'raw': 18310513.0}, + 'totals': {'display': '111674336.0', 'raw': 111674336.0} }] }, result) @@ -1087,63 +1228,63 @@ def test_pivot_second_dimension_and_transpose_with_all_levels_totals(self): 'data': [{ '$d$metrics': {'raw': 'Wins'}, '$d$state': {'display': 'Texas', 'raw': '1'}, - '1996-01-01': {'raw': 1.0}, - '2000-01-01': {'raw': 1.0}, - '2004-01-01': {'raw': 1.0}, - '2008-01-01': {'raw': 1.0}, - '2012-01-01': {'raw': 1.0}, - '2016-01-01': {'raw': 1.0}, - 'totals': {'raw': ''} + '1996-01-01': {'display': 1.0, 'raw': 1.0}, + '2000-01-01': {'display': 1.0, 'raw': 1.0}, + '2004-01-01': {'display': 1.0, 'raw': 1.0}, + '2008-01-01': {'display': 1.0, 'raw': 1.0}, + '2012-01-01': {'display': 1.0, 'raw': 1.0}, + '2016-01-01': {'display': 1.0, 'raw': 1.0}, + 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Wins'}, '$d$state': {'display': 'California', 'raw': '2'}, - '1996-01-01': {'raw': 1.0}, - '2000-01-01': {'raw': 1.0}, - '2004-01-01': {'raw': 1.0}, - '2008-01-01': {'raw': 1.0}, - '2012-01-01': {'raw': 1.0}, - '2016-01-01': {'raw': 1.0}, - 'totals': {'raw': ''} + '1996-01-01': {'display': 1.0, 'raw': 1.0}, + '2000-01-01': {'display': 1.0, 'raw': 1.0}, + '2004-01-01': {'display': 1.0, 'raw': 1.0}, + '2008-01-01': {'display': 1.0, 'raw': 1.0}, + '2012-01-01': {'display': 1.0, 'raw': 1.0}, + '2016-01-01': {'display': 1.0, 'raw': 1.0}, + 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Wins'}, '$d$state': {'raw': 'Totals'}, - '1996-01-01': {'raw': 2.0}, - '2000-01-01': {'raw': 2.0}, - '2004-01-01': {'raw': 2.0}, - '2008-01-01': {'raw': 2.0}, - '2012-01-01': {'raw': 2.0}, - '2016-01-01': {'raw': 2.0}, - 'totals': {'raw': 12.0} + '1996-01-01': {'display': 2.0, 'raw': 2.0}, + '2000-01-01': {'display': 2.0, 'raw': 2.0}, + '2004-01-01': {'display': 2.0, 'raw': 2.0}, + '2008-01-01': {'display': 2.0, 'raw': 2.0}, + '2012-01-01': {'display': 2.0, 'raw': 2.0}, + '2016-01-01': {'display': 2.0, 'raw': 2.0}, + 'totals': {'display': '12.0', 'raw': 12.0} }, { '$d$metrics': {'raw': 'Votes'}, '$d$state': {'display': 'Texas', 'raw': '1'}, - '1996-01-01': {'raw': 5574387.0}, - '2000-01-01': {'raw': 6233385.0}, - '2004-01-01': {'raw': 7359621.0}, - '2008-01-01': {'raw': 8007961.0}, - '2012-01-01': {'raw': 7877967.0}, - '2016-01-01': {'raw': 5072915.0}, - 'totals': {'raw': ''} + '1996-01-01': {'display': 5574387.0, 'raw': 5574387.0}, + '2000-01-01': {'display': 6233385.0, 'raw': 6233385.0}, + '2004-01-01': {'display': 7359621.0, 'raw': 7359621.0}, + '2008-01-01': {'display': 8007961.0, 'raw': 8007961.0}, + '2012-01-01': {'display': 7877967.0, 'raw': 7877967.0}, + '2016-01-01': {'display': 5072915.0, 'raw': 5072915.0}, + 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Votes'}, '$d$state': {'display': 'California', 'raw': '2'}, - '1996-01-01': {'raw': 9646062.0}, - '2000-01-01': {'raw': 10428632.0}, - '2004-01-01': {'raw': 12255311.0}, - '2008-01-01': {'raw': 13286254.0}, - '2012-01-01': {'raw': 12694243.0}, - '2016-01-01': {'raw': 13237598.0}, - 'totals': {'raw': ''} + '1996-01-01': {'display': 9646062.0, 'raw': 9646062.0}, + '2000-01-01': {'display': 10428632.0, 'raw': 10428632.0}, + '2004-01-01': {'display': 12255311.0, 'raw': 12255311.0}, + '2008-01-01': {'display': 13286254.0, 'raw': 13286254.0}, + '2012-01-01': {'display': 12694243.0, 'raw': 12694243.0}, + '2016-01-01': {'display': 13237598.0, 'raw': 13237598.0}, + 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Votes'}, '$d$state': {'raw': 'Totals'}, - '1996-01-01': {'raw': 15220449.0}, - '2000-01-01': {'raw': 16662017.0}, - '2004-01-01': {'raw': 19614932.0}, - '2008-01-01': {'raw': 21294215.0}, - '2012-01-01': {'raw': 20572210.0}, - '2016-01-01': {'raw': 18310513.0}, - 'totals': {'raw': 111674336.0} + '1996-01-01': {'display': 15220449.0, 'raw': 15220449.0}, + '2000-01-01': {'display': 16662017.0, 'raw': 16662017.0}, + '2004-01-01': {'display': 19614932.0, 'raw': 19614932.0}, + '2008-01-01': {'display': 21294215.0, 'raw': 21294215.0}, + '2012-01-01': {'display': 20572210.0, 'raw': 20572210.0}, + '2016-01-01': {'display': 18310513.0, 'raw': 18310513.0}, + 'totals': {'display': '111674336.0', 'raw': 111674336.0} }] }, result) From 143ce2bfd2c70d87b800e2c659bdb47d1b814478 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 8 Aug 2018 15:10:18 +0200 Subject: [PATCH 109/123] bumped version to dev45 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 523085f5..e8df5816 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev43' +__version__ = '1.0.0.dev45' From d9976e6a4c481008571f9a1d5cd86164c9da2410 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 8 Aug 2018 17:14:27 +0200 Subject: [PATCH 110/123] Fixed reacttable to use the same formatting as datatablesjs --- fireant/formats.py | 21 +- fireant/slicer/widgets/reacttable.py | 36 +- .../tests/slicer/widgets/test_datatables.py | 82 +-- fireant/tests/slicer/widgets/test_pandas.py | 4 +- .../tests/slicer/widgets/test_reacttable.py | 535 ++++++++---------- fireant/tests/test_formats.py | 106 +++- 6 files changed, 369 insertions(+), 415 deletions(-) diff --git a/fireant/formats.py b/fireant/formats.py index 611b5e06..b05016a6 100644 --- a/fireant/formats.py +++ b/fireant/formats.py @@ -1,13 +1,11 @@ -import locale +import numpy as np +import pandas as pd from datetime import ( date, datetime, time, ) -import numpy as np -import pandas as pd - INFINITY = "Infinity" NULL_VALUE = 'null' TOTALS_VALUE = 'totals' @@ -116,23 +114,22 @@ def metric_display(value, prefix=None, suffix=None, precision=None): if pd.isnull(value) or value in {np.inf, -np.inf}: return '' + if isinstance(value, bool): + value = str(value).lower() + if isinstance(value, float): if precision is not None: - float_format = '%d' if precision == 0 else '%.{}f'.format(precision) - value = locale.format(float_format, value, grouping=True) + value = '{:,.{precision}f}'.format(value, precision=precision) elif value.is_integer(): - float_format = '%d' - value = locale.format(float_format, value, grouping=True) + value = '{:,.0f}'.format(value) else: - float_format = '%f' # Stripping trailing zeros is necessary because %f can add them if no precision is set - value = locale.format(float_format, value, grouping=True).rstrip('.0') + value = '{:,f}'.format(value).rstrip('.0') if isinstance(value, int): - float_format = '%d' - value = locale.format(float_format, value, grouping=True) + value = '{:,.0f}'.format(value) return '{prefix}{value}{suffix}'.format( prefix=prefix or '', diff --git a/fireant/slicer/widgets/reacttable.py b/fireant/slicer/widgets/reacttable.py index 28a6f7fb..8b193894 100644 --- a/fireant/slicer/widgets/reacttable.py +++ b/fireant/slicer/widgets/reacttable.py @@ -6,6 +6,8 @@ from fireant.formats import ( RAW_VALUE, TOTALS_VALUE, + metric_display, + metric_value, ) from fireant.utils import ( format_dimension_key, @@ -69,26 +71,6 @@ def fillna_index(index, value): return index.fillna(value) -def display_value(value, prefix=None, suffix=None, precision=None): - if isinstance(value, bool): - value = str(value).lower() - - def format_numeric_value(value): - if precision is not None and isinstance(value, float): - return '{:.{precision}f}'.format(value, precision=precision) - - if isinstance(value, int): - return '{:,}'.format(value) - - return '{}'.format(value) - - return '{prefix}{value}{suffix}'.format( - prefix=(prefix or ""), - suffix=(suffix or ""), - value=format_numeric_value(value), - ) - - class ReferenceItem: def __init__(self, item, reference): if reference is None: @@ -371,17 +353,15 @@ def transform_data(data_frame, item_map, dimension_display_values): # Add the values to the row for key, value in series.iteritems(): - # pd.Series casts everything to float, cast it back to int if it's an int - if np.int64 == data_frame[key].dtype: - value = int(value) - - data = {RAW_VALUE: value} + value = metric_value(value) + data = {RAW_VALUE: metric_value(value)} # Try to find a display value for the item item = item_map.get(key[0] if isinstance(key, tuple) else key) - display = display_value(value, item.prefix, item.suffix, item.precision) \ - if item is not None else \ - value + display = metric_display(value, + getattr(item, 'prefix', None), + getattr(item, 'suffix', None), + getattr(item, 'precision', None)) if display is not None: data['display'] = display diff --git a/fireant/tests/slicer/widgets/test_datatables.py b/fireant/tests/slicer/widgets/test_datatables.py index 7b86fd8b..17c41703 100644 --- a/fireant/tests/slicer/widgets/test_datatables.py +++ b/fireant/tests/slicer/widgets/test_datatables.py @@ -41,7 +41,7 @@ def test_single_metric(self): 'render': {'_': 'value', 'display': 'display'}, }], 'data': [{ - 'votes': {'value': 111674336, 'display': '111674336'} + 'votes': {'value': 111674336, 'display': '111,674,336'} }], }, result) @@ -60,7 +60,7 @@ def test_multiple_metrics(self): 'render': {'_': 'value', 'display': 'display'}, }], 'data': [{ - 'votes': {'value': 111674336, 'display': '111674336'}, + 'votes': {'value': 111674336, 'display': '111,674,336'}, 'wins': {'value': 12, 'display': '12'}, }], }, result) @@ -81,7 +81,7 @@ def test_multiple_metrics_reversed(self): }], 'data': [{ 'wins': {'value': 12, 'display': '12'}, - 'votes': {'value': 111674336, 'display': '111674336'}, + 'votes': {'value': 111674336, 'display': '111,674,336'}, }], }, result) @@ -135,22 +135,22 @@ def test_time_series_dim_with_operation(self): 'render': {'_': 'value', 'display': 'display'}, }], 'data': [{ - 'cumsum(votes)': {'display': '15220449', 'value': 15220449}, + 'cumsum(votes)': {'display': '15,220,449', 'value': 15220449}, 'timestamp': {'value': '1996-01-01'} }, { - 'cumsum(votes)': {'display': '31882466', 'value': 31882466}, + 'cumsum(votes)': {'display': '31,882,466', 'value': 31882466}, 'timestamp': {'value': '2000-01-01'} }, { - 'cumsum(votes)': {'display': '51497398', 'value': 51497398}, + 'cumsum(votes)': {'display': '51,497,398', 'value': 51497398}, 'timestamp': {'value': '2004-01-01'} }, { - 'cumsum(votes)': {'display': '72791613', 'value': 72791613}, + 'cumsum(votes)': {'display': '72,791,613', 'value': 72791613}, 'timestamp': {'value': '2008-01-01'} }, { - 'cumsum(votes)': {'display': '93363823', 'value': 93363823}, + 'cumsum(votes)': {'display': '93,363,823', 'value': 93363823}, 'timestamp': {'value': '2012-01-01'} }, { - 'cumsum(votes)': {'display': '111674336', 'value': 111674336}, + 'cumsum(votes)': {'display': '111,674,336', 'value': 111674336}, 'timestamp': {'value': '2016-01-01'} }], }, result) @@ -664,38 +664,38 @@ def test_pivoted_multi_dims_time_series_and_uni(self): 'data': [{ 'timestamp': {'value': '1996-01-01'}, 'votes': { - '1': {'display': '5574387', 'value': 5574387}, - '2': {'display': '9646062', 'value': 9646062} + '1': {'display': '5,574,387', 'value': 5574387}, + '2': {'display': '9,646,062', 'value': 9646062} } }, { 'timestamp': {'value': '2000-01-01'}, 'votes': { - '1': {'display': '6233385', 'value': 6233385}, - '2': {'display': '10428632', 'value': 10428632} + '1': {'display': '6,233,385', 'value': 6233385}, + '2': {'display': '10,428,632', 'value': 10428632} } }, { 'timestamp': {'value': '2004-01-01'}, 'votes': { - '1': {'display': '7359621', 'value': 7359621}, - '2': {'display': '12255311', 'value': 12255311} + '1': {'display': '7,359,621', 'value': 7359621}, + '2': {'display': '12,255,311', 'value': 12255311} } }, { 'timestamp': {'value': '2008-01-01'}, 'votes': { - '1': {'display': '8007961', 'value': 8007961}, - '2': {'display': '13286254', 'value': 13286254} + '1': {'display': '8,007,961', 'value': 8007961}, + '2': {'display': '13,286,254', 'value': 13286254} } }, { 'timestamp': {'value': '2012-01-01'}, 'votes': { - '1': {'display': '7877967', 'value': 7877967}, - '2': {'display': '12694243', 'value': 12694243} + '1': {'display': '7,877,967', 'value': 7877967}, + '2': {'display': '12,694,243', 'value': 12694243} } }, { 'timestamp': {'value': '2016-01-01'}, 'votes': { - '1': {'display': '5072915', 'value': 5072915}, - '2': {'display': '13237598', 'value': 13237598} + '1': {'display': '5,072,915', 'value': 5072915}, + '2': {'display': '13,237,598', 'value': 13237598} } }], }, result) @@ -732,53 +732,53 @@ def test_time_series_ref(self): 'data': [{ 'timestamp': {'value': '2000-01-01'}, 'state': {'display': 'Texas', 'value': 1}, - 'votes': {'display': '6233385', 'value': 6233385.}, - 'votes_eoe': {'display': '5574387', 'value': 5574387.}, + 'votes': {'display': '6,233,385', 'value': 6233385.}, + 'votes_eoe': {'display': '5,574,387', 'value': 5574387.}, }, { 'timestamp': {'value': '2000-01-01'}, 'state': {'display': 'California', 'value': 2}, - 'votes': {'display': '10428632', 'value': 10428632.}, - 'votes_eoe': {'display': '9646062', 'value': 9646062.}, + 'votes': {'display': '10,428,632', 'value': 10428632.}, + 'votes_eoe': {'display': '9,646,062', 'value': 9646062.}, }, { 'timestamp': {'value': '2004-01-01'}, 'state': {'display': 'Texas', 'value': 1}, - 'votes': {'display': '7359621', 'value': 7359621.}, - 'votes_eoe': {'display': '6233385', 'value': 6233385.}, + 'votes': {'display': '7,359,621', 'value': 7359621.}, + 'votes_eoe': {'display': '6,233,385', 'value': 6233385.}, }, { 'timestamp': {'value': '2004-01-01'}, 'state': {'display': 'California', 'value': 2}, - 'votes': {'display': '12255311', 'value': 12255311.}, - 'votes_eoe': {'display': '10428632', 'value': 10428632.}, + 'votes': {'display': '12,255,311', 'value': 12255311.}, + 'votes_eoe': {'display': '10,428,632', 'value': 10428632.}, }, { 'timestamp': {'value': '2008-01-01'}, 'state': {'display': 'Texas', 'value': 1}, - 'votes': {'display': '8007961', 'value': 8007961.}, - 'votes_eoe': {'display': '7359621', 'value': 7359621.}, + 'votes': {'display': '8,007,961', 'value': 8007961.}, + 'votes_eoe': {'display': '7,359,621', 'value': 7359621.}, }, { 'timestamp': {'value': '2008-01-01'}, 'state': {'display': 'California', 'value': 2}, - 'votes': {'display': '13286254', 'value': 13286254.}, - 'votes_eoe': {'display': '12255311', 'value': 12255311.}, + 'votes': {'display': '13,286,254', 'value': 13286254.}, + 'votes_eoe': {'display': '12,255,311', 'value': 12255311.}, }, { 'timestamp': {'value': '2012-01-01'}, 'state': {'display': 'Texas', 'value': 1}, - 'votes': {'display': '7877967', 'value': 7877967.}, - 'votes_eoe': {'display': '8007961', 'value': 8007961.}, + 'votes': {'display': '7,877,967', 'value': 7877967.}, + 'votes_eoe': {'display': '8,007,961', 'value': 8007961.}, }, { 'timestamp': {'value': '2012-01-01'}, 'state': {'display': 'California', 'value': 2}, - 'votes': {'display': '12694243', 'value': 12694243.}, - 'votes_eoe': {'display': '13286254', 'value': 13286254.}, + 'votes': {'display': '12,694,243', 'value': 12694243.}, + 'votes_eoe': {'display': '13,286,254', 'value': 13286254.}, }, { 'timestamp': {'value': '2016-01-01'}, 'state': {'display': 'Texas', 'value': 1}, - 'votes': {'display': '5072915', 'value': 5072915.}, - 'votes_eoe': {'display': '7877967', 'value': 7877967.}, + 'votes': {'display': '5,072,915', 'value': 5072915.}, + 'votes_eoe': {'display': '7,877,967', 'value': 7877967.}, }, { 'timestamp': {'value': '2016-01-01'}, 'state': {'display': 'California', 'value': 2}, - 'votes': {'display': '13237598', 'value': 13237598.}, - 'votes_eoe': {'display': '12694243', 'value': 12694243.}, + 'votes': {'display': '13,237,598', 'value': 13237598.}, + 'votes_eoe': {'display': '12,694,243', 'value': 12694243.}, }], }, result) diff --git a/fireant/tests/slicer/widgets/test_pandas.py b/fireant/tests/slicer/widgets/test_pandas.py index 27a77d42..361a4ad7 100644 --- a/fireant/tests/slicer/widgets/test_pandas.py +++ b/fireant/tests/slicer/widgets/test_pandas.py @@ -219,7 +219,7 @@ def test_metric_format(self): .transform(cont_dim_df / 3, slicer, [slicer.dimensions.timestamp], []) expected = cont_dim_df.copy()[[fm('votes')]] - expected[fm('votes')] = ['${0:.2f}€'.format(x) + expected[fm('votes')] = ['${0:,.2f}€'.format(x) for x in expected[fm('votes')] / 3] expected.index.names = ['Timestamp'] expected.columns = ['Votes'] @@ -289,4 +289,4 @@ def test_inf_in_metrics_with_precision_zero(self): expected.columns = ['Wins'] expected.columns.name = 'Metrics' - pandas.testing.assert_frame_equal(expected, result) \ No newline at end of file + pandas.testing.assert_frame_equal(expected, result) diff --git a/fireant/tests/slicer/widgets/test_reacttable.py b/fireant/tests/slicer/widgets/test_reacttable.py index ddd2b7c4..243011b8 100644 --- a/fireant/tests/slicer/widgets/test_reacttable.py +++ b/fireant/tests/slicer/widgets/test_reacttable.py @@ -2,7 +2,6 @@ from fireant.slicer.widgets.reacttable import ( ReactTable, - display_value, ) from fireant.tests.slicer.mocks import ( CumSum, @@ -467,52 +466,52 @@ def test_time_series_ref(self): '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2000-01-01'}, '$m$votes': {'display': '6,233,385', 'raw': 6233385}, - '$m$votes_eoe': {'display': '5574387.0', 'raw': 5574387.0} + '$m$votes_eoe': {'display': '5,574,387', 'raw': 5574387} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2000-01-01'}, '$m$votes': {'display': '10,428,632', 'raw': 10428632}, - '$m$votes_eoe': {'display': '9646062.0', 'raw': 9646062.0} + '$m$votes_eoe': {'display': '9,646,062', 'raw': 9646062} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2004-01-01'}, '$m$votes': {'display': '7,359,621', 'raw': 7359621}, - '$m$votes_eoe': {'display': '6233385.0', 'raw': 6233385.0} + '$m$votes_eoe': {'display': '6,233,385', 'raw': 6233385} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2004-01-01'}, '$m$votes': {'display': '12,255,311', 'raw': 12255311}, - '$m$votes_eoe': {'display': '10428632.0', 'raw': 10428632.0} + '$m$votes_eoe': {'display': '10,428,632', 'raw': 10428632} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2008-01-01'}, '$m$votes': {'display': '8,007,961', 'raw': 8007961}, - '$m$votes_eoe': {'display': '7359621.0', 'raw': 7359621.0} + '$m$votes_eoe': {'display': '7,359,621', 'raw': 7359621} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2008-01-01'}, '$m$votes': {'display': '13,286,254', 'raw': 13286254}, - '$m$votes_eoe': {'display': '12255311.0', 'raw': 12255311.0} + '$m$votes_eoe': {'display': '12,255,311', 'raw': 12255311} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2012-01-01'}, '$m$votes': {'display': '7,877,967', 'raw': 7877967}, - '$m$votes_eoe': {'display': '8007961.0', 'raw': 8007961.0} + '$m$votes_eoe': {'display': '8,007,961', 'raw': 8007961} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2012-01-01'}, '$m$votes': {'display': '12,694,243', 'raw': 12694243}, - '$m$votes_eoe': {'display': '13286254.0', 'raw': 13286254.0} + '$m$votes_eoe': {'display': '13,286,254', 'raw': 13286254} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2016-01-01'}, '$m$votes': {'display': '5,072,915', 'raw': 5072915}, - '$m$votes_eoe': {'display': '7877967.0', 'raw': 7877967.0} + '$m$votes_eoe': {'display': '7,877,967', 'raw': 7877967} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2016-01-01'}, '$m$votes': {'display': '13,237,598', 'raw': 13237598}, - '$m$votes_eoe': {'display': '12694243.0', 'raw': 12694243.0} + '$m$votes_eoe': {'display': '12,694,243', 'raw': 12694243} }] }, result) @@ -538,72 +537,72 @@ def test_time_series_ref_multiple_metrics(self): '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2000-01-01'}, '$m$votes': {'display': '6,233,385', 'raw': 6233385}, - '$m$votes_eoe': {'display': '5574387.0', 'raw': 5574387.0}, + '$m$votes_eoe': {'display': '5,574,387', 'raw': 5574387}, '$m$wins': {'display': '1', 'raw': 1}, - '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} + '$m$wins_eoe': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2000-01-01'}, '$m$votes': {'display': '10,428,632', 'raw': 10428632}, - '$m$votes_eoe': {'display': '9646062.0', 'raw': 9646062.0}, + '$m$votes_eoe': {'display': '9,646,062', 'raw': 9646062}, '$m$wins': {'display': '1', 'raw': 1}, - '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} + '$m$wins_eoe': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2004-01-01'}, '$m$votes': {'display': '7,359,621', 'raw': 7359621}, - '$m$votes_eoe': {'display': '6233385.0', 'raw': 6233385.0}, + '$m$votes_eoe': {'display': '6,233,385', 'raw': 6233385}, '$m$wins': {'display': '1', 'raw': 1}, - '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} + '$m$wins_eoe': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2004-01-01'}, '$m$votes': {'display': '12,255,311', 'raw': 12255311}, - '$m$votes_eoe': {'display': '10428632.0', 'raw': 10428632.0}, + '$m$votes_eoe': {'display': '10,428,632', 'raw': 10428632}, '$m$wins': {'display': '1', 'raw': 1}, - '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} + '$m$wins_eoe': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2008-01-01'}, '$m$votes': {'display': '8,007,961', 'raw': 8007961}, - '$m$votes_eoe': {'display': '7359621.0', 'raw': 7359621.0}, + '$m$votes_eoe': {'display': '7,359,621', 'raw': 7359621}, '$m$wins': {'display': '1', 'raw': 1}, - '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} + '$m$wins_eoe': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2008-01-01'}, '$m$votes': {'display': '13,286,254', 'raw': 13286254}, - '$m$votes_eoe': {'display': '12255311.0', 'raw': 12255311.0}, + '$m$votes_eoe': {'display': '12,255,311', 'raw': 12255311}, '$m$wins': {'display': '1', 'raw': 1}, - '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} + '$m$wins_eoe': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2012-01-01'}, '$m$votes': {'display': '7,877,967', 'raw': 7877967}, - '$m$votes_eoe': {'display': '8007961.0', 'raw': 8007961.0}, + '$m$votes_eoe': {'display': '8,007,961', 'raw': 8007961}, '$m$wins': {'display': '1', 'raw': 1}, - '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} + '$m$wins_eoe': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2012-01-01'}, '$m$votes': {'display': '12,694,243', 'raw': 12694243}, - '$m$votes_eoe': {'display': '13286254.0', 'raw': 13286254.0}, + '$m$votes_eoe': {'display': '13,286,254', 'raw': 13286254}, '$m$wins': {'display': '1', 'raw': 1}, - '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} + '$m$wins_eoe': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'Texas', 'raw': '1'}, '$d$timestamp': {'raw': '2016-01-01'}, '$m$votes': {'display': '5,072,915', 'raw': 5072915}, - '$m$votes_eoe': {'display': '7877967.0', 'raw': 7877967.0}, + '$m$votes_eoe': {'display': '7,877,967', 'raw': 7877967}, '$m$wins': {'display': '1', 'raw': 1}, - '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} + '$m$wins_eoe': {'display': '1', 'raw': 1} }, { '$d$state': {'display': 'California', 'raw': '2'}, '$d$timestamp': {'raw': '2016-01-01'}, '$m$votes': {'display': '13,237,598', 'raw': 13237598}, - '$m$votes_eoe': {'display': '12694243.0', 'raw': 12694243.0}, + '$m$votes_eoe': {'display': '12,694,243', 'raw': 12694243}, '$m$wins': {'display': '1', 'raw': 1}, - '$m$wins_eoe': {'display': '1.0', 'raw': 1.0} + '$m$wins_eoe': {'display': '1', 'raw': 1} }] }, result) @@ -618,9 +617,9 @@ def test_transpose(self): {'Header': 'Republican', 'accessor': 'r'}], 'data': [{ '$d$metrics': {'raw': 'Wins'}, - 'd': {'display': 6, 'raw': 6}, - 'i': {'display': 0, 'raw': 0}, - 'r': {'display': 6, 'raw': 6} + 'd': {'display': '6', 'raw': 6}, + 'i': {'display': '0', 'raw': 0}, + 'r': {'display': '6', 'raw': 6} }] }, result) @@ -634,28 +633,28 @@ def test_pivot_second_dimension_with_one_metric(self): {'Header': 'California', 'accessor': '2'}], 'data': [{ '$d$timestamp': {'raw': '1996-01-01'}, - '1': {'display': 1, 'raw': 1}, - '2': {'display': 1, 'raw': 1} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} }, { '$d$timestamp': {'raw': '2000-01-01'}, - '1': {'display': 1, 'raw': 1}, - '2': {'display': 1, 'raw': 1} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} }, { '$d$timestamp': {'raw': '2004-01-01'}, - '1': {'display': 1, 'raw': 1}, - '2': {'display': 1, 'raw': 1} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} }, { '$d$timestamp': {'raw': '2008-01-01'}, - '1': {'display': 1, 'raw': 1}, - '2': {'display': 1, 'raw': 1} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} }, { '$d$timestamp': {'raw': '2012-01-01'}, - '1': {'display': 1, 'raw': 1}, - '2': {'display': 1, 'raw': 1} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} }, { '$d$timestamp': {'raw': '2016-01-01'}, - '1': {'display': 1, 'raw': 1}, - '2': {'display': 1, 'raw': 1} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} }] }, result) @@ -778,16 +777,16 @@ def test_pivot_second_dimension_with_multiple_metrics_and_references(self): '2': {'display': '10,428,632', 'raw': 10428632} }, '$m$votes_eoe': { - '1': {'display': '5574387.0', 'raw': 5574387.0}, - '2': {'display': '9646062.0', 'raw': 9646062.0} + '1': {'display': '5,574,387', 'raw': 5574387}, + '2': {'display': '9,646,062', 'raw': 9646062} }, '$m$wins': { '1': {'display': '1', 'raw': 1}, '2': {'display': '1', 'raw': 1} }, '$m$wins_eoe': { - '1': {'display': '1.0', 'raw': 1.0}, - '2': {'display': '1.0', 'raw': 1.0} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} } }, { '$d$timestamp': {'raw': '2004-01-01'}, @@ -796,16 +795,16 @@ def test_pivot_second_dimension_with_multiple_metrics_and_references(self): '2': {'display': '12,255,311', 'raw': 12255311} }, '$m$votes_eoe': { - '1': {'display': '6233385.0', 'raw': 6233385.0}, - '2': {'display': '10428632.0', 'raw': 10428632.0} + '1': {'display': '6,233,385', 'raw': 6233385}, + '2': {'display': '10,428,632', 'raw': 10428632} }, '$m$wins': { '1': {'display': '1', 'raw': 1}, '2': {'display': '1', 'raw': 1} }, '$m$wins_eoe': { - '1': {'display': '1.0', 'raw': 1.0}, - '2': {'display': '1.0', 'raw': 1.0} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} } }, { '$d$timestamp': {'raw': '2008-01-01'}, @@ -814,16 +813,16 @@ def test_pivot_second_dimension_with_multiple_metrics_and_references(self): '2': {'display': '13,286,254', 'raw': 13286254} }, '$m$votes_eoe': { - '1': {'display': '7359621.0', 'raw': 7359621.0}, - '2': {'display': '12255311.0', 'raw': 12255311.0} + '1': {'display': '7,359,621', 'raw': 7359621}, + '2': {'display': '12,255,311', 'raw': 12255311} }, '$m$wins': { '1': {'display': '1', 'raw': 1}, '2': {'display': '1', 'raw': 1} }, '$m$wins_eoe': { - '1': {'display': '1.0', 'raw': 1.0}, - '2': {'display': '1.0', 'raw': 1.0} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} } }, { '$d$timestamp': {'raw': '2012-01-01'}, @@ -832,16 +831,16 @@ def test_pivot_second_dimension_with_multiple_metrics_and_references(self): '2': {'display': '12,694,243', 'raw': 12694243} }, '$m$votes_eoe': { - '1': {'display': '8007961.0', 'raw': 8007961.0}, - '2': {'display': '13286254.0', 'raw': 13286254.0} + '1': {'display': '8,007,961', 'raw': 8007961}, + '2': {'display': '13,286,254', 'raw': 13286254} }, '$m$wins': { '1': {'display': '1', 'raw': 1}, '2': {'display': '1', 'raw': 1} }, '$m$wins_eoe': { - '1': {'display': '1.0', 'raw': 1.0}, - '2': {'display': '1.0', 'raw': 1.0} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} } }, { '$d$timestamp': {'raw': '2016-01-01'}, @@ -850,16 +849,16 @@ def test_pivot_second_dimension_with_multiple_metrics_and_references(self): '2': {'display': '13,237,598', 'raw': 13237598} }, '$m$votes_eoe': { - '1': {'display': '7877967.0', 'raw': 7877967.0}, - '2': {'display': '12694243.0', 'raw': 12694243.0} + '1': {'display': '7,877,967', 'raw': 7877967}, + '2': {'display': '12,694,243', 'raw': 12694243} }, '$m$wins': { '1': {'display': '1', 'raw': 1}, '2': {'display': '1', 'raw': 1} }, '$m$wins_eoe': { - '1': {'display': '1.0', 'raw': 1.0}, - '2': {'display': '1.0', 'raw': 1.0} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1} } }] }, result) @@ -883,17 +882,17 @@ def test_pivot_single_dimension_as_rows_single_metric_metrics_automatically_pivo {'Header': 'Hillary Clinton', 'accessor': '11'}], 'data': [{ '$d$metrics': {'raw': 'Wins'}, - '1': {'display': 2, 'raw': 2}, - '10': {'display': 2, 'raw': 2}, - '11': {'display': 0, 'raw': 0}, - '2': {'display': 0, 'raw': 0}, - '3': {'display': 0, 'raw': 0}, - '4': {'display': 4, 'raw': 4}, - '5': {'display': 0, 'raw': 0}, - '6': {'display': 0, 'raw': 0}, - '7': {'display': 4, 'raw': 4}, - '8': {'display': 0, 'raw': 0}, - '9': {'display': 0, 'raw': 0} + '1': {'display': '2', 'raw': 2}, + '10': {'display': '2', 'raw': 2}, + '11': {'display': '0', 'raw': 0}, + '2': {'display': '0', 'raw': 0}, + '3': {'display': '0', 'raw': 0}, + '4': {'display': '4', 'raw': 4}, + '5': {'display': '0', 'raw': 0}, + '6': {'display': '0', 'raw': 0}, + '7': {'display': '4', 'raw': 4}, + '8': {'display': '0', 'raw': 0}, + '9': {'display': '0', 'raw': 0} }] }, result) @@ -916,17 +915,17 @@ def test_pivot_single_dimension_as_rows_single_metric_and_transpose_set_to_true( {'Header': 'Hillary Clinton', 'accessor': '11'}], 'data': [{ '$d$metrics': {'raw': 'Wins'}, - '1': {'display': 2, 'raw': 2}, - '10': {'display': 2, 'raw': 2}, - '11': {'display': 0, 'raw': 0}, - '2': {'display': 0, 'raw': 0}, - '3': {'display': 0, 'raw': 0}, - '4': {'display': 4, 'raw': 4}, - '5': {'display': 0, 'raw': 0}, - '6': {'display': 0, 'raw': 0}, - '7': {'display': 4, 'raw': 4}, - '8': {'display': 0, 'raw': 0}, - '9': {'display': 0, 'raw': 0} + '1': {'display': '2', 'raw': 2}, + '10': {'display': '2', 'raw': 2}, + '11': {'display': '0', 'raw': 0}, + '2': {'display': '0', 'raw': 0}, + '3': {'display': '0', 'raw': 0}, + '4': {'display': '4', 'raw': 4}, + '5': {'display': '0', 'raw': 0}, + '6': {'display': '0', 'raw': 0}, + '7': {'display': '4', 'raw': 4}, + '8': {'display': '0', 'raw': 0}, + '9': {'display': '0', 'raw': 0} }] }, result) @@ -950,30 +949,30 @@ def test_pivot_single_dimension_as_rows_multiple_metrics(self): {'Header': 'Hillary Clinton', 'accessor': '11'}], 'data': [{ '$d$metrics': {'raw': 'Wins'}, - '1': {'display': 2, 'raw': 2}, - '10': {'display': 2, 'raw': 2}, - '11': {'display': 0, 'raw': 0}, - '2': {'display': 0, 'raw': 0}, - '3': {'display': 0, 'raw': 0}, - '4': {'display': 4, 'raw': 4}, - '5': {'display': 0, 'raw': 0}, - '6': {'display': 0, 'raw': 0}, - '7': {'display': 4, 'raw': 4}, - '8': {'display': 0, 'raw': 0}, - '9': {'display': 0, 'raw': 0} + '1': {'display': '2', 'raw': 2}, + '10': {'display': '2', 'raw': 2}, + '11': {'display': '0', 'raw': 0}, + '2': {'display': '0', 'raw': 0}, + '3': {'display': '0', 'raw': 0}, + '4': {'display': '4', 'raw': 4}, + '5': {'display': '0', 'raw': 0}, + '6': {'display': '0', 'raw': 0}, + '7': {'display': '4', 'raw': 4}, + '8': {'display': '0', 'raw': 0}, + '9': {'display': '0', 'raw': 0} }, { '$d$metrics': {'raw': 'Votes'}, - '1': {'display': 7579518, 'raw': 7579518}, - '10': {'display': 13438835, 'raw': 13438835}, - '11': {'display': 4871678, 'raw': 4871678}, - '2': {'display': 6564547, 'raw': 6564547}, - '3': {'display': 1076384, 'raw': 1076384}, - '4': {'display': 18403811, 'raw': 18403811}, - '5': {'display': 8294949, 'raw': 8294949}, - '6': {'display': 9578189, 'raw': 9578189}, - '7': {'display': 24227234, 'raw': 24227234}, - '8': {'display': 9491109, 'raw': 9491109}, - '9': {'display': 8148082, 'raw': 8148082} + '1': {'display': '7,579,518', 'raw': 7579518}, + '10': {'display': '13,438,835', 'raw': 13438835}, + '11': {'display': '4,871,678', 'raw': 4871678}, + '2': {'display': '6,564,547', 'raw': 6564547}, + '3': {'display': '1,076,384', 'raw': 1076384}, + '4': {'display': '18,403,811', 'raw': 18403811}, + '5': {'display': '8,294,949', 'raw': 8294949}, + '6': {'display': '9,578,189', 'raw': 9578189}, + '7': {'display': '24,227,234', 'raw': 24227234}, + '8': {'display': '9,491,109', 'raw': 9491109}, + '9': {'display': '8,148,082', 'raw': 8148082} }] }, result) @@ -1035,74 +1034,74 @@ def test_pivot_multi_dims_with_all_levels_totals(self): 'data': [{ '$d$timestamp': {'raw': '1996-01-01'}, '$m$votes': { - '1': {'display': '5574387.0', 'raw': 5574387.0}, - '2': {'display': '9646062.0', 'raw': 9646062.0}, - 'totals': {'display': '15220449.0', 'raw': 15220449.0} + '1': {'display': '5,574,387', 'raw': 5574387}, + '2': {'display': '9,646,062', 'raw': 9646062}, + 'totals': {'display': '15,220,449', 'raw': 15220449} }, '$m$wins': { - '1': {'display': '1.0', 'raw': 1.0}, - '2': {'display': '1.0', 'raw': 1.0}, - 'totals': {'display': '2.0', 'raw': 2.0} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1}, + 'totals': {'display': '2', 'raw': 2} } }, { '$d$timestamp': {'raw': '2000-01-01'}, '$m$votes': { - '1': {'display': '6233385.0', 'raw': 6233385.0}, - '2': {'display': '10428632.0', 'raw': 10428632.0}, - 'totals': {'display': '16662017.0', 'raw': 16662017.0} + '1': {'display': '6,233,385', 'raw': 6233385}, + '2': {'display': '10,428,632', 'raw': 10428632}, + 'totals': {'display': '16,662,017', 'raw': 16662017} }, '$m$wins': { - '1': {'display': '1.0', 'raw': 1.0}, - '2': {'display': '1.0', 'raw': 1.0}, - 'totals': {'display': '2.0', 'raw': 2.0} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1}, + 'totals': {'display': '2', 'raw': 2} } }, { '$d$timestamp': {'raw': '2004-01-01'}, '$m$votes': { - '1': {'display': '7359621.0', 'raw': 7359621.0}, - '2': {'display': '12255311.0', 'raw': 12255311.0}, - 'totals': {'display': '19614932.0', 'raw': 19614932.0} + '1': {'display': '7,359,621', 'raw': 7359621}, + '2': {'display': '12,255,311', 'raw': 12255311}, + 'totals': {'display': '19,614,932', 'raw': 19614932} }, '$m$wins': { - '1': {'display': '1.0', 'raw': 1.0}, - '2': {'display': '1.0', 'raw': 1.0}, - 'totals': {'display': '2.0', 'raw': 2.0} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1}, + 'totals': {'display': '2', 'raw': 2} } }, { '$d$timestamp': {'raw': '2008-01-01'}, '$m$votes': { - '1': {'display': '8007961.0', 'raw': 8007961.0}, - '2': {'display': '13286254.0', 'raw': 13286254.0}, - 'totals': {'display': '21294215.0', 'raw': 21294215.0} + '1': {'display': '8,007,961', 'raw': 8007961}, + '2': {'display': '13,286,254', 'raw': 13286254}, + 'totals': {'display': '21,294,215', 'raw': 21294215} }, '$m$wins': { - '1': {'display': '1.0', 'raw': 1.0}, - '2': {'display': '1.0', 'raw': 1.0}, - 'totals': {'display': '2.0', 'raw': 2.0} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1}, + 'totals': {'display': '2', 'raw': 2} } }, { '$d$timestamp': {'raw': '2012-01-01'}, '$m$votes': { - '1': {'display': '7877967.0', 'raw': 7877967.0}, - '2': {'display': '12694243.0', 'raw': 12694243.0}, - 'totals': {'display': '20572210.0', 'raw': 20572210.0} + '1': {'display': '7,877,967', 'raw': 7877967}, + '2': {'display': '12,694,243', 'raw': 12694243}, + 'totals': {'display': '20,572,210', 'raw': 20572210} }, '$m$wins': { - '1': {'display': '1.0', 'raw': 1.0}, - '2': {'display': '1.0', 'raw': 1.0}, - 'totals': {'display': '2.0', 'raw': 2.0} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1}, + 'totals': {'display': '2', 'raw': 2} } }, { '$d$timestamp': {'raw': '2016-01-01'}, '$m$votes': { - '1': {'display': '5072915.0', 'raw': 5072915.0}, - '2': {'display': '13237598.0', 'raw': 13237598.0}, - 'totals': {'display': '18310513.0', 'raw': 18310513.0} + '1': {'display': '5,072,915', 'raw': 5072915}, + '2': {'display': '13,237,598', 'raw': 13237598}, + 'totals': {'display': '18,310,513', 'raw': 18310513} }, '$m$wins': { - '1': {'display': '1.0', 'raw': 1.0}, - '2': {'display': '1.0', 'raw': 1.0}, - 'totals': {'display': '2.0', 'raw': 2.0} + '1': {'display': '1', 'raw': 1}, + '2': {'display': '1', 'raw': 1}, + 'totals': {'display': '2', 'raw': 2} } }, { '$d$timestamp': {'raw': 'Totals'}, @@ -1110,14 +1109,14 @@ def test_pivot_multi_dims_with_all_levels_totals(self): '1': {'display': '', 'raw': ''}, '2': {'display': '', 'raw': ''}, 'totals': { - 'display': '111674336.0', - 'raw': 111674336.0 + 'display': '111,674,336', + 'raw': 111674336 } }, '$m$wins': { '1': {'display': '', 'raw': ''}, '2': {'display': '', 'raw': ''}, - 'totals': {'display': '12.0', 'raw': 12.0} + 'totals': {'display': '12', 'raw': 12} } }] }, result) @@ -1145,63 +1144,63 @@ def test_pivot_first_dimension_and_transpose_with_all_levels_totals(self): 'data': [{ '$d$metrics': {'raw': 'Wins'}, '$d$state': {'display': 'Texas', 'raw': '1'}, - '1996-01-01': {'display': 1.0, 'raw': 1.0}, - '2000-01-01': {'display': 1.0, 'raw': 1.0}, - '2004-01-01': {'display': 1.0, 'raw': 1.0}, - '2008-01-01': {'display': 1.0, 'raw': 1.0}, - '2012-01-01': {'display': 1.0, 'raw': 1.0}, - '2016-01-01': {'display': 1.0, 'raw': 1.0}, + '1996-01-01': {'display': '1', 'raw': 1}, + '2000-01-01': {'display': '1', 'raw': 1}, + '2004-01-01': {'display': '1', 'raw': 1}, + '2008-01-01': {'display': '1', 'raw': 1}, + '2012-01-01': {'display': '1', 'raw': 1}, + '2016-01-01': {'display': '1', 'raw': 1}, 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Wins'}, '$d$state': {'display': 'California', 'raw': '2'}, - '1996-01-01': {'display': 1.0, 'raw': 1.0}, - '2000-01-01': {'display': 1.0, 'raw': 1.0}, - '2004-01-01': {'display': 1.0, 'raw': 1.0}, - '2008-01-01': {'display': 1.0, 'raw': 1.0}, - '2012-01-01': {'display': 1.0, 'raw': 1.0}, - '2016-01-01': {'display': 1.0, 'raw': 1.0}, + '1996-01-01': {'display': '1', 'raw': 1}, + '2000-01-01': {'display': '1', 'raw': 1}, + '2004-01-01': {'display': '1', 'raw': 1}, + '2008-01-01': {'display': '1', 'raw': 1}, + '2012-01-01': {'display': '1', 'raw': 1}, + '2016-01-01': {'display': '1', 'raw': 1}, 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Wins'}, '$d$state': {'raw': 'Totals'}, - '1996-01-01': {'display': 2.0, 'raw': 2.0}, - '2000-01-01': {'display': 2.0, 'raw': 2.0}, - '2004-01-01': {'display': 2.0, 'raw': 2.0}, - '2008-01-01': {'display': 2.0, 'raw': 2.0}, - '2012-01-01': {'display': 2.0, 'raw': 2.0}, - '2016-01-01': {'display': 2.0, 'raw': 2.0}, - 'totals': {'display': '12.0', 'raw': 12.0} + '1996-01-01': {'display': '2', 'raw': 2}, + '2000-01-01': {'display': '2', 'raw': 2}, + '2004-01-01': {'display': '2', 'raw': 2}, + '2008-01-01': {'display': '2', 'raw': 2}, + '2012-01-01': {'display': '2', 'raw': 2}, + '2016-01-01': {'display': '2', 'raw': 2}, + 'totals': {'display': '12', 'raw': 12} }, { '$d$metrics': {'raw': 'Votes'}, '$d$state': {'display': 'Texas', 'raw': '1'}, - '1996-01-01': {'display': 5574387.0, 'raw': 5574387.0}, - '2000-01-01': {'display': 6233385.0, 'raw': 6233385.0}, - '2004-01-01': {'display': 7359621.0, 'raw': 7359621.0}, - '2008-01-01': {'display': 8007961.0, 'raw': 8007961.0}, - '2012-01-01': {'display': 7877967.0, 'raw': 7877967.0}, - '2016-01-01': {'display': 5072915.0, 'raw': 5072915.0}, + '1996-01-01': {'display': '5,574,387', 'raw': 5574387}, + '2000-01-01': {'display': '6,233,385', 'raw': 6233385}, + '2004-01-01': {'display': '7,359,621', 'raw': 7359621}, + '2008-01-01': {'display': '8,007,961', 'raw': 8007961}, + '2012-01-01': {'display': '7,877,967', 'raw': 7877967}, + '2016-01-01': {'display': '5,072,915', 'raw': 5072915}, 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Votes'}, '$d$state': {'display': 'California', 'raw': '2'}, - '1996-01-01': {'display': 9646062.0, 'raw': 9646062.0}, - '2000-01-01': {'display': 10428632.0, 'raw': 10428632.0}, - '2004-01-01': {'display': 12255311.0, 'raw': 12255311.0}, - '2008-01-01': {'display': 13286254.0, 'raw': 13286254.0}, - '2012-01-01': {'display': 12694243.0, 'raw': 12694243.0}, - '2016-01-01': {'display': 13237598.0, 'raw': 13237598.0}, + '1996-01-01': {'display': '9,646,062', 'raw': 9646062}, + '2000-01-01': {'display': '10,428,632', 'raw': 10428632}, + '2004-01-01': {'display': '12,255,311', 'raw': 12255311}, + '2008-01-01': {'display': '13,286,254', 'raw': 13286254}, + '2012-01-01': {'display': '12,694,243', 'raw': 12694243}, + '2016-01-01': {'display': '13,237,598', 'raw': 13237598}, 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Votes'}, '$d$state': {'raw': 'Totals'}, - '1996-01-01': {'display': 15220449.0, 'raw': 15220449.0}, - '2000-01-01': {'display': 16662017.0, 'raw': 16662017.0}, - '2004-01-01': {'display': 19614932.0, 'raw': 19614932.0}, - '2008-01-01': {'display': 21294215.0, 'raw': 21294215.0}, - '2012-01-01': {'display': 20572210.0, 'raw': 20572210.0}, - '2016-01-01': {'display': 18310513.0, 'raw': 18310513.0}, - 'totals': {'display': '111674336.0', 'raw': 111674336.0} + '1996-01-01': {'display': '15,220,449', 'raw': 15220449}, + '2000-01-01': {'display': '16,662,017', 'raw': 16662017}, + '2004-01-01': {'display': '19,614,932', 'raw': 19614932}, + '2008-01-01': {'display': '21,294,215', 'raw': 21294215}, + '2012-01-01': {'display': '20,572,210', 'raw': 20572210}, + '2016-01-01': {'display': '18,310,513', 'raw': 18310513}, + 'totals': {'display': '111,674,336', 'raw': 111674336} }] }, result) @@ -1228,148 +1227,62 @@ def test_pivot_second_dimension_and_transpose_with_all_levels_totals(self): 'data': [{ '$d$metrics': {'raw': 'Wins'}, '$d$state': {'display': 'Texas', 'raw': '1'}, - '1996-01-01': {'display': 1.0, 'raw': 1.0}, - '2000-01-01': {'display': 1.0, 'raw': 1.0}, - '2004-01-01': {'display': 1.0, 'raw': 1.0}, - '2008-01-01': {'display': 1.0, 'raw': 1.0}, - '2012-01-01': {'display': 1.0, 'raw': 1.0}, - '2016-01-01': {'display': 1.0, 'raw': 1.0}, + '1996-01-01': {'display': '1', 'raw': 1}, + '2000-01-01': {'display': '1', 'raw': 1}, + '2004-01-01': {'display': '1', 'raw': 1}, + '2008-01-01': {'display': '1', 'raw': 1}, + '2012-01-01': {'display': '1', 'raw': 1}, + '2016-01-01': {'display': '1', 'raw': 1}, 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Wins'}, '$d$state': {'display': 'California', 'raw': '2'}, - '1996-01-01': {'display': 1.0, 'raw': 1.0}, - '2000-01-01': {'display': 1.0, 'raw': 1.0}, - '2004-01-01': {'display': 1.0, 'raw': 1.0}, - '2008-01-01': {'display': 1.0, 'raw': 1.0}, - '2012-01-01': {'display': 1.0, 'raw': 1.0}, - '2016-01-01': {'display': 1.0, 'raw': 1.0}, + '1996-01-01': {'display': '1', 'raw': 1}, + '2000-01-01': {'display': '1', 'raw': 1}, + '2004-01-01': {'display': '1', 'raw': 1}, + '2008-01-01': {'display': '1', 'raw': 1}, + '2012-01-01': {'display': '1', 'raw': 1}, + '2016-01-01': {'display': '1', 'raw': 1}, 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Wins'}, '$d$state': {'raw': 'Totals'}, - '1996-01-01': {'display': 2.0, 'raw': 2.0}, - '2000-01-01': {'display': 2.0, 'raw': 2.0}, - '2004-01-01': {'display': 2.0, 'raw': 2.0}, - '2008-01-01': {'display': 2.0, 'raw': 2.0}, - '2012-01-01': {'display': 2.0, 'raw': 2.0}, - '2016-01-01': {'display': 2.0, 'raw': 2.0}, - 'totals': {'display': '12.0', 'raw': 12.0} + '1996-01-01': {'display': '2', 'raw': 2}, + '2000-01-01': {'display': '2', 'raw': 2}, + '2004-01-01': {'display': '2', 'raw': 2}, + '2008-01-01': {'display': '2', 'raw': 2}, + '2012-01-01': {'display': '2', 'raw': 2}, + '2016-01-01': {'display': '2', 'raw': 2}, + 'totals': {'display': '12', 'raw': 12} }, { '$d$metrics': {'raw': 'Votes'}, '$d$state': {'display': 'Texas', 'raw': '1'}, - '1996-01-01': {'display': 5574387.0, 'raw': 5574387.0}, - '2000-01-01': {'display': 6233385.0, 'raw': 6233385.0}, - '2004-01-01': {'display': 7359621.0, 'raw': 7359621.0}, - '2008-01-01': {'display': 8007961.0, 'raw': 8007961.0}, - '2012-01-01': {'display': 7877967.0, 'raw': 7877967.0}, - '2016-01-01': {'display': 5072915.0, 'raw': 5072915.0}, + '1996-01-01': {'display': '5,574,387', 'raw': 5574387}, + '2000-01-01': {'display': '6,233,385', 'raw': 6233385}, + '2004-01-01': {'display': '7,359,621', 'raw': 7359621}, + '2008-01-01': {'display': '8,007,961', 'raw': 8007961}, + '2012-01-01': {'display': '7,877,967', 'raw': 7877967}, + '2016-01-01': {'display': '5,072,915', 'raw': 5072915}, 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Votes'}, '$d$state': {'display': 'California', 'raw': '2'}, - '1996-01-01': {'display': 9646062.0, 'raw': 9646062.0}, - '2000-01-01': {'display': 10428632.0, 'raw': 10428632.0}, - '2004-01-01': {'display': 12255311.0, 'raw': 12255311.0}, - '2008-01-01': {'display': 13286254.0, 'raw': 13286254.0}, - '2012-01-01': {'display': 12694243.0, 'raw': 12694243.0}, - '2016-01-01': {'display': 13237598.0, 'raw': 13237598.0}, + '1996-01-01': {'display': '9,646,062', 'raw': 9646062}, + '2000-01-01': {'display': '10,428,632', 'raw': 10428632}, + '2004-01-01': {'display': '12,255,311', 'raw': 12255311}, + '2008-01-01': {'display': '13,286,254', 'raw': 13286254}, + '2012-01-01': {'display': '12,694,243', 'raw': 12694243}, + '2016-01-01': {'display': '13,237,598', 'raw': 13237598}, 'totals': {'display': '', 'raw': ''} }, { '$d$metrics': {'raw': 'Votes'}, '$d$state': {'raw': 'Totals'}, - '1996-01-01': {'display': 15220449.0, 'raw': 15220449.0}, - '2000-01-01': {'display': 16662017.0, 'raw': 16662017.0}, - '2004-01-01': {'display': 19614932.0, 'raw': 19614932.0}, - '2008-01-01': {'display': 21294215.0, 'raw': 21294215.0}, - '2012-01-01': {'display': 20572210.0, 'raw': 20572210.0}, - '2016-01-01': {'display': 18310513.0, 'raw': 18310513.0}, - 'totals': {'display': '111674336.0', 'raw': 111674336.0} + '1996-01-01': {'display': '15,220,449', 'raw': 15220449}, + '2000-01-01': {'display': '16,662,017', 'raw': 16662017}, + '2004-01-01': {'display': '19,614,932', 'raw': 19614932}, + '2008-01-01': {'display': '21,294,215', 'raw': 21294215}, + '2012-01-01': {'display': '20,572,210', 'raw': 20572210}, + '2016-01-01': {'display': '18,310,513', 'raw': 18310513}, + 'totals': {'display': '111,674,336', 'raw': 111674336} }] }, result) - - -class ReactTableDisplayValueFormat(TestCase): - def test_str_value_no_formats(self): - display = display_value('abcdef') - self.assertEqual('abcdef', display) - - def test_bool_true_value_no_formats(self): - display = display_value(True) - self.assertEqual('true', display) - - def test_bool_false_value_no_formats(self): - display = display_value(False) - self.assertEqual('false', display) - - def test_int_value_no_formats(self): - display = display_value(12345) - self.assertEqual('12,345', display) - - def test_decimal_value_no_formats(self): - display = display_value(12345.123456789) - self.assertEqual('12345.123456789', display) - - def test_str_value_with_prefix(self): - display = display_value('abcdef', prefix='$') - self.assertEqual('$abcdef', display) - - def test_bool_true_value_with_prefix(self): - display = display_value(True, prefix='$') - self.assertEqual('$true', display) - - def test_bool_false_value_with_prefix(self): - display = display_value(False, prefix='$') - self.assertEqual('$false', display) - - def test_int_value_with_prefix(self): - display = display_value(12345, prefix='$') - self.assertEqual('$12,345', display) - - def test_decimal_value_with_prefix(self): - display = display_value(12345.123456789, prefix='$') - self.assertEqual('$12345.123456789', display) - - def test_str_value_with_suffix(self): - display = display_value('abcdef', suffix='€') - self.assertEqual('abcdef€', display) - - def test_bool_true_value_with_suffix(self): - display = display_value(True, suffix='€') - self.assertEqual('true€', display) - - def test_bool_false_value_with_suffix(self): - display = display_value(False, suffix='€') - self.assertEqual('false€', display) - - def test_int_value_with_suffix(self): - display = display_value(12345, suffix='€') - self.assertEqual('12,345€', display) - - def test_decimal_value_with_suffix(self): - display = display_value(12345.123456789, suffix='€') - self.assertEqual('12345.123456789€', display) - - def test_str_value_with_precision(self): - display = display_value('abcdef', precision=2) - self.assertEqual('abcdef', display) - - def test_bool_true_value_with_precision(self): - display = display_value(True, precision=2) - self.assertEqual('true', display) - - def test_bool_false_value_with_precision(self): - display = display_value(False, precision=2) - self.assertEqual('false', display) - - def test_int_value_with_precision(self): - display = display_value(12345, precision=2) - self.assertEqual('12,345', display) - - def test_decimal_value_with_precision_0(self): - display = display_value(12345.123456789, precision=0) - self.assertEqual('12345', display) - - def test_decimal_value_with_precision_2(self): - display = display_value(12345.123456789, precision=2) - self.assertEqual('12345.12', display) diff --git a/fireant/tests/test_formats.py b/fireant/tests/test_formats.py index e1ae49bd..54a885f8 100644 --- a/fireant/tests/test_formats.py +++ b/fireant/tests/test_formats.py @@ -59,33 +59,97 @@ def test_timestamp_data_point_is_returned_as_string_iso_no_time(self): class DisplayValueTests(TestCase): - def test_precision_default(self): - result = formats.metric_display(0.123456789) - self.assertEqual('0.123457', result) + def test_str_value_no_formats(self): + display = formats.metric_display('abcdef') + self.assertEqual('abcdef', display) - def test_zero_precision(self): - result = formats.metric_display(0.123456789, precision=0) - self.assertEqual('0', result) + def test_bool_true_value_no_formats(self): + display = formats.metric_display(True) + self.assertEqual('true', display) - def test_precision(self): - result = formats.metric_display(0.123456789, precision=2) - self.assertEqual('0.12', result) + def test_bool_false_value_no_formats(self): + display = formats.metric_display(False) + self.assertEqual('false', display) - def test_precision_zero(self): - result = formats.metric_display(0.0) - self.assertEqual('0', result) + def test_int_value_no_formats(self): + display = formats.metric_display(12345) + self.assertEqual('12,345', display) - def test_precision_trim_trailing_zeros(self): - result = formats.metric_display(1.01) - self.assertEqual('1.01', result) + def test_decimal_value_no_formats(self): + display = formats.metric_display(12345.123456789) + self.assertEqual('12,345.123457', display) + + def test_str_value_with_prefix(self): + display = formats.metric_display('abcdef', prefix='$') + self.assertEqual('$abcdef', display) + + def test_bool_true_value_with_prefix(self): + display = formats.metric_display(True, prefix='$') + self.assertEqual('$true', display) + + def test_bool_false_value_with_prefix(self): + display = formats.metric_display(False, prefix='$') + self.assertEqual('$false', display) + + def test_int_value_with_prefix(self): + display = formats.metric_display(12345, prefix='$') + self.assertEqual('$12,345', display) + + def test_decimal_value_with_prefix(self): + display = formats.metric_display(12345.123456789, prefix='$') + self.assertEqual('$12,345.123457', display) + + def test_str_value_with_suffix(self): + display = formats.metric_display('abcdef', suffix='€') + self.assertEqual('abcdef€', display) + + def test_bool_true_value_with_suffix(self): + display = formats.metric_display(True, suffix='€') + self.assertEqual('true€', display) + + def test_bool_false_value_with_suffix(self): + display = formats.metric_display(False, suffix='€') + self.assertEqual('false€', display) - def test_prefix(self): - result = formats.metric_display(0.12, prefix='$') - self.assertEqual('$0.12', result) + def test_int_value_with_suffix(self): + display = formats.metric_display(12345, suffix='€') + self.assertEqual('12,345€', display) - def test_suffix(self): - result = formats.metric_display(0.12, suffix='€') - self.assertEqual('0.12€', result) + def test_decimal_value_with_suffix(self): + display = formats.metric_display(12345.123456789, suffix='€') + self.assertEqual('12,345.123457€', display) + + def test_str_value_with_precision(self): + display = formats.metric_display('abcdef', precision=2) + self.assertEqual('abcdef', display) + + def test_bool_true_value_with_precision(self): + display = formats.metric_display(True, precision=2) + self.assertEqual('true', display) + + def test_bool_false_value_with_precision(self): + display = formats.metric_display(False, precision=2) + self.assertEqual('false', display) + + def test_int_value_with_precision(self): + display = formats.metric_display(12345, precision=2) + self.assertEqual('12,345', display) + + def test_decimal_value_with_precision_0(self): + display = formats.metric_display(12345.123456789, precision=0) + self.assertEqual('12,345', display) + + def test_decimal_value_with_precision_2(self): + display = formats.metric_display(12345.123456789, precision=2) + self.assertEqual('12,345.12', display) + + def test_decimal_value_with_precision_9(self): + display = formats.metric_display(12345.123456789, precision=9) + self.assertEqual('12,345.123456789', display) + + def test_decimal_value_with_precision_trim_trailing_zeros(self): + result = formats.metric_display(1.01) + self.assertEqual('1.01', result) class CoerceTypeTests(TestCase): From fb4c8b73f79b117bce10d095c586cab492971d27 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 8 Aug 2018 17:18:34 +0200 Subject: [PATCH 111/123] bumped version to dev46 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index e8df5816..e8e3f4d6 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev45' +__version__ = '1.0.0.dev46' From c8dd245bb513cd2326c7d9a939a43a2216a9f2ca Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 10 Aug 2018 11:23:24 +0200 Subject: [PATCH 112/123] Fixed the formatting of NaNs, Infs, negative USD values --- fireant/formats.py | 22 ++++++++++++---- fireant/slicer/widgets/reacttable.py | 15 ++++++++--- fireant/tests/slicer/widgets/test_pandas.py | 2 +- fireant/tests/test_formats.py | 29 ++++++++++++++++++--- 4 files changed, 54 insertions(+), 14 deletions(-) diff --git a/fireant/formats.py b/fireant/formats.py index b05016a6..42de942d 100644 --- a/fireant/formats.py +++ b/fireant/formats.py @@ -1,13 +1,15 @@ -import numpy as np -import pandas as pd from datetime import ( date, datetime, time, ) -INFINITY = "Infinity" +import numpy as np +import pandas as pd + +INF_VALUE = "Inf" NULL_VALUE = 'null' +NAN_VALUE = 'NaN' TOTALS_VALUE = 'totals' RAW_VALUE = 'raw' @@ -111,12 +113,22 @@ def metric_display(value, prefix=None, suffix=None, precision=None): :return: A formatted string containing the display value for the metric. """ - if pd.isnull(value) or value in {np.inf, -np.inf}: - return '' + if pd.isnull(value): + value = NULL_VALUE + + if value in {np.inf, -np.inf}: + value = INF_VALUE + + if value in (NAN_VALUE, INF_VALUE, NULL_VALUE): + return value if isinstance(value, bool): value = str(value).lower() + if prefix == '$' and isinstance(value, (float, int)) and value < 0: + value = -value + prefix = '-$' + if isinstance(value, float): if precision is not None: value = '{:,.{precision}f}'.format(value, precision=precision) diff --git a/fireant/slicer/widgets/reacttable.py b/fireant/slicer/widgets/reacttable.py index 8b193894..cde28228 100644 --- a/fireant/slicer/widgets/reacttable.py +++ b/fireant/slicer/widgets/reacttable.py @@ -4,6 +4,8 @@ import pandas as pd from fireant.formats import ( + INF_VALUE, + NAN_VALUE, RAW_VALUE, TOTALS_VALUE, metric_display, @@ -120,7 +122,6 @@ class ReactTable(Pandas): ; ``` """ - def __init__(self, metric, *metrics: Metric, pivot=(), transpose=False, max_columns=None): super(ReactTable, self).__init__(metric, *metrics, pivot=pivot, @@ -171,7 +172,13 @@ def format_data_frame(data_frame, dimensions): for i, dimension in enumerate(dimensions): if isinstance(dimension, DatetimeDimension): date_format = DATE_FORMATS.get(dimension.interval, DATE_FORMATS[daily]) - data_frame.index = map_index_level(data_frame.index, i, lambda dt: dt.strftime(date_format)) + + def format_datetime(dt): + if pd.isnull(dt): + return TOTALS_VALUE + return dt.strftime(date_format) + + data_frame.index = map_index_level(data_frame.index, i, format_datetime) data_frame.index = fillna_index(data_frame.index, TOTALS_VALUE) data_frame.columns.name = metrics_dimension_key @@ -401,8 +408,8 @@ def transform(self, data_frame, slicer, dimensions, references): item_map[TOTALS_VALUE] = TotalsItem df = data_frame[df_dimension_columns + df_metric_columns].copy() \ - .fillna(value='NaN') \ - .replace([np.inf, -np.inf], 'Inf') + .fillna(value=NAN_VALUE) \ + .replace([np.inf, -np.inf], INF_VALUE) dimension_display_values = self.map_display_values(df, dimensions) self.format_data_frame(df, dimensions) diff --git a/fireant/tests/slicer/widgets/test_pandas.py b/fireant/tests/slicer/widgets/test_pandas.py index 361a4ad7..7f51a227 100644 --- a/fireant/tests/slicer/widgets/test_pandas.py +++ b/fireant/tests/slicer/widgets/test_pandas.py @@ -285,7 +285,7 @@ def test_inf_in_metrics_with_precision_zero(self): expected = cat_dim_df_with_nan.copy()[[fm('wins')]] expected.index = pd.Index(['Democrat', 'Independent', 'Republican'], name='Party') - expected['$m$wins'] = ['6', '0', ''] + expected['$m$wins'] = ['6', '0', 'Inf'] expected.columns = ['Wins'] expected.columns.name = 'Metrics' diff --git a/fireant/tests/test_formats.py b/fireant/tests/test_formats.py index 54a885f8..4aa6a80c 100644 --- a/fireant/tests/test_formats.py +++ b/fireant/tests/test_formats.py @@ -1,13 +1,13 @@ +from datetime import ( + date, + datetime, +) from unittest import ( TestCase, ) import numpy as np import pandas as pd -from datetime import ( - date, - datetime, -) from fireant import formats @@ -151,6 +151,27 @@ def test_decimal_value_with_precision_trim_trailing_zeros(self): result = formats.metric_display(1.01) self.assertEqual('1.01', result) + def test_nan_format_no_formatting(self): + display = formats.metric_display('NaN', prefix='$', suffix='€', precision=2) + self.assertEqual('NaN', display) + + def test_inf_format_no_formatting(self): + display = formats.metric_display('Inf', prefix='$', suffix='€', precision=2) + self.assertEqual('Inf', display) + + def test_null_format_no_formatting(self): + display = formats.metric_display('null', prefix='$', suffix='€', precision=2) + self.assertEqual('null', display) + + def test_negative_usd_float_value(self): + display = formats.metric_display(-0.12, prefix='$') + self.assertEqual('-$0.12', display) + + def test_negative_usd_int_value(self): + display = formats.metric_display(-12, prefix='$') + self.assertEqual('-$12', display) + + class CoerceTypeTests(TestCase): def allow_literal_nan(self): From 948e95dca2e8811a5b2572f8f7a47622f6f17e73 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 10 Aug 2018 11:51:58 +0200 Subject: [PATCH 113/123] bumped version to dev47 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index e8e3f4d6..7aa9be60 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev46' +__version__ = '1.0.0.dev47' From 486c512d550bd13900efd56ee8a4e6522d25bb24 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 23 Aug 2018 12:53:31 -0400 Subject: [PATCH 114/123] Added support for sorting by column to pandas, reacttable, and csv widgets --- fireant/slicer/widgets/pandas.py | 34 ++- fireant/slicer/widgets/reacttable.py | 9 +- fireant/tests/slicer/widgets/test_pandas.py | 242 ++++++++++++++++++++ 3 files changed, 278 insertions(+), 7 deletions(-) diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index 26a74296..93753c53 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -18,10 +18,13 @@ class Pandas(TransformableWidget): - def __init__(self, metric, *metrics: Metric, pivot=(), transpose=False, max_columns=None): + def __init__(self, metric, *metrics: Metric, pivot=(), transpose=False, sort=None, ascending=None, + max_columns=None): super(Pandas, self).__init__(metric, *metrics) self.pivot = pivot self.transpose = transpose + self.sort = sort + self.ascending = ascending self.max_columns = min(max_columns, HARD_MAX_COLUMNS) \ if max_columns is not None \ else HARD_MAX_COLUMNS @@ -79,8 +82,7 @@ def transform(self, data_frame, slicer, dimensions, references): return self.pivot_data_frame(result, [d.label or d.key for d in self.pivot], self.transpose) - @staticmethod - def pivot_data_frame(data_frame, pivot=(), transpose=False): + def pivot_data_frame(self, data_frame, pivot=(), transpose=False): """ Pivot and transpose the data frame. Dimensions including in the `pivot` arg will be unshifted to columns. If `transpose` is True the data frame will be transposed. If there is only index level in the data frame (ie. one @@ -98,7 +100,7 @@ def pivot_data_frame(data_frame, pivot=(), transpose=False): The shifted/transposed data frame """ if not (pivot or transpose): - return data_frame + return self.sort_data_frame(data_frame) # NOTE: Don't pivot a single dimension data frame. This turns the data frame into a series and pivots the # metrics anyway. Instead, transpose the data frame. @@ -115,7 +117,29 @@ def pivot_data_frame(data_frame, pivot=(), transpose=False): data_frame.name = data_frame.columns.levels[0][0] # capture the name of the metrics column data_frame.columns = data_frame.columns.droplevel(0) # drop the metrics level - return data_frame.fillna('') + data_frame.fillna('', inplace=True) + + return self.sort_data_frame(data_frame) + + def sort_data_frame(self, data_frame): + if self.sort is None: + return data_frame + + # reset the index so all columns can be sorted together + index_names = data_frame.index.names + unsorted = data_frame.reset_index() + + column_names = list(unsorted.columns) + + ascending = self.ascending \ + if self.ascending is not None \ + else True + + return unsorted \ + .sort_values([column_names[abs(column)] + for column in self.sort], + ascending=ascending) \ + .set_index(index_names) def _replace_display_values_in_index(self, dimension, result): """ diff --git a/fireant/slicer/widgets/reacttable.py b/fireant/slicer/widgets/reacttable.py index cde28228..fa27b4a1 100644 --- a/fireant/slicer/widgets/reacttable.py +++ b/fireant/slicer/widgets/reacttable.py @@ -122,17 +122,22 @@ class ReactTable(Pandas): ; ``` """ - def __init__(self, metric, *metrics: Metric, pivot=(), transpose=False, max_columns=None): + + def __init__(self, metric, *metrics: Metric, pivot=(), transpose=False, sort=None, ascending=None, + max_columns=None): super(ReactTable, self).__init__(metric, *metrics, pivot=pivot, transpose=transpose, + sort=sort, + ascending=ascending, max_columns=max_columns) def __repr__(self): return '{}({})'.format(self.__class__.__name__, ','.join(str(m) for m in self.items)) - def map_display_values(self, df, dimensions): + @staticmethod + def map_display_values(df, dimensions): """ WRITEME diff --git a/fireant/tests/slicer/widgets/test_pandas.py b/fireant/tests/slicer/widgets/test_pandas.py index 7f51a227..51bf58dd 100644 --- a/fireant/tests/slicer/widgets/test_pandas.py +++ b/fireant/tests/slicer/widgets/test_pandas.py @@ -290,3 +290,245 @@ def test_inf_in_metrics_with_precision_zero(self): expected.columns.name = 'Metrics' pandas.testing.assert_frame_equal(expected, result) + + +class PandasTransformerSortTests(TestCase): + def test_multiple_metrics_sort_index_asc(self): + result = Pandas(slicer.metrics.wins, sort=[0]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) + + expected = cont_dim_df.copy()[[fm('wins')]] + expected.index.names = ['Timestamp'] + expected.columns = ['Wins'] + expected.columns.name = 'Metrics' + + expected = expected.sort_index() + + pandas.testing.assert_frame_equal(expected, result) + + def test_multiple_metrics_sort_index_desc(self): + result = Pandas(slicer.metrics.wins, sort=[0], ascending=[False]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) + + expected = cont_dim_df.copy()[[fm('wins')]] + expected.index.names = ['Timestamp'] + expected.columns = ['Wins'] + expected.columns.name = 'Metrics' + + expected = expected.sort_index(ascending=False) + + pandas.testing.assert_frame_equal(expected, result) + + def test_multiple_metrics_sort_value_asc(self): + result = Pandas(slicer.metrics.wins, sort=[1]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) + + expected = cont_dim_df.copy()[[fm('wins')]] + expected.index.names = ['Timestamp'] + expected.columns = ['Wins'] + expected.columns.name = 'Metrics' + + expected = expected.sort_values(['Wins']) + + pandas.testing.assert_frame_equal(expected, result) + + def test_multiple_metrics_sort_value_desc(self): + result = Pandas(slicer.metrics.wins, sort=[1], ascending=[False]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) + + expected = cont_dim_df.copy()[[fm('wins')]] + expected.index.names = ['Timestamp'] + expected.columns = ['Wins'] + expected.columns.name = 'Metrics' + + expected = expected.sort_values(['Wins'], ascending=False) + + pandas.testing.assert_frame_equal(expected, result) + + def test_multiple_metrics_sort_index_and_value(self): + result = Pandas(slicer.metrics.wins, sort=[-0, 1]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) + + expected = cont_dim_df.copy()[[fm('wins')]] + expected.index.names = ['Timestamp'] + expected.columns = ['Wins'] + expected.columns.name = 'Metrics' + + expected = expected.reset_index() + expected = expected.sort_values(['Timestamp', 'Wins'], ascending=[True, False]) + expected = expected.set_index('Timestamp') + + pandas.testing.assert_frame_equal(expected, result) + + def test_pivoted_multi_dims_time_series_and_uni_with_sort_index_asc(self): + result = Pandas(slicer.metrics.votes, pivot=[slicer.dimensions.state], sort=[0]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) + + expected = cont_uni_dim_df.copy() \ + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=True)[[fm('votes')]] + expected = expected.unstack(level=[1]) + expected.index.names = ['Timestamp'] + expected.columns = ['California', 'Texas'] + expected.columns.names = ['State'] + + expected = expected.sort_index() + + pandas.testing.assert_frame_equal(expected, result) + + def test_pivoted_multi_dims_time_series_and_uni_with_sort_index_desc(self): + result = Pandas(slicer.metrics.votes, pivot=[slicer.dimensions.state], sort=[0], ascending=[False]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) + + expected = cont_uni_dim_df.copy() \ + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=True)[[fm('votes')]] + expected = expected.unstack(level=[1]) + expected.index.names = ['Timestamp'] + expected.columns = ['California', 'Texas'] + expected.columns.names = ['State'] + + expected = expected.sort_index(ascending=False) + + pandas.testing.assert_frame_equal(expected, result) + + def test_pivoted_multi_dims_time_series_and_uni_with_sort_first_metric_asc(self): + result = Pandas(slicer.metrics.votes, pivot=[slicer.dimensions.state], sort=[1]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) + + expected = cont_uni_dim_df.copy() \ + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=True)[[fm('votes')]] + expected = expected.unstack(level=[1]) + expected.index.names = ['Timestamp'] + expected.columns = ['California', 'Texas'] + expected.columns.names = ['State'] + + expected = expected.reset_index() + expected = expected.sort_values(['California']) + expected = expected.set_index('Timestamp') + + pandas.testing.assert_frame_equal(expected, result) + + def test_pivoted_multi_dims_time_series_and_uni_with_sort_first_metric_desc(self): + result = Pandas(slicer.metrics.votes, pivot=[slicer.dimensions.state], sort=[1], ascending=[False]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) + + expected = cont_uni_dim_df.copy() \ + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=True)[[fm('votes')]] + expected = expected.unstack(level=[1]) + expected.index.names = ['Timestamp'] + expected.columns = ['California', 'Texas'] + expected.columns.names = ['State'] + + expected = expected.reset_index() + expected = expected.sort_values(['California'], ascending=[False]) + expected = expected.set_index('Timestamp') + + pandas.testing.assert_frame_equal(expected, result) + + def test_pivoted_multi_dims_time_series_and_uni_with_sort_second_metric_asc(self): + result = Pandas(slicer.metrics.votes, pivot=[slicer.dimensions.state], sort=[2]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) + + expected = cont_uni_dim_df.copy() \ + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=True)[[fm('votes')]] + expected = expected.unstack(level=[1]) + expected.index.names = ['Timestamp'] + expected.columns = ['California', 'Texas'] + expected.columns.names = ['State'] + + expected = expected.reset_index() + expected = expected.sort_values(['Texas'], ascending=[True]) + expected = expected.set_index('Timestamp') + + pandas.testing.assert_frame_equal(expected, result) + + def test_pivoted_multi_dims_time_series_and_uni_with_sort_second_metric_desc(self): + result = Pandas(slicer.metrics.votes, pivot=[slicer.dimensions.state], sort=[2], ascending=[False]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) + + expected = cont_uni_dim_df.copy() \ + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=True)[[fm('votes')]] + expected = expected.unstack(level=[1]) + expected.index.names = ['Timestamp'] + expected.columns = ['California', 'Texas'] + expected.columns.names = ['State'] + + expected = expected.reset_index() + expected = expected.sort_values(['Texas'], ascending=[False]) + expected = expected.set_index('Timestamp') + + pandas.testing.assert_frame_equal(expected, result) + + def test_pivoted_multi_dims_time_series_and_uni_with_sort_index_and_columns(self): + result = Pandas(slicer.metrics.votes, pivot=[slicer.dimensions.state], sort=[0, 2], ascending=[True, False]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) + + expected = cont_uni_dim_df.copy() \ + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=True)[[fm('votes')]] + expected = expected.unstack(level=[1]) + expected.index.names = ['Timestamp'] + expected.columns = ['California', 'Texas'] + expected.columns.names = ['State'] + + expected = expected.reset_index() + expected = expected.sort_values(['Timestamp', 'Texas'], ascending=[True, False]) + expected = expected.set_index('Timestamp') + + pandas.testing.assert_frame_equal(expected, result) + + def test_multi_dims_time_series_and_cat_sort_index_level_0_asc(self): + result = Pandas(slicer.metrics.wins, sort=[0]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) + + expected = cont_uni_dim_df.copy() \ + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=False)[[fm('wins')]] + expected.index.names = ['Timestamp', 'State'] + expected.columns = ['Wins'] + expected.columns.name = 'Metrics' + + expected = expected.reset_index() + expected = expected.sort_values(['Timestamp']) + expected = expected.set_index(['Timestamp', 'State']) + + pandas.testing.assert_frame_equal(expected, result) + + def test_pivoted_multi_dims_time_series_and_cat_sort_index_level_1_desc(self): + result = Pandas(slicer.metrics.wins, sort=[1], ascending=[False]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) + + expected = cont_uni_dim_df.copy() \ + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=False)[[fm('wins')]] + expected.index.names = ['Timestamp', 'State'] + expected.columns = ['Wins'] + expected.columns.name = 'Metrics' + + expected = expected.reset_index() + expected = expected.sort_values(['State'], ascending=[False]) + expected = expected.set_index(['Timestamp', 'State']) + + pandas.testing.assert_frame_equal(expected, result) + + def test_pivoted_multi_dims_time_series_and_cat_sort_index_and_values(self): + result = Pandas(slicer.metrics.wins, sort=[0, 2], ascending=[False, True]) \ + .transform(cont_uni_dim_df, slicer, [slicer.dimensions.timestamp, slicer.dimensions.state], []) + + expected = cont_uni_dim_df.copy() \ + .set_index(fd('state_display'), append=True) \ + .reset_index(fd('state'), drop=False)[[fm('wins')]] + expected.index.names = ['Timestamp', 'State'] + expected.columns = ['Wins'] + expected.columns.name = 'Metrics' + + expected = expected.reset_index() + expected = expected.sort_values(['Timestamp', 'Wins'], ascending=[False, True]) + expected = expected.set_index(['Timestamp', 'State']) + + pandas.testing.assert_frame_equal(expected, result) From 35acfa69292a7c04790737111caa1c3a0345fad5 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 23 Aug 2018 14:19:35 -0400 Subject: [PATCH 115/123] Fixed the replacement of display values on multiindex data frames in the pandas transformer --- fireant/slicer/widgets/pandas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index 26a74296..28b17d35 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -123,8 +123,9 @@ def _replace_display_values_in_index(self, dimension, result): """ if isinstance(result.index, pd.MultiIndex): df_key = format_dimension_key(dimension.key) + level = result.index.names.index(df_key) values = [dimension.display_values.get(x, x) - for x in result.index.get_level_values(df_key)] + for x in result.index.levels[level]] result.index.set_levels(level=df_key, levels=values, inplace=True) From a6329e0984b4554465b366c2a5d6b573aef5b54c Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 23 Aug 2018 14:22:13 -0400 Subject: [PATCH 116/123] Remove code that would change the column accessor to data for a data frame that drops the metrics column index level when only a single metric is used. The changed accessor value was incorrect, the dropped metric name index level value is not reflected in the data. --- fireant/slicer/widgets/reacttable.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/fireant/slicer/widgets/reacttable.py b/fireant/slicer/widgets/reacttable.py index fa27b4a1..9f3d5fc5 100644 --- a/fireant/slicer/widgets/reacttable.py +++ b/fireant/slicer/widgets/reacttable.py @@ -301,13 +301,6 @@ def _make_columns(columns_frame, previous_levels=()): column['columns'] = _make_columns(next_level_df, levels) else: - # If there is no group, then this is a leaf, or a column header on the bottom row of the table - # head. These are effectively the actual columns in the table. All leaf column header definitions - # require an accessor for how to acccess data the for that column - if hasattr(data_frame, 'name'): - # If the metrics column index level was dropped (due to there being a single metric), then the - # index level name will be set as the data frame's name. - levels += (data_frame.name,) column['accessor'] = '.'.join(levels) if is_totals: From 90b9456811b25a063efcc312c1c7ee3344e67bff Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Mon, 27 Aug 2018 13:12:38 +0200 Subject: [PATCH 117/123] Bumped pypika to 0.15.2 and version to dev48 --- fireant/__init__.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 7aa9be60..3e5caaee 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev47' +__version__ = '1.0.0.dev48' diff --git a/requirements.txt b/requirements.txt index bc723c45..b233b96a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ six pandas==0.22.0 -pypika==0.14.9 +pypika==0.15.2 pymysql==0.8.0 toposort==1.5 typing==3.6.2 From d0ccce78c63c1734ac830dd52a725f5205afab84 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 29 Aug 2018 16:59:19 +0200 Subject: [PATCH 118/123] Added handle for empty sort array in pandas transformer --- fireant/slicer/widgets/pandas.py | 2 +- fireant/tests/slicer/widgets/test_pandas.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index bf062825..7cc43a5b 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -122,7 +122,7 @@ def pivot_data_frame(self, data_frame, pivot=(), transpose=False): return self.sort_data_frame(data_frame) def sort_data_frame(self, data_frame): - if self.sort is None: + if not self.sort: return data_frame # reset the index so all columns can be sorted together diff --git a/fireant/tests/slicer/widgets/test_pandas.py b/fireant/tests/slicer/widgets/test_pandas.py index 51bf58dd..0180cdaa 100644 --- a/fireant/tests/slicer/widgets/test_pandas.py +++ b/fireant/tests/slicer/widgets/test_pandas.py @@ -532,3 +532,14 @@ def test_pivoted_multi_dims_time_series_and_cat_sort_index_and_values(self): expected = expected.set_index(['Timestamp', 'State']) pandas.testing.assert_frame_equal(expected, result) + + def test_empty_sort_array_is_ignored(self): + result = Pandas(slicer.metrics.wins, sort=[]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) + + expected = cont_dim_df.copy()[[fm('wins')]] + expected.index.names = ['Timestamp'] + expected.columns = ['Wins'] + expected.columns.name = 'Metrics' + + pandas.testing.assert_frame_equal(expected, result) From 029f157351cb2b3d10a56872e82a2505e3adc935 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 29 Aug 2018 17:06:56 +0200 Subject: [PATCH 119/123] Fixed react table transformer to sort with NaN values in the column by moving the fillna part after the sort --- fireant/slicer/widgets/pandas.py | 5 ++--- fireant/slicer/widgets/reacttable.py | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index bf062825..070f32e9 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -135,10 +135,9 @@ def sort_data_frame(self, data_frame): if self.ascending is not None \ else True + sort_columns = [column_names[abs(column)] for column in self.sort] return unsorted \ - .sort_values([column_names[abs(column)] - for column in self.sort], - ascending=ascending) \ + .sort_values(sort_columns, ascending=ascending) \ .set_index(index_names) def _replace_display_values_in_index(self, dimension, result): diff --git a/fireant/slicer/widgets/reacttable.py b/fireant/slicer/widgets/reacttable.py index 9f3d5fc5..3ebe88a9 100644 --- a/fireant/slicer/widgets/reacttable.py +++ b/fireant/slicer/widgets/reacttable.py @@ -405,15 +405,15 @@ def transform(self, data_frame, slicer, dimensions, references): # Add an extra item to map the totals key to it's label item_map[TOTALS_VALUE] = TotalsItem - df = data_frame[df_dimension_columns + df_metric_columns].copy() \ - .fillna(value=NAN_VALUE) \ - .replace([np.inf, -np.inf], INF_VALUE) + df = data_frame[df_dimension_columns + df_metric_columns].copy() dimension_display_values = self.map_display_values(df, dimensions) self.format_data_frame(df, dimensions) dimension_keys = [format_dimension_key(dimension.key) for dimension in self.pivot] - df = self.pivot_data_frame(df, dimension_keys, self.transpose) + df = self.pivot_data_frame(df, dimension_keys, self.transpose) \ + .fillna(value=NAN_VALUE) \ + .replace([np.inf, -np.inf], INF_VALUE) dimension_columns = self.transform_dimension_column_headers(df, dimensions) metric_columns = self.transform_metric_column_headers(df, item_map, dimension_display_values) From c73438bfe8d083a270b2510991794039f0d0f24d Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 29 Aug 2018 17:15:30 +0200 Subject: [PATCH 120/123] Fixed a case where if a null was returned for a display value, the high charts transformer would raise an unhandled exception --- fireant/slicer/widgets/helpers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/fireant/slicer/widgets/helpers.py b/fireant/slicer/widgets/helpers.py index 49bb89f3..7648efe3 100644 --- a/fireant/slicer/widgets/helpers.py +++ b/fireant/slicer/widgets/helpers.py @@ -1,6 +1,11 @@ +import numpy as np import pandas as pd from fireant import utils +from fireant.formats import ( + INF_VALUE, + NAN_VALUE, +) from ..references import reference_label @@ -33,7 +38,9 @@ def extract_display_values(dimensions, data_frame): display_values[key] = data_frame[f_display_key] \ .groupby(level=key) \ - .first() + .first() \ + .fillna(value=NAN_VALUE) \ + .replace([np.inf, -np.inf], INF_VALUE) return display_values From e82fa31f0a5fea5e1b46d1a5f2b2c752c1530a86 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Wed, 29 Aug 2018 17:34:20 +0200 Subject: [PATCH 121/123] Bumped version to dev49 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index 3e5caaee..a51cab93 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev48' +__version__ = '1.0.0.dev49' From 1a4d87283d63761f20f6fb267fe5461533a87965 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 30 Aug 2018 13:59:05 +0200 Subject: [PATCH 122/123] Changed pandas transformer to filter out invalid column indices in the sort parameter --- fireant/slicer/widgets/pandas.py | 8 +++++++- fireant/tests/slicer/widgets/test_pandas.py | 12 ++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/fireant/slicer/widgets/pandas.py b/fireant/slicer/widgets/pandas.py index 39977ef5..ad698e5a 100644 --- a/fireant/slicer/widgets/pandas.py +++ b/fireant/slicer/widgets/pandas.py @@ -135,7 +135,13 @@ def sort_data_frame(self, data_frame): if self.ascending is not None \ else True - sort_columns = [column_names[abs(column)] for column in self.sort] + sort_columns = [column_names[column] + for column in self.sort + if column < len(column_names)] + + if not sort_columns: + return data_frame + return unsorted \ .sort_values(sort_columns, ascending=ascending) \ .set_index(index_names) diff --git a/fireant/tests/slicer/widgets/test_pandas.py b/fireant/tests/slicer/widgets/test_pandas.py index 0180cdaa..cc4fe8d2 100644 --- a/fireant/tests/slicer/widgets/test_pandas.py +++ b/fireant/tests/slicer/widgets/test_pandas.py @@ -543,3 +543,15 @@ def test_empty_sort_array_is_ignored(self): expected.columns.name = 'Metrics' pandas.testing.assert_frame_equal(expected, result) + + + def test_sort_value_greater_than_number_of_columns_is_ignored(self): + result = Pandas(slicer.metrics.wins, sort=[5]) \ + .transform(cont_dim_df, slicer, [slicer.dimensions.timestamp], []) + + expected = cont_dim_df.copy()[[fm('wins')]] + expected.index.names = ['Timestamp'] + expected.columns = ['Wins'] + expected.columns.name = 'Metrics' + + pandas.testing.assert_frame_equal(expected, result) From 607c659b788dfbfcc1d65cd840ea1a9029fb9c6b Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Thu, 30 Aug 2018 14:29:29 +0200 Subject: [PATCH 123/123] bumped version to dev50 --- fireant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fireant/__init__.py b/fireant/__init__.py index a51cab93..b49a2b90 100644 --- a/fireant/__init__.py +++ b/fireant/__init__.py @@ -5,4 +5,4 @@ # noinspection PyUnresolvedReferences from .slicer.widgets import * -__version__ = '1.0.0.dev49' +__version__ = '1.0.0.dev50'