From fdf2037ca7cc6d0a4e2b1187c7ae399adebc163f Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 12 Aug 2016 14:54:56 +0200 Subject: [PATCH 1/4] Refactored transformers into groups on the Slicer instead of all in the slicer manager so that each group has its own manager on the slicer --- README.rst | 24 ++- docs/1_setup.rst | 27 ++- docs/2_slicer.rst | 186 ++++++------------ docs/3_dashboard.rst | 44 +---- fireant/dashboards/schemas.py | 13 +- fireant/settings.py | 1 + fireant/slicer/managers.py | 37 ++-- fireant/slicer/schemas.py | 5 +- fireant/slicer/transformers/__init__.py | 7 +- fireant/slicer/transformers/bundles.py | 32 +++ fireant/slicer/transformers/datatables.py | 41 ++-- fireant/slicer/transformers/highcharts.py | 29 ++- fireant/slicer/transformers/notebook.py | 12 ++ fireant/tests/dashboards/test_widgets.py | 23 +-- fireant/tests/slicer/test_manager.py | 15 ++ fireant/tests/slicer/transformers/test_csv.py | 6 +- .../slicer/transformers/test_datatables.py | 14 +- .../slicer/transformers/test_highcharts.py | 32 +-- setup.py | 9 +- 19 files changed, 281 insertions(+), 276 deletions(-) create mode 100644 fireant/slicer/transformers/bundles.py create mode 100644 fireant/slicer/transformers/notebook.py create mode 100644 fireant/tests/slicer/test_manager.py diff --git a/README.rst b/README.rst index 743b76ea..3db17a49 100644 --- a/README.rst +++ b/README.rst @@ -103,15 +103,27 @@ To configure a slicer, instantiate a |ClassSlicer| with a list of |ClassMetric| Querying Data and Rendering Charts ---------------------------------- -Once a slicer is configured, it is ready to be used. Each slicer comes with a |ClassSlicerManager| which exposes an interface for executing queries and transforming the results. Each function in the manager uses the same signature. The principal function is ``data`` and all othe functions call this function first. The additional functions provide a transformation to the data. +Once a slicer is configured, it is ready to be used. Each slicer comes with a |ClassSlicerManager| and several |ClassTransformerManager| which expose an interface for executing queries and transforming the results. Each function in the manager uses the same signature. The principal function is ``data`` and all othe functions call this function first. The additional functions provide a transformation to the data. + +The notebook transformer bundle includes different functions for use in Jupyter_ notebooks. Other formats return results in JSON format. .. _manager_api_start: * ``my_slicer.manager.data`` - A Pandas_ data frame indexed by the selected dimensions. -* ``my_slicer.manager.line_chart`` - A Highcharts_ line chart. -* ``my_slicer.manager.bar_chart`` - A Highcharts_ bar chart. -* ``my_slicer.manager.row_index_table`` - A Datatables_ row-indexed table. -* ``my_slicer.manager.column_index_table`` - A Datatables_ column-indexed table. + +* ``my_slicer.notebook.row_index_table`` - A Datatables_ row-indexed table. +* ``my_slicer.notebook.column_index_table`` - A Datatables_ column-indexed table. + +* ``my_slicer.notebook.line_chart`` - A Matplotlib_ line chart. (Requires [matplotlib] dependency) +* ``my_slicer.notebook.column_chart`` - A Matplotlib_ column chart. (Requires [matplotlib] dependency) +* ``my_slicer.notebook.bar_chart`` - A Matplotlib_ bar chart. (Requires [matplotlib] dependency) + +* ``my_slicer.highcharts.line_chart`` - A Highcharts_ line chart. +* ``my_slicer.highcharts.column_chart`` - A Highcharts_ column chart. +* ``my_slicer.highcharts.bar_chart`` - A Highcharts_ bar chart. + +* ``my_slicer.datatables.row_index_table`` - A Datatables_ row-indexed table. +* ``my_slicer.datatables.column_index_table`` - A Datatables_ column-indexed table. .. code-block:: python @@ -251,6 +263,8 @@ limitations under the License. .. _PyPika: https://github.com/kayak/pypika/ .. _Pandas: http://pandas.pydata.org/ +.. _Jupyter: http://jupyter.org/ +.. _Matplotlib: http://matplotlib.org/ .. _Highcharts: http://www.highcharts.com/ .. _Datatables: https://datatables.net/ diff --git a/docs/1_setup.rst b/docs/1_setup.rst index 6547afdc..b7369fb6 100644 --- a/docs/1_setup.rst +++ b/docs/1_setup.rst @@ -6,22 +6,34 @@ Installation and Setup :end-before: _installation_end: +Database Connector add-ons +-------------------------- + + By default, |Brand| does not include any database drivers. You can optionally include one or provide your own. Vertica -""""""" .. code-block:: bash pip install fireant[vertica] +Transformer add-ons +------------------- + +There are also optional transformer packages which give access to different widget libraries. This only applies to transformers that require additional packages. All other transformers are included by default. + + +matplotlib + +.. code-block:: bash + + pip install fireant[matplotlib] + -Once you have added |Brand| to your project, you must provide some additional settings. A database connection is -required in order to execute queries. Currently, only Vertica is supported via ``vertica_python``, however future plans -include support for various other databases such as MySQL and Oracle. +Once you have added |Brand| to your project, you must provide some additional settings. A database connection is required in order to execute queries. Currently, only Vertica is supported via ``vertica_python``, however future plans include support for various other databases such as MySQL and Oracle. -To configure a database, instantiate a subclass of |ClassDatabase| and set it in ``fireant.settings``. This -must be only set once. At the present, only one database connection is supported. +To configure a database, instantiate a subclass of |ClassDatabase| and set it in ``fireant.settings``. This must be only set once. At the present, only one database connection is supported. .. code-block:: python @@ -79,8 +91,7 @@ Instead of using one of the built in database connectors, you can provide your o password='password123', ) -In a custom database connector, the ``connect`` function must be overridden to provide a ``connection`` to the database. -The ``round_date`` function must also be overridden since there is no common way to round dates in SQL databases. +In a custom database connector, the ``connect`` function must be overridden to provide a ``connection`` to the database. The ``round_date`` function must also be overridden since there is no common way to round dates in SQL databases. diff --git a/docs/2_slicer.rst b/docs/2_slicer.rst index 2958cc9a..96e56c37 100644 --- a/docs/2_slicer.rst +++ b/docs/2_slicer.rst @@ -5,80 +5,50 @@ The Slicer :start-after: _appendix_start: :end-before: _appendix_end: -The |FeatureSlicer| is the core component of |Brand| which defines a schema and an API. With a small amount of -configuration, it becomes a powerful tool which can be used to quickly query data from a database and transform it into -a chart, table, or other widget. The slicer can be used directly in python in Notebooks or in a python shell and -provides a rich API for building quick, ad hoc queries. The |FeatureSlicer| underlies the |FeatureWidgetGroup| feature, -providing a simple abstraction for building complex reports and dashboards. +The |FeatureSlicer| is the core component of |Brand| which defines a schema and an API. With a small amount of configuration, it becomes a powerful tool which can be used to quickly query data from a database and transform it into a chart, table, or other widget. The slicer can be used directly in python in Notebooks or in a python shell and provides a rich API for building quick, ad hoc queries. The |FeatureSlicer| underlies the |FeatureWidgetGroup| feature, providing a simple abstraction for building complex reports and dashboards. Metrics ------- -Creating a |FeatureSlicer| involves extending the |ClassSlicer| class with a table, a set of |ClassMetric|, and -a set of |ClassDimension|. The table identifies which table the |FeatureSlicer| will primarily be accessing, -although additional tables can be joined as well to support many different possibilities. Next, a list of -|ClassMetric| must be supplied which represent the data that should be accessible through the slicer. These can be -defined as columns in your table as well as arithmetic expressions or aggregate SQL functions. |ClassMetric| -definitions are expressed using PyPika_. +Creating a |FeatureSlicer| involves extending the |ClassSlicer| class with a table, a set of |ClassMetric|, and a set of |ClassDimension|. The table identifies which table the |FeatureSlicer| will primarily be accessing, although additional tables can be joined as well to support many different possibilities. Next, a list of |ClassMetric| must be supplied which represent the data that should be accessible through the slicer. These can be defined as columns in your table as well as arithmetic expressions or aggregate SQL functions. |ClassMetric| definitions are expressed using PyPika_. .. note:: - When defining a |ClassMetric|, it is important to note that all queries executed by fireant are aggregated over - the dimensions (via a ``GROUP BY`` clause in the SQL query) and therefore are required to use aggregation functions. - By default, a |ClassMetric| will use the ``SUM`` function and it's ``key``. A custom definition is commonly required - and must use a SQL aggregate function over any columns. + When defining a |ClassMetric|, it is important to note that all queries executed by fireant are aggregated over the dimensions (via a ``GROUP BY`` clause in the SQL query) and therefore are required to use aggregation functions. By default, a |ClassMetric| will use the ``SUM`` function and it's ``key``. A custom definition is commonly required and must use a SQL aggregate function over any columns. Dimensions ---------- -Next a set of |ClassDimension| must be provided. A |ClassDimension| is a value that the data can be grouped by when -aggregated. For example in a line chart, a ``TIMESTAMP`` would be commonly used as |ClassDimension| to represent the -X-Axis in the chart. Subsequently, additional |ClassDimension| could be used to plot multiple curves for the same value -in comparison. When rendering tables, dimensions function as the indices and with multiple |ClassDimension| can be -used to create row indexes, which display a row in the table for each unique combination across all of the dimensions, -or column indexes, which pivot the table all but the first dimensions to make unique columns for side-by-side -comparison. +Next a set of |ClassDimension| must be provided. A |ClassDimension| is a value that the data can be grouped by when aggregated. For example in a line chart, a ``TIMESTAMP`` would be commonly used as |ClassDimension| to represent the X-Axis in the chart. Subsequently, additional |ClassDimension| could be used to plot multiple curves for the same value in comparison. When rendering tables, dimensions function as the indices and with multiple |ClassDimension| can be used to create row indexes, which display a row in the table for each unique combination across all of the dimensions, or column indexes, which pivot the table all but the first dimensions to make unique columns for side-by-side comparison. There are different types of dimensions and choosing one depends on some properties of the column. Categorical Dimensions """""""""""""""""""""" -A categorical dimension represents a column that contains one value from a finite set, such as a member of an -enumeration. This is the most common type of dimension. For example, a `color` column could be used that contains -values such as `red`, `green`, and `blue`. Aggregating metrics with this dimension will give a data set grouped by -each color. +A categorical dimension represents a column that contains one value from a finite set, such as a member of an enumeration. This is the most common type of dimension. For example, a `color` column could be used that contains values such as `red`, `green`, and `blue`. Aggregating metrics with this dimension will give a data set grouped by each color. Continuous Dimensions """"""""""""""""""""" -For dimensions that do not have a finite set of values, a continuous dimension can be used. This is especially useful -for numeric values that can be viewed with varied precision, such as a decimal value. Continuous Dimensions require -an additional parameter for an ``Interval`` which groups values into discrete segments. In a numerical example, values -could be grouped by increments of 5. +For dimensions that do not have a finite set of values, a continuous dimension can be used. This is especially useful for numeric values that can be viewed with varied precision, such as a decimal value. Continuous Dimensions require an additional parameter for an ``Interval`` which groups values into discrete segments. In a numerical example, values could be grouped by increments of 5. Date Dimensions """"""""""""""" -Date/Time Dimensions are a special type of continuous dimension which contain some predefined intervals: hours, days, -weeks, months, quarters, and years. In any widget that displays time series data, a date/time dimension. +Date/Time Dimensions are a special type of continuous dimension which contain some predefined intervals: hours, days, weeks, months, quarters, and years. In any widget that displays time series data, a date/time dimension. Unique Dimensions """"""""""""""""" -Lastly, a unique dimension represents a column that has one or more identifier columns and optionally a display label -column. This is useful when your data contains a significant number of values that cannot be represented by a small -list of categories and is akin to using a foreign key in a SQL table. In conjunction with a join on a foreign key, a -display value can be selected from a second table and used when rendering your widgets. +Lastly, a unique dimension represents a column that has one or more identifier columns and optionally a display label column. This is useful when your data contains a significant number of values that cannot be represented by a small list of categories and is akin to using a foreign key in a SQL table. In conjunction with a join on a foreign key, a display value can be selected from a second table and used when rendering your widgets. .. warning:: - If the column your |FeatureDimension| uses contains ``null`` values, it is advised to define the dimension using the - ``COALESE`` function in order to specify some label for that value. |Brand| makes use of advanced queries that - could lead to collisions with null values. + If the column your |FeatureDimension| uses contains ``null`` values, it is advised to define the dimension using the ``COALESE`` function in order to specify some label for that value. |Brand| makes use of advanced queries that could lead to collisions with null values. .. _config_slicer_start: @@ -86,41 +56,25 @@ display value can be selected from a second table and used when rendering your w Configuring a Slicer -------------------- -Here is a concrete example of a |FeatureSlicer| configuration. It includes a parameter ``metrics`` which is a list of -|ClassMetric|. Below that is the list of |ClassDimension| with a |ClassDateDimension|, |ClassCatDimension| and -|ClassUniqueDimension|. +Here is a concrete example of a |FeatureSlicer| configuration. It includes a parameter ``metrics`` which is a list of |ClassMetric|. Below that is the list of |ClassDimension| with a |ClassDateDimension|, |ClassCatDimension| and |ClassUniqueDimension|. .. include:: ../README.rst :start-after: _slicer_example_start: :end-before: _slicer_example_end: -In our example, the first couple of metrics pass ``key`` and ``label`` parameters. The key is a unique identifier for -the |FeatureSlicer| and cannot be shared by other |FeatureSlicer| elements. The label is used when transforming the -data into widgets to represent the field. The last three metrics also provide a ``definition`` parameter which is a -PyPika_ expression used to select the data from the database. When a ``definition`` parameter is not supplied, the key -of the metric is wrapped in a ``Sum`` function as a default. The metric for ``impressions`` will get the definition -``fn.Sum(analytics.impressions)``. +In our example, the first couple of metrics pass ``key`` and ``label`` parameters. The key is a unique identifier for the |FeatureSlicer| and cannot be shared by other |FeatureSlicer| elements. The label is used when transforming the data into widgets to represent the field. The last three metrics also provide a ``definition`` parameter which is a PyPika_ expression used to select the data from the database. When a ``definition`` parameter is not supplied, the key of the metric is wrapped in a ``Sum`` function as a default. The metric for ``impressions`` will get the definition ``fn.Sum(analytics.impressions)``. -Here a few dimensions as also defined. A |ClassDateDimension| is used with a custom definition which maps to the ``dt`` -column in the database. The Device dimension uses the column with the same name as the key ``device`` as a default. -There are three possible values for a device: 'desktop', 'tablet', or 'mobile', so a |ClassCatDimension| is a good fit. -Last there is a |ClassUniqueDimension| which uses the column ``account_id`` as an identifier but the column -``account_name`` as a display label. Both columns will be included in the query. +Here a few dimensions as also defined. A |ClassDateDimension| is used with a custom definition which maps to the ``dt`` column in the database. The Device dimension uses the column with the same name as the key ``device`` as a default. There are three possible values for a device: 'desktop', 'tablet', or 'mobile', so a |ClassCatDimension| is a good fit. Last there is a |ClassUniqueDimension| which uses the column ``account_id`` as an identifier but the column ``account_name`` as a display label. Both columns will be included in the query. .. _config_slicer_end: Columns from Joined Tables """""""""""""""""""""""""" -Commonly data from a secondary table is required. These tables can be joined in the query so that their columns become -available for use in metric and dimension definitions. Joins must be defined in the slicer in order to use them, and -metrics and dimensions must also define which joins they require, so that they can be added to the query when used. +Commonly data from a secondary table is required. These tables can be joined in the query so that their columns become available for use in metric and dimension definitions. Joins must be defined in the slicer in order to use them, and metrics and dimensions must also define which joins they require, so that they can be added to the query when used. -A join requires three parameters, a *key*, a *table*, and a *criterion*. The *key* is used for reference when using the -join on a metric or dimension, the *table* is the table which is being joined. The *criterion* is a PyPika_ -expression which defines how to join the tables, more concretely an equality condition of when to join rows of each -table. +A join requires three parameters, a *key*, a *table*, and a *criterion*. The *key* is used for reference when using the join on a metric or dimension, the *table* is the table which is being joined. The *criterion* is a PyPika_ expression which defines how to join the tables, more concretely an equality condition of when to join rows of each table. .. code-block:: python @@ -150,17 +104,12 @@ table. -Using the Slicer Manager ------------------------- +Slicer and Transformer Managers +------------------------------- -After defining a |FeatureSlicer|, you are reading to query data into charts, tables, and widgets. Each of these -components is created via a |FeatureTransformer|. A |FeatureTransformer| is a thin layer in between the raw data and -the presentation. |Brand| includes a suite of prebuilt transformers but it is also possible to create your own for -custom output formats. When requesting data, a transformer must be selected. Requests can also include multiple -transformers but that will be covered in a later section. +The |FeatureSlicer| expose different managers for different types of request. The primary one is the Slicer manager which exposes a ``data`` function which returns the query results as Pandas_ data frame. Transformer managers provide the additional functionality of converting your data into a specified format. There are several transformers available by default as well as optional ones which require additional python dependencies. -The slicer contains a manager class, |ClassSlicerManager| which offers a method for each transformer and a ``data`` method -which forgoes transformation and returns a Pandas_ data frame. +The ``notebook`` transformer manager is the default one which is intended for use in Jupyter_ notebooks. In this tutorial it will be used exclusively. All transformer managers expose different methods for different types of results, but the methods always have the same signature. .. include:: ../README.rst :start-after: _manager_api_start: @@ -170,79 +119,71 @@ which forgoes transformation and returns a Pandas_ data frame. Getting Raw Data ---------------- -Now it is possible to put all the pieces together and start using the |FeatureSlicer|. Each of the manager functions has -the following signature containing ``tuple`` or ``list`` of |ClassMetric|, |ClassDimension|. The other parameters -will be introduced in a later section. For now, only metrics and dimensions are required to start fetching data. +Now it is possible to put all the pieces together and start using the |FeatureSlicer|. Each of the manager functions has the following signature containing ``tuple`` or ``list`` of |ClassMetric|, |ClassDimension|. The other parameters will be introduced in a later section. For now, only metrics and dimensions are required to start fetching data. -The ``metrics`` parameter is always a list of ``str`` matching the ``key`` of the desired metrics defined in the slicer. -The ``dimensions`` parameter is a list of mixed types but most often a ``str`` referencing the keys of the desired -dimensions. Continuous dimensions can also optionally specify an interval. DateDimensions by default use the -interval ``DatetimeDimension.day``. +The ``metrics`` parameter is always a list of ``str`` matching the ``key`` of the desired metrics defined in the slicer. The ``dimensions`` parameter is a list of mixed types but most often a ``str`` referencing the keys of the desired dimensions. Continuous dimensions can also optionally specify an interval. DateDimensions by default use the interval ``DatetimeDimension.day``. -When calling a |ClassSlicerManager| function, the ``tuple`` of metrics should contain string values matching the -``name`` of a |ClassMetric| or |ClassDimension| selected in the configuration. +When calling a |ClassSlicerManager| function, the ``tuple`` of metrics should contain string values matching the ``name`` of a |ClassMetric| or |ClassDimension| selected in the configuration. -Line Charts ------------ +Highcharts Line Charts +---------------------- -Below is an example of how to request a Highcharts_ line chart from the |FeatureSlicer|. The return value will be a -``dict`` which can be transformed into a JSON and used directly by Highcharts_. The example chart includes six lines: -Clicks (Desktop), Clicks (Tablet), Clicks (Mobile) and Conversions (Desktop), Conversions (Tablet), Conversions -(Mobile). +Below is an example of how to request a Highcharts_ line chart from the |FeatureSlicer|. The return value will be a ``dict`` which can be serialized into a JSON and used directly by Highcharts_. The example chart includes six lines: Clicks (Desktop), Clicks (Tablet), Clicks (Mobile) and Conversions (Desktop), Conversions (Tablet), Conversions (Mobile). -The ``metrics`` and ``dimensions`` parameters are a ``list`` or ``tuple`` of ``str`` matching the key of an element -configured in the |FeatureSlicer|. +The ``metrics`` and ``dimensions`` parameters are a ``list`` or ``tuple`` of ``str`` matching the key of an element configured in the |FeatureSlicer|. .. note:: - Line Charts *require* a |ClassContDimension| or a |ClassDateDimension| as the first selected dimension since it is - used as the X-Axis. Subsequent dimensions can be used and will be split into different lines in the chart for - comparisons across a dimension. + Line Charts *require* a |ClassContDimension| or a |ClassDateDimension| as the first selected dimension since it is used as the X-Axis. Subsequent dimensions can be used and will be split into different lines in the chart for comparisons across a dimension. .. code-block:: python - slicer.manager.line_chart( + result = slicer.highcharts.line_chart( metrics=['clicks', 'conversions'], dimensions=['date', 'device_type'] ) -When using a |ClassDateDimension|, the default interval is a day. To change the interval, pass a ``tuple`` instead of -a ``string`` as a parameter with the first element matching the metric key in the |FeatureSlicer| and the second element -as a ``DatetimeDimension``. +When using a |ClassDateDimension|, the default interval is a day. To change the interval, pass a ``tuple`` instead of a ``string`` as a parameter with the first element matching the metric key in the |FeatureSlicer| and the second element as a ``DatetimeDimension``. .. code-block:: python - slicer.manager.line_chart( + result = slicer.highcharts.line_chart( metrics=['clicks', 'conversions'], dimensions=[('date', DatetimeDimension.year), 'device_type'] ) -Column and Bar Charts ---------------------- +The result can then be serialized to JSON: -Similar to Line Charts, column and bar charts are also given in the Highcharts_ format. The return value from the -|FeatureSlicer| is practically the same for the two, only they are rendered in slightly different formats. Namely, -Bar charts are oriented horizontally and Column charts vertically. +.. code-block:: python + + import json + + json.dumps(result) + + +Highcharts Column and Bar Charts +-------------------------------- + +Column charts and bar charts are also available in the Highcharts_ transformer. The output format is the same, except for the ``chart_type`` option. .. note:: - Column and Bar charts require *one* or *two* dimensions, preferably of type |ClassCatDimension|, but this is not - required. + Column and Bar charts require *one* or *two* dimensions, preferably of type |ClassCatDimension|, but this is not required. .. code-block:: python - slicer.manager.bar_chart( + slicer.highcharts.bar_chart( metrics=['clicks', 'conversions'], dimensions=[('date', DatetimeDimension.year), 'device_type'] ) .. code-block:: python - slicer.manager.column_chart( + slicer.highcharts.column_chart( metrics=['clicks', 'conversions'], dimensions=['device_type'], ) @@ -250,28 +191,23 @@ Bar charts are oriented horizontally and Column charts vertically. Tables ------ -Tables don't have any requirements as to the number or types of dimensions and generally can display any type of -|FeatureSlicer| result. The return value is given in Datatables_ format. +Tables don't have any requirements as to the number or types of dimensions and generally can display any type of |FeatureSlicer| result. The return value is given in Datatables_ format. -In a *row-indexed* Table, the rows of the table each will have a unique combination of values of the dimensions. Below -is an example that will give the following columns: Day, Device Type, Clicks, Conversions. +In a *row-indexed* Table, the rows of the table each will have a unique combination of values of the dimensions. Below is an example that will give the following columns: Day, Device Type, Clicks, Conversions. .. code-block:: python - slicer.manager.row_index_table( + slicer.datatables.row_index_table( metrics=['clicks', 'conversions'], dimensions=['date', 'device_type'] ) -A *column-indexed* table will contain only one index column and display a metrics column for each combination of -subsequent dimensions. For example with the same parameters as above, the result will include seven columns: Day, -Clicks (Desktop), Clicks (Tablet), Clicks (Mobile) and Conversions (Desktop), Conversions (Tablet), and Conversions -(Mobile). +A *column-indexed* table will contain only one index column and display a metrics column for each combination of subsequent dimensions. For example with the same parameters as above, the result will include seven columns: Day, Clicks (Desktop), Clicks (Tablet), Clicks (Mobile) and Conversions (Desktop), Conversions (Tablet), and Conversions (Mobile). .. code-block:: python - slicer.manager.column_index_table( + slicer.datatables.column_index_table( metrics=['clicks', 'conversions'], dimensions=['date', 'device_type'] ) @@ -280,15 +216,11 @@ Clicks (Desktop), Clicks (Tablet), Clicks (Mobile) and Conversions (Desktop), Co Filtering Data -------------- -So far all of the examples will display a result that queries all of the data in the database table. In many cases it is -also useful to narrow the window of data to display in the result. Filters are used for this purpose. Filters are -expressions that refer to a defined metric or dimension and some criteria. For example, it might be desirable to only -display data for `desktop` devices. +So far all of the examples will display a result that queries all of the data in the database table. In many cases it is also useful to narrow the window of data to display in the result. Filters are used for this purpose. Filters are expressions that refer to a defined metric or dimension and some criteria. For example, it might be desirable to only display data for `desktop` devices. .. note:: - Filters can be used for either metrics or dimensions. Filtering metrics is synonymous to the ``HAVING`` clause in - SQL whereas dimensions is synoymous with the ``WHERE`` clause. + Filters can be used for either metrics or dimensions. Filtering metrics is synonymous to the ``HAVING`` clause in SQL whereas dimensions is synoymous with the ``WHERE`` clause. Equalities and Inequalities """"""""""""""""""""""""""" @@ -300,7 +232,7 @@ The most basic type of filtering uses a equality/inequality expression such as ` from fireant.slicer import EqualityFilter, EqualityOperator # Only desktop data - slicer.manager.column_index_table( + slicer.notebook.column_index_table( metrics=['clicks', 'conversions'], dimensions=['date'], dimension_filters=[EqualityFilter('device_type', EqualityOperator.eq, 'desktop')], @@ -311,7 +243,7 @@ The most basic type of filtering uses a equality/inequality expression such as ` from fireant.slicer import EqualityFilter, EqualityOperator # Only data for days where clicks were greater than 100 - slicer.manager.column_index_table( + slicer.notebook.column_index_table( metrics=['clicks', 'conversions'], dimensions=['date'], metric_filters=[EqualityFilter('clicks', EqualityOperator.gt, 100)], @@ -327,7 +259,7 @@ When a column should be equal to one of a set of values, a `Contains` filter can from fireant.slicer import ContainsFilter - slicer.manager.column_index_table( + slicer.notebook.column_index_table( metrics=['clicks', 'conversions'], dimensions=['date'], dimension_filters=[ContainsFilter('device_type', ['desktop', 'mobile'])], @@ -343,7 +275,7 @@ Windows from fireant.slicer import RangeFilter - slicer.manager.column_index_table( + slicer.notebook.column_index_table( metrics=['clicks', 'conversions'], dimensions=['date'], dimension_filters=[RangeFilter('date', date.today() - timedelta(days=60), date.today())], @@ -358,7 +290,7 @@ For pattern matching a `Fuzzy` can be used which parallels ``LIKE`` expressions from fireant.slicer import WildcardFilter - slicer.manager.column_index_table( + slicer.notebook.column_index_table( metrics=['clicks', 'conversions'], dimensions=['date'], dimension_filters=[WildcardFilter('account', 'abc%')], @@ -368,9 +300,7 @@ For pattern matching a `Fuzzy` can be used which parallels ``LIKE`` expressions Comparing Data to Previous Values --------------------------------- -In some cases it is useful to compare current numbers to previous values such as in a Week-over-Week report. A -|FeatureReference| can be used to achieve this. |FeatureReference| is a built-in function which can be chosen from the -subclasses of |ClassReference|. +In some cases it is useful to compare current numbers to previous values such as in a Week-over-Week report. A |FeatureReference| can be used to achieve this. |FeatureReference| is a built-in function which can be chosen from the subclasses of |ClassReference|. A |FeatureReference| can be used as a fixed comparison, a change in value (delta), or a change in value as a percentage. @@ -390,7 +320,7 @@ For each |FeatureReference|, there are the following variations: from fireant.slicer import WoW, DeltaMoM, DeltaQoQ - slicer.manager.column_index_table( + slicer.notebook.column_index_table( metrics=['clicks', 'conversions'], dimensions=['date'], references=[WoW('date'), DeltaMoM('date'), DeltaQoQ('date')], diff --git a/docs/3_dashboard.rst b/docs/3_dashboard.rst index ae4e7e05..80552376 100644 --- a/docs/3_dashboard.rst +++ b/docs/3_dashboard.rst @@ -5,23 +5,14 @@ Dashboards and Reports :start-after: _appendix_start: :end-before: _appendix_end: -|Brand| offers higher level features that leverage the |FeatureSlicer| for more complex requests in order to create -dashboards and reports. A |FeatureWidgetGroup| is a layer of abstraction on top of the |FeatureSlicer| which -facilitates the configuration of groups of widgets such as charts and tables. The |FeatureWidgetGroup| then provides an -API for querying data for the whole group. +|Brand| offers higher level features that leverage the |FeatureSlicer| for more complex requests in order to create dashboards and reports. A |FeatureWidgetGroup| is a layer of abstraction on top of the |FeatureSlicer| which facilitates the configuration of groups of widgets such as charts and tables. The |FeatureWidgetGroup| then provides an API for querying data for the whole group. Widgets ------- -Each |FeatureWidgetGroup| contains one or more |FeatureWidget| which represents one chart, table or other type of UI -component. Each |FeatureWidget| will display a subset of the data from the combined result of a |FeatureSlicer| request -made by the |FeatureWidgetGroup|. Widgets define which metrics they display and dimensions, filters, references, and -operations are applied to the entire group. +Each |FeatureWidgetGroup| contains one or more |FeatureWidget| which represents one chart, table or other type of UI component. Each |FeatureWidget| will display a subset of the data from the combined result of a |FeatureSlicer| request made by the |FeatureWidgetGroup|. Widgets define which metrics they display and dimensions, filters, references, and operations are applied to the entire group. -Additionally, widgets can display a subset of values for a dimension. This is useful in reports where a several widgets -display the same metrics but one widget per dimension value is used. For example, a chart could display the metrics -*clicks* and *conversions* with a |ClassCatDimension| for *device* so that there is one chart for each type of device, -desktop, tablet, and mobile. +Additionally, widgets can display a subset of values for a dimension. This is useful in reports where a several widgets display the same metrics but one widget per dimension value is used. For example, a chart could display the metrics *clicks* and *conversions* with a |ClassCatDimension| for *device* so that there is one chart for each type of device, desktop, tablet, and mobile. .. _config_dashboard_start: @@ -30,16 +21,11 @@ desktop, tablet, and mobile. Configuring a Widget Group -------------------------- -Here is a concrete example of a |FeatureWidgetGroup| configuration. The |FeatureWidgetGroup| at a minimum defines a -|ClassSlicer| and a list containing *at least one* |ClassWidget|. +Here is a concrete example of a |FeatureWidgetGroup| configuration. The |FeatureWidgetGroup| at a minimum defines a |ClassSlicer| and a list containing *at least one* |ClassWidget|. -In addition, dimensions, filters, references, and operations can be defined. Any of these are applied to all widgets -in the widget group and are implicitly and always present in any request of the |ClassWidgetGroupManager|. +In addition, dimensions, filters, references, and operations can be defined. Any of these are applied to all widgets in the widget group and are implicitly and always present in any request of the |ClassWidgetGroupManager|. -Sometimes it is necessary to include these, for example a Line Chart widget will always require a continuous dimension -as its first dimension, in the |ClassWidgetGroup| one can be defined in order to avoid having to constantly pass it as a -parameter to the |ClassWidgetGroupManager|. Dimensions, filters, references, and operations behave exactly the same -in the |ClassWidgetGroup| schema as they do in the |ClassSlicerManager| as well as in the |ClassWidgetGroupManager| API. +Sometimes it is necessary to include these, for example a Line Chart widget will always require a continuous dimension as its first dimension, in the |ClassWidgetGroup| one can be defined in order to avoid having to constantly pass it as a parameter to the |ClassWidgetGroupManager|. Dimensions, filters, references, and operations behave exactly the same in the |ClassWidgetGroup| schema as they do in the |ClassSlicerManager| as well as in the |ClassWidgetGroupManager| API. .. code-block:: python @@ -71,28 +57,18 @@ in the |ClassWidgetGroup| schema as they do in the |ClassSlicerManager| as well Using the Widget Group Manager ------------------------------ -The |ClassWidgetGroupManager| provides an API similar to that of the |ClassSlicerManager| but combines the requirements -of all of its widgets into one slicer request and then transforms the resulting data appropriately for each widget. The -``render`` function accepts the same parameters as the functions of the |ClassSlicerManager| except for ``metrics``, -but additionally will prepend any parameters provided to the |ClassWidgetGroup| constructor. +The |ClassWidgetGroupManager| provides an API similar to that of the |ClassSlicerManager| but combines the requirements of all of its widgets into one slicer request and then transforms the resulting data appropriately for each widget. The ``render`` function accepts the same parameters as the functions of the |ClassSlicerManager| except for ``metrics``, but additionally will prepend any parameters provided to the |ClassWidgetGroup| constructor. -The ``render`` function of the |ClassWidgetGroupManager| will return an array containing python ``dict`` containing -the data for each widget in the respective format. The ``render`` function can be called without any parameters, which -will use only what is defined in the |ClassWidgetGroup| in the request. +The ``render`` function of the |ClassWidgetGroupManager| will return an array containing python ``dict`` containing the data for each widget in the respective format. The ``render`` function can be called without any parameters, which will use only what is defined in the |ClassWidgetGroup| in the request. - -The following example will return an array containing data for the widgets defined about, each in the Highcharts_ line -chart format. Each of them will have the *date* dimension, the *date range filter*, and the *Week over Week* reference -applied to them. + The following example will return an array containing data for the widgets defined about, each in the Highcharts_ line chart format. Each of them will have the *date* dimension, the *date range filter*, and the *Week over Week* reference applied to them. .. code-block:: python result = my_widget_group.manager.render() -Additional dimensions, filters, references, and operations can be added to the request in the render function and -behave similarly to the |ClassSlicerManager| API, except that anything defined in the |ClassWidgetGroupManager| -constructor will be prepended to the parameters provided in the API. +Additional dimensions, filters, references, and operations can be added to the request in the render function and behave similarly to the |ClassSlicerManager| API, except that anything defined in the |ClassWidgetGroupManager| constructor will be prepended to the parameters provided in the API. .. code-block:: python diff --git a/fireant/dashboards/schemas.py b/fireant/dashboards/schemas.py index 06b8ca92..5b7ae6a6 100644 --- a/fireant/dashboards/schemas.py +++ b/fireant/dashboards/schemas.py @@ -1,8 +1,7 @@ # coding: utf-8 from fireant.dashboards.managers import WidgetGroupManager -from fireant.slicer.transformers import DataTablesTransformer, HighchartsTransformer, HighchartsColumnTransformer, \ - TableIndex +from fireant.slicer.transformers import * class Widget(object): @@ -23,35 +22,35 @@ class LineChartWidget(Widget): """ The `LineChartWidget` class represents a Highcharts line chart. """ - transformer = HighchartsTransformer() + transformer = HighchartsLineTransformer() class BarChartWidget(Widget): """ The `BarChartWidget` class represents a Highcharts bar chart. """ - transformer = HighchartsColumnTransformer(HighchartsColumnTransformer.bar) + transformer = HighchartsBarTransformer() class ColumnChartWidget(Widget): """ The `ColumnChartWidget` class represents a Highcharts column chart. """ - transformer = HighchartsColumnTransformer(HighchartsColumnTransformer.column) + transformer = HighchartsColumnTransformer() class RowIndexTableWidget(Widget): """ The `RowIndexTableWidget` class represents datatables.js data table with row-indexed dimensions. """ - transformer = DataTablesTransformer(TableIndex.row_index) + transformer = DataTablesRowIndexTransformer() class ColumnIndexTableWidget(Widget): """ The `ColumnIndexTableWidget` class represents datatables.js data table with column-indexed dimensions. """ - transformer = DataTablesTransformer(TableIndex.column_index) + transformer = DataTablesColumnIndexTransformer() class WidgetGroup(object): diff --git a/fireant/settings.py b/fireant/settings.py index 03aa578b..6598d2a3 100644 --- a/fireant/settings.py +++ b/fireant/settings.py @@ -1,3 +1,4 @@ # coding: utf-8 database = None +transformers = None diff --git a/fireant/slicer/managers.py b/fireant/slicer/managers.py index 4ac8a397..0111265d 100644 --- a/fireant/slicer/managers.py +++ b/fireant/slicer/managers.py @@ -5,15 +5,6 @@ from pypika import functions as fn from .queries import QueryManager -from .transformers import (TableIndex, HighchartsTransformer, HighchartsColumnTransformer, DataTablesTransformer) - -web_transformers = { - 'line_chart': HighchartsTransformer(), - 'column_chart': HighchartsColumnTransformer(HighchartsColumnTransformer.column), - 'bar_chart': HighchartsColumnTransformer(HighchartsColumnTransformer.bar), - 'row_index_table': DataTablesTransformer(TableIndex.row_index), - 'column_index_table': DataTablesTransformer(TableIndex.column_index), -} class SlicerException(Exception): @@ -21,17 +12,12 @@ class SlicerException(Exception): class SlicerManager(QueryManager): - def __init__(self, slicer, transformers=None): + def __init__(self, slicer): """ :param slicer: - :param transformers: """ self.slicer = slicer - # Creates a function on the slicer for each transformer - for tx_key, tx in (transformers or web_transformers).items(): - setattr(self, tx_key, functools.partial(self._get_and_transform_data, tx)) - def _get_and_transform_data(self, tx, metrics=tuple(), dimensions=tuple(), metric_filters=tuple(), dimension_filters=tuple(), references=tuple(), operations=tuple()): @@ -275,3 +261,24 @@ def _default_dimension_definition(self, key): def _default_metric_definition(self, key): return fn.Sum(self.slicer.table.field(key)) + + +class TransformerManager(object): + def __init__(self, manager, transformers): + self._manager = manager + + # Creates a function on the slicer for each transformer + for tx_key, tx in transformers.items(): + setattr(self, tx_key, functools.partial(self._get_and_transform_data, tx)) + + def _get_and_transform_data(self, tx, metrics=tuple(), dimensions=tuple(), + metric_filters=tuple(), dimension_filters=tuple(), + references=tuple(), operations=tuple()): + # Loads data and transforms it with a given transformer. + df = self._manager.data(metrics=metrics, dimensions=dimensions, + metric_filters=metric_filters, dimension_filters=dimension_filters, + references=references, operations=operations) + + display_schema = self._manager.get_display_schema(metrics, dimensions, references) + + return tx.transform(df, display_schema) diff --git a/fireant/slicer/schemas.py b/fireant/slicer/schemas.py index 7b7e83cc..cef6fadb 100644 --- a/fireant/slicer/schemas.py +++ b/fireant/slicer/schemas.py @@ -1,6 +1,7 @@ # coding: utf-8 from fireant import settings -from fireant.slicer.managers import SlicerManager +from fireant.slicer import transformers +from fireant.slicer.managers import SlicerManager, TransformerManager from pypika import JoinType from pypika.terms import Mod @@ -165,6 +166,8 @@ def __init__(self, table, metrics=tuple(), dimensions=tuple(), joins=tuple()): self.joins = {join.key: join for join in joins} self.manager = SlicerManager(self) + for name, bundle in transformers.bundles.items(): + setattr(self, name, TransformerManager(self.manager, bundle)) class EqualityOperator(object): diff --git a/fireant/slicer/transformers/__init__.py b/fireant/slicer/transformers/__init__.py index 7b780a8c..6107e4b3 100644 --- a/fireant/slicer/transformers/__init__.py +++ b/fireant/slicer/transformers/__init__.py @@ -1,5 +1,8 @@ # coding: utf-8 from .base import Transformer, TransformationException -from .datatables import TableIndex, DataTablesTransformer, CSVTransformer -from .highcharts import HighchartsTransformer, HighchartsColumnTransformer \ No newline at end of file +from .datatables import (DataTablesRowIndexTransformer, DataTablesColumnIndexTransformer, CSVRowIndexTransformer, + CSVColumnIndexTransformer) +from .highcharts import HighchartsLineTransformer, HighchartsColumnTransformer, HighchartsBarTransformer +from .notebook import MatplotlibTransformer, PandasTransformer +from .bundles import bundles \ No newline at end of file diff --git a/fireant/slicer/transformers/bundles.py b/fireant/slicer/transformers/bundles.py new file mode 100644 index 00000000..61dd032f --- /dev/null +++ b/fireant/slicer/transformers/bundles.py @@ -0,0 +1,32 @@ +# coding: utf-8 +from . import (HighchartsLineTransformer, HighchartsColumnTransformer, HighchartsBarTransformer, + DataTablesRowIndexTransformer, DataTablesColumnIndexTransformer, PandasTransformer) + +notebook_tx = { + 'row_index': PandasTransformer(), + 'column_index': PandasTransformer(), +} + +try: + from fireant.slicer.transformers.notebook import MatplotlibTransformer + + notebook_tx['line'] = MatplotlibTransformer() + notebook_tx['column'] = MatplotlibTransformer() + notebook_tx['bar'] = MatplotlibTransformer() + +except ImportError: + # Matplotlib not installed + pass + +bundles = { + 'notebook': notebook_tx, + 'highcharts': { + 'line': HighchartsLineTransformer(), + 'column ': HighchartsColumnTransformer(), + 'bar': HighchartsBarTransformer(), + }, + 'datatables': { + 'row_index': DataTablesRowIndexTransformer(), + 'column_index': DataTablesColumnIndexTransformer(), + }, +} diff --git a/fireant/slicer/transformers/datatables.py b/fireant/slicer/transformers/datatables.py index 48c1401e..afaf7848 100644 --- a/fireant/slicer/transformers/datatables.py +++ b/fireant/slicer/transformers/datatables.py @@ -5,12 +5,7 @@ from .base import Transformer -class TableIndex(object): - column_index = 'col' - row_index = 'row' - - -def format_data_point(value): +def _format_data_point(value): if isinstance(value, str): return value if isinstance(value, pd.Timestamp): @@ -23,9 +18,8 @@ def format_data_point(value): return value -class DataTablesTransformer(Transformer): - def __init__(self, table_type=TableIndex.row_index): - self.table_type = table_type +class DataTablesRowIndexTransformer(Transformer): + table_type = 'row' def transform(self, data_frame, display_schema): dim_ordinal = {name: ordinal @@ -65,7 +59,7 @@ def _prepare_data_frame(self, data_frame, dimensions): # Replaces invalid values and unstacks the data frame for column_index tables. data_frame = data_frame.replace([np.inf, -np.inf], np.nan) - if 1 < len(dimensions) and self.table_type == TableIndex.column_index: + if 1 < len(dimensions) and self.table_type == 'column': unstack_levels = list(range( len(dimensions[0]['id_fields']), len(data_frame.index.levels)) @@ -75,12 +69,12 @@ def _prepare_data_frame(self, data_frame, dimensions): return data_frame def _render_index_levels(self, idx, dim_ordinal, display_schema): - row_dimensions = display_schema['dimensions'][:None if self.table_type == TableIndex.row_index else 1] + row_dimensions = display_schema['dimensions'][:None if self.table_type == 'row' else 1] for dimension in row_dimensions: key = dimension['label'] if not isinstance(idx, tuple): - value = format_data_point(idx) + value = _format_data_point(idx) elif 1 < len(dimension['id_fields']) or 'label_field' in dimension: fields = dimension['id_fields'] + [dimension.get('label_field')] @@ -90,7 +84,7 @@ def _render_index_levels(self, idx, dim_ordinal, display_schema): else: id_field = dimension['id_fields'][0] - value = format_data_point(idx[dim_ordinal[id_field]]) + value = _format_data_point(idx[dim_ordinal[id_field]]) if value is None: value = 'Total' @@ -106,14 +100,14 @@ def _render_column_levels(self, row, dim_ordinal, display_schema, reference_key= for idx, value in row.iteritems(): label = self._format_series_labels(idx, dim_ordinal, display_schema) label = self._format_reference_label(display_schema, label, reference_key) - yield label, format_data_point(value) + yield label, _format_data_point(value) else: # Single level columns for col in row.index: label = display_schema['metrics'][col] label = self._format_reference_label(display_schema, label, reference_key) - yield label, format_data_point(row[col]) + yield label, _format_data_point(row[col]) def _format_series_labels(self, idx, dim_ordinal, display_schema): metric, dimensions = idx[0], idx[1:] @@ -163,10 +157,7 @@ def _format_dimension_label(idx, dim_ordinal, dimension): return dimension_label -class CSVTransformer(DataTablesTransformer): - def __init__(self, table_type=TableIndex.row_index): - self.table_type = table_type - +class CSVRowIndexTransformer(DataTablesRowIndexTransformer): def transform(self, data_frame, display_schema): dim_ordinal = {name: ordinal for ordinal, name in enumerate(data_frame.index.names)} @@ -179,7 +170,7 @@ def transform(self, data_frame, display_schema): csv_df = self._format_index(csv_df, dim_ordinal, display_schema) - row_dimensions = display_schema['dimensions'][:None if self.table_type == TableIndex.row_index else 1] + row_dimensions = display_schema['dimensions'][:None if self.table_type == 'row' else 1] return csv_df.to_csv(index_label=[dimension['label'] for dimension in row_dimensions]) @@ -200,7 +191,7 @@ def _format_index(self, csv_df, dim_ordinal, display_schema): return csv_df def _format_columns(self, data_frame, dim_ordinal, display_schema): - if 1 < len(display_schema['dimensions']) and self.table_type == TableIndex.column_index: + if 1 < len(display_schema['dimensions']) and self.table_type == 'column': csv_df = self._prepare_data_frame(data_frame, display_schema['dimensions']) csv_df.columns = pd.Index([self._format_series_labels(column, dim_ordinal, display_schema) @@ -210,3 +201,11 @@ def _format_columns(self, data_frame, dim_ordinal, display_schema): return data_frame.rename( columns=lambda metric: display_schema['metrics'].get(metric, metric) ) + + +class DataTablesColumnIndexTransformer(DataTablesRowIndexTransformer): + table_type = 'column' + + +class CSVColumnIndexTransformer(DataTablesColumnIndexTransformer, CSVRowIndexTransformer): + pass diff --git a/fireant/slicer/transformers/highcharts.py b/fireant/slicer/transformers/highcharts.py index 56175481..6f8ecc35 100644 --- a/fireant/slicer/transformers/highcharts.py +++ b/fireant/slicer/transformers/highcharts.py @@ -5,28 +5,27 @@ from .base import Transformer, TransformationException -def format_data_point(value): +def _format_data_point(value): if isinstance(value, str): return value if isinstance(value, pd.Timestamp): return int(value.asm8) // int(1e6) if np.isnan(value): return None + if np.isnan(value): + return None if isinstance(value, np.int64): # Cannot serialize np.int64 to json return int(value) return value -class HighchartsTransformer(Transformer): +class HighchartsLineTransformer(Transformer): """ Transforms data frames into Highcharts format for several chart types, particularly line or bar charts. """ - line = 'line' - - def __init__(self, chart_type=line): - self.chart_type = chart_type + chart_type = 'line' def transform(self, data_frame, display_schema): self._validate_dimensions(data_frame, display_schema['dimensions']) @@ -170,7 +169,7 @@ def _format_dimension_label(dim_ordinal, dimension, idx): def _format_data(self, column): if isinstance(column, float): - return [format_data_point(column)] + return [_format_data_point(column)] return [self._format_point(key, value) for key, value in column.iteritems() @@ -178,7 +177,7 @@ def _format_data(self, column): @staticmethod def _format_point(x, y): - return (format_data_point(x), format_data_point(y)) + return (_format_data_point(x), _format_data_point(y)) def _unstack_levels(self, dimensions, dim_ordinal): for dimension in dimensions: @@ -189,17 +188,11 @@ def _unstack_levels(self, dimensions, dim_ordinal): yield dim_ordinal[dimension['label_field']] -class HighchartsColumnTransformer(HighchartsTransformer): - column = 'column' - bar = 'bar' - - def __init__(self, chart_type=column): - super(HighchartsColumnTransformer, self).__init__(chart_type) - +class HighchartsColumnTransformer(HighchartsLineTransformer): def _make_series_item(self, idx, item, dim_ordinal, display_schema, y_axis, reference): return { 'name': self._format_label(idx, dim_ordinal, display_schema, reference), - 'data': [format_data_point(x) + 'data': [_format_data_point(x) for x in item if not np.isnan(x)], 'yAxis': y_axis @@ -248,3 +241,7 @@ def _make_categories(data_frame, dim_ordinal, display_schema): return data_frame.index.get_level_values(label_field).unique().tolist() return [] + + +class HighchartsBarTransformer(HighchartsColumnTransformer): + chart_type = 'bar' diff --git a/fireant/slicer/transformers/notebook.py b/fireant/slicer/transformers/notebook.py new file mode 100644 index 00000000..624e25d8 --- /dev/null +++ b/fireant/slicer/transformers/notebook.py @@ -0,0 +1,12 @@ +# coding: utf-8 +from . import Transformer + + +class MatplotlibTransformer(Transformer): + def transform(self, data_frame, display_schema): + return data_frame.plot() + + +class PandasTransformer(Transformer): + def transform(self, data_frame, display_schema): + return data_frame.plot() diff --git a/fireant/tests/dashboards/test_widgets.py b/fireant/tests/dashboards/test_widgets.py index 5d4b85a6..e10f5c70 100644 --- a/fireant/tests/dashboards/test_widgets.py +++ b/fireant/tests/dashboards/test_widgets.py @@ -2,8 +2,9 @@ from unittest import TestCase from fireant.dashboards import * -from fireant.slicer.transformers import (HighchartsTransformer, HighchartsColumnTransformer, TableIndex, - DataTablesTransformer) +from fireant.slicer.transformers import (HighchartsLineTransformer, HighchartsColumnTransformer, + HighchartsBarTransformer, + DataTablesRowIndexTransformer, DataTablesColumnIndexTransformer) class DashboardTests(TestCase): @@ -15,28 +16,28 @@ def _widget_test(self, widget_type, metrics, transformer_type): return widget def test_line_chart(self): - self._widget_test(LineChartWidget, ['clicks', 'conversions'], HighchartsTransformer()) + self._widget_test(LineChartWidget, ['clicks', 'conversions'], HighchartsLineTransformer()) def test_bar_chart(self): - tx = HighchartsColumnTransformer(HighchartsColumnTransformer.bar) + tx = HighchartsBarTransformer() widget = self._widget_test(BarChartWidget, ['clicks', 'conversions'], tx) - self.assertEqual(HighchartsColumnTransformer.bar, widget.transformer.chart_type) + self.assertEqual(HighchartsBarTransformer.chart_type, widget.transformer.chart_type) def test_column_chart(self): - tx = HighchartsColumnTransformer(HighchartsColumnTransformer.column) + tx = HighchartsColumnTransformer() widget = self._widget_test(ColumnChartWidget, ['clicks', 'conversions'], tx) - self.assertEqual(HighchartsColumnTransformer.column, widget.transformer.chart_type) + self.assertEqual(HighchartsColumnTransformer.chart_type, widget.transformer.chart_type) def test_row_index_table(self): - tx = DataTablesTransformer(TableIndex.row_index) + tx = DataTablesRowIndexTransformer() widget = self._widget_test(RowIndexTableWidget, ['clicks', 'conversions'], tx) - self.assertEqual(TableIndex.row_index, widget.transformer.table_type) + self.assertEqual(DataTablesRowIndexTransformer.table_type, widget.transformer.table_type) def test_column_index_table(self): - tx = DataTablesTransformer(TableIndex.column_index) + tx = DataTablesColumnIndexTransformer() widget = self._widget_test(ColumnIndexTableWidget, ['clicks', 'conversions'], tx) - self.assertEqual(TableIndex.column_index, widget.transformer.table_type) + self.assertEqual(DataTablesColumnIndexTransformer.table_type, widget.transformer.table_type) diff --git a/fireant/tests/slicer/test_manager.py b/fireant/tests/slicer/test_manager.py new file mode 100644 index 00000000..6bd8c9be --- /dev/null +++ b/fireant/tests/slicer/test_manager.py @@ -0,0 +1,15 @@ +# coding: utf-8 +from unittest import TestCase + +from fireant.slicer import Slicer +from pypika import Table + + +class ManagerInitializationTests(TestCase): + def test_transformers(self): + slicer = Slicer(Table('test')) + + self.assertTrue(hasattr(slicer, 'manager')) + self.assertTrue(hasattr(slicer, 'notebook')) + self.assertTrue(hasattr(slicer, 'highcharts')) + self.assertTrue(hasattr(slicer, 'datatables')) diff --git a/fireant/tests/slicer/transformers/test_csv.py b/fireant/tests/slicer/transformers/test_csv.py index ff3196a0..7cfe58dd 100644 --- a/fireant/tests/slicer/transformers/test_csv.py +++ b/fireant/tests/slicer/transformers/test_csv.py @@ -1,11 +1,11 @@ # coding: utf-8 -from fireant.slicer.transformers import TableIndex, CSVTransformer +from fireant.slicer.transformers import CSVRowIndexTransformer, CSVColumnIndexTransformer from fireant.tests.slicer.transformers.base import BaseTransformerTests class CSVRowIndexTransformerTests(BaseTransformerTests): maxDiff = None - csv_tx = CSVTransformer(TableIndex.row_index) + csv_tx = CSVRowIndexTransformer() def test_no_dims_single_metric(self): df = self.no_dims_multi_metric_df @@ -196,7 +196,7 @@ def test_rollup_cont_cat_cat_dim_multi_metric(self): class CSVColumnIndexTransformerTests(CSVRowIndexTransformerTests): maxDiff = None - csv_tx = CSVTransformer(TableIndex.column_index) + csv_tx = CSVColumnIndexTransformer() def test_cont_cat_dim_single_metric(self): df = self.cont_cat_dims_single_metric_df diff --git a/fireant/tests/slicer/transformers/test_datatables.py b/fireant/tests/slicer/transformers/test_datatables.py index 164472e5..c8224e85 100644 --- a/fireant/tests/slicer/transformers/test_datatables.py +++ b/fireant/tests/slicer/transformers/test_datatables.py @@ -5,13 +5,13 @@ import numpy as np import pandas as pd -from fireant.slicer.transformers import TableIndex, DataTablesTransformer +from fireant.slicer.transformers import DataTablesRowIndexTransformer, DataTablesColumnIndexTransformer from fireant.slicer.transformers import datatables from fireant.tests.slicer.transformers.base import BaseTransformerTests class DataTablesRowIndexTransformerTests(BaseTransformerTests): - dt_tx = DataTablesTransformer(TableIndex.row_index) + dt_tx = DataTablesRowIndexTransformer() def _evaluate_table(self, result, num_rows=1): self.assertSetEqual({'draw', 'recordsTotal', 'recordsFiltered', 'data'}, set(result.keys())) @@ -192,7 +192,7 @@ def test_rollup_cont_cat_cat_dim_multi_metric(self): class DataTablesColumnIndexTransformerTests(BaseTransformerTests): - dt_tx = DataTablesTransformer(TableIndex.column_index) + dt_tx = DataTablesColumnIndexTransformer() def _evaluate_table(self, result, num_rows=1): self.assertSetEqual({'draw', 'recordsTotal', 'recordsFiltered', 'data'}, set(result.keys())) @@ -423,19 +423,19 @@ def test_rollup_cont_cat_cat_dim_multi_metric(self): class DatatablesUtilityTests(TestCase): def test_nan_data_point(self): # Needs to be cast to python int - result = datatables.format_data_point(np.nan) + result = datatables._format_data_point(np.nan) self.assertIsNone(result) def test_str_data_point(self): - result = datatables.format_data_point('abc') + result = datatables._format_data_point('abc') self.assertEqual('abc', result) def test_int64_data_point(self): # Needs to be cast to python int - result = datatables.format_data_point(np.int64(1)) + result = datatables._format_data_point(np.int64(1)) self.assertEqual(int(1), result) def test_datetime_data_point(self): # Needs to be converted to milliseconds - result = datatables.format_data_point(pd.Timestamp(date(2000, 1, 1))) + result = datatables._format_data_point(pd.Timestamp(date(2000, 1, 1))) self.assertEqual('2000-01-01T00:00:00', result) diff --git a/fireant/tests/slicer/transformers/test_highcharts.py b/fireant/tests/slicer/transformers/test_highcharts.py index 158d3afa..3dd22035 100644 --- a/fireant/tests/slicer/transformers/test_highcharts.py +++ b/fireant/tests/slicer/transformers/test_highcharts.py @@ -4,21 +4,21 @@ import numpy as np import pandas as pd - -from fireant.slicer.transformers import HighchartsTransformer, TransformationException, HighchartsColumnTransformer +from fireant.slicer.transformers import (HighchartsLineTransformer, TransformationException, + HighchartsColumnTransformer, + HighchartsBarTransformer) from fireant.slicer.transformers import highcharts from fireant.tests.slicer.transformers.base import BaseTransformerTests -class HighChartsLineTransformerTests(BaseTransformerTests): +class HighchartsLineTransformerTests(BaseTransformerTests): """ Line charts work with the following requests: 1-cont-dim, *-metric 1-cont-dim, *-dim, *-metric """ - hc_tx = HighchartsTransformer() - type = HighchartsTransformer.line + hc_tx = HighchartsLineTransformer() def evaluate_result(self, df, result): result_data = [series['data'] for series in result['series']] @@ -33,7 +33,7 @@ def evaluate_chart_options(self, result, num_series=1, xaxis_type='linear', dash self.assertSetEqual({'text'}, set(result['title'].keys())) self.assertIsNone(result['title']['text']) - self.assertEqual(self.type, result['chart']['type']) + self.assertEqual(HighchartsLineTransformer.chart_type, result['chart']['type']) self.assertSetEqual({'type'}, set(result['xAxis'].keys())) self.assertEqual(xaxis_type, result['xAxis']['type']) @@ -223,14 +223,14 @@ def test_require_at_least_one_dimension(self): self.hc_tx.transform(df, self.no_dims_multi_metric_schema) -class HighChartsColumnTransformerTests(BaseTransformerTests): +class HighchartsColumnTransformerTests(BaseTransformerTests): """ Bar and Column charts work with the following requests: 1-dim, *-metric 2-dim, 1-metric """ - type = HighchartsColumnTransformer.column + type = HighchartsColumnTransformer.chart_type def evaluate_chart_options(self, result, n_results=1, categories=None): self.assertSetEqual({'title', 'series', 'chart', 'tooltip', 'xAxis', 'yAxis'}, set(result.keys())) @@ -252,7 +252,7 @@ def evaluate_chart_options(self, result, n_results=1, categories=None): @classmethod def setUpClass(cls): - cls.hc_tx = HighchartsColumnTransformer(cls.type) + cls.hc_tx = HighchartsColumnTransformer() def evaluate_result(self, df, result): result_data = [series['data'] for series in result['series']] @@ -356,21 +356,25 @@ def test_require_no_more_than_one_dimension_with_multi_metrics(self): self.hc_tx.transform(df, self.cat_cat_dims_multi_metric_schema) -class HighChartsBarTransformerTests(HighChartsColumnTransformerTests): - type = HighchartsColumnTransformer.bar +class HighchartsBarTransformerTests(HighchartsColumnTransformerTests): + type = HighchartsBarTransformer.chart_type + + @classmethod + def setUpClass(cls): + cls.hc_tx = HighchartsBarTransformer() class HighchartsUtilityTests(TestCase): def test_str_data_point(self): - result = highcharts.format_data_point('abc') + result = highcharts._format_data_point('abc') self.assertEqual('abc', result) def test_int64_data_point(self): # Needs to be cast to python int - result = highcharts.format_data_point(np.int64(1)) + result = highcharts._format_data_point(np.int64(1)) self.assertEqual(int(1), result) def test_datetime_data_point(self): # Needs to be converted to milliseconds - result = highcharts.format_data_point(pd.Timestamp(date(2000, 1, 1))) + result = highcharts._format_data_point(pd.Timestamp(date(2000, 1, 1))) self.assertEqual(946684800000, result) diff --git a/setup.py b/setup.py index 5ddaf1fe..42dc3d62 100644 --- a/setup.py +++ b/setup.py @@ -30,9 +30,9 @@ def readme(): include_package_data=True, # Details - url="https://github.com/kayak/fireant", + url='https://github.com/kayak/fireant', - description="A data analysis tool for Python and Jupyter Notebooks", + description='A data analysis tool for Python and Jupyter Notebooks', long_description=readme(), classifiers=[ @@ -61,8 +61,9 @@ def readme(): 'mock', ], extras_require={ - 'vertica': ["vertica-python>=0.6"], + 'vertica': ['vertica-python>=0.6'], + 'matplotlib': ['matplotlib', 'seaborn'], }, - test_suite="fireant.tests", + test_suite='fireant.tests', ) From ed1ff806505c54b2ef1386a1c919ad5620750745 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Fri, 12 Aug 2016 16:55:03 +0200 Subject: [PATCH 2/4] Added some stubs for notebook transformers --- fireant/slicer/managers.py | 4 +- fireant/slicer/transformers/__init__.py | 2 +- fireant/slicer/transformers/bundles.py | 25 ++++++------ fireant/slicer/transformers/notebook.py | 51 ++++++++++++++++++++++++- fireant/tests/slicer/test_manager.py | 4 +- setup.py | 2 +- 6 files changed, 69 insertions(+), 19 deletions(-) diff --git a/fireant/slicer/managers.py b/fireant/slicer/managers.py index 0111265d..03ec52e4 100644 --- a/fireant/slicer/managers.py +++ b/fireant/slicer/managers.py @@ -97,8 +97,8 @@ def get_query_schema(self, metrics=None, dimensions=None, 'joins': schema_joins, 'metrics': schema_metrics, 'dimensions': schema_dimensions, - 'mfilters': self._filters_schema(self.slicer.metrics, metric_filters or [], self._default_metric_definition, - element_label='metric'), + 'mfilters': self._filters_schema(self.slicer.metrics, metric_filters or [], + self._default_metric_definition, element_label='metric'), 'dfilters': self._filters_schema(self.slicer.dimensions, dimension_filters or [], self._default_dimension_definition), 'references': self._references_schema(references, dimensions or [], schema_dimensions), diff --git a/fireant/slicer/transformers/__init__.py b/fireant/slicer/transformers/__init__.py index 6107e4b3..a5e8ade1 100644 --- a/fireant/slicer/transformers/__init__.py +++ b/fireant/slicer/transformers/__init__.py @@ -4,5 +4,5 @@ from .datatables import (DataTablesRowIndexTransformer, DataTablesColumnIndexTransformer, CSVRowIndexTransformer, CSVColumnIndexTransformer) from .highcharts import HighchartsLineTransformer, HighchartsColumnTransformer, HighchartsBarTransformer -from .notebook import MatplotlibTransformer, PandasTransformer +from .notebook import PlotlyTransformer, PandasTransformer from .bundles import bundles \ No newline at end of file diff --git a/fireant/slicer/transformers/bundles.py b/fireant/slicer/transformers/bundles.py index 61dd032f..b8e1d5c4 100644 --- a/fireant/slicer/transformers/bundles.py +++ b/fireant/slicer/transformers/bundles.py @@ -3,30 +3,31 @@ DataTablesRowIndexTransformer, DataTablesColumnIndexTransformer, PandasTransformer) notebook_tx = { - 'row_index': PandasTransformer(), - 'column_index': PandasTransformer(), + 'row_index_table': PandasTransformer(), + 'column_index_table': PandasTransformer(), } try: - from fireant.slicer.transformers.notebook import MatplotlibTransformer + from fireant.slicer.transformers.notebook import PlotlyTransformer - notebook_tx['line'] = MatplotlibTransformer() - notebook_tx['column'] = MatplotlibTransformer() - notebook_tx['bar'] = MatplotlibTransformer() + notebook_tx['line_chart'] = PlotlyTransformer() + notebook_tx['column_chart'] = PlotlyTransformer() + notebook_tx['bar_chart'] = PlotlyTransformer() except ImportError: # Matplotlib not installed pass bundles = { - 'notebook': notebook_tx, + # Unfinished + # 'notebook': notebook_tx, 'highcharts': { - 'line': HighchartsLineTransformer(), - 'column ': HighchartsColumnTransformer(), - 'bar': HighchartsBarTransformer(), + 'line_chart': HighchartsLineTransformer(), + 'column_chart ': HighchartsColumnTransformer(), + 'bar_chart': HighchartsBarTransformer(), }, 'datatables': { - 'row_index': DataTablesRowIndexTransformer(), - 'column_index': DataTablesColumnIndexTransformer(), + 'row_index_table': DataTablesRowIndexTransformer(), + 'column_index_table': DataTablesColumnIndexTransformer(), }, } diff --git a/fireant/slicer/transformers/notebook.py b/fireant/slicer/transformers/notebook.py index 624e25d8..918edc9c 100644 --- a/fireant/slicer/transformers/notebook.py +++ b/fireant/slicer/transformers/notebook.py @@ -1,12 +1,59 @@ # coding: utf-8 +import numpy as np +import pandas as pd + from . import Transformer -class MatplotlibTransformer(Transformer): +class PlotlyTransformer(Transformer): + # FIXME Unfinished + def transform(self, data_frame, display_schema): - return data_frame.plot() + if isinstance(data_frame.index, pd.MultiIndex): + data_frame = self._reorder_index_levels(data_frame, display_schema) + + dim_ordinal = {name: ordinal + for ordinal, name in enumerate(data_frame.index.names)} + return self._prepare_data_frame(data_frame, dim_ordinal, display_schema['dimensions']) + + def _prepare_data_frame(self, data_frame, dim_ordinal, dimensions): + # Replaces invalid values and unstacks the data frame for line charts. + + # Force all fields to be float (Safer for highcharts) + data_frame = data_frame.astype(np.float).replace([np.inf, -np.inf], np.nan) + + # Unstack multi-indices + if 1 < len(dimensions): + # We need to unstack all of the dimensions here after the first dimension, which is the first dimension in + # the dimensions list, not necessarily the one in the dataframe + unstack_levels = list(self._unstack_levels(dimensions[1:], dim_ordinal)) + data_frame = data_frame.unstack(level=unstack_levels) + + return data_frame + + def _unstack_levels(self, dimensions, dim_ordinal): + for dimension in dimensions: + for id_field in dimension['id_fields']: + yield dim_ordinal[id_field] + + if 'label_field' in dimension: + yield dim_ordinal[dimension['label_field']] + + def _reorder_index_levels(self, data_frame, display_schema): + dimension_orders = [id_field + for d in display_schema['dimensions'] + for id_field in + (d['id_fields'] + ( + [d['label_field']] + if 'label_field' in d + else []))] + reordered = data_frame.reorder_levels(data_frame.index.names.index(level) + for level in dimension_orders) + return reordered class PandasTransformer(Transformer): + # FIXME Unfinished + def transform(self, data_frame, display_schema): return data_frame.plot() diff --git a/fireant/tests/slicer/test_manager.py b/fireant/tests/slicer/test_manager.py index 6bd8c9be..64a82dfb 100644 --- a/fireant/tests/slicer/test_manager.py +++ b/fireant/tests/slicer/test_manager.py @@ -10,6 +10,8 @@ def test_transformers(self): slicer = Slicer(Table('test')) self.assertTrue(hasattr(slicer, 'manager')) - self.assertTrue(hasattr(slicer, 'notebook')) self.assertTrue(hasattr(slicer, 'highcharts')) self.assertTrue(hasattr(slicer, 'datatables')) + + # False until the feature can be fully added + self.assertFalse(hasattr(slicer, 'notebook')) diff --git a/setup.py b/setup.py index 42dc3d62..b37d5ed9 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ def readme(): ], extras_require={ 'vertica': ['vertica-python>=0.6'], - 'matplotlib': ['matplotlib', 'seaborn'], + 'plotly': ['plotly'], }, test_suite='fireant.tests', From 5fba6565779a8bd4fc9a4b07de1f2a45aab39afb Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 16 Aug 2016 10:43:48 +0200 Subject: [PATCH 3/4] Added more test coverage to slicer manager --- fireant/dashboards/managers.py | 2 +- fireant/slicer/managers.py | 18 ++-- fireant/slicer/transformers/bundles.py | 7 +- .../tests/dashboards/test_dashboard_api.py | 4 +- fireant/tests/slicer/test_manager.py | 87 ++++++++++++++++- fireant/tests/slicer/test_slicer_api.py | 96 +++++++++---------- 6 files changed, 147 insertions(+), 67 deletions(-) diff --git a/fireant/dashboards/managers.py b/fireant/dashboards/managers.py index 2f5fe154..ad54e0ae 100644 --- a/fireant/dashboards/managers.py +++ b/fireant/dashboards/managers.py @@ -22,7 +22,7 @@ def render(self, dimensions=None, metric_filters=None, dimension_filters=None, r references=enabled_references, operations=enabled_operations ) - display_schema = self.widget_group.slicer.manager.get_display_schema( + display_schema = self.widget_group.slicer.manager.display_schema( metrics=enabled_metrics, dimensions=enabled_dimensions ) diff --git a/fireant/slicer/managers.py b/fireant/slicer/managers.py index 03ec52e4..7a662a3b 100644 --- a/fireant/slicer/managers.py +++ b/fireant/slicer/managers.py @@ -25,7 +25,7 @@ def _get_and_transform_data(self, tx, metrics=tuple(), dimensions=tuple(), df = self.data(metrics=metrics, dimensions=dimensions, metric_filters=metric_filters, dimension_filters=dimension_filters, references=references, operations=operations) - display_schema = self.get_display_schema(metrics, dimensions, references) + display_schema = self.display_schema(metrics, dimensions, references) return tx.transform(df, display_schema) def data(self, metrics=tuple(), dimensions=tuple(), @@ -64,9 +64,9 @@ def data(self, metrics=tuple(), dimensions=tuple(), raise SlicerException('Unable to execute queries until a database is configured. Please import ' '`fireant.settings` and set some value to `settings.database`.') - query_schema = self.get_query_schema(metrics=metrics, dimensions=dimensions, - metric_filters=metric_filters, dimension_filters=dimension_filters, - references=references, operations=operations) + query_schema = self.query_schema(metrics=metrics, dimensions=dimensions, + metric_filters=metric_filters, dimension_filters=dimension_filters, + references=references, operations=operations) data_frame = self._query_data(**query_schema) @@ -81,9 +81,9 @@ def data(self, metrics=tuple(), dimensions=tuple(), return data_frame - def get_query_schema(self, metrics=None, dimensions=None, - metric_filters=None, dimension_filters=None, - references=None, operations=None): + def query_schema(self, metrics=None, dimensions=None, + metric_filters=None, dimension_filters=None, + references=None, operations=None): if not metrics: raise ValueError('Invalid slicer request. At least one metric is required to build a query.') @@ -108,7 +108,7 @@ def get_query_schema(self, metrics=None, dimensions=None, for dimension in operation.schemas(self.slicer)], } - def get_display_schema(self, metrics=None, dimensions=None, references=None): + def display_schema(self, metrics=None, dimensions=None, references=None): """ Builds a display schema for @@ -279,6 +279,6 @@ def _get_and_transform_data(self, tx, metrics=tuple(), dimensions=tuple(), metric_filters=metric_filters, dimension_filters=dimension_filters, references=references, operations=operations) - display_schema = self._manager.get_display_schema(metrics, dimensions, references) + display_schema = self._manager.display_schema(metrics, dimensions, references) return tx.transform(df, display_schema) diff --git a/fireant/slicer/transformers/bundles.py b/fireant/slicer/transformers/bundles.py index b8e1d5c4..8ca0d912 100644 --- a/fireant/slicer/transformers/bundles.py +++ b/fireant/slicer/transformers/bundles.py @@ -1,6 +1,7 @@ # coding: utf-8 from . import (HighchartsLineTransformer, HighchartsColumnTransformer, HighchartsBarTransformer, - DataTablesRowIndexTransformer, DataTablesColumnIndexTransformer, PandasTransformer) + DataTablesRowIndexTransformer, DataTablesColumnIndexTransformer, PandasTransformer, + CSVRowIndexTransformer, CSVColumnIndexTransformer) notebook_tx = { 'row_index_table': PandasTransformer(), @@ -23,11 +24,13 @@ # 'notebook': notebook_tx, 'highcharts': { 'line_chart': HighchartsLineTransformer(), - 'column_chart ': HighchartsColumnTransformer(), + 'column_chart': HighchartsColumnTransformer(), 'bar_chart': HighchartsBarTransformer(), }, 'datatables': { 'row_index_table': DataTablesRowIndexTransformer(), 'column_index_table': DataTablesColumnIndexTransformer(), + 'row_index_csv': CSVRowIndexTransformer(), + 'column_index_csv': CSVColumnIndexTransformer(), }, } diff --git a/fireant/tests/dashboards/test_dashboard_api.py b/fireant/tests/dashboards/test_dashboard_api.py index 6296c710..80674c4d 100644 --- a/fireant/tests/dashboards/test_dashboard_api.py +++ b/fireant/tests/dashboards/test_dashboard_api.py @@ -56,7 +56,7 @@ def setUpClass(cls): cls.mock_dataframe = MagicMock() cls.mock_display_schema = {'OK'} cls.test_slicer.manager.data = Mock(return_value=cls.mock_dataframe) - cls.test_slicer.manager.get_display_schema = Mock(return_value=cls.mock_display_schema) + cls.test_slicer.manager.display_schema = Mock(return_value=cls.mock_display_schema) def assert_slicer_queried(self, metrics, dimensions=None, mfilters=None, dfilters=None, references=None, operations=None): @@ -68,7 +68,7 @@ def assert_slicer_queried(self, metrics, dimensions=None, mfilters=None, dfilter references=references or [], operations=operations or [], ) - self.test_slicer.manager.get_display_schema.assert_called_with( + self.test_slicer.manager.display_schema.assert_called_with( metrics=metrics, dimensions=dimensions or [], ) diff --git a/fireant/tests/slicer/test_manager.py b/fireant/tests/slicer/test_manager.py index 64a82dfb..06e30816 100644 --- a/fireant/tests/slicer/test_manager.py +++ b/fireant/tests/slicer/test_manager.py @@ -1,17 +1,94 @@ # coding: utf-8 from unittest import TestCase +from mock import patch, MagicMock + from fireant.slicer import Slicer +from fireant.slicer.managers import SlicerManager +from fireant.slicer.transformers import * from pypika import Table class ManagerInitializationTests(TestCase): + def setUp(self): + self.slicer = Slicer(Table('test')) + def test_transformers(self): - slicer = Slicer(Table('test')) + self.assertTrue(hasattr(self.slicer, 'manager')) + + self.assertTrue(hasattr(self.slicer, 'highcharts')) + self.assertTrue(hasattr(self.slicer.highcharts, 'line_chart')) + self.assertTrue(hasattr(self.slicer.highcharts, 'column_chart')) + self.assertTrue(hasattr(self.slicer.highcharts, 'bar_chart')) - self.assertTrue(hasattr(slicer, 'manager')) - self.assertTrue(hasattr(slicer, 'highcharts')) - self.assertTrue(hasattr(slicer, 'datatables')) + self.assertTrue(hasattr(self.slicer, 'datatables')) + self.assertTrue(hasattr(self.slicer.datatables, 'row_index_table')) + self.assertTrue(hasattr(self.slicer.datatables, 'column_index_table')) + self.assertTrue(hasattr(self.slicer.datatables, 'row_index_csv')) + self.assertTrue(hasattr(self.slicer.datatables, 'column_index_csv')) # False until the feature can be fully added - self.assertFalse(hasattr(slicer, 'notebook')) + self.assertFalse(hasattr(self.slicer, 'notebook')) + + @patch('fireant.settings.database') + def test_data(self, mock_db): + self.slicer.manager.query_schema = MagicMock() + self.slicer.manager._query_data = MagicMock() + + mock_args = {'metrics': 0, 'dimensions': 1, + 'metric_filters': 2, 'dimension_filters': 3, + 'references': 4, 'operations': 5,} + self.slicer.manager.query_schema.return_value = {'a': 1, 'b': 2} + self.slicer.manager._query_data.return_value = 'OK' + + result = self.slicer.manager.data(**mock_args) + + self.assertEqual('OK', result) + self.slicer.manager.query_schema.assert_called_once_with(**mock_args) + + @patch.object(SlicerManager, 'display_schema') + @patch.object(SlicerManager, 'data') + def _test_transform(self, test_func, mock_transform, mock_sm_data, mock_sm_ds): + mock_df, mock_schema, mock_return = 'dataframe', 'schema', 'OK' + mock_sm_data.return_value = mock_df + mock_sm_ds.return_value = mock_schema + mock_transform.return_value = mock_return + + mock_args = {'metrics': 0, 'dimensions': 1, + 'metric_filters': 2, 'dimension_filters': 3, + 'references': 4, 'operations': 5,} + + result = test_func(**mock_args) + + self.assertEqual(mock_return, result) + mock_sm_data.assert_called_once_with(**mock_args) + mock_sm_ds.assert_called_once_with(mock_args['metrics'], mock_args['dimensions'], mock_args['references']) + mock_transform.assert_called_once_with(mock_df, mock_schema) + + @patch.object(HighchartsLineTransformer, 'transform') + def test_transform_highcharts_line_chart(self, mock_transform): + self._test_transform(self.slicer.highcharts.line_chart, mock_transform) + + @patch.object(HighchartsColumnTransformer, 'transform') + def test_transform_highcharts_column_chart(self, mock_transform): + self._test_transform(self.slicer.highcharts.column_chart, mock_transform) + + @patch.object(HighchartsBarTransformer, 'transform') + def test_transform_highcharts_bar_chart(self, mock_transform): + self._test_transform(self.slicer.highcharts.bar_chart, mock_transform) + + @patch.object(DataTablesRowIndexTransformer, 'transform') + def test_transform_datatables_row_index_table(self, mock_transform): + self._test_transform(self.slicer.datatables.row_index_table, mock_transform) + + @patch.object(DataTablesColumnIndexTransformer, 'transform') + def test_transform_datatables_col_index_table(self, mock_transform): + self._test_transform(self.slicer.datatables.column_index_table, mock_transform) + + @patch.object(CSVRowIndexTransformer, 'transform') + def test_transform_datatables_row_index_table(self, mock_transform): + self._test_transform(self.slicer.datatables.row_index_csv, mock_transform) + + @patch.object(CSVColumnIndexTransformer, 'transform') + def test_transform_datatables_col_index_table(self, mock_transform): + self._test_transform(self.slicer.datatables.column_index_csv, mock_transform) diff --git a/fireant/tests/slicer/test_slicer_api.py b/fireant/tests/slicer/test_slicer_api.py index be78ca6a..7c8ebc12 100644 --- a/fireant/tests/slicer/test_slicer_api.py +++ b/fireant/tests/slicer/test_slicer_api.py @@ -79,7 +79,7 @@ def setUpClass(cls): class SlicerSchemaMetricTests(SlicerSchemaTests): def test_metric_with_default_definition(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], ) @@ -90,7 +90,7 @@ def test_metric_with_default_definition(self): self.assertEqual('SUM("test"."foo")', str(query_schema['metrics']['foo'])) def test_metric_with_custom_definition(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['bar'], ) @@ -103,7 +103,7 @@ def test_metric_with_custom_definition(self): class SlicerSchemaDimensionTests(SlicerSchemaTests): def test_date_dimension_default_interval(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['date'], ) @@ -118,7 +118,7 @@ def test_date_dimension_default_interval(self): self.assertEqual('ROUND("test"."dt",\'DD\')', str(query_schema['dimensions']['date'])) def test_date_dimension_custom_interval(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], # TODO This could be improved by using an object dimensions=[('date', DatetimeDimension.week)], @@ -134,7 +134,7 @@ def test_date_dimension_custom_interval(self): self.assertEqual('ROUND("test"."dt",\'WW\')', str(query_schema['dimensions']['date'])) def test_numeric_dimension_default_interval(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['clicks'], ) @@ -149,7 +149,7 @@ def test_numeric_dimension_default_interval(self): self.assertEqual('MOD("test"."clicks"+0,1)', str(query_schema['dimensions']['clicks'])) def test_numeric_dimension_custom_interval(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], # TODO This could be improved by using an object dimensions=[('clicks', 100, 25)], @@ -165,7 +165,7 @@ def test_numeric_dimension_custom_interval(self): self.assertEqual('MOD("test"."clicks"+25,100)', str(query_schema['dimensions']['clicks'])) def test_categorical_dimension(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], ) @@ -180,7 +180,7 @@ def test_categorical_dimension(self): self.assertEqual('"test"."locale"', str(query_schema['dimensions']['locale'])) def test_unique_dimension_single_id(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['account'], ) @@ -196,7 +196,7 @@ def test_unique_dimension_single_id(self): self.assertEqual('"test"."account_name"', str(query_schema['dimensions']['account_label'])) def test_unique_dimension_composite_id(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['keyword'], ) @@ -214,7 +214,7 @@ def test_unique_dimension_composite_id(self): self.assertEqual('"test"."keyword_name"', str(query_schema['dimensions']['keyword_label'])) def test_multiple_metrics_and_dimensions(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo', 'bar'], dimensions=[('date', DatetimeDimension.month), ('clicks', 50, 100), 'locale', 'account'], ) @@ -236,7 +236,7 @@ def test_multiple_metrics_and_dimensions(self): class SlicerSchemaFilterTests(SlicerSchemaTests): def test_dimension_filter_eq(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], dimension_filters=[EqualityFilter('locale', EqualityOperator.eq, 'en')], @@ -254,7 +254,7 @@ def test_dimension_filter_eq(self): self.assertListEqual(['"test"."locale"=\'en\''], [str(f) for f in query_schema['dfilters']]) def test_dimension_filter_ne(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], dimension_filters=[ @@ -274,7 +274,7 @@ def test_dimension_filter_ne(self): self.assertListEqual(['"test"."locale"<>\'en\''], [str(f) for f in query_schema['dfilters']]) def test_dimension_filter_in(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], dimension_filters=[ @@ -294,7 +294,7 @@ def test_dimension_filter_in(self): self.assertListEqual(['"test"."locale" IN (\'en\',\'es\',\'de\')'], [str(f) for f in query_schema['dfilters']]) def test_dimension_filter_like(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], dimension_filters=[ @@ -314,7 +314,7 @@ def test_dimension_filter_like(self): self.assertListEqual(['"test"."locale" LIKE \'e%\''], [str(f) for f in query_schema['dfilters']]) def test_dimension_filter_gt(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], dimension_filters=[ @@ -334,7 +334,7 @@ def test_dimension_filter_gt(self): self.assertListEqual(['"test"."dt">\'2000-01-01\''], [str(f) for f in query_schema['dfilters']]) def test_dimension_filter_lt(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], dimension_filters=[ @@ -354,7 +354,7 @@ def test_dimension_filter_lt(self): self.assertListEqual(['"test"."dt"<\'2000-01-01\''], [str(f) for f in query_schema['dfilters']]) def test_dimension_filter_gte(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], dimension_filters=[ @@ -374,7 +374,7 @@ def test_dimension_filter_gte(self): self.assertListEqual(['"test"."dt">=\'2000-01-01\''], [str(f) for f in query_schema['dfilters']]) def test_dimension_filter_lte(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], dimension_filters=[ @@ -394,7 +394,7 @@ def test_dimension_filter_lte(self): self.assertListEqual(['"test"."dt"<=\'2000-01-01\''], [str(f) for f in query_schema['dfilters']]) def test_dimension_filter_daterange(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], dimension_filters=[ @@ -412,7 +412,7 @@ def test_dimension_filter_daterange(self): [str(f) for f in query_schema['dfilters']]) def test_unique_dimension_filter_single_id(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['account'], dimension_filters=[EqualityFilter('account', EqualityOperator.eq, 1)], @@ -431,7 +431,7 @@ def test_unique_dimension_filter_single_id(self): self.assertListEqual(['"test"."account_id"=1'], [str(f) for f in query_schema['dfilters']]) def test_unique_dimension_filter_composite_id(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['keyword'], dimension_filters=[EqualityFilter('keyword', EqualityOperator.eq, (1, 'broad'))], @@ -452,7 +452,7 @@ def test_unique_dimension_filter_composite_id(self): [str(f) for f in query_schema['dfilters']]) def test_unique_dimension_filter_label(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['account'], dimension_filters=[WildcardFilter('account.label', 'nam%')], @@ -471,7 +471,7 @@ def test_unique_dimension_filter_label(self): self.assertListEqual(['"test"."account_name" LIKE \'nam%\''], [str(f) for f in query_schema['dfilters']]) def test_metric_filter_eq(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], metric_filters=[EqualityFilter('foo', EqualityOperator.eq, 0)], @@ -489,7 +489,7 @@ def test_metric_filter_eq(self): self.assertListEqual(['SUM("test"."foo")=0'], [str(f) for f in query_schema['mfilters']]) def test_metric_filter_ne(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], metric_filters=[ @@ -509,7 +509,7 @@ def test_metric_filter_ne(self): self.assertListEqual(['SUM("test"."foo")<>0'], [str(f) for f in query_schema['mfilters']]) def test_metric_filter_gt(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], metric_filters=[ @@ -529,7 +529,7 @@ def test_metric_filter_gt(self): self.assertListEqual(['SUM("test"."foo")>100'], [str(f) for f in query_schema['mfilters']]) def test_metric_filter_lt(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], metric_filters=[ @@ -549,7 +549,7 @@ def test_metric_filter_lt(self): self.assertListEqual(['SUM("test"."foo")<100'], [str(f) for f in query_schema['mfilters']]) def test_metric_filter_gte(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], metric_filters=[ @@ -569,7 +569,7 @@ def test_metric_filter_gte(self): self.assertListEqual(['SUM("test"."foo")>=100'], [str(f) for f in query_schema['mfilters']]) def test_metric_filter_lte(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], metric_filters=[ @@ -590,7 +590,7 @@ def test_metric_filter_lte(self): def test_invalid_dimensions_raise_exception(self): with self.assertRaises(SlicerException): - self.test_slicer.manager.get_query_schema( + self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], dimension_filters=[ @@ -600,7 +600,7 @@ def test_invalid_dimensions_raise_exception(self): def test_invalid_metrics_raise_exception(self): with self.assertRaises(SlicerException): - self.test_slicer.manager.get_query_schema( + self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], metric_filters=[ @@ -610,7 +610,7 @@ def test_invalid_metrics_raise_exception(self): def test_metrics_dont_work_for_dimensions(self): with self.assertRaises(SlicerException): - self.test_slicer.manager.get_query_schema( + self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], dimension_filters=[ @@ -620,7 +620,7 @@ def test_metrics_dont_work_for_dimensions(self): def test_dimensions_dont_work_for_metrics(self): with self.assertRaises(SlicerException): - self.test_slicer.manager.get_query_schema( + self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], metric_filters=[ @@ -631,7 +631,7 @@ def test_dimensions_dont_work_for_metrics(self): class SlicerSchemaReferenceTests(SlicerSchemaTests): def _reference_test_with_date(self, reference): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['date'], references=[reference], @@ -683,7 +683,7 @@ def test_reference_yoy_p_with_date(self): def test_reference_missing_dimension(self): # Reference dimension is required in order to use a reference with it with self.assertRaises(SlicerException): - self.test_slicer.manager.get_query_schema( + self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=[], references=[WoW('date')], @@ -692,14 +692,14 @@ def test_reference_missing_dimension(self): def test_reference_wrong_dimension_type(self): # Reference dimension is required in order to use a reference with it with self.assertRaises(SlicerException): - self.test_slicer.manager.get_query_schema( + self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['locale'], references=[WoW('locale')], ) def test_rollup_operation(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['date', 'locale', 'account'], operations=[Rollup(['locale', 'account'])], @@ -719,7 +719,7 @@ def test_rollup_operation(self): class SlicerDisplaySchemaTests(SlicerSchemaTests): def test_metric_with_default_definition(self): - display_schema = self.test_slicer.manager.get_display_schema( + display_schema = self.test_slicer.manager.display_schema( metrics=['foo'], ) @@ -733,7 +733,7 @@ def test_metric_with_default_definition(self): ) def test_metric_with_custom_definition(self): - display_schema = self.test_slicer.manager.get_display_schema( + display_schema = self.test_slicer.manager.display_schema( metrics=['bar'], ) @@ -747,7 +747,7 @@ def test_metric_with_custom_definition(self): ) def test_date_dimension_default_interval(self): - display_schema = self.test_slicer.manager.get_display_schema( + display_schema = self.test_slicer.manager.display_schema( metrics=['foo'], dimensions=['date'], ) @@ -765,7 +765,7 @@ def test_date_dimension_default_interval(self): ) def test_numeric_dimension_default_interval(self): - display_schema = self.test_slicer.manager.get_display_schema( + display_schema = self.test_slicer.manager.display_schema( metrics=['foo'], dimensions=['clicks'], ) @@ -783,7 +783,7 @@ def test_numeric_dimension_default_interval(self): ) def test_categorical_dimension(self): - display_schema = self.test_slicer.manager.get_display_schema( + display_schema = self.test_slicer.manager.display_schema( metrics=['foo'], dimensions=['locale'], ) @@ -802,7 +802,7 @@ def test_categorical_dimension(self): ) def test_unique_dimension_single_id(self): - display_schema = self.test_slicer.manager.get_display_schema( + display_schema = self.test_slicer.manager.display_schema( metrics=['foo'], dimensions=['account'], ) @@ -820,7 +820,7 @@ def test_unique_dimension_single_id(self): ) def test_unique_dimension_composite_id(self): - display_schema = self.test_slicer.manager.get_display_schema( + display_schema = self.test_slicer.manager.display_schema( metrics=['foo'], dimensions=['keyword'], ) @@ -839,7 +839,7 @@ def test_unique_dimension_composite_id(self): ) def test_multiple_metrics_and_dimensions(self): - display_schema = self.test_slicer.manager.get_display_schema( + display_schema = self.test_slicer.manager.display_schema( metrics=['foo', 'bar'], dimensions=[('date', DatetimeDimension.month), ('clicks', 50, 100), 'locale', 'account'], ) @@ -865,7 +865,7 @@ def test_multiple_metrics_and_dimensions(self): ) def test_reference_wow_with_date(self): - display_schema = self.test_slicer.manager.get_display_schema( + display_schema = self.test_slicer.manager.display_schema( metrics=['foo'], dimensions=['date'], references=[WoW('date')], @@ -886,7 +886,7 @@ def test_reference_wow_with_date(self): class SlicerSchemaJoinTests(SlicerSchemaTests): def test_metric_with_join(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['piddle'], ) @@ -901,7 +901,7 @@ def test_metric_with_join(self): self.assertEqual('SUM("join"."piddle")', str(query_schema['metrics']['piddle'])) def test_metric_with_complex_join(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['paddle'], ) @@ -916,7 +916,7 @@ def test_metric_with_complex_join(self): self.assertEqual('SUM("join"."paddle"+"test"."foo")', str(query_schema['metrics']['paddle'])) def test_dimension_with_join(self): - query_schema = self.test_slicer.manager.get_query_schema( + query_schema = self.test_slicer.manager.query_schema( metrics=['foo'], dimensions=['blah'], ) From 7f77141a681357a4ab0c1c3e6b091266246211a4 Mon Sep 17 00:00:00 2001 From: Timothy Heys Date: Tue, 16 Aug 2016 11:09:47 +0200 Subject: [PATCH 4/4] Added test for nan data point to highcharts --- fireant/slicer/transformers/highcharts.py | 4 ---- fireant/tests/slicer/transformers/test_highcharts.py | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/fireant/slicer/transformers/highcharts.py b/fireant/slicer/transformers/highcharts.py index 6f8ecc35..ca70cf5c 100644 --- a/fireant/slicer/transformers/highcharts.py +++ b/fireant/slicer/transformers/highcharts.py @@ -12,8 +12,6 @@ def _format_data_point(value): return int(value.asm8) // int(1e6) if np.isnan(value): return None - if np.isnan(value): - return None if isinstance(value, np.int64): # Cannot serialize np.int64 to json return int(value) @@ -240,8 +238,6 @@ def _make_categories(data_frame, dim_ordinal, display_schema): label_field = category_dimension['label_field'] return data_frame.index.get_level_values(label_field).unique().tolist() - return [] - class HighchartsBarTransformer(HighchartsColumnTransformer): chart_type = 'bar' diff --git a/fireant/tests/slicer/transformers/test_highcharts.py b/fireant/tests/slicer/transformers/test_highcharts.py index 3dd22035..f73258fd 100644 --- a/fireant/tests/slicer/transformers/test_highcharts.py +++ b/fireant/tests/slicer/transformers/test_highcharts.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd + from fireant.slicer.transformers import (HighchartsLineTransformer, TransformationException, HighchartsColumnTransformer, HighchartsBarTransformer) @@ -378,3 +379,8 @@ def test_datetime_data_point(self): # Needs to be converted to milliseconds result = highcharts._format_data_point(pd.Timestamp(date(2000, 1, 1))) self.assertEqual(946684800000, result) + + def test_nan_data_point(self): + # Needs to be cast to python int + result = highcharts._format_data_point(np.nan) + self.assertIsNone(result)