Merge pull request #43 from lyft/merge_apache_d1b81e0

Merge apache d1b81e0

UPDATING.md

@@ -44,6 +44,11 @@ creation of permissions set `FAB_UPDATE_PERMS = False` on config.
 which adds missing non-nullable fields and uniqueness constraints to the metrics
 and sql_metrics tables. Depending on the integrity of the data, manual
 intervention may be required.
+* [7616](https://github.com/apache/incubator-superset/pull/7616): this bug fix
+changes time_compare deltas to correctly evaluate to the number of days prior
+instead of number of days in the future. It will change the data for advanced
+analytics time_compare so `1 year` from 5/1/2019 will be calculated as 365 days
+instead of 366 days.
 
 ## Superset 0.32.0
 

requirements.txt

@@ -75,7 +75,7 @@ selenium==3.141.0
 simplejson==3.15.0
 six==1.11.0               # via bleach, cryptography, flask-jwt-extended, flask-talisman, isodate, jsonschema, pathlib2, polyline, prison, pydruid, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json
 sqlalchemy-utils==0.33.11
-sqlalchemy==1.3.1
+sqlalchemy==1.3.5
 sqlparse==0.2.4
 urllib3==1.24.3           # via requests, selenium
 vine==1.1.4               # via amqp

setup.py

@@ -102,7 +102,7 @@ def get_git_sha():
         'retry>=0.9.2',
         'selenium>=3.141.0',
         'simplejson>=3.15.0',
-        'sqlalchemy>=1.3.1,<2.0',
+        'sqlalchemy>=1.3.5,<2.0',
         'sqlalchemy-utils>=0.33.2',
         'sqlparse',
         'wtforms-json',

superset/assets/src/SqlLab/main.less

@@ -243,7 +243,7 @@ div.Workspace {
     .schemaPane {
         flex: 0 0 300px;
         max-width: 300px;
-        transition: all .3s ease-in-out;
+        transition: transform .3s ease-in-out;
     }
 
     .queryPane {
@@ -255,13 +255,18 @@ div.Workspace {
         transform: translateX(0);
     }
 
-    .schemaPane-enter-active, .schemaPane-exit-active {
-        transform: translateX(-50%);
+    .schemaPane-exit-active {
+        transform: translateX(-120%);
+    }
+
+    .schemaPane-enter-active {
+        transform: translateX(0);
+        max-width: 300px;
     }
 
     .schemaPane-enter, .schemaPane-exit-done {
-        transform: translateX(-100%);
         max-width: 0;
+        transform: translateX(-120%);
         overflow: hidden;
     }
 

superset/assets/src/components/TableSelector.jsx

@@ -130,7 +130,12 @@ export default class TableSelector extends React.PureComponent {
           `${encodeURIComponent(schema)}/${encodeURIComponent(substr)}/${forceRefresh}/`);
        return SupersetClient.get({ endpoint })
         .then(({ json }) => {
-          const filterOptions = createFilterOptions({ options: json.options });
+          const filterOptions = createFilterOptions({
+            options: json.options.map(o => ({
+              value: o.value.table,
+              label: o.label,
+            })),
+          });
           this.setState(() => ({
             filterOptions,
             tableLoading: false,

superset/data/birth_names.py

@@ -19,6 +19,7 @@
 
 import pandas as pd
 from sqlalchemy import DateTime, String
+from sqlalchemy.sql import column
 
 from superset import db, security_manager
 from superset.connectors.sqla.models import SqlMetric, TableColumn
@@ -64,15 +65,18 @@ def load_birth_names():
     obj.filter_select_enabled = True
 
     if not any(col.column_name == 'num_california' for col in obj.columns):
+        col_state = str(column('state').compile(db.engine))
+        col_num = str(column('num').compile(db.engine))
         obj.columns.append(TableColumn(
             column_name='num_california',
-            expression="CASE WHEN state = 'CA' THEN num ELSE 0 END",
+            expression=f"CASE WHEN {col_state} = 'CA' THEN {col_num} ELSE 0 END",
         ))
 
     if not any(col.metric_name == 'sum__num' for col in obj.metrics):
+        col = str(column('num').compile(db.engine))
         obj.metrics.append(SqlMetric(
             metric_name='sum__num',
-            expression='SUM(num)',
+            expression=f'SUM({col})',
         ))
 
     db.session.merge(obj)

superset/data/country_map.py

@@ -18,6 +18,7 @@
 
 import pandas as pd
 from sqlalchemy import BigInteger, Date, String
+from sqlalchemy.sql import column
 
 from superset import db
 from superset.connectors.sqla.models import SqlMetric
@@ -69,9 +70,10 @@ def load_country_map_data():
     obj.main_dttm_col = 'dttm'
     obj.database = utils.get_or_create_main_db()
     if not any(col.metric_name == 'avg__2004' for col in obj.metrics):
+        col = str(column('2004').compile(db.engine))
         obj.metrics.append(SqlMetric(
             metric_name='avg__2004',
-            expression='AVG(2004)',
+            expression=f'AVG({col})',
         ))
     db.session.merge(obj)
     db.session.commit()

superset/data/energy.py

@@ -20,6 +20,7 @@
 
 import pandas as pd
 from sqlalchemy import Float, String
+from sqlalchemy.sql import column
 
 from superset import db
 from superset.connectors.sqla.models import SqlMetric
@@ -54,9 +55,10 @@ def load_energy():
     tbl.database = utils.get_or_create_main_db()
 
     if not any(col.metric_name == 'sum__value' for col in tbl.metrics):
+        col = str(column('value').compile(db.engine))
         tbl.metrics.append(SqlMetric(
             metric_name='sum__value',
-            expression='SUM(value)',
+            expression=f'SUM({col})',
         ))
 
     db.session.merge(tbl)

superset/data/world_bank.py

@@ -22,6 +22,7 @@
 
 import pandas as pd
 from sqlalchemy import DateTime, String
+from sqlalchemy.sql import column
 
 from superset import db
 from superset.connectors.sqla.models import SqlMetric
@@ -75,9 +76,11 @@ def load_world_bank_health_n_pop():
     ]
     for m in metrics:
         if not any(col.metric_name == m for col in tbl.metrics):
+            aggr_func = m[:3]
+            col = str(column(m[5:]).compile(db.engine))
             tbl.metrics.append(SqlMetric(
                 metric_name=m,
-                expression=f'{m[:3]}({m[5:]})',
+                expression=f'{aggr_func}({col})',
             ))
 
     db.session.merge(tbl)

superset/utils/core.py

@@ -298,7 +298,7 @@ def default(self, o):
             return json.JSONEncoder.default(self, o)
 
 
-def parse_human_timedelta(s: str):
+def parse_human_timedelta(s: str) -> timedelta:
     """
     Returns ``datetime.datetime`` from natural language time deltas
 
@@ -312,6 +312,19 @@ def parse_human_timedelta(s: str):
     return d - dttm
 
 
+def parse_past_timedelta(delta_str: str) -> timedelta:
+    """
+    Takes a delta like '1 year' and finds the timedelta for that period in
+    the past, then represents that past timedelta in positive terms.
+
+    parse_human_timedelta('1 year') find the timedelta 1 year in the future.
+    parse_past_timedelta('1 year') returns -datetime.timedelta(-365)
+    or datetime.timedelta(365).
+    """
+    return -parse_human_timedelta(
+        delta_str if delta_str.startswith('-') else f'-{delta_str}')
+
+
 class JSONEncodedDict(TypeDecorator):
     """Represents an immutable structure as a json-encoded string."""
 
@@ -1003,9 +1016,9 @@ def get_since_until(time_range: Optional[str] = None,
         until = parse_human_datetime(until) if until else relative_end
 
     if time_shift:
-        time_shift = parse_human_timedelta(time_shift)
-        since = since if since is None else (since - time_shift)  # noqa: T400
-        until = until if until is None else (until - time_shift)  # noqa: T400
+        time_delta = parse_past_timedelta(time_shift)
+        since = since if since is None else (since - time_delta)  # noqa: T400
+        until = until if until is None else (until - time_delta)  # noqa: T400
 
     if since and until and since > until:
         raise ValueError(_('From date cannot be larger than to date'))

superset/viz.py

@@ -283,7 +283,7 @@ def query_obj(self):
                                              since=form_data.get('since'),
                                              until=form_data.get('until'))
         time_shift = form_data.get('time_shift', '')
-        self.time_shift = utils.parse_human_timedelta(time_shift)
+        self.time_shift = utils.parse_past_timedelta(time_shift)
         from_dttm = None if since is None else (since - self.time_shift)
         to_dttm = None if until is None else (until - self.time_shift)
         if from_dttm and to_dttm and from_dttm > to_dttm:
@@ -1212,7 +1212,7 @@ def run_extra_queries(self):
 
         for option in time_compare:
             query_object = self.query_obj()
-            delta = utils.parse_human_timedelta(option)
+            delta = utils.parse_past_timedelta(option)
             query_object['inner_from_dttm'] = query_object['from_dttm']
             query_object['inner_to_dttm'] = query_object['to_dttm']
 

tests/utils_tests.py

@@ -36,6 +36,7 @@
     merge_request_params,
     parse_human_timedelta,
     parse_js_uri_path_item,
+    parse_past_timedelta,
     validate_json,
     zlib_compress,
     zlib_decompress_to_string,
@@ -119,9 +120,21 @@ def test_base_json_conv(self):
         assert isinstance(base_json_conv(uuid.uuid4()), str) is True
 
     @patch('superset.utils.core.datetime')
-    def test_parse_human_timedelta(self, mock_now):
-        mock_now.return_value = datetime(2016, 12, 1)
+    def test_parse_human_timedelta(self, mock_datetime):
+        mock_datetime.now.return_value = datetime(2019, 4, 1)
+        mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
         self.assertEquals(parse_human_timedelta('now'), timedelta(0))
+        self.assertEquals(parse_human_timedelta('1 year'), timedelta(366))
+        self.assertEquals(parse_human_timedelta('-1 year'), timedelta(-365))
+
+    @patch('superset.utils.core.datetime')
+    def test_parse_past_timedelta(self, mock_datetime):
+        mock_datetime.now.return_value = datetime(2019, 4, 1)
+        mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
+        self.assertEquals(parse_past_timedelta('1 year'), timedelta(365))
+        self.assertEquals(parse_past_timedelta('-1 year'), timedelta(365))
+        self.assertEquals(parse_past_timedelta('52 weeks'), timedelta(364))
+        self.assertEquals(parse_past_timedelta('1 month'), timedelta(31))
 
     def test_zlib_compression(self):
         json_str = '{"test": 1}'