diff --git a/src/cr/cube/crunch_cube.py b/src/cr/cube/crunch_cube.py index 2be4f8b72..d36a2ca9e 100644 --- a/src/cr/cube/crunch_cube.py +++ b/src/cr/cube/crunch_cube.py @@ -314,69 +314,18 @@ def margin(self, axis=None, weighted=True, include_missing=False, ]) """ - def hs_dims_for_den(hs_dims, axis): - if axis is None or hs_dims is None: - return None - if isinstance(axis, int): - axis = [axis] - return [dim for dim in hs_dims if dim not in axis] - - table = self._counts(weighted).raw_cube_array - new_axis = self._adjust_axis(axis) - index = tuple( - None if i in new_axis else slice(None) - for i, _ in enumerate(table.shape) + den = self._denominator( + axis, weighted, include_missing, + include_transforms_for_dims, prune, ) - # Calculate denominator. Only include those H&S dimensions, across - # which we DON'T sum. These H&S are needed because of the shape, when - # dividing. Those across dims which are summed across MUST NOT be - # included, because they would change the result. - hs_dims = hs_dims_for_den(include_transforms_for_dims, axis) - den = self._apply_missings_and_insertions( - table, hs_dims, include_missing=include_missing - ) + # Calculate "margin" from denominator + margin = self._drop_mr_cat_dims(den, fix_valids=include_missing) - # Apply correct mask (based on the as_array shape) - arr = self._as_array( - include_transforms_for_dims=hs_dims, - include_missing=include_missing - ) + if margin.shape[0] == 1 and len(margin.shape) > 1 and self.ndim < 3: + margin = margin.reshape(margin.shape[1:]) - # ---prune array if pruning was requested--- - if prune: - arr = self._prune_body(arr, transforms=hs_dims) - - arr = self._drop_mr_cat_dims(arr, fix_valids=include_missing) - - if isinstance(arr, np.ma.core.MaskedArray): - inflate_ind = tuple( - ( - None - if ( - d.dimension_type == DT.MR_CAT or - n <= 1 or - len(d.elements()) <= 1 - ) else - slice(None) - ) - for d, n in zip(self._all_dimensions, table.shape) - ) - mask = np.logical_or( - np.zeros(den.shape, dtype=bool), - arr.mask[inflate_ind], - ) - den = np.ma.masked_array(den, mask) - - if (self.ndim != 1 or axis is None or - axis == 0 and len(self._all_dimensions) == 1): - # Special case for 1D cube wigh MR, for "Table" direction - den = np.sum(den, axis=new_axis)[index] - - den = self._drop_mr_cat_dims(den, fix_valids=include_missing) - if den.shape[0] == 1 and len(den.shape) > 1 and self.ndim < 3: - den = den.reshape(den.shape[1:]) - return den + return margin @lazyproperty def missing(self): @@ -572,36 +521,20 @@ def proportions(self, axis=None, weighted=True, ]) """ - def hs_dims_for_den(hs_dims, axis): - if axis is None or hs_dims is None: - return None - if isinstance(axis, int): - axis = [axis] - return [dim for dim in hs_dims if dim not in axis] - - table = self._measure(weighted).raw_cube_array - new_axis = self._adjust_axis(axis) - index = tuple( - None if i in new_axis else slice(None) - for i, _ in enumerate(table.shape) - ) - - # Calculate denominator. Only include those H&S dimensions, across - # which we DON'T sum. These H&S are needed because of the shape, when - # dividing. Those across dims which are summed across MUST NOT be - # included, because they would change the result. - hs_dims = hs_dims_for_den(include_transforms_for_dims, axis) - den = self._apply_missings_and_insertions(table, hs_dims) - den = np.sum(den, axis=new_axis)[index] - # Calculate numerator from table (include all H&S dimensions). num = self._apply_missings_and_insertions( - table, include_transforms_for_dims + self._measure(weighted).raw_cube_array, + include_transforms_for_dims, + ) + # Always use unpruned denominator (bases), because pruning is based on + # unweighted bases explicitly + den = self._denominator( + axis, weighted, False, include_transforms_for_dims, False ) res = self._drop_mr_cat_dims(num / den) - # Apply correct mask (based on the as_array shape) + # Apply correct pruning mask (based on the as_array shape) arr = self.as_array( prune=prune, include_transforms_for_dims=include_transforms_for_dims, @@ -928,6 +861,32 @@ def _cube_dict(self): '(str) or dict.' % type(self._cube_response_arg).__name__ ) + def _denominator(self, axis, weighted, include_missing, + include_transforms_for_dims, prune=False): + table = self._counts(weighted).raw_cube_array + new_axis = self._adjust_axis(axis) + index = tuple( + None if i in new_axis else slice(None) + for i, _ in enumerate(table.shape) + ) + + # Calculate denominator. Only include those H&S dimensions, across + # which we DON'T sum. These H&S are needed because of the shape, when + # dividing. Those across dims which are summed across MUST NOT be + # included, because they would change the result. + if prune: + # Always prune only based on _unweighted_ counts + mask = self._counts(False).raw_cube_array == 0 + table = np.ma.masked_array(table, mask) + hs_dims = self._hs_dims_for_den(include_transforms_for_dims, axis) + den = self._apply_missings_and_insertions( + table, hs_dims, include_missing=include_missing + ) + try: + return np.sum(den, axis=new_axis)[index] + except np.AxisError: + return den + def _drop_mr_cat_dims(self, array, fix_valids=False): """Return ndarray reflecting *array* with MR_CAT dims dropped. @@ -991,6 +950,14 @@ def _fix_valid_indices(cls, valid_indices, insertion_index, dim): valid_indices[dim] = indices.tolist() return valid_indices + @staticmethod + def _hs_dims_for_den(hs_dims, axis): + if axis is None or hs_dims is None: + return None + if isinstance(axis, int): + axis = [axis] + return [dim for dim in hs_dims if dim not in axis] + def _inserted_dim_inds(self, transform_dims, axis): dim_ind = axis if self.ndim < 3 else axis + 1 if not transform_dims or dim_ind not in transform_dims: @@ -1030,6 +997,10 @@ def _is_axis_allowed(self, axis): In case the calculation is requested over CA items dimension, it is not valid. It's valid in all other cases. """ + if not self.dimensions: + # In case of no dimensions any direction is not allowed + return False + if axis is None: # If table direction was requested, we must ensure that each slice # doesn't have the CA items dimension (thus the [-2:] part). It's @@ -1300,16 +1271,25 @@ def _update_result(self, result, insertions, dimension_index, """Insert subtotals into resulting ndarray.""" # TODO: valid_indices should be a tuple as a parameter and as a return # value + masked = type(result) == np.ma.core.MaskedArray + if masked: + mask = result.mask for j, (ind_insertion, value) in enumerate(insertions): result = np.insert( result, ind_insertion + j + 1, value, axis=dimension_index ) + if masked: + mask = np.insert( + mask, ind_insertion + j + 1, False, axis=dimension_index + ) valid_indices = ( valid_indices and self._fix_valid_indices( valid_indices, ind_insertion + j, dimension_index ) ) + if masked: + result = np.ma.masked_array(result, mask) return result, valid_indices diff --git a/src/cr/cube/cube_slice.py b/src/cr/cube/cube_slice.py index f6518457a..9a15a8e81 100644 --- a/src/cr/cube/cube_slice.py +++ b/src/cr/cube/cube_slice.py @@ -213,6 +213,82 @@ def prune_dimension_labels(labels, prune_indices): ] return labels + def margin(self, axis=None, weighted=True, include_missing=False, + include_transforms_for_dims=None, prune=False): + """Return ndarray representing slice margin across selected axis. + + A margin (or basis) can be calculated for a contingency table, provided + that the dimensions of the desired directions are marginable. The + dimensions are marginable if they represent mutualy exclusive data, + such as true categorical data. For array types the items dimensions are + not marginable. Requesting a margin across these dimensions + (e.g. slice.margin(axis=0) for a categorical array cube slice) will + produce an error. For multiple response slices, the implicit convention + is that the provided direction scales to the selections dimension of the + slice. These cases produce meaningful data, but of a slightly different + shape (e.g. slice.margin(0) for a MR x CAT slice will produce 2D ndarray + (variable dimensions are never collapsed!)). + + :param axis: Axis across which to sum. Can be 0 (columns margin), + 1 (rows margin) and None (table margin). If requested across + variables dimension (e.g. requesting 0 margin for CA array) it will + produce an error. + :param weighted: Weighted or unweighted counts. + :param include_missing: Include missing categories or not. + :param include_transforms_for_dims: Indices of dimensions for which to + include transformations + :param prune: Perform pruning based on unweighted counts. + :returns: (weighed or unweighted counts) summed across provided axis. + For multiple response types, items dimensions are not collapsed. + """ + + cube_axis = self._calculate_correct_axis_for_cube(axis) + hs_dims = self._hs_dims_for_cube(include_transforms_for_dims) + + margin = self._cube.margin( + axis=cube_axis, weighted=weighted, + include_missing=include_missing, + include_transforms_for_dims=hs_dims, + prune=prune, + ) + + margin = self._extract_slice_result_from_cube(margin) + if prune: + return self._fixup_margin_pruning(margin, axis) + return margin + + def _fix_pruning_for_2D_margin_with_mr(self, margin, axis): + if self.mr_dim_ind != axis and axis is not None: + return margin + opposite_axis = self._calculate_correct_axis_for_cube( + 1 - self.mr_dim_ind) + opposite_mask = self._extract_slice_result_from_cube( + self._cube.margin(axis=opposite_axis, prune=True) + ).mask + if axis == 0 and len(margin.shape) > 1: + opposite_mask = opposite_mask[:, None] + mask = np.logical_or(margin.mask, opposite_mask) + return np.ma.masked_array(margin, mask) + + def _fix_pruning_for_margin_with_only_empty_hs(self, margin): + mask = margin.mask + if not np.all(mask): + # --Special case when a margin is not pruned all the way + # --(because of H&S), but the oposite margin _is_ completely + # --pruned, so there's nothing to show actually. + return np.ma.masked_array(margin.data, np.ones(margin.shape)) + + return margin + + def _fixup_margin_pruning(self, margin, axis): + if self.has_mr and not self.has_ca: + return self._fix_pruning_for_2D_margin_with_mr(margin, axis) + + if self._is_empty: + return self._fix_pruning_for_margin_with_only_empty_hs(margin) + + return margin + @lazyproperty def mr_dim_ind(self): """Get the correct index of the MR dimension in the cube slice.""" @@ -396,7 +472,59 @@ def _call_cube_method(self, method, *args, **kwargs): if not self.ca_as_0th: result = result[-2:] return result - return self._update_result(result) + return self._extract_slice_result_from_cube(result) + + def _calculate_correct_axis_for_cube(self, axis): + """Return correct axis for cube, based on ndim. + + If cube has 3 dimensions, increase axis by 1. This will translate the + default 0 (cols direction) and 1 (rows direction) to actual 1 + (cols direction) and 2 (rows direction). This is needed because the + 0th dimension of the 3D cube is only used to slice across. The actual + margins need to be calculated for each slice separately, and since + they're implemented as an ndarray, the direction needs to be increased + by one. For the value of `None`, don't modify the axis parameter. + + :param axis: 0, 1, or None. Axis that will be passed to self._cube + methods. If the cube is 3D, the axis is typically + increased by 1, to represent correct measure direction. + :returns: int or None, representing the updated axis to pass to cube + """ + + if self._cube.ndim < 3: + if self.ca_as_0th and axis is None: + # Special case for CA slices (in multitables). In this case, + # we need to calculate a measurement across CA categories + # dimension (and not across items, because it's not + # allowed). The value for the axis parameter of None, would + # imply both cat and items dimensions, and we don't want that. + return 1 + return axis + + # Expected usage of the 'axis' parameter from CubeSlice is 0, 1, or + # None. CrunchCube handles all other logic. The only 'smart' thing + # about the handling here, is that the axes are increased for 3D cubes. + # This way the 3Dness is hidden from the user and he still sees 2D + # crosstabs, with col and row axes (0 and 1), which are transformed to + # corresponding numbers in case of 3D cubes (namely 1 and 2). In the + # case of None, we need to analyze across all valid dimensions, and the + # CrunchCube takes care of that (no need to update axis if it's None). + # If the user provides a tuple, it's considered that he "knows" what + # he's doing, and the axis argument is not updated in this case. + if isinstance(axis, int): + axis += 1 + return axis + + def _hs_dims_for_cube(self, hs_dims): + if self._cube.ndim < 3: + return hs_dims + + # Keep the 2D illusion for the user. If a user sees a 2D slice, he + # still needs to be able to address both dimensions (for which he + # wants the H&S included) as 0 and 1. Since these are offset by a 0 + # dimension in a 3D case, inside the cr.cube, we need to increase + # the indexes of the required dims. + return [d + 1 for d in hs_dims] if hs_dims is not None else None def _intersperse_hs_in_std_res(self, hs_dims, res): for dim, inds in enumerate(self.inserted_hs_indices()): @@ -406,6 +534,11 @@ def _intersperse_hs_in_std_res(self, hs_dims, res): res = np.insert(res, i, np.nan, axis=(dim - self.ndim)) return res + @lazyproperty + def _is_empty(self): + array = self.as_array(weighted=False, prune=True) + return np.all(array.mask) + def _prepare_index_baseline(self, axis): # First get the margin of the opposite direction of the index axis. # We need this in order to end up with the right shape of the @@ -453,64 +586,37 @@ def _update_args(self, kwargs): # If cube is 2D it doesn't actually have slices (itself is a slice). # In this case we don't need to convert any arguments, but just # pass them to the underlying cube (which is the slice). - if self.ca_as_0th: - axis = kwargs.get('axis', False) - if axis is None: - # Special case for CA slices (in multitables). In this case, - # we need to calculate a measurement across CA categories - # dimension (and not across items, because it's not - # allowed). The value for the axis parameter of None, would - # incur the items dimension, and we don't want that. - kwargs['axis'] = 1 return kwargs # Handling API methods that include 'axis' parameter - axis = kwargs.get('axis') - # Expected usage of the 'axis' parameter from CubeSlice is 0, 1, or - # None. CrunchCube handles all other logic. The only 'smart' thing - # about the handling here, is that the axes are increased for 3D cubes. - # This way the 3Dness is hidden from the user and he still sees 2D - # crosstabs, with col and row axes (0 and 1), which are transformed to - # corresponding numbers in case of 3D cubes (namely 1 and 2). In the - # case of None, we need to analyze across all valid dimensions, and the - # CrunchCube takes care of that (no need to update axis if it's None). - # If the user provides a tuple, it's considered that he "knows" what - # he's doing, and the axis argument is not updated in this case. - if isinstance(axis, int): - kwargs['axis'] += 1 - - # Handling API methods that include H&S parameter + axis = self._calculate_correct_axis_for_cube(kwargs.get('axis')) + if axis: + kwargs['axis'] = axis - # For most cr.cube methods, we use the 'include_transforms_for_dims' - # parameter name. For some, namely the prune_indices, we use the - # 'transforms'. These are parameters that tell to the cr.cube "which - # dimensions to include the H&S for". The only point of this parameter - # (from the perspective of the cr.exporter) is to exclude the 0th - # dimension's H&S in the case of 3D cubes. hs_dims_key = ( 'transforms' in kwargs and 'transforms' or 'hs_dims' in kwargs and 'hs_dims' or 'include_transforms_for_dims' ) - hs_dims = kwargs.get(hs_dims_key) - if isinstance(hs_dims, list): - # Keep the 2D illusion for the user. If a user sees a 2D slice, he - # still needs to be able to address both dimensions (for which he - # wants the H&S included) as 0 and 1. Since these are offset by a 0 - # dimension in a 3D case, inside the cr.cube, we need to increase - # the indexes of the required dims. - kwargs[hs_dims_key] = [dim + 1 for dim in hs_dims] + hs_dims = self._hs_dims_for_cube(kwargs.get(hs_dims_key)) + if hs_dims: + kwargs[hs_dims_key] = hs_dims return kwargs - def _update_result(self, result): + def _extract_slice_result_from_cube(self, result): if (self._cube.ndim < 3 and not self.ca_as_0th or len(result) - 1 < self._index): return result - result = result[self._index] - if isinstance(result, tuple): - return np.array(result) - elif not isinstance(result, np.ndarray): - result = np.array([result]) - return result + masked = type(result) == np.ma.core.MaskedArray + slice_result = result[self._index] + if isinstance(slice_result, tuple): + return np.array(slice_result) + elif not isinstance(slice_result, np.ndarray): + if masked: + return np.ma.masked_array( + slice_result, result.mask[self._index] + ) + return np.array([slice_result]) + return slice_result diff --git a/src/cr/cube/measures/scale_means.py b/src/cr/cube/measures/scale_means.py index 5671a8f33..1538af119 100644 --- a/src/cr/cube/measures/scale_means.py +++ b/src/cr/cube/measures/scale_means.py @@ -20,6 +20,7 @@ import numpy as np from cr.cube.util import lazyproperty +from cr.cube.enum import DIMENSION_TYPE as DT class ScaleMeans(object): @@ -65,7 +66,13 @@ def margin(self, axis): raise ValueError(msg) dimension_index = 1 - axis - margin = self._slice.margin(axis) + margin = self._slice.margin(axis=axis) + if len(margin.shape) > 1: + index = [ + 0 if d.dimension_type == DT.MR else slice(None) + for d in self._slice.dimensions + ] + margin = margin[index] total = np.sum(margin) values = self.values[dimension_index] diff --git a/src/cr/cube/util.py b/src/cr/cube/util.py index 15c59253d..93293cbd0 100644 --- a/src/cr/cube/util.py +++ b/src/cr/cube/util.py @@ -179,7 +179,7 @@ def decorating_function(user_function, len=len, iter=iter, tuple=tuple, queue_appendleft, queue_pop = queue.appendleft, queue.pop @functools.wraps(user_function) - def wrapper(*args, **kwds): + def wrapper(*args, **kwds): # pragma: no cover # cache key records both positional and keyword args key = args if kwds: diff --git a/tests/fixtures/ca-subvar-hs-x-ca-cat-x-cat-colpct.json b/tests/fixtures/ca-subvar-hs-x-ca-cat-x-cat-colpct.json new file mode 100644 index 000000000..c6fb2aa00 --- /dev/null +++ b/tests/fixtures/ca-subvar-hs-x-ca-cat-x-cat-colpct.json @@ -0,0 +1,858 @@ +{ + "query": { + "dimensions": [ + { + "each": 1 + }, + { + "variable": "0000d8" + }, + { + "variable": "000067" + } + ], + "measures": { + "count": { + "args": [], + "function": "cube_count" + } + }, + "weight": "http://127.0.0.1:8080/datasets/230d454e1b5942a99f870fdb0a31c8ca/variables/0000d0/" + }, + "query_environment": { + "filter": [] + }, + "result": { + "counts": [ + 214, + 1, + 79, + 12, + 0, + 6, + 0, + 123, + 0, + 218, + 51, + 0, + 23, + 0, + 40, + 1, + 93, + 79, + 0, + 58, + 0, + 17, + 1, + 47, + 111, + 0, + 314, + 0, + 35, + 1, + 51, + 39, + 0, + 27, + 0, + 5, + 2, + 7, + 2, + 0, + 5, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17, + 0, + 18, + 39, + 0, + 98, + 0, + 27, + 1, + 78, + 79, + 0, + 174, + 0, + 51, + 1, + 96, + 59, + 0, + 75, + 0, + 287, + 1, + 246, + 77, + 0, + 56, + 0, + 46, + 1, + 48, + 36, + 0, + 28, + 0, + 6, + 2, + 9, + 4, + 0, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 434, + 6, + 495, + 294, + 0, + 433, + 0 + ], + "dimensions": [ + { + "derived": true, + "references": { + "alias": "abolitionists", + "description": "Do you have a favorable or an unfavorable opinion of the following abolitionists?", + "name": "Abolitionists", + "notes": "A categorical array variable, where one item has no responses", + "subreferences": [ + { + "alias": "douglass", + "description": "fav_ppdem", + "name": "Frederick Douglass" + }, + { + "alias": "brown", + "description": "fav_pprep", + "name": "John Brown" + }, + { + "alias": "truth", + "name": "Sojourner Truth" + } + ], + "view": { + "column_width": null, + "include_missing": false, + "show_counts": false, + "transform": { + "insertions": [ + { + "anchor": 2, + "args": [ + 0, + 2 + ], + "function": "subtotal", + "name": "favorable" + }, + { + "anchor": 4, + "args": [ + 3, + 4 + ], + "function": "subtotal", + "name": "unfavorable" + } + ] + } + } + }, + "type": { + "class": "enum", + "elements": [ + { + "id": 1, + "missing": false, + "value": { + "derived": false, + "id": "0061", + "references": { + "alias": "douglass", + "description": "fav_ppdem", + "name": "Frederick Douglass" + }, + "type": { + "categories": [ + { + "id": 0, + "missing": false, + "name": "Very favorable", + "numeric_value": 0, + "selected": false + }, + { + "id": 2, + "missing": false, + "name": "Somewhat favorable", + "numeric_value": 2, + "selected": false + }, + { + "id": 3, + "missing": false, + "name": "Somewhat unfavorable", + "numeric_value": 3, + "selected": false + }, + { + "id": 4, + "missing": false, + "name": "Very unfavorable", + "numeric_value": 4, + "selected": false + }, + { + "id": 5, + "missing": false, + "name": "Don't know", + "numeric_value": 5, + "selected": false + }, + { + "id": 32766, + "missing": true, + "name": "skipped", + "numeric_value": 32766, + "selected": false + }, + { + "id": 32767, + "missing": true, + "name": "not asked", + "numeric_value": 32767, + "selected": false + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 2, + "missing": false, + "value": { + "derived": false, + "id": "0062", + "references": { + "alias": "brown", + "description": "fav_pprep", + "name": "John Brown" + }, + "type": { + "categories": [ + { + "id": 0, + "missing": false, + "name": "Very favorable", + "numeric_value": 0, + "selected": false + }, + { + "id": 2, + "missing": false, + "name": "Somewhat favorable", + "numeric_value": 2, + "selected": false + }, + { + "id": 3, + "missing": false, + "name": "Somewhat unfavorable", + "numeric_value": 3, + "selected": false + }, + { + "id": 4, + "missing": false, + "name": "Very unfavorable", + "numeric_value": 4, + "selected": false + }, + { + "id": 5, + "missing": false, + "name": "Don't know", + "numeric_value": 5, + "selected": false + }, + { + "id": 32766, + "missing": true, + "name": "skipped", + "numeric_value": 32766, + "selected": false + }, + { + "id": 32767, + "missing": true, + "name": "not asked", + "numeric_value": 32767, + "selected": false + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 3, + "missing": false, + "value": { + "derived": false, + "id": "fb4492a07d5142f9a9a49de9c07ce8a1", + "references": { + "alias": "truth", + "name": "Sojourner Truth" + }, + "type": { + "categories": [ + { + "id": 0, + "missing": false, + "name": "Very favorable", + "numeric_value": 0, + "selected": false + }, + { + "id": 2, + "missing": false, + "name": "Somewhat favorable", + "numeric_value": 2, + "selected": false + }, + { + "id": 3, + "missing": false, + "name": "Somewhat unfavorable", + "numeric_value": 3, + "selected": false + }, + { + "id": 4, + "missing": false, + "name": "Very unfavorable", + "numeric_value": 4, + "selected": false + }, + { + "id": 5, + "missing": false, + "name": "Don't know", + "numeric_value": 5, + "selected": false + }, + { + "id": 32766, + "missing": true, + "name": "skipped", + "numeric_value": 32766, + "selected": false + }, + { + "id": 32767, + "missing": true, + "name": "not asked", + "numeric_value": 32767, + "selected": false + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + } + ], + "subtype": { + "class": "variable" + } + } + }, + { + "derived": false, + "references": { + "alias": "abolitionists", + "description": "Do you have a favorable or an unfavorable opinion of the following abolitionists?", + "name": "Abolitionists", + "notes": "A categorical array variable, where one item has no responses", + "subreferences": [ + { + "alias": "douglass", + "description": "fav_ppdem", + "name": "Frederick Douglass" + }, + { + "alias": "brown", + "description": "fav_pprep", + "name": "John Brown" + }, + { + "alias": "truth", + "name": "Sojourner Truth" + } + ], + "view": { + "column_width": null, + "include_missing": false, + "show_counts": false, + "transform": { + "insertions": [ + { + "anchor": 2, + "args": [ + 0, + 2 + ], + "function": "subtotal", + "name": "favorable" + }, + { + "anchor": 4, + "args": [ + 3, + 4 + ], + "function": "subtotal", + "name": "unfavorable" + } + ] + } + } + }, + "type": { + "categories": [ + { + "id": 0, + "missing": false, + "name": "Very favorable", + "numeric_value": 0, + "selected": false + }, + { + "id": 2, + "missing": false, + "name": "Somewhat favorable", + "numeric_value": 2, + "selected": false + }, + { + "id": 3, + "missing": false, + "name": "Somewhat unfavorable", + "numeric_value": 3, + "selected": false + }, + { + "id": 4, + "missing": false, + "name": "Very unfavorable", + "numeric_value": 4, + "selected": false + }, + { + "id": 5, + "missing": false, + "name": "Don't know", + "numeric_value": 5, + "selected": false + }, + { + "id": 32766, + "missing": true, + "name": "skipped", + "numeric_value": 32766, + "selected": false + }, + { + "id": 32767, + "missing": true, + "name": "not asked", + "numeric_value": 32767, + "selected": false + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false, + "subvariables": [ + "0061", + "0062", + "fb4492a07d5142f9a9a49de9c07ce8a1" + ] + } + }, + { + "derived": false, + "references": { + "alias": "food_groups", + "description": "Four of the five USDA food groups", + "name": "Food groups", + "notes": "A categorical variable where the missing categories are interspersed throughout the non-missing categories", + "view": { + "column_width": null, + "include_missing": false, + "show_counts": false, + "show_numeric_values": false, + "transform": { + "insertions": [] + } + } + }, + "type": { + "categories": [ + { + "id": 0, + "missing": false, + "name": "Vegetables", + "numeric_value": 0 + }, + { + "id": 32766, + "missing": true, + "name": "Don't know", + "numeric_value": 32766 + }, + { + "id": 2, + "missing": false, + "name": "Fruit", + "numeric_value": 2 + }, + { + "id": 5, + "missing": false, + "name": "Grain", + "numeric_value": 5 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + }, + { + "id": 4, + "missing": false, + "name": "Meat", + "numeric_value": 4 + }, + { + "id": 32767, + "missing": true, + "name": "Not asked", + "numeric_value": 32767 + } + ], + "class": "categorical", + "ordinal": false + } + } + ], + "element": "crunch:cube", + "measures": { + "count": { + "data": [ + 175.7706778019, + 1.4714274949, + 69.2301852478, + 9.1734717817, + 0, + 5.9102247713, + 0, + 99.950123551, + 0, + 204.7343967189, + 59.4610994313, + 0, + 27.6822811897, + 0, + 31.1109507937, + 1.2887348151, + 88.7050070294, + 77.4922188913, + 0, + 65.524300784, + 0, + 17.6688253748, + 0.8441290249, + 44.2243736828, + 115.9351379606, + 0, + 322.7776480468, + 0, + 45.9089532773, + 1.3364538264, + 69.6643528395, + 53.425324945, + 0, + 42.0097332312, + 0, + 6.3656880013, + 4.9989584402, + 9.030895117, + 2.6873057241, + 0, + 7.617120206, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11.4532735643, + 0, + 19.9127149409, + 42.1005010748, + 0, + 97.5318868163, + 0, + 27.5214697515, + 0.8441290249, + 90.5818751885, + 95.9350215001, + 0, + 189.0270756122, + 0, + 43.6048806298, + 1.2887348151, + 93.3367744269, + 54.9627061409, + 0, + 75.202999163, + 0, + 229.5601354504, + 1.4714274949, + 206.1009639531, + 68.2007510768, + 0, + 64.892139029, + 0, + 58.6753661709, + 1.3364538264, + 65.9588886546, + 52.4895658577, + 0, + 42.4251930577, + 0, + 5.9600932333, + 4.9989584402, + 9.6979934714, + 4.4860130839, + 0, + 2.4420145507, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 376.7752188001, + 9.9397036014, + 485.5892106354, + 318.174558734, + 0, + 471.5213082289, + 0 + ], + "metadata": { + "derived": true, + "references": {}, + "type": { + "class": "numeric", + "integer": false, + "missing_reasons": { + "No Data": -1 + }, + "missing_rules": {} + } + }, + "n_missing": 14 + } + }, + "missing": 14, + "n": 1662 + } +} diff --git a/tests/fixtures/ca-subvar-x-ca-cat-x-cat.json b/tests/fixtures/ca-subvar-x-ca-cat-x-cat.json new file mode 100644 index 000000000..092f2ff5c --- /dev/null +++ b/tests/fixtures/ca-subvar-x-ca-cat-x-cat.json @@ -0,0 +1,2275 @@ +{ + "query": { + "dimensions": [ + { + "each": "https://app.crunch.io/api/datasets/3aefb7da75f643c9a36ed70882465c3d/variables/000317/" + }, + { + "variable": "https://app.crunch.io/api/datasets/3aefb7da75f643c9a36ed70882465c3d/variables/000317/" + }, + { + "variable": "https://app.crunch.io/api/datasets/3aefb7da75f643c9a36ed70882465c3d/variables/47a800ed909a417c96d1f3bdc584b420/" + } + ], + "measures": { + "count": { + "args": [], + "function": "cube_count" + } + }, + "weight": "https://app.crunch.io/api/datasets/3aefb7da75f643c9a36ed70882465c3d/variables/43dd76dfe4ce4dbabdbe4a6792d1635d/" + }, + "query_environment": { + "filter": [] + }, + "result": { + "counts": [ + 5268, + 6780, + 0, + 0, + 0, + 1567, + 1538, + 0, + 0, + 0, + 696, + 587, + 0, + 0, + 0, + 379, + 313, + 0, + 0, + 0, + 715, + 637, + 0, + 0, + 0, + 3051, + 2748, + 0, + 0, + 0, + 1644, + 1082, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2507, + 3466, + 0, + 0, + 0, + 1050, + 1124, + 0, + 0, + 0, + 553, + 559, + 0, + 0, + 0, + 367, + 382, + 0, + 0, + 0, + 828, + 1035, + 0, + 0, + 0, + 2843, + 2982, + 0, + 0, + 0, + 5172, + 4137, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7715, + 7645, + 0, + 0, + 0, + 1749, + 2447, + 0, + 0, + 0, + 518, + 785, + 0, + 0, + 0, + 231, + 258, + 0, + 0, + 0, + 297, + 395, + 0, + 0, + 0, + 505, + 619, + 0, + 0, + 0, + 2305, + 1536, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1471, + 1576, + 0, + 0, + 0, + 553, + 475, + 0, + 0, + 0, + 293, + 240, + 0, + 0, + 0, + 162, + 165, + 0, + 0, + 0, + 396, + 431, + 0, + 0, + 0, + 1059, + 1065, + 0, + 0, + 0, + 9386, + 9733, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6656, + 7009, + 0, + 0, + 0, + 679, + 584, + 0, + 0, + 0, + 278, + 259, + 0, + 0, + 0, + 155, + 152, + 0, + 0, + 0, + 333, + 307, + 0, + 0, + 0, + 689, + 654, + 0, + 0, + 0, + 4530, + 4720, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 170, + 268, + 0, + 0, + 0, + 145, + 253, + 0, + 0, + 0, + 145, + 171, + 0, + 0, + 0, + 70, + 140, + 0, + 0, + 0, + 190, + 433, + 0, + 0, + 0, + 871, + 1210, + 0, + 0, + 0, + 11729, + 11210, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 73, + 31, + 0, + 0, + 0, + 66, + 29, + 0, + 0, + 0, + 42, + 22, + 0, + 0, + 0, + 30, + 16, + 0, + 0, + 0, + 87, + 23, + 0, + 0, + 0, + 185, + 152, + 0, + 0, + 0, + 12837, + 13412, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 781, + 737, + 0, + 0, + 0, + 180, + 147, + 0, + 0, + 0, + 104, + 69, + 0, + 0, + 0, + 73, + 33, + 0, + 0, + 0, + 177, + 125, + 0, + 0, + 0, + 152, + 78, + 0, + 0, + 0, + 11853, + 12496, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 449, + 275, + 0, + 0, + 0, + 70, + 61, + 0, + 0, + 0, + 45, + 32, + 0, + 0, + 0, + 31, + 30, + 0, + 0, + 0, + 121, + 104, + 0, + 0, + 0, + 70, + 65, + 0, + 0, + 0, + 12534, + 13118, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 138, + 108, + 0, + 0, + 0, + 67, + 54, + 0, + 0, + 0, + 33, + 26, + 0, + 0, + 0, + 33, + 24, + 0, + 0, + 0, + 97, + 80, + 0, + 0, + 0, + 184, + 140, + 0, + 0, + 0, + 12768, + 13253, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7, + 22, + 0, + 0, + 0, + 6, + 21, + 0, + 0, + 0, + 3, + 13, + 0, + 0, + 0, + 2, + 24, + 0, + 0, + 0, + 9, + 50, + 0, + 0, + 0, + 61, + 156, + 0, + 0, + 0, + 13232, + 13399, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 46, + 214, + 0, + 0, + 0, + 49, + 226, + 0, + 0, + 0, + 45, + 139, + 0, + 0, + 0, + 38, + 64, + 0, + 0, + 0, + 59, + 118, + 0, + 0, + 0, + 387, + 374, + 0, + 0, + 0, + 12696, + 12550, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "dimensions": [ + { + "derived": true, + "references": { + "alias": "usage_grid", + "description": "When was the last time you used this app?", + "name": "[Brand] Usage", + "subreferences": [ + { + "alias": "usage_Instagram", + "name": "Instagram" + }, + { + "alias": "usage_Snapchat", + "name": "Snapchat" + }, + { + "alias": "usage_YouTube", + "name": "YouTube" + }, + { + "alias": "usage_Twitter", + "name": "Twitter" + }, + { + "alias": "usage_WhatsApp", + "name": "WhatsApp" + }, + { + "alias": "usage_Musical_ly", + "name": "Musical.ly" + }, + { + "alias": "usage_BigoLive", + "name": "Bigo Live" + }, + { + "alias": "usage_Line", + "name": "Line" + }, + { + "alias": "usage_BBM", + "name": "BBM" + }, + { + "alias": "usage_Path", + "name": "Path" + }, + { + "alias": "usage_MixChannel", + "name": "MixChannel" + }, + { + "alias": "usage_SNOW", + "name": "SNOW" + } + ], + "view": { + "column_width": null, + "include_missing": false, + "show_counts": false + } + }, + "type": { + "class": "enum", + "elements": [ + { + "id": 1, + "missing": false, + "value": { + "derived": false, + "id": "0001", + "references": { + "alias": "usage_Instagram", + "name": "Instagram" + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "In the past day", + "numeric_value": 1 + }, + { + "id": 2, + "missing": false, + "name": "In the past week, but not in the last day", + "numeric_value": 2 + }, + { + "id": 3, + "missing": false, + "name": "In the past month, but not in the past week", + "numeric_value": 3 + }, + { + "id": 4, + "missing": false, + "name": "In the past 3 months, but not in the last month", + "numeric_value": 4 + }, + { + "id": 5, + "missing": false, + "name": "Over 3 months ago", + "numeric_value": 5 + }, + { + "id": 6, + "missing": false, + "name": "Never", + "numeric_value": 6 + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 2, + "missing": false, + "value": { + "derived": false, + "id": "0002", + "references": { + "alias": "usage_Snapchat", + "name": "Snapchat" + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "In the past day", + "numeric_value": 1 + }, + { + "id": 2, + "missing": false, + "name": "In the past week, but not in the last day", + "numeric_value": 2 + }, + { + "id": 3, + "missing": false, + "name": "In the past month, but not in the past week", + "numeric_value": 3 + }, + { + "id": 4, + "missing": false, + "name": "In the past 3 months, but not in the last month", + "numeric_value": 4 + }, + { + "id": 5, + "missing": false, + "name": "Over 3 months ago", + "numeric_value": 5 + }, + { + "id": 6, + "missing": false, + "name": "Never", + "numeric_value": 6 + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 3, + "missing": false, + "value": { + "derived": false, + "id": "0003", + "references": { + "alias": "usage_YouTube", + "name": "YouTube" + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "In the past day", + "numeric_value": 1 + }, + { + "id": 2, + "missing": false, + "name": "In the past week, but not in the last day", + "numeric_value": 2 + }, + { + "id": 3, + "missing": false, + "name": "In the past month, but not in the past week", + "numeric_value": 3 + }, + { + "id": 4, + "missing": false, + "name": "In the past 3 months, but not in the last month", + "numeric_value": 4 + }, + { + "id": 5, + "missing": false, + "name": "Over 3 months ago", + "numeric_value": 5 + }, + { + "id": 6, + "missing": false, + "name": "Never", + "numeric_value": 6 + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 4, + "missing": false, + "value": { + "derived": false, + "id": "0004", + "references": { + "alias": "usage_Twitter", + "name": "Twitter" + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "In the past day", + "numeric_value": 1 + }, + { + "id": 2, + "missing": false, + "name": "In the past week, but not in the last day", + "numeric_value": 2 + }, + { + "id": 3, + "missing": false, + "name": "In the past month, but not in the past week", + "numeric_value": 3 + }, + { + "id": 4, + "missing": false, + "name": "In the past 3 months, but not in the last month", + "numeric_value": 4 + }, + { + "id": 5, + "missing": false, + "name": "Over 3 months ago", + "numeric_value": 5 + }, + { + "id": 6, + "missing": false, + "name": "Never", + "numeric_value": 6 + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 5, + "missing": false, + "value": { + "derived": false, + "id": "0005", + "references": { + "alias": "usage_WhatsApp", + "name": "WhatsApp" + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "In the past day", + "numeric_value": 1 + }, + { + "id": 2, + "missing": false, + "name": "In the past week, but not in the last day", + "numeric_value": 2 + }, + { + "id": 3, + "missing": false, + "name": "In the past month, but not in the past week", + "numeric_value": 3 + }, + { + "id": 4, + "missing": false, + "name": "In the past 3 months, but not in the last month", + "numeric_value": 4 + }, + { + "id": 5, + "missing": false, + "name": "Over 3 months ago", + "numeric_value": 5 + }, + { + "id": 6, + "missing": false, + "name": "Never", + "numeric_value": 6 + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 6, + "missing": false, + "value": { + "derived": false, + "id": "0006", + "references": { + "alias": "usage_Musical_ly", + "name": "Musical.ly" + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "In the past day", + "numeric_value": 1 + }, + { + "id": 2, + "missing": false, + "name": "In the past week, but not in the last day", + "numeric_value": 2 + }, + { + "id": 3, + "missing": false, + "name": "In the past month, but not in the past week", + "numeric_value": 3 + }, + { + "id": 4, + "missing": false, + "name": "In the past 3 months, but not in the last month", + "numeric_value": 4 + }, + { + "id": 5, + "missing": false, + "name": "Over 3 months ago", + "numeric_value": 5 + }, + { + "id": 6, + "missing": false, + "name": "Never", + "numeric_value": 6 + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 7, + "missing": false, + "value": { + "derived": false, + "id": "0007", + "references": { + "alias": "usage_BigoLive", + "name": "Bigo Live" + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "In the past day", + "numeric_value": 1 + }, + { + "id": 2, + "missing": false, + "name": "In the past week, but not in the last day", + "numeric_value": 2 + }, + { + "id": 3, + "missing": false, + "name": "In the past month, but not in the past week", + "numeric_value": 3 + }, + { + "id": 4, + "missing": false, + "name": "In the past 3 months, but not in the last month", + "numeric_value": 4 + }, + { + "id": 5, + "missing": false, + "name": "Over 3 months ago", + "numeric_value": 5 + }, + { + "id": 6, + "missing": false, + "name": "Never", + "numeric_value": 6 + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 8, + "missing": false, + "value": { + "derived": false, + "id": "0008", + "references": { + "alias": "usage_Line", + "name": "Line" + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "In the past day", + "numeric_value": 1 + }, + { + "id": 2, + "missing": false, + "name": "In the past week, but not in the last day", + "numeric_value": 2 + }, + { + "id": 3, + "missing": false, + "name": "In the past month, but not in the past week", + "numeric_value": 3 + }, + { + "id": 4, + "missing": false, + "name": "In the past 3 months, but not in the last month", + "numeric_value": 4 + }, + { + "id": 5, + "missing": false, + "name": "Over 3 months ago", + "numeric_value": 5 + }, + { + "id": 6, + "missing": false, + "name": "Never", + "numeric_value": 6 + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 9, + "missing": false, + "value": { + "derived": false, + "id": "0009", + "references": { + "alias": "usage_BBM", + "name": "BBM" + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "In the past day", + "numeric_value": 1 + }, + { + "id": 2, + "missing": false, + "name": "In the past week, but not in the last day", + "numeric_value": 2 + }, + { + "id": 3, + "missing": false, + "name": "In the past month, but not in the past week", + "numeric_value": 3 + }, + { + "id": 4, + "missing": false, + "name": "In the past 3 months, but not in the last month", + "numeric_value": 4 + }, + { + "id": 5, + "missing": false, + "name": "Over 3 months ago", + "numeric_value": 5 + }, + { + "id": 6, + "missing": false, + "name": "Never", + "numeric_value": 6 + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 10, + "missing": false, + "value": { + "derived": false, + "id": "0010", + "references": { + "alias": "usage_Path", + "name": "Path" + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "In the past day", + "numeric_value": 1 + }, + { + "id": 2, + "missing": false, + "name": "In the past week, but not in the last day", + "numeric_value": 2 + }, + { + "id": 3, + "missing": false, + "name": "In the past month, but not in the past week", + "numeric_value": 3 + }, + { + "id": 4, + "missing": false, + "name": "In the past 3 months, but not in the last month", + "numeric_value": 4 + }, + { + "id": 5, + "missing": false, + "name": "Over 3 months ago", + "numeric_value": 5 + }, + { + "id": 6, + "missing": false, + "name": "Never", + "numeric_value": 6 + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 11, + "missing": false, + "value": { + "derived": false, + "id": "0011", + "references": { + "alias": "usage_MixChannel", + "name": "MixChannel" + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "In the past day", + "numeric_value": 1 + }, + { + "id": 2, + "missing": false, + "name": "In the past week, but not in the last day", + "numeric_value": 2 + }, + { + "id": 3, + "missing": false, + "name": "In the past month, but not in the past week", + "numeric_value": 3 + }, + { + "id": 4, + "missing": false, + "name": "In the past 3 months, but not in the last month", + "numeric_value": 4 + }, + { + "id": 5, + "missing": false, + "name": "Over 3 months ago", + "numeric_value": 5 + }, + { + "id": 6, + "missing": false, + "name": "Never", + "numeric_value": 6 + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 12, + "missing": false, + "value": { + "derived": false, + "id": "0012", + "references": { + "alias": "usage_SNOW", + "name": "SNOW" + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "In the past day", + "numeric_value": 1 + }, + { + "id": 2, + "missing": false, + "name": "In the past week, but not in the last day", + "numeric_value": 2 + }, + { + "id": 3, + "missing": false, + "name": "In the past month, but not in the past week", + "numeric_value": 3 + }, + { + "id": 4, + "missing": false, + "name": "In the past 3 months, but not in the last month", + "numeric_value": 4 + }, + { + "id": 5, + "missing": false, + "name": "Over 3 months ago", + "numeric_value": 5 + }, + { + "id": 6, + "missing": false, + "name": "Never", + "numeric_value": 6 + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false + } + } + } + ], + "subtype": { + "class": "variable" + } + } + }, + { + "derived": false, + "references": { + "alias": "usage_grid", + "description": "When was the last time you used this app?", + "name": "[Brand] Usage", + "subreferences": [ + { + "alias": "usage_Instagram", + "name": "Instagram" + }, + { + "alias": "usage_Snapchat", + "name": "Snapchat" + }, + { + "alias": "usage_YouTube", + "name": "YouTube" + }, + { + "alias": "usage_Twitter", + "name": "Twitter" + }, + { + "alias": "usage_WhatsApp", + "name": "WhatsApp" + }, + { + "alias": "usage_Musical_ly", + "name": "Musical.ly" + }, + { + "alias": "usage_BigoLive", + "name": "Bigo Live" + }, + { + "alias": "usage_Line", + "name": "Line" + }, + { + "alias": "usage_BBM", + "name": "BBM" + }, + { + "alias": "usage_Path", + "name": "Path" + }, + { + "alias": "usage_MixChannel", + "name": "MixChannel" + }, + { + "alias": "usage_SNOW", + "name": "SNOW" + } + ], + "view": { + "column_width": null, + "include_missing": false, + "show_counts": false + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "In the past day", + "numeric_value": 1 + }, + { + "id": 2, + "missing": false, + "name": "In the past week, but not in the last day", + "numeric_value": 2 + }, + { + "id": 3, + "missing": false, + "name": "In the past month, but not in the past week", + "numeric_value": 3 + }, + { + "id": 4, + "missing": false, + "name": "In the past 3 months, but not in the last month", + "numeric_value": 4 + }, + { + "id": 5, + "missing": false, + "name": "Over 3 months ago", + "numeric_value": 5 + }, + { + "id": 6, + "missing": false, + "name": "Never", + "numeric_value": 6 + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null, + "selected": false + } + ], + "class": "categorical", + "ordinal": false, + "subvariables": [ + "0001", + "0002", + "0003", + "0004", + "0005", + "0006", + "0007", + "0008", + "0009", + "0010", + "0011", + "0012" + ] + } + }, + { + "derived": false, + "references": { + "alias": "gender2cat", + "description": "Are you male or female?", + "discarded": false, + "format": { + "summary": { + "digits": 0 + } + }, + "name": "Gender", + "notes": "", + "view": { + "column_width": null, + "include_missing": false, + "show_counts": false, + "show_numeric_values": false + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Male", + "numeric_value": null + }, + { + "id": 2, + "missing": false, + "name": "Female", + "numeric_value": null + }, + { + "id": 9, + "missing": true, + "name": "not asked", + "numeric_value": null + }, + { + "id": 8, + "missing": true, + "name": "skipped", + "numeric_value": null + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + ], + "element": "crunch:cube", + "measures": { + "count": { + "data": [ + 807.0319999999988, + 1149.3788999999965, + 0, + 0, + 0, + 280.9202, + 358.6191000000008, + 0, + 0, + 0, + 133.6142000000001, + 145.79300000000012, + 0, + 0, + 0, + 111.15220000000004, + 74.77919999999993, + 0, + 0, + 0, + 192.56540000000007, + 193.5960000000001, + 0, + 0, + 0, + 1053.9514000000024, + 821.4441999999982, + 0, + 0, + 0, + 235.04890000000037, + 238.06810000000004, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 625.773800000004, + 725.8424000000006, + 0, + 0, + 0, + 196.39420000000044, + 253.04080000000062, + 0, + 0, + 0, + 143.1135000000002, + 139.87830000000002, + 0, + 0, + 0, + 103.97980000000003, + 102.3926, + 0, + 0, + 0, + 336.4187000000001, + 289.93580000000037, + 0, + 0, + 0, + 1098.7115000000008, + 1157.0831999999973, + 0, + 0, + 0, + 309.8928000000005, + 313.50540000000035, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1382.0630999999867, + 1156.0773999999985, + 0, + 0, + 0, + 718.7748000000033, + 812.2358999999988, + 0, + 0, + 0, + 164.57770000000028, + 340.17870000000045, + 0, + 0, + 0, + 100.49040000000005, + 118.52589999999995, + 0, + 0, + 0, + 97.64699999999999, + 151.82100000000005, + 0, + 0, + 0, + 202.42190000000028, + 232.7614000000003, + 0, + 0, + 0, + 148.30940000000015, + 170.0782, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2814.2842999999325, + 2981.6785000000764, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 729.6135000000082, + 799.6292999999986, + 0, + 0, + 0, + 161.3057999999995, + 232.31700000000043, + 0, + 0, + 0, + 73.82529999999994, + 87.36209999999991, + 0, + 0, + 0, + 53.73309999999996, + 33.98349999999998, + 0, + 0, + 0, + 101.9765, + 79.87939999999985, + 0, + 0, + 0, + 271.08889999999957, + 209.41950000000017, + 0, + 0, + 0, + 1422.7411999999774, + 1539.0877000000148, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 36.90299999999999, + 15.435600000000012, + 0, + 0, + 0, + 31.35889999999999, + 22.248699999999996, + 0, + 0, + 0, + 16.66149999999999, + 23.691299999999995, + 0, + 0, + 0, + 31.752099999999984, + 11.416000000000007, + 0, + 0, + 0, + 8.833900000000014, + 74.94549999999984, + 0, + 0, + 0, + 314.37400000000036, + 469.2866000000012, + 0, + 0, + 0, + 2374.400899999955, + 2364.6548000000375, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2814.2842999999325, + 2981.6785000000764, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2814.2842999999325, + 2981.6785000000764, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2814.2842999999325, + 2981.6785000000764, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2814.2842999999325, + 2981.6785000000764, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2814.2842999999325, + 2981.6785000000764, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2814.2842999999325, + 2981.6785000000764, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "metadata": { + "derived": true, + "references": {}, + "type": { + "class": "numeric", + "integer": false, + "missing_reasons": { + "No Data": -1 + }, + "missing_rules": {} + } + }, + "n_missing": 21209 + } + }, + "missing": 21209, + "n": 27005 + } +} diff --git a/tests/fixtures/mr-x-cat-prune-with-sig.json b/tests/fixtures/mr-x-cat-prune-with-sig.json new file mode 100644 index 000000000..95031a2da --- /dev/null +++ b/tests/fixtures/mr-x-cat-prune-with-sig.json @@ -0,0 +1,1235 @@ +{ + "element": "shoji:view", + "self": "https://app.crunch.io/api/datasets/ddc1b8a25c454689911d3d7a59c97aee/cube/?filter=%5B%5D&query=%7B%22dimensions%22:%5B%7B%22each%22:%22https:%2F%2Fapp.crunch.io%2Fapi%2Fdatasets%2Fddc1b8a25c454689911d3d7a59c97aee%2Fvariables%2F0000db%2F%22%7D,%7B%22function%22:%22as_selected%22,%22args%22:%5B%7B%22variable%22:%22https:%2F%2Fapp.crunch.io%2Fapi%2Fdatasets%2Fddc1b8a25c454689911d3d7a59c97aee%2Fvariables%2F0000db%2F%22%7D%5D%7D,%7B%22variable%22:%22https:%2F%2Fapp.crunch.io%2Fapi%2Fdatasets%2Fddc1b8a25c454689911d3d7a59c97aee%2Fvariables%2F0000bd%2F%22%7D%5D,%22measures%22:%7B%22count%22:%7B%22function%22:%22cube_count%22,%22args%22:%5B%5D%7D%7D,%22weight%22:null%7D", + "value": { + "query": { + "measures": { + "count": { + "function": "cube_count", + "args": [] + } + }, + "dimensions": [ + { + "each": "https://app.crunch.io/api/datasets/ddc1b8a25c454689911d3d7a59c97aee/variables/0000db/" + }, + { + "function": "as_selected", + "args": [ + { + "variable": "https://app.crunch.io/api/datasets/ddc1b8a25c454689911d3d7a59c97aee/variables/0000db/" + } + ] + }, + { + "variable": "https://app.crunch.io/api/datasets/ddc1b8a25c454689911d3d7a59c97aee/variables/0000bd/" + } + ], + "weight": null + }, + "query_environment": { + "filter": [] + }, + "result": { + "dimensions": [ + { + "derived": true, + "references": { + "subreferences": [ + { + "alias": "france", + "name": "France", + "description": "multrace_1" + }, + { + "alias": "bdr", + "name": "West Germany", + "description": "multrace_2" + }, + { + "alias": "italy", + "name": "Italy", + "description": "multrace_3" + }, + { + "alias": "belgium", + "name": "Belgium", + "description": "multrace_4" + }, + { + "alias": "holland", + "name": "the Netherlands", + "description": "multrace_5" + }, + { + "alias": "luxembourg", + "name": "Luxembourg", + "description": "multrace_8" + }, + { + "alias": "other", + "name": "Other", + "description": "multrace_97" + }, + { + "alias": "ec98", + "name": "Don't know", + "description": "multrace_98" + }, + { + "alias": "ec99", + "name": "None of these", + "description": "multrace_99" + } + ], + "notes": "A multiple response variable", + "description": "Which of the following members of the ECSC have you lived in? (select all that apply)", + "name": "Founding Members of the European Coal and Steel Community", + "alias": "ec" + }, + "type": { + "subtype": { + "class": "variable" + }, + "elements": [ + { + "id": 1, + "value": { + "derived": false, + "references": { + "alias": "france", + "name": "France", + "description": "multrace_1" + }, + "id": "00a5", + "type": { + "ordinal": false, + "class": "categorical", + "categories": [ + { + "numeric_value": 1, + "selected": true, + "id": 1, + "missing": false, + "name": "Selected" + }, + { + "numeric_value": 0, + "missing": false, + "id": 0, + "name": "Other" + }, + { + "numeric_value": null, + "missing": true, + "id": -1, + "name": "No Data" + } + ] + } + }, + "missing": false + }, + { + "id": 2, + "value": { + "derived": false, + "references": { + "alias": "bdr", + "name": "West Germany", + "description": "multrace_2" + }, + "id": "00a6", + "type": { + "ordinal": false, + "class": "categorical", + "categories": [ + { + "numeric_value": 1, + "selected": true, + "id": 1, + "missing": false, + "name": "Selected" + }, + { + "numeric_value": 0, + "missing": false, + "id": 0, + "name": "Other" + }, + { + "numeric_value": null, + "missing": true, + "id": -1, + "name": "No Data" + } + ] + } + }, + "missing": false + }, + { + "id": 3, + "value": { + "derived": false, + "references": { + "alias": "italy", + "name": "Italy", + "description": "multrace_3" + }, + "id": "00a7", + "type": { + "ordinal": false, + "class": "categorical", + "categories": [ + { + "numeric_value": 1, + "selected": true, + "id": 1, + "missing": false, + "name": "Selected" + }, + { + "numeric_value": 0, + "missing": false, + "id": 0, + "name": "Other" + }, + { + "numeric_value": null, + "missing": true, + "id": -1, + "name": "No Data" + } + ] + } + }, + "missing": false + }, + { + "id": 4, + "value": { + "derived": false, + "references": { + "alias": "belgium", + "name": "Belgium", + "description": "multrace_4" + }, + "id": "00a8", + "type": { + "ordinal": false, + "class": "categorical", + "categories": [ + { + "numeric_value": 1, + "selected": true, + "id": 1, + "missing": false, + "name": "Selected" + }, + { + "numeric_value": 0, + "missing": false, + "id": 0, + "name": "Other" + }, + { + "numeric_value": null, + "missing": true, + "id": -1, + "name": "No Data" + } + ] + } + }, + "missing": false + }, + { + "id": 5, + "value": { + "derived": false, + "references": { + "alias": "holland", + "name": "the Netherlands", + "description": "multrace_5" + }, + "id": "00a9", + "type": { + "ordinal": false, + "class": "categorical", + "categories": [ + { + "numeric_value": 1, + "selected": true, + "id": 1, + "missing": false, + "name": "Selected" + }, + { + "numeric_value": 0, + "missing": false, + "id": 0, + "name": "Other" + }, + { + "numeric_value": null, + "missing": true, + "id": -1, + "name": "No Data" + } + ] + } + }, + "missing": false + }, + { + "id": 6, + "value": { + "derived": false, + "references": { + "alias": "luxembourg", + "name": "Luxembourg", + "description": "multrace_8" + }, + "id": "00aa", + "type": { + "ordinal": false, + "class": "categorical", + "categories": [ + { + "numeric_value": 1, + "selected": true, + "id": 1, + "missing": false, + "name": "Selected" + }, + { + "numeric_value": 0, + "missing": false, + "id": 0, + "name": "Other" + }, + { + "numeric_value": null, + "missing": true, + "id": -1, + "name": "No Data" + } + ] + } + }, + "missing": false + }, + { + "id": 7, + "value": { + "derived": false, + "references": { + "alias": "other", + "name": "Other", + "description": "multrace_97" + }, + "id": "00ab", + "type": { + "ordinal": false, + "class": "categorical", + "categories": [ + { + "numeric_value": 1, + "selected": true, + "id": 1, + "missing": false, + "name": "Selected" + }, + { + "numeric_value": 0, + "missing": false, + "id": 0, + "name": "Other" + }, + { + "numeric_value": null, + "missing": true, + "id": -1, + "name": "No Data" + } + ] + } + }, + "missing": false + }, + { + "id": 8, + "value": { + "derived": false, + "references": { + "alias": "ec98", + "name": "Don't know", + "description": "multrace_98" + }, + "id": "00ac", + "type": { + "ordinal": false, + "class": "categorical", + "categories": [ + { + "numeric_value": 1, + "selected": true, + "id": 1, + "missing": false, + "name": "Selected" + }, + { + "numeric_value": 0, + "missing": false, + "id": 0, + "name": "Other" + }, + { + "numeric_value": null, + "missing": true, + "id": -1, + "name": "No Data" + } + ] + } + }, + "missing": false + }, + { + "id": 9, + "value": { + "derived": false, + "references": { + "alias": "ec99", + "name": "None of these", + "description": "multrace_99" + }, + "id": "00ad", + "type": { + "ordinal": false, + "class": "categorical", + "categories": [ + { + "numeric_value": 1, + "selected": true, + "id": 1, + "missing": false, + "name": "Selected" + }, + { + "numeric_value": 0, + "missing": false, + "id": 0, + "name": "Other" + }, + { + "numeric_value": null, + "missing": true, + "id": -1, + "name": "No Data" + } + ] + } + }, + "missing": false + } + ], + "class": "enum" + } + }, + { + "references": { + "subreferences": [ + { + "alias": "france", + "name": "France", + "description": "multrace_1" + }, + { + "alias": "bdr", + "name": "West Germany", + "description": "multrace_2" + }, + { + "alias": "italy", + "name": "Italy", + "description": "multrace_3" + }, + { + "alias": "belgium", + "name": "Belgium", + "description": "multrace_4" + }, + { + "alias": "holland", + "name": "the Netherlands", + "description": "multrace_5" + }, + { + "alias": "luxembourg", + "name": "Luxembourg", + "description": "multrace_8" + }, + { + "alias": "other", + "name": "Other", + "description": "multrace_97" + }, + { + "alias": "ec98", + "name": "Don't know", + "description": "multrace_98" + }, + { + "alias": "ec99", + "name": "None of these", + "description": "multrace_99" + } + ], + "notes": "A multiple response variable", + "alias": "ec", + "description": "Which of the following members of the ECSC have you lived in? (select all that apply)", + "name": "Founding Members of the European Coal and Steel Community" + }, + "derived": true, + "type": { + "ordinal": false, + "subvariables": [ + "00a5", + "00a6", + "00a7", + "00a8", + "00a9", + "00aa", + "00ab", + "00ac", + "00ad" + ], + "class": "categorical", + "categories": [ + { + "numeric_value": 1, + "selected": true, + "id": 1, + "missing": false, + "name": "Selected" + }, + { + "numeric_value": 0, + "missing": false, + "id": 0, + "name": "Other" + }, + { + "numeric_value": null, + "missing": true, + "id": -1, + "name": "No Data" + } + ] + } + }, + { + "references": { + "alias": "offal", + "notes": "A categorical variable where one of the categories has no responses", + "name": "Offal", + "description": "Mike's favorite food" + }, + "derived": false, + "type": { + "ordinal": false, + "class": "categorical", + "categories": [ + { + "numeric_value": 1, + "missing": false, + "id": 1, + "name": "Liver" + }, + { + "numeric_value": 2, + "missing": false, + "id": 2, + "name": "Kidney" + }, + { + "numeric_value": 3, + "missing": false, + "id": 3, + "name": "Heart" + }, + { + "numeric_value": 4, + "missing": false, + "id": 4, + "name": "Pancreas" + }, + { + "numeric_value": 5, + "missing": false, + "id": 5, + "name": "Thymus" + }, + { + "numeric_value": 6, + "missing": false, + "id": 6, + "name": "Snout" + }, + { + "numeric_value": 7, + "missing": false, + "id": 7, + "name": "Lung" + }, + { + "numeric_value": 8, + "missing": false, + "id": 8, + "name": "Tongue" + }, + { + "numeric_value": 32766, + "missing": true, + "id": 32766, + "name": "Don't know" + }, + { + "numeric_value": 32767, + "missing": true, + "id": 32767, + "name": "Not asked" + }, + { + "numeric_value": null, + "missing": true, + "id": -1, + "name": "No Data" + } + ] + } + } + ], + "missing": 1104, + "measures": { + "count": { + "data": [ + 144, + 48, + 0, + 24, + 124, + 43, + 30, + 9, + 0, + 23, + 0, + 49, + 21, + 0, + 11, + 21, + 20, + 10, + 4, + 0, + 10, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 32, + 13, + 0, + 10, + 16, + 7, + 4, + 0, + 0, + 4, + 0, + 161, + 56, + 0, + 25, + 129, + 56, + 36, + 13, + 0, + 29, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 13, + 8, + 0, + 4, + 3, + 9, + 6, + 2, + 0, + 1, + 0, + 180, + 61, + 0, + 31, + 142, + 54, + 34, + 11, + 0, + 32, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 9, + 3, + 0, + 0, + 2, + 2, + 2, + 1, + 0, + 9, + 0, + 184, + 66, + 0, + 35, + 143, + 61, + 38, + 12, + 0, + 24, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 2, + 4, + 0, + 2, + 6, + 3, + 0, + 1, + 0, + 0, + 0, + 191, + 65, + 0, + 33, + 139, + 60, + 40, + 12, + 0, + 33, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 190, + 69, + 0, + 35, + 145, + 63, + 40, + 13, + 0, + 32, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 193, + 69, + 0, + 35, + 145, + 63, + 40, + 13, + 0, + 33, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 3, + 1, + 0, + 0, + 2, + 2, + 0, + 1, + 0, + 1, + 0, + 190, + 68, + 0, + 35, + 143, + 61, + 40, + 12, + 0, + 32, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 193, + 69, + 0, + 35, + 145, + 63, + 40, + 13, + 0, + 33, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0 + ], + "n_missing": 1104, + "metadata": { + "references": {}, + "derived": true, + "type": { + "integer": true, + "missing_rules": {}, + "missing_reasons": { + "No Data": -1 + }, + "class": "numeric" + } + } + } + }, + "element": "crunch:cube", + "counts": [ + 144, + 48, + 0, + 24, + 124, + 43, + 30, + 9, + 0, + 23, + 0, + 49, + 21, + 0, + 11, + 21, + 20, + 10, + 4, + 0, + 10, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 32, + 13, + 0, + 10, + 16, + 7, + 4, + 0, + 0, + 4, + 0, + 161, + 56, + 0, + 25, + 129, + 56, + 36, + 13, + 0, + 29, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 13, + 8, + 0, + 4, + 3, + 9, + 6, + 2, + 0, + 1, + 0, + 180, + 61, + 0, + 31, + 142, + 54, + 34, + 11, + 0, + 32, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 9, + 3, + 0, + 0, + 2, + 2, + 2, + 1, + 0, + 9, + 0, + 184, + 66, + 0, + 35, + 143, + 61, + 38, + 12, + 0, + 24, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 2, + 4, + 0, + 2, + 6, + 3, + 0, + 1, + 0, + 0, + 0, + 191, + 65, + 0, + 33, + 139, + 60, + 40, + 12, + 0, + 33, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 190, + 69, + 0, + 35, + 145, + 63, + 40, + 13, + 0, + 32, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 193, + 69, + 0, + 35, + 145, + 63, + 40, + 13, + 0, + 33, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 3, + 1, + 0, + 0, + 2, + 2, + 0, + 1, + 0, + 1, + 0, + 190, + 68, + 0, + 35, + 143, + 61, + 40, + 12, + 0, + 32, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 193, + 69, + 0, + 35, + 145, + 63, + 40, + 13, + 0, + 33, + 0, + 361, + 101, + 0, + 115, + 202, + 126, + 77, + 18, + 1, + 70, + 0 + ], + "n": 1662 + } + } +} diff --git a/tests/fixtures/network-live-tv.json b/tests/fixtures/network-live-tv.json new file mode 100644 index 000000000..12f848178 --- /dev/null +++ b/tests/fixtures/network-live-tv.json @@ -0,0 +1,2104 @@ +{ + "query": { + "dimensions": [ + { + "each": 1 + }, + { + "args": [ + { + "variable": "afda2064ac8140a4960eac27fd43f253" + } + ], + "function": "as_selected" + }, + { + "each": 3 + }, + { + "args": [ + { + "variable": "62adad17f78a4fcdaed5a852c92bbcde" + } + ], + "function": "as_selected" + } + ], + "measures": { + "count": { + "args": [], + "function": "cube_count" + } + }, + "weight": null + }, + "query_environment": { + "filter": [] + }, + "result": { + "counts": [ + 5905, + 12760, + 4, + 7122, + 11543, + 4, + 12972, + 5693, + 4, + 16285, + 2380, + 4, + 5543, + 13122, + 4, + 3335, + 15330, + 4, + 12447, + 74514, + 34, + 13962, + 72999, + 34, + 23911, + 63050, + 34, + 67271, + 19690, + 34, + 8651, + 78310, + 34, + 5739, + 81222, + 34, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 6098, + 11225, + 3, + 6663, + 10660, + 3, + 11759, + 5564, + 3, + 14144, + 3179, + 3, + 5018, + 12305, + 3, + 3929, + 13394, + 3, + 12254, + 76049, + 35, + 14421, + 73882, + 35, + 25124, + 63179, + 35, + 69412, + 18891, + 35, + 9176, + 79127, + 35, + 5145, + 83158, + 35, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 3973, + 11001, + 1, + 4383, + 10591, + 1, + 7839, + 7135, + 1, + 14469, + 505, + 1, + 3179, + 11795, + 1, + 1937, + 13037, + 1, + 14379, + 76273, + 37, + 16701, + 73951, + 37, + 29044, + 61608, + 37, + 69087, + 21565, + 37, + 11015, + 79637, + 37, + 7137, + 83515, + 37, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 9222, + 40217, + 15, + 10664, + 38775, + 15, + 17267, + 32172, + 15, + 48428, + 1011, + 15, + 7730, + 41709, + 15, + 3953, + 45486, + 15, + 9130, + 47057, + 23, + 10420, + 45767, + 23, + 19616, + 36571, + 23, + 35128, + 21059, + 23, + 6464, + 49723, + 23, + 5121, + 51066, + 23, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 5213, + 20387, + 8, + 5608, + 19992, + 8, + 9149, + 16451, + 8, + 25166, + 434, + 8, + 4535, + 21065, + 8, + 2345, + 23255, + 8, + 13139, + 66887, + 30, + 15476, + 64550, + 30, + 27734, + 52292, + 30, + 58390, + 21636, + 30, + 9659, + 70367, + 30, + 6729, + 73297, + 30, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 12627, + 31293, + 12, + 14252, + 29668, + 12, + 24468, + 19452, + 12, + 37381, + 6539, + 12, + 10210, + 33710, + 12, + 7137, + 36783, + 12, + 5725, + 55981, + 26, + 6832, + 54874, + 26, + 12415, + 49291, + 26, + 46175, + 15531, + 26, + 3984, + 57722, + 26, + 1937, + 59769, + 26, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 11745, + 25670, + 9, + 13649, + 23766, + 9, + 22861, + 14554, + 9, + 31583, + 5832, + 9, + 9615, + 27800, + 9, + 5642, + 31773, + 9, + 6607, + 61604, + 29, + 7435, + 60776, + 29, + 14022, + 54189, + 29, + 51973, + 16238, + 29, + 4579, + 63632, + 29, + 3432, + 64779, + 29, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 3912, + 17022, + 3, + 4520, + 16414, + 3, + 8853, + 12081, + 3, + 20492, + 442, + 3, + 3737, + 17197, + 3, + 2305, + 18629, + 3, + 14440, + 70252, + 35, + 16564, + 68128, + 35, + 28030, + 56662, + 35, + 63064, + 21628, + 35, + 10457, + 74235, + 35, + 6769, + 77923, + 35, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 8096, + 8963, + 8, + 8563, + 8496, + 8, + 10184, + 6875, + 8, + 14530, + 2529, + 8, + 6259, + 10800, + 8, + 3942, + 13117, + 8, + 10256, + 78311, + 30, + 12521, + 76046, + 30, + 26699, + 61868, + 30, + 69026, + 19541, + 30, + 7935, + 80632, + 30, + 5132, + 83435, + 30, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851 + ], + "dimensions": [ + { + "derived": true, + "references": { + "alias": "draft abbreviated Services used to watch TV yesterday (12 months combined)", + "name": "draft abbreviated Services used to watch TV yesterday (12 months combined)", + "subreferences": [ + { + "alias": "Amazon Prime Video-afda2064ac8140a4960eac27fd43f253", + "name": "Amazon Prime Video", + "view": { + "entity_name": "Amazon Prime Video", + "origin_entity": "9", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Hulu-afda2064ac8140a4960eac27fd43f253", + "name": "Hulu", + "view": { + "entity_name": "Hulu", + "origin_entity": "8", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Live TV (free-to-air)-afda2064ac8140a4960eac27fd43f253", + "name": "Live TV (free-to-air)", + "view": { + "entity_name": "Live TV (free-to-air)", + "origin_entity": "3", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Live TV through cable provider-afda2064ac8140a4960eac27fd43f253", + "name": "Live TV through cable provider", + "view": { + "entity_name": "Live TV through cable provider", + "origin_entity": "1", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Live TV via satellite provider-afda2064ac8140a4960eac27fd43f253", + "name": "Live TV via satellite provider", + "view": { + "entity_name": "Live TV via satellite provider", + "origin_entity": "2", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Netflix-afda2064ac8140a4960eac27fd43f253", + "name": "Netflix", + "view": { + "entity_name": "Netflix", + "origin_entity": "7", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "other etc-afda2064ac8140a4960eac27fd43f253", + "name": "other etc", + "view": { + "entity_name": "Pay-per-view", + "origin_entity": "21", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Via DVR (e.g., TiVo, or \u2018On Demand\u2019 services)-afda2064ac8140a4960eac27fd43f253", + "name": "Via DVR (e.g., TiVo, or \u2018On Demand\u2019 services)", + "view": { + "entity_name": "Via DVR (e.g., TiVo, or \u2018On Demand\u2019 services)", + "origin_entity": "5", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "YouTube/ Vevo-afda2064ac8140a4960eac27fd43f253", + "name": "YouTube/ Vevo", + "view": { + "entity_name": "YouTube/ Vevo", + "origin_entity": "15", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + } + ] + }, + "type": { + "class": "enum", + "elements": [ + { + "id": 1, + "missing": false, + "value": { + "derived": false, + "id": "0001", + "references": { + "alias": "Amazon Prime Video-afda2064ac8140a4960eac27fd43f253", + "name": "Amazon Prime Video", + "view": { + "entity_name": "Amazon Prime Video", + "origin_entity": "9", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 2, + "missing": false, + "value": { + "derived": false, + "id": "0002", + "references": { + "alias": "Hulu-afda2064ac8140a4960eac27fd43f253", + "name": "Hulu", + "view": { + "entity_name": "Hulu", + "origin_entity": "8", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 3, + "missing": false, + "value": { + "derived": false, + "id": "0003", + "references": { + "alias": "Live TV (free-to-air)-afda2064ac8140a4960eac27fd43f253", + "name": "Live TV (free-to-air)", + "view": { + "entity_name": "Live TV (free-to-air)", + "origin_entity": "3", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 4, + "missing": false, + "value": { + "derived": false, + "id": "0004", + "references": { + "alias": "Live TV through cable provider-afda2064ac8140a4960eac27fd43f253", + "name": "Live TV through cable provider", + "view": { + "entity_name": "Live TV through cable provider", + "origin_entity": "1", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 5, + "missing": false, + "value": { + "derived": false, + "id": "0005", + "references": { + "alias": "Live TV via satellite provider-afda2064ac8140a4960eac27fd43f253", + "name": "Live TV via satellite provider", + "view": { + "entity_name": "Live TV via satellite provider", + "origin_entity": "2", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 6, + "missing": false, + "value": { + "derived": false, + "id": "0006", + "references": { + "alias": "Netflix-afda2064ac8140a4960eac27fd43f253", + "name": "Netflix", + "view": { + "entity_name": "Netflix", + "origin_entity": "7", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 7, + "missing": false, + "value": { + "derived": false, + "id": "0007", + "references": { + "alias": "other etc-afda2064ac8140a4960eac27fd43f253", + "name": "other etc", + "view": { + "entity_name": "Pay-per-view", + "origin_entity": "21", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 8, + "missing": false, + "value": { + "derived": false, + "id": "0008", + "references": { + "alias": "Via DVR (e.g., TiVo, or \u2018On Demand\u2019 services)-afda2064ac8140a4960eac27fd43f253", + "name": "Via DVR (e.g., TiVo, or \u2018On Demand\u2019 services)", + "view": { + "entity_name": "Via DVR (e.g., TiVo, or \u2018On Demand\u2019 services)", + "origin_entity": "5", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 9, + "missing": false, + "value": { + "derived": false, + "id": "0009", + "references": { + "alias": "YouTube/ Vevo-afda2064ac8140a4960eac27fd43f253", + "name": "YouTube/ Vevo", + "view": { + "entity_name": "YouTube/ Vevo", + "origin_entity": "15", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + } + ], + "subtype": { + "class": "variable" + } + } + }, + { + "derived": true, + "references": { + "alias": "draft abbreviated Services used to watch TV yesterday (12 months combined)", + "name": "draft abbreviated Services used to watch TV yesterday (12 months combined)", + "subreferences": [ + { + "alias": "Amazon Prime Video-afda2064ac8140a4960eac27fd43f253", + "name": "Amazon Prime Video", + "view": { + "entity_name": "Amazon Prime Video", + "origin_entity": "9", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Hulu-afda2064ac8140a4960eac27fd43f253", + "name": "Hulu", + "view": { + "entity_name": "Hulu", + "origin_entity": "8", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Live TV (free-to-air)-afda2064ac8140a4960eac27fd43f253", + "name": "Live TV (free-to-air)", + "view": { + "entity_name": "Live TV (free-to-air)", + "origin_entity": "3", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Live TV through cable provider-afda2064ac8140a4960eac27fd43f253", + "name": "Live TV through cable provider", + "view": { + "entity_name": "Live TV through cable provider", + "origin_entity": "1", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Live TV via satellite provider-afda2064ac8140a4960eac27fd43f253", + "name": "Live TV via satellite provider", + "view": { + "entity_name": "Live TV via satellite provider", + "origin_entity": "2", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Netflix-afda2064ac8140a4960eac27fd43f253", + "name": "Netflix", + "view": { + "entity_name": "Netflix", + "origin_entity": "7", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "other etc-afda2064ac8140a4960eac27fd43f253", + "name": "other etc", + "view": { + "entity_name": "Pay-per-view", + "origin_entity": "21", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Via DVR (e.g., TiVo, or \u2018On Demand\u2019 services)-afda2064ac8140a4960eac27fd43f253", + "name": "Via DVR (e.g., TiVo, or \u2018On Demand\u2019 services)", + "view": { + "entity_name": "Via DVR (e.g., TiVo, or \u2018On Demand\u2019 services)", + "origin_entity": "5", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "YouTube/ Vevo-afda2064ac8140a4960eac27fd43f253", + "name": "YouTube/ Vevo", + "view": { + "entity_name": "YouTube/ Vevo", + "origin_entity": "15", + "origin_metric": "services_used_any_year", + "origin_source": "tv_survey" + } + } + ] + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false, + "subvariables": [ + "0001", + "0002", + "0003", + "0004", + "0005", + "0006", + "0007", + "0008", + "0009" + ] + } + }, + { + "derived": true, + "references": { + "alias": "draft abbreviated Devices used to watch TV yesterday (12 months combined)", + "name": "draft abbreviated Devices used to watch TV yesterday (12 months combined)", + "subreferences": [ + { + "alias": "Cell phone (Devices used to watch TV yesterday (12 months combined))-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Cell phone (Devices used to watch TV yesterday (12 months combined))", + "view": { + "entity_name": "Cell phone", + "origin_entity": "10", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Desktop/laptop-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Desktop/laptop", + "view": { + "entity_name": "Desktop/laptop", + "origin_entity": "11", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Other etc-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Other etc", + "view": { + "entity_name": "Other", + "origin_entity": "12", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "TV (regular or smart)-62adad17f78a4fcdaed5a852c92bbcde", + "name": "TV (regular or smart)", + "view": { + "entity_name": "Regular TV ", + "origin_entity": "1", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Tablet-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Tablet", + "view": { + "entity_name": "Tablet", + "origin_entity": "9", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Via gaming console-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Via gaming console", + "view": { + "entity_name": "Via gaming console (e.g., PS3, Xbox)", + "origin_entity": "3", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + } + ] + }, + "type": { + "class": "enum", + "elements": [ + { + "id": 1, + "missing": false, + "value": { + "derived": false, + "id": "0001", + "references": { + "alias": "Cell phone (Devices used to watch TV yesterday (12 months combined))-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Cell phone (Devices used to watch TV yesterday (12 months combined))", + "view": { + "entity_name": "Cell phone", + "origin_entity": "10", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 2, + "missing": false, + "value": { + "derived": false, + "id": "0002", + "references": { + "alias": "Desktop/laptop-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Desktop/laptop", + "view": { + "entity_name": "Desktop/laptop", + "origin_entity": "11", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 3, + "missing": false, + "value": { + "derived": false, + "id": "0003", + "references": { + "alias": "Other etc-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Other etc", + "view": { + "entity_name": "Other", + "origin_entity": "12", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 4, + "missing": false, + "value": { + "derived": false, + "id": "0004", + "references": { + "alias": "TV (regular or smart)-62adad17f78a4fcdaed5a852c92bbcde", + "name": "TV (regular or smart)", + "view": { + "entity_name": "Regular TV ", + "origin_entity": "1", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 5, + "missing": false, + "value": { + "derived": false, + "id": "0005", + "references": { + "alias": "Tablet-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Tablet", + "view": { + "entity_name": "Tablet", + "origin_entity": "9", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + }, + { + "id": 6, + "missing": false, + "value": { + "derived": false, + "id": "0006", + "references": { + "alias": "Via gaming console-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Via gaming console", + "view": { + "entity_name": "Via gaming console (e.g., PS3, Xbox)", + "origin_entity": "3", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false + } + } + } + ], + "subtype": { + "class": "variable" + } + } + }, + { + "derived": true, + "references": { + "alias": "draft abbreviated Devices used to watch TV yesterday (12 months combined)", + "name": "draft abbreviated Devices used to watch TV yesterday (12 months combined)", + "subreferences": [ + { + "alias": "Cell phone (Devices used to watch TV yesterday (12 months combined))-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Cell phone (Devices used to watch TV yesterday (12 months combined))", + "view": { + "entity_name": "Cell phone", + "origin_entity": "10", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Desktop/laptop-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Desktop/laptop", + "view": { + "entity_name": "Desktop/laptop", + "origin_entity": "11", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Other etc-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Other etc", + "view": { + "entity_name": "Other", + "origin_entity": "12", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "TV (regular or smart)-62adad17f78a4fcdaed5a852c92bbcde", + "name": "TV (regular or smart)", + "view": { + "entity_name": "Regular TV ", + "origin_entity": "1", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Tablet-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Tablet", + "view": { + "entity_name": "Tablet", + "origin_entity": "9", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + }, + { + "alias": "Via gaming console-62adad17f78a4fcdaed5a852c92bbcde", + "name": "Via gaming console", + "view": { + "entity_name": "Via gaming console (e.g., PS3, Xbox)", + "origin_entity": "3", + "origin_metric": "devices_used_any_year", + "origin_source": "tv_survey" + } + } + ] + }, + "type": { + "categories": [ + { + "id": 1, + "missing": false, + "name": "Selected", + "numeric_value": 1, + "selected": true + }, + { + "id": 0, + "missing": false, + "name": "Other", + "numeric_value": 0 + }, + { + "id": -1, + "missing": true, + "name": "No Data", + "numeric_value": null + } + ], + "class": "categorical", + "ordinal": false, + "subvariables": [ + "0001", + "0002", + "0003", + "0004", + "0005", + "0006" + ] + } + } + ], + "element": "crunch:cube", + "measures": { + "count": { + "data": [ + 5905, + 12760, + 4, + 7122, + 11543, + 4, + 12972, + 5693, + 4, + 16285, + 2380, + 4, + 5543, + 13122, + 4, + 3335, + 15330, + 4, + 12447, + 74514, + 34, + 13962, + 72999, + 34, + 23911, + 63050, + 34, + 67271, + 19690, + 34, + 8651, + 78310, + 34, + 5739, + 81222, + 34, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 6098, + 11225, + 3, + 6663, + 10660, + 3, + 11759, + 5564, + 3, + 14144, + 3179, + 3, + 5018, + 12305, + 3, + 3929, + 13394, + 3, + 12254, + 76049, + 35, + 14421, + 73882, + 35, + 25124, + 63179, + 35, + 69412, + 18891, + 35, + 9176, + 79127, + 35, + 5145, + 83158, + 35, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 3973, + 11001, + 1, + 4383, + 10591, + 1, + 7839, + 7135, + 1, + 14469, + 505, + 1, + 3179, + 11795, + 1, + 1937, + 13037, + 1, + 14379, + 76273, + 37, + 16701, + 73951, + 37, + 29044, + 61608, + 37, + 69087, + 21565, + 37, + 11015, + 79637, + 37, + 7137, + 83515, + 37, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 9222, + 40217, + 15, + 10664, + 38775, + 15, + 17267, + 32172, + 15, + 48428, + 1011, + 15, + 7730, + 41709, + 15, + 3953, + 45486, + 15, + 9130, + 47057, + 23, + 10420, + 45767, + 23, + 19616, + 36571, + 23, + 35128, + 21059, + 23, + 6464, + 49723, + 23, + 5121, + 51066, + 23, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 5213, + 20387, + 8, + 5608, + 19992, + 8, + 9149, + 16451, + 8, + 25166, + 434, + 8, + 4535, + 21065, + 8, + 2345, + 23255, + 8, + 13139, + 66887, + 30, + 15476, + 64550, + 30, + 27734, + 52292, + 30, + 58390, + 21636, + 30, + 9659, + 70367, + 30, + 6729, + 73297, + 30, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 12627, + 31293, + 12, + 14252, + 29668, + 12, + 24468, + 19452, + 12, + 37381, + 6539, + 12, + 10210, + 33710, + 12, + 7137, + 36783, + 12, + 5725, + 55981, + 26, + 6832, + 54874, + 26, + 12415, + 49291, + 26, + 46175, + 15531, + 26, + 3984, + 57722, + 26, + 1937, + 59769, + 26, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 11745, + 25670, + 9, + 13649, + 23766, + 9, + 22861, + 14554, + 9, + 31583, + 5832, + 9, + 9615, + 27800, + 9, + 5642, + 31773, + 9, + 6607, + 61604, + 29, + 7435, + 60776, + 29, + 14022, + 54189, + 29, + 51973, + 16238, + 29, + 4579, + 63632, + 29, + 3432, + 64779, + 29, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 3912, + 17022, + 3, + 4520, + 16414, + 3, + 8853, + 12081, + 3, + 20492, + 442, + 3, + 3737, + 17197, + 3, + 2305, + 18629, + 3, + 14440, + 70252, + 35, + 16564, + 68128, + 35, + 28030, + 56662, + 35, + 63064, + 21628, + 35, + 10457, + 74235, + 35, + 6769, + 77923, + 35, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851, + 8096, + 8963, + 8, + 8563, + 8496, + 8, + 10184, + 6875, + 8, + 14530, + 2529, + 8, + 6259, + 10800, + 8, + 3942, + 13117, + 8, + 10256, + 78311, + 30, + 12521, + 76046, + 30, + 26699, + 61868, + 30, + 69026, + 19541, + 30, + 7935, + 80632, + 30, + 5132, + 83435, + 30, + 37, + 97, + 111851, + 20, + 114, + 111851, + 40, + 94, + 111851, + 101, + 33, + 111851, + 20, + 114, + 111851, + 12, + 122, + 111851 + ], + "metadata": { + "derived": true, + "references": {}, + "type": { + "class": "numeric", + "integer": true, + "missing_reasons": { + "No Data": -1 + }, + "missing_rules": {} + } + }, + "n_missing": 112023 + } + }, + "missing": 112023, + "n": 217649 + } +} diff --git a/tests/fixtures/num-by-cat-hs.json b/tests/fixtures/num-by-cat-hs.json new file mode 100644 index 000000000..13f89550d --- /dev/null +++ b/tests/fixtures/num-by-cat-hs.json @@ -0,0 +1,306 @@ +{ + "element": "shoji:view", + "self": "/api/datasets/06a0204a79f04304bb296dd24c356841/cube/?filter=%5B%5D&query=%7B%22dimensions%22:%5B%7B%22function%22:%22bin%22,%22args%22:%5B%7B%22variable%22:%22http:%2F%2Flocal.crunch.io:8080%2Fapi%2Fdatasets%2F06a0204a79f04304bb296dd24c356841%2Fvariables%2F000002%2F%22%7D%5D%7D,%7B%22variable%22:%22http:%2F%2Flocal.crunch.io:8080%2Fapi%2Fdatasets%2F06a0204a79f04304bb296dd24c356841%2Fvariables%2F000000%2F%22%7D%5D,%22measures%22:%7B%22count%22:%7B%22function%22:%22cube_count%22,%22args%22:%5B%5D%7D%7D,%22weight%22:null%7D", + "value": { + "query": { + "measures": { + "count": { + "function": "cube_count", + "args": [] + } + }, + "dimensions": [ + { + "function": "bin", + "args": [ + { + "variable": "/api/datasets/06a0204a79f04304bb296dd24c356841/variables/000002/" + } + ] + }, + { + "variable": "/api/datasets/06a0204a79f04304bb296dd24c356841/variables/000000/" + } + ], + "weight": null + }, + "query_environment": { + "filter": [] + }, + "result": { + "dimensions": [ + { + "references": { + "alias": "z", + "format": { + "data": { + "digits": 2 + }, + "summary": { + "digits": 2 + } + }, + "name": "z" + }, + "derived": true, + "type": { + "subtype": { + "missing_rules": {}, + "missing_reasons": { + "No Data": -1 + }, + "class": "numeric" + }, + "elements": [ + { + "id": -1, + "value": { + "?": -1 + }, + "missing": true + }, + { + "id": 1, + "value": [ + 1.0, + 1.5 + ], + "missing": false + }, + { + "id": 2, + "value": [ + 1.5, + 2.0 + ], + "missing": false + }, + { + "id": 3, + "value": [ + 2.0, + 2.5 + ], + "missing": false + }, + { + "id": 4, + "value": [ + 2.5, + 3.0 + ], + "missing": false + }, + { + "id": 5, + "value": [ + 3.0, + 3.5 + ], + "missing": false + } + ], + "class": "enum" + } + }, + { + "derived": false, + "references": { + "alias": "x", + "view": { + "show_counts": false, + "show_numeric_values": false, + "transform": { + "insertions": [ + { + "function": "subtotal", + "args": [ + 1 + ], + "name": "top", + "anchor": "top" + }, + { + "function": "subtotal", + "args": [ + 2 + ], + "name": "mid", + "anchor": 3 + }, + { + "function": "subtotal", + "args": [ + 9 + ], + "name": "bot", + "anchor": "bottom" + } + ] + }, + "include_missing": false, + "column_width": null + }, + "name": "x" + }, + "type": { + "ordinal": false, + "class": "categorical", + "categories": [ + { + "numeric_value": 1, + "missing": false, + "id": 1, + "name": "red" + }, + { + "numeric_value": 2, + "missing": false, + "id": 2, + "name": "green" + }, + { + "numeric_value": 3, + "missing": false, + "id": 3, + "name": "blue" + }, + { + "numeric_value": 4, + "missing": false, + "id": 4, + "name": "4" + }, + { + "numeric_value": 8, + "missing": true, + "id": 8, + "name": "8" + }, + { + "numeric_value": 9, + "missing": false, + "id": 9, + "name": "9" + }, + { + "numeric_value": null, + "missing": true, + "id": -1, + "name": "No Data" + } + ] + } + } + ], + "missing": 5, + "measures": { + "count": { + "data": [ + 1, + 1, + 0, + 1, + 0, + 1, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0 + ], + "n_missing": 5, + "metadata": { + "references": {}, + "derived": true, + "type": { + "integer": true, + "missing_rules": {}, + "missing_reasons": { + "No Data": -1 + }, + "class": "numeric" + } + } + } + }, + "element": "crunch:cube", + "counts": [ + 1, + 1, + 0, + 1, + 0, + 1, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0 + ], + "n": 6 + } + } +} diff --git a/tests/integration/test_crunch_cube.py b/tests/integration/test_crunch_cube.py index f2838cc22..8145f0dbf 100644 --- a/tests/integration/test_crunch_cube.py +++ b/tests/integration/test_crunch_cube.py @@ -2041,3 +2041,43 @@ def test_univ_mr_with_hs_does_not_crash(self): # in the result). H&S shouldn't be in the MR variable, but there # are cases when there are. assert True + + def test_ca_x_cat_pruning_bases(self): + """Assert that pruning is based on unweighted bases.""" + cube = CrunchCube(CR.CA_SUBVAR_X_CA_CAT_X_CAT) + cs = cube.slices[3] + + # Expectation is that the mask values are all False + # (i.e. not masked for pruning) + expected = np.zeros((6, 2), dtype=bool) + + # Assert that both weighted and unweighted proportions + # are not pruned + actual = cs.proportions(weighted=True, prune=True).mask + np.testing.assert_array_equal(actual, expected) + actual = cs.proportions(weighted=False, prune=True).mask + np.testing.assert_array_equal(actual, expected) + actual = cs.proportions(axis=0, weighted=True, prune=True).mask + np.testing.assert_array_equal(actual, expected) + actual = cs.proportions(axis=1, weighted=True, prune=True).mask + np.testing.assert_array_equal(actual, expected) + + # Assert that both weighted and unweighted + # summaries are not pruned + expected = False + actual = cs.margin(weighted=True, prune=True).mask + assert actual == expected + actual = cs.margin(weighted=False, prune=True).mask + assert actual == expected + + expected = np.zeros((2,), dtype=bool) + actual = cs.margin(axis=0, weighted=True, prune=True).mask + np.testing.assert_array_equal(actual, expected) + actual = cs.margin(axis=0, weighted=False, prune=True).mask + np.testing.assert_array_equal(actual, expected) + + expected = np.zeros((6,), dtype=bool) + actual = cs.margin(axis=1, weighted=True, prune=True).mask + np.testing.assert_array_equal(actual, expected) + actual = cs.margin(axis=1, weighted=False, prune=True).mask + np.testing.assert_array_equal(actual, expected) diff --git a/tests/integration/test_cube_slice.py b/tests/integration/test_cube_slice.py index ade019939..52c4758b6 100644 --- a/tests/integration/test_cube_slice.py +++ b/tests/integration/test_cube_slice.py @@ -5,6 +5,7 @@ import numpy as np from cr.cube.crunch_cube import CrunchCube +from cr.cube.util import compress_pruned from ..fixtures import CR # ---mnemonic: CR = 'cube-response'--- @@ -25,6 +26,50 @@ def it_provides_a_console_friendly_repr_for_a_slice(self): "C 5 3" ) + def it_prunes_margin_with_only_empty_hs(self): + cube = CrunchCube(CR.CA_SUBVAR_HS_X_CA_CAT_X_CAT_COLPCT) + cube_slice = cube.slices[2] + + total_margin = cube_slice.margin( + prune=True, include_transforms_for_dims=[0, 1]) + assert type(total_margin) is np.ma.core.MaskedConstant + + column_margin = cube_slice.margin( + axis=0, prune=True, include_transforms_for_dims=[0, 1]) + assert all(column_margin.mask) + + row_margin = cube_slice.margin( + axis=1, prune=True, include_transforms_for_dims=[0, 1]) + assert all(row_margin.mask) + + def it_prunes_mr_x_cat_pvals_correctly(self): + cube = CrunchCube(CR.MR_X_CAT_PRUNE_WITH_SIG) + cube_slice = cube.slices[0] + + col_margin = compress_pruned(cube_slice.margin(axis=0, prune=True)) + assert col_margin.shape == (7, 7) + + total_margin = compress_pruned(cube_slice.margin(prune=True)) + assert total_margin.shape == (7,) + + def it_prunes_mr_x_mr_margins_correctly(self): + cube_slice = CrunchCube(CR.NETWORK_LIVE_TV).slices[0] + row_margin = compress_pruned(cube_slice.margin(axis=1, prune=True)) + np.testing.assert_array_almost_equal( + row_margin, + np.array([ + [18665, 18665, 18665, 18665, 18665, 18665], + [17323, 17323, 17323, 17323, 17323, 17323], + [14974, 14974, 14974, 14974, 14974, 14974], + [49439, 49439, 49439, 49439, 49439, 49439], + [25600, 25600, 25600, 25600, 25600, 25600], + [43920, 43920, 43920, 43920, 43920, 43920], + [37415, 37415, 37415, 37415, 37415, 37415], + [20934, 20934, 20934, 20934, 20934, 20934], + [17059, 17059, 17059, 17059, 17059, 17059], + ]), + ) + def it_knows_zscore_for_MR_X_MR_X_CAT(self): cube = CrunchCube(CR.MR_X_MR_X_CAT) slice_ = cube.slices[0] diff --git a/tests/integration/test_dimension.py b/tests/integration/test_dimension.py index 741aee1a4..ff6267625 100644 --- a/tests/integration/test_dimension.py +++ b/tests/integration/test_dimension.py @@ -94,7 +94,7 @@ def test_labels_for_categoricals(self): labels = dimension.labels(include_missing=True) assert labels == ['Cat', 'Mouse', 'Iguana'] - def test_labels_for_numericals(self): + def test_labels_and_elements_for_numericals(self): dimension_dict = { 'type': { 'class': 'enum', @@ -134,6 +134,12 @@ def test_labels_for_numericals(self): labels = dimension.labels(include_cat_ids=True) assert labels == [('smallish', 0), ('kinda big', 1)] + elements = dimension.elements(include_missing=False) + assert [e.label for e in elements] == ['smallish', 'kinda big'] + + elements = dimension.elements(include_missing=True) + assert [e.label for e in elements] == ['smallish', 'kinda big', ''] + def test_subtotals_indices_two_subtotals(self): dimension = CrunchCube(CR.ECON_BLAME_WITH_HS_MISSING).dimensions[0] hs_indices = dimension.hs_indices diff --git a/tests/integration/test_headers_and_subtotals.py b/tests/integration/test_headers_and_subtotals.py index 7b03e24eb..96c26171c 100644 --- a/tests/integration/test_headers_and_subtotals.py +++ b/tests/integration/test_headers_and_subtotals.py @@ -841,3 +841,13 @@ def test_ca_x_mr_proportions_pruning(self): ]) actual = cube.proportions(axis=None, prune=True)[0].mask np.testing.assert_array_equal(actual, expected) + + def test_num_x_cat_hs_margin_pruned(self): + cube = CrunchCube(CR.NUM_BY_CAT_HS) + expected = np.array( + [False, True, True, False, False, True, True, False] + ) + actual = cube.margin( + axis=0, prune=True, include_transforms_for_dims=[0, 1] + ).mask + np.testing.assert_array_equal(actual, expected) diff --git a/tests/integration/test_multiple_response.py b/tests/integration/test_multiple_response.py index f22e7fe88..eb821e0d8 100644 --- a/tests/integration/test_multiple_response.py +++ b/tests/integration/test_multiple_response.py @@ -48,7 +48,7 @@ def test_proportions_simple_mr(): def test_proportions_simple_mr_prune(): cube = CrunchCube(CR.SIMPLE_MR) - expected = np.array([0.6, 0.6666667]) + expected = np.array([0.6, 0.6666667, 0.]) actual = np.ma.compressed(cube.proportions(prune=True)) np.testing.assert_almost_equal(actual, expected) @@ -359,7 +359,7 @@ def test_array_x_mr_by_cell(): def test_simple_mr_margin_by_col(): cube = CrunchCube(CR.SIMPLE_MR) - expected = np.array([3, 4, 0]) + expected = np.array([5, 6, 6]) actual = cube.margin(axis=0) np.testing.assert_array_equal(actual, expected) @@ -568,31 +568,7 @@ def test_cat_x_mr_x_cat_by_cell(): np.testing.assert_almost_equal(actual, expected) -def test_mr_props_pruned(): - cube = CrunchCube(CR.PROMPTED_AWARENESS) - expected = np.array([ - 9.70083312e-01, 9.53131845e-01, 9.64703914e-01, - 9.59703205e-01, 9.37891446e-01, 8.84137923e-01, - 7.77056917e-01, 7.15135296e-01, 9.03057657e-01, - 8.67103783e-01, 8.38011719e-01, 8.60897234e-01, - 7.68101070e-01, 7.59030477e-01, 8.66127931e-01, - 6.89111039e-01, 7.39338305e-01, 1.89895586e-01, - 1.95866187e-01, 8.90452848e-01, 6.10278144e-01, - 6.35237428e-01, 6.54874171e-01, 6.89736947e-01, - 2.31607423e-01, 4.44608376e-01, 6.06987388e-01, - 4.16165746e-01, 2.06262071e-01, 2.08512519e-01, - 1.59533129e-01, 1.86245154e-01, 1.01661334e-01, - 1.82235674e-01, 7.30060936e-01, 4.45912391e-01, - 4.87037442e-01, 1.29527814e-01, 4.95486986e-01, - 2.84392427e-01, 3.93962082e-01, 3.91279968e-01, - 8.96639874e-02, 9.50985735e-04, 1.35477929e-01, - 1.86531215e-01, - ]) - actual = np.ma.compressed(cube.proportions(prune=True)) - np.testing.assert_almost_equal(actual, expected) - - -def test_mr_counts_not_pruned(): +def test_mr_counts_not_pruned_and_pruned(): cube = CrunchCube(CR.PROMPTED_AWARENESS) expected = np.array([ 224833, 221990, 223560, 222923, 217586, 206164, 183147, 167720, @@ -606,37 +582,15 @@ def test_mr_counts_not_pruned(): actual = cube.as_array(weighted=False) np.testing.assert_array_equal(actual, expected) - -def test_mr_counts_pruned(): - cube = CrunchCube(CR.PROMPTED_AWARENESS) - expected = np.array([ - 224833, 221990, 223560, 222923, 217586, 206164, 183147, 167720, - 209355, 201847, 193826, 198744, 180015, 174349, 200050, 160769, - 167969, 43193, 44339, 207539, 135973, 146002, 146789, 160692, - 53995, 95741, 135700, 91878, 48465, 48929, 35189, 42764, - 21194, 41422, 167652, 95676, 111961, 26137, 111760, 60761, - 87645, 85306, 18873, 178, 30461, 42843, - ]) - # Use unweighted, because of the whole numbers (for comparison) + # Now test the pruned version. Note that the zero entries are not removed. + # This might seem counter-intuitive, but is actually the right thing to do. + # The criterion for pruning is the denominator, which is the combined value + # of selected + non-selected (which is not zero in this case). actual = cube.as_array(weighted=False, prune=True) - np.testing.assert_array_equal(actual[~actual.mask], expected) - - pruned_expected = [ - np.array( - [False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, - False, False, True, True, False, False, False, False, False, - False, False, False]) - ] - pruned = cube._prune_indices() - assert len(pruned) == len(pruned_expected) - for i, actual in enumerate(pruned): - np.testing.assert_array_equal(pruned[i], pruned_expected[i]) + np.testing.assert_array_equal(actual, expected) -def test_mr_props_not_pruned(): +def test_mr_props_not_pruned_and_pruned(): cube = CrunchCube(CR.PROMPTED_AWARENESS) expected = np.array([ 9.70083312e-01, 9.53131845e-01, 9.64703914e-01, @@ -659,6 +613,13 @@ def test_mr_props_not_pruned(): actual = cube.proportions() np.testing.assert_almost_equal(actual, expected) + # Now test the pruned version. Note that the zero entries are not removed. + # This might seem counter-intuitive, but is actually the right thing to do. + # The criterion for pruning is the denominator, which is the combined value + # of selected + non-selected (which is not zero in this case). + actual = cube.proportions(prune=True) + np.testing.assert_almost_equal(actual, expected) + def test_mr_x_cat_x_mr_pruned_rows(): cube = CrunchCube(CR.MR_X_CAT_X_MR) @@ -776,8 +737,8 @@ def test_mr_x_num_cols_margin_pruned_unweighted(): def test_mr_x_num_cols_margin_pruned_weighted(): cube = CrunchCube(CR.BBC_NEWS) expected = np.array( - [1709.607711404295, 1438.5504956329391, 1556.0764946283794, - 1419.8513591680107, 0, 0, 0, 0] + [1709.607711404295, 1438.5504956329391, + 1556.0764946283794, 1419.8513591680107, 0., 0., 0., 0.] ) margin = np.ma.compress_cols( cube.margin(axis=0, weighted=True, prune=True) diff --git a/tests/integration/test_scale_means.py b/tests/integration/test_scale_means.py index 745b35b15..ed412227b 100644 --- a/tests/integration/test_scale_means.py +++ b/tests/integration/test_scale_means.py @@ -67,11 +67,10 @@ def test_ca_x_mr(): assert_scale_means_equal(actual, expected[0]) # Test ScaleMeans marginal - with pytest.raises(ValueError): - # Items dimension doesn't have means - actual = slice_.scale_means_margin(0) - actual = slice_.scale_means_margin(1) - assert actual == 0.5066265060240964 + actual = slice_.scale_means_margin(axis=0) + assert actual is None + actual = slice_.scale_means_margin(axis=1) + assert actual == 1.504548211036992 def test_cat_x_ca_cat_x_items(): @@ -98,10 +97,10 @@ def test_cat_x_ca_cat_x_items(): assert_scale_means_equal(actual, expected[0]) # Test ScaleMeans marginal - actual = slice_.scale_means_margin(0) + actual = slice_.scale_means_margin(axis=0) assert actual is None - actual = slice_.scale_means_margin(1) - assert actual == 2.4413145539906105 + with pytest.raises(ValueError): + slice_.scale_means_margin(axis=1) def test_cat_x_cat(): @@ -142,7 +141,7 @@ def test_cat_x_mr(): actual = slice_.scale_means_margin(0) assert actual is None actual = slice_.scale_means_margin(1) - assert actual == 2.75 + assert actual == 2.5323565323565322 def test_mr_x_cat(): @@ -336,9 +335,7 @@ def test_cat_x_cat_pruning_and_hs(): def test_cat_x_cat_scale_means_margin(): cs = CrunchCube(SM.CAT_X_CAT_SM_MARGIN).slices[0] expected = 2.6846246973365617 - actual = cs.scale_means_margin(1) - assert actual == expected + assert cs.scale_means_margin(1) == expected expected = 2.536319612590799 - actual = cs.scale_means_margin(0) - assert actual == expected + assert cs.scale_means_margin(0) == expected diff --git a/tests/unit/test_crunch_cube.py b/tests/unit/test_crunch_cube.py index e6487e34f..1af28e206 100644 --- a/tests/unit/test_crunch_cube.py +++ b/tests/unit/test_crunch_cube.py @@ -311,6 +311,9 @@ def adjust_fixture(self, request): # ---9 - MR x MR x CAT--- ((DT.MR, DT.MR, DT.CAT), ((0, True), (1, True), (2, True), (None, True), ((1, 2), True))), + # ---10 - No dimensions--- + ((), + ((0, False), (1, False), (None, False))), ]) def allowed_fixture(self, request): dimension_types, axis_cases = request.param diff --git a/tests/unit/test_cube_slice.py b/tests/unit/test_cube_slice.py index 34f9d7c1b..ed10796ad 100644 --- a/tests/unit/test_cube_slice.py +++ b/tests/unit/test_cube_slice.py @@ -15,6 +15,16 @@ class DescribeCubeSlice(object): + def it_can_calculate_correct_axis_for_cube(self, axis_fixture, cube_): + axis, ndim, ca_as_0th, expected_value = axis_fixture + cube_.ndim = ndim + slice_ = CubeSlice(cube_, None) + slice_.ca_as_0th = ca_as_0th + + updated_axis = slice_._calculate_correct_axis_for_cube(axis) + + assert updated_axis == expected_value + def it_can_calculate_std_res(self, std_res_fixture, cube_, dim_types_prop_): dim_types, expected_value = std_res_fixture dim_types_prop_.return_value = dim_types @@ -27,6 +37,17 @@ def it_can_calculate_std_res(self, std_res_fixture, cube_, dim_types_prop_): slice_, counts, total, colsum, rowsum ) + def it_can_extract_slice_from_cube_result(self, extract_fixture, cube_): + result, ndim, ca_as_0th, index, expected_value = extract_fixture + cube_.ndim = ndim + slice_ = CubeSlice(cube_, None) + slice_.ca_as_0th = ca_as_0th + slice_._index = index + + extracted_result = slice_._extract_slice_result_from_cube(result) + + np.testing.assert_array_equal(extracted_result, expected_value) + def it_provides_a_default_repr(self): slice_ = CubeSlice(None, None) repr_ = repr(slice_) @@ -57,8 +78,66 @@ def it_knows_whether_its_a_double_mr( assert is_double_mr is expected_value + def it_updates_hs_dims_arguments(self, hs_fixture, cube_): + hs_dims, ndim, expected_value = hs_fixture + cube_.ndim = ndim + slice_ = CubeSlice(cube_, None) + + updated_hs_dims = slice_._hs_dims_for_cube(hs_dims) + + assert updated_hs_dims == expected_value + # fixtures ------------------------------------------------------- + @pytest.fixture(params=[ + (0, 1, False, 0), + (1, 1, False, 1), + (None, 1, False, None), + (0, 2, False, 0), + (1, 2, False, 1), + (None, 2, False, None), + (0, 3, False, 1), + (1, 3, False, 2), + (None, 3, False, None), + (0, 0, False, 0), + (None, 2, True, 1), + ]) + def axis_fixture(self, request): + axis, ndim, ca_as_0th, expected_value = request.param + return axis, ndim, ca_as_0th, expected_value + + @pytest.fixture(params=[ + # Expect same value as result, since cube has < 3 dimensions + (1, 1, False, 0, 1), + (1, 2, False, 1, 1), + # Expect same value as result, since there's nothing to index, even + # though the ndim is == 3 + ([1], 3, False, 1, 1), + # Expect slicing to take place + ([0, 1, 2], 3, False, 0, 0), + ([0, 1, 2], 3, False, 1, 1), + ([0, 1, 2], 3, False, 2, 2), + # Return entire result if not capable to index correctly + ([0, 1, 2], 3, False, 3, [0, 1, 2]), + # Check that it converts extracted tuples + ((0, 1, (1, 2)), 3, False, 2, [1, 2]), + ]) + def extract_fixture(self, request): + result, ndim, ca_as_0th, index, expected = request.param + expected_value = np.array(expected) + return result, ndim, ca_as_0th, index, expected_value + + @pytest.fixture(params=[ + ([0, 1], 1, [0, 1]), + ([0, 1], 2, [0, 1]), + ([0, 1], 3, [1, 2]), + (None, 2, None), + (None, 3, None), + ]) + def hs_fixture(self, request): + axis, ndim, expected_value = request.param + return axis, ndim, expected_value + @pytest.fixture(params=[ ((DT.CAT,), (DT.CAT,)), ((DT.CA_SUBVAR, DT.CA_CAT), (DT.CA_SUBVAR, DT.CA_CAT)), @@ -255,7 +334,10 @@ def test_margin(self): # Assert arguments are passed correctly cs.margin(axis=0) # Expect axis to be increased by 1, because 3D - cs._cube.margin.assert_called_once_with(axis=1) + cs._cube.margin.assert_called_once_with( + axis=1, include_missing=False, include_transforms_for_dims=None, + prune=False, weighted=True, + ) # Assert correct slice is returned when index is set assert cs.margin() == array[1] @@ -368,7 +450,10 @@ def test_axis_for_ca_as_0th(self): cube.margin.return_value = np.array([0, 1, 2]) cs = CubeSlice(cube, 0, ca_as_0th=True) cs.margin(axis=None) - cube.margin.assert_called_once_with(axis=1) + cube.margin.assert_called_once_with( + axis=1, include_missing=False, include_transforms_for_dims=None, + prune=False, weighted=True + ) def test_update_hs_dims(self): '''Test if H&S dims are updated for 3D cubes.'''