Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

REF/BUG/API: factorizing categorical data #19938

Merged
merged 14 commits into from Mar 15, 2018
Merged
Changes from 2 commits
Commits
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

@@ -746,6 +746,8 @@ Categorical
- Bug in :meth:`Series.astype` and ``Categorical.astype()`` where an existing categorical data does not get updated (:issue:`10696`, :issue:`18593`)
- Bug in :class:`Index` constructor with ``dtype=CategoricalDtype(...)`` where ``categories`` and ``ordered`` are not maintained (issue:`19032`)
- Bug in :class:`Series` constructor with scalar and ``dtype=CategoricalDtype(...)`` where ``categories`` and ``ordered`` are not maintained (issue:`19565`)
- Bug in :func:`pandas.factorize` returning the unique codes for the ``uniques``. This now returns a ``Categorical`` with the same dtype as the input (:issue:`19721`)
- Bug in :func:`pandas.factorize` including an item for missing values in the ``uniques`` return value (:issue:`19721`)

Datetimelike
^^^^^^^^^^^^
@@ -438,15 +438,45 @@ def isin(comps, values):
return f(comps, values)


def _factorize_array(values, check_nulls, na_sentinel=-1, size_hint=None):
"""Factorize an array-like to labels and uniques.
This doesn't do any coercion of types or unboxing before factorization.
Parameters
----------
values : ndarray
check_nulls : bool
Whether to check for nulls in the hashtable's 'get_labels' method.
na_sentinel : int, default -1
size_hint : int, optional
Passsed through to the hashtable's 'get_labels' method
Returns
-------
labels, uniques : ndarray
"""
(hash_klass, vec_klass), values = _get_data_algo(values, _hashtables)

table = hash_klass(size_hint or len(values))
uniques = vec_klass()
labels = table.get_labels(values, uniques, 0, na_sentinel, check_nulls)

labels = _ensure_platform_int(labels)
uniques = uniques.to_array()
return labels, uniques


@deprecate_kwarg(old_arg_name='order', new_arg_name=None)
def factorize(values, sort=False, order=None, na_sentinel=-1, size_hint=None):
"""
Encode input values as an enumerated type or categorical variable
Parameters
----------
values : ndarray (1-d)
Sequence
values : Sequence
ndarrays must be 1-D. Sequences that aren't pandas objects are
coereced to ndarrays before factorization.
sort : boolean, default False
Sort by values
na_sentinel : int, default -1
@@ -461,26 +491,36 @@ def factorize(values, sort=False, order=None, na_sentinel=-1, size_hint=None):
Series
note: an array of Periods will ignore sort as it returns an always sorted
PeriodIndex
PeriodIndex.
"""
# Implementation notes: This method is responsible for 3 things
# 1.) coercing data to array-like (ndarray, Index, extension array)
# 2.) factorizing labels and uniques
# 3.) Maybe boxing the output in an Index
#
# Step 2 is dispatched to extension types (like Categorical). They are
# responsible only for factorization and sorting if necessary. All
# data coercion and boxing should happen here.

values = _ensure_arraylike(values)
original = values
values, dtype, _ = _ensure_data(values)
(hash_klass, vec_klass), values = _get_data_algo(values, _hashtables)

table = hash_klass(size_hint or len(values))
uniques = vec_klass()
check_nulls = not is_integer_dtype(original)
labels = table.get_labels(values, uniques, 0, na_sentinel, check_nulls)

labels = _ensure_platform_int(labels)
uniques = uniques.to_array()

if sort and len(uniques) > 0:
from pandas.core.sorting import safe_sort
uniques, labels = safe_sort(uniques, labels, na_sentinel=na_sentinel,
assume_unique=True)
if is_categorical_dtype(values):
values = getattr(values, '_values', values)
labels, uniques = values.factorize(sort=sort)
dtype = original.dtype

This comment has been minimized.

Copy link
@jreback

jreback Mar 4, 2018

Contributor

see my comment below, but you might simpy dispatch on categricals and just return, mixing the impl is really confusing here.

else:

This comment has been minimized.

Copy link
@jreback

jreback Mar 13, 2018

Contributor

shouldn't this actually be a check on the values if they have a .factorize() method (or check is_extension_array)? instead of specifically checking for categorical? (of course categorical will pass these checks). as this will then make pd.factorize(an_extension_array) work?

This comment has been minimized.

Copy link
@TomAugspurger

TomAugspurger Mar 13, 2018

Author Contributor

This PR is just a bugfix for categorical. But the structure will be very similar (I'll just change is_categorical_dtype to is_extension_array_dtype.)

I'll implement EA.factorize today hopefully, but have to get things like unique and argsort working first.

values, dtype, _ = _ensure_data(values)
check_nulls = not is_integer_dtype(original)
labels, uniques = _factorize_array(values, check_nulls,
na_sentinel=na_sentinel,
size_hint=size_hint)

if sort and len(uniques) > 0:
from pandas.core.sorting import safe_sort
uniques, labels = safe_sort(uniques, labels,
na_sentinel=na_sentinel,
assume_unique=True)

uniques = _reconstruct_data(uniques, dtype, original)

@@ -7,6 +7,7 @@
from pandas import compat
from pandas.compat import u, lzip
from pandas._libs import lib, algos as libalgos
from pandas._libs.tslib import iNaT

from pandas.core.dtypes.generic import (
ABCSeries, ABCIndexClass, ABCCategoricalIndex)
@@ -2068,6 +2069,64 @@ def unique(self):
take_codes = sorted(take_codes)
return cat.set_categories(cat.categories.take(take_codes))

def factorize(self, sort=False, na_sentinel=-1):
"""Encode the Categorical as an enumerated type.
Parameters
----------
sort : boolean, default False
Sort by values
na_sentinel: int, default -1
Value to mark "not found"
Returns
-------
labels : ndarray
An integer NumPy array that's an indexer into the original
Categorical
uniques : Categorical
A Categorical whose values are the unique values and
whose dtype matches the original CategoricalDtype. Note that if
there any unobserved categories in ``self`` will not be present
in ``uniques.values``. They will be present in
``uniques.categories``
Examples
--------
>>> cat = pd.Categorical(['a', 'a', 'c'], categories=['a', 'b', 'c'])
>>> labels, uniques = cat.factorize()
>>> labels
(array([0, 0, 1]),
>>> uniques
[a, c]
Categories (3, object): [a, b, c])
Missing values are handled
>>> labels, uniques = pd.factorize(pd.Categorical(['a', 'b', None]))
>>> labels
array([ 0, 1, -1])
>>> uniques
[a, b]
Categories (2, object): [a, b]
"""
from pandas.core.algorithms import _factorize_array, take_1d

codes = self.codes.astype('int64')
# We set missing codes, normally -1, to iNaT so that the

This comment has been minimized.

Copy link
@jreback

jreback Mar 4, 2018

Contributor

put the astype after the comment, looks awkward otherwise

why do you think you need to do this? the point of the na_sentinel is to select the missing values

This comment has been minimized.

Copy link
@TomAugspurger

TomAugspurger Mar 5, 2018

Author Contributor

na_sentinel controls the missing marker for the output. We're modifying the input, since the Int64HashTable sees that they're missing, instead of the value -1.

This comment has been minimized.

Copy link
@jreback

jreback Mar 7, 2018

Contributor

this should be fixed generally, e.g. we should have a way to pass in the missing value. can you fix

This comment has been minimized.

Copy link
@TomAugspurger

TomAugspurger Mar 7, 2018

Author Contributor

I'm not familiar with the hashtable code, but at a glance it looks like the null condition is included in the class definition template?

{{py:

# name, dtype, null_condition, float_group
dtypes = [('Float64', 'float64', 'val != val', True),
          ('UInt64', 'uint64', 'False', False),
          ('Int64', 'int64', 'val == iNaT', False)]

I'm not sure how to pass expressions down to cython as a parameter.

Anyway, do we actually need this to be parameterized? Do we have other cases where we've needed to pass the null condition down?

# Int64HashTable treats them as missing values.
codes[codes == -1] = iNaT
labels, uniques = _factorize_array(codes, check_nulls=True,
na_sentinel=na_sentinel)
uniques = self._constructor(self.categories.take(uniques),
categories=self.categories,
ordered=self.ordered)
if sort:

This comment has been minimized.

Copy link
@jreback

jreback Mar 4, 2018

Contributor

I would remove the sort from here and just do it in factorize part (after the else), otherwise logic is in multiple places here

order = uniques.argsort()
labels = take_1d(order, labels, fill_value=na_sentinel)
uniques = uniques.take(order)
return labels, uniques

def equals(self, other):
"""
Returns True if categorical arrays are equal.
@@ -0,0 +1,49 @@
import pytest
import numpy as np

import pandas as pd
import pandas.util.testing as tm


This comment has been minimized.

Copy link
@jreback

jreback Mar 4, 2018

Contributor

you should prob pull the categorical tests out of test_algos.py then

This comment has been minimized.

Copy link
@TomAugspurger

TomAugspurger Mar 5, 2018

Author Contributor

For better or worse, test_algos.py didn't have any tests for Categorical.

This comment has been minimized.

Copy link
@jreback

jreback Mar 7, 2018

Contributor

hmm, yeah i guess just lots of tests for unique.

@pytest.mark.parametrize('ordered', [True, False])
@pytest.mark.parametrize('categories', [
['b', 'a', 'c'],
['a', 'b', 'c', 'd'],
])
def test_factorize(categories, ordered):
cat = pd.Categorical(['b', 'b', 'a', 'c', None],
categories=categories,
ordered=ordered)
labels, uniques = pd.factorize(cat)
expected_labels = np.array([0, 0, 1, 2, -1], dtype='int64')
expected_uniques = pd.Categorical(['b', 'a', 'c'],
categories=categories,
ordered=ordered)

tm.assert_numpy_array_equal(labels, expected_labels)
tm.assert_categorical_equal(uniques, expected_uniques)


def test_factorized_sort():
cat = pd.Categorical(['b', 'b', None, 'a'])
labels, uniques = pd.factorize(cat, sort=True)
expected_labels = np.array([1, 1, -1, 0], dtype='int64')
expected_uniques = pd.Categorical(['a', 'b'])

tm.assert_numpy_array_equal(labels, expected_labels)
tm.assert_categorical_equal(uniques, expected_uniques)


def test_factorized_sort_ordered():
cat = pd.Categorical(['b', 'b', None, 'a'],
categories=['c', 'b', 'a'],
ordered=True)

labels, uniques = pd.factorize(cat, sort=True)
expected_labels = np.array([0, 0, -1, 1], dtype='int64')
expected_uniques = pd.Categorical(['b', 'a'],
categories=['c', 'b', 'a'],
ordered=True)

tm.assert_numpy_array_equal(labels, expected_labels)
tm.assert_categorical_equal(uniques, expected_uniques)
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.