Merge pull request #7373 from charris/add-bitwise_and-identity

ENH: Add bitwise_and identity

doc/release/1.12.0-notes.rst

@@ -76,10 +76,10 @@ The following functions are changed: ``sum``, ``product``,
 ``nansum``, ``nanprod``, ``nanmean``, ``nanmedian``, ``nanvar``,
 ``nanstd``
 
-
-DeprecationWarning to error
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
+``bitwise_and`` identity changed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The previous identity was 1, it is now -1. See entry in `Improvements`_ for
+more explanation.
 
 FutureWarning to changed behavior
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -111,7 +111,7 @@ Improvements
 ============
 
 *np.loadtxt* now supports a single integer as ``usecol`` argument
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Instead of using ``usecol=(n,)`` to read the nth column of a file
 it is now allowed to use ``usecol=n``. Also the error message is
 more user friendly when a non-integer is passed as a column index.
@@ -120,6 +120,14 @@ Additional estimators for ``histogram``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Added 'doane' and 'sqrt' estimators to ``histogram`` via the ``bins`` argument.
 
+``bitwise_and`` identity changed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The previous identity was 1 with the result that all bits except the LSB were
+masked out when the reduce method was used.  The new identity is -1, which
+should work properly on twos complement machines as all bits will be set to
+one.
+
+
 Changes
 =======
 

numpy/core/code_generators/generate_umath.py

@@ -13,6 +13,7 @@
 Zero = "PyUFunc_Zero"
 One = "PyUFunc_One"
 None_ = "PyUFunc_None"
+AllOnes = "PyUFunc_MinusOne"
 ReorderableNone = "PyUFunc_ReorderableNone"
 
 # Sentinel value to specify using the full type description in the
@@ -493,7 +494,7 @@ def english_upper(s):
           TD(flts, f="logaddexp2", astype={'e':'f'})
           ),
 'bitwise_and':
-    Ufunc(2, 1, One,
+    Ufunc(2, 1, AllOnes,
           docstrings.get('numpy.core.umath.bitwise_and'),
           None,
           TD(bints),
@@ -507,7 +508,7 @@ def english_upper(s):
           TD(O, f='PyNumber_Or'),
           ),
 'bitwise_xor':
-    Ufunc(2, 1, None,
+    Ufunc(2, 1, Zero,
           docstrings.get('numpy.core.umath.bitwise_xor'),
           None,
           TD(bints),

numpy/core/include/numpy/ufuncobject.h

@@ -250,16 +250,22 @@ typedef struct _tagPyUFuncObject {
 #define NPY_LOOP_END_THREADS
 #endif
 
+/*
+ * UFunc has unit of 0, and the order of operations can be reordered
+ * This case allows reduction with multiple axes at once.
+ */
+#define PyUFunc_Zero 0
 /*
  * UFunc has unit of 1, and the order of operations can be reordered
  * This case allows reduction with multiple axes at once.
  */
 #define PyUFunc_One 1
 /*
- * UFunc has unit of 0, and the order of operations can be reordered
- * This case allows reduction with multiple axes at once.
+ * UFunc has unit of -1, and the order of operations can be reordered
+ * This case allows reduction with multiple axes at once. Intended for
+ * bitwise_and reduction.
  */
-#define PyUFunc_Zero 0
+#define PyUFunc_MinusOne 2
 /*
  * UFunc has no unit, and the order of operations cannot be reordered.
  * This case does not allow reduction with multiple axes at once.

numpy/core/src/umath/ufunc_object.c

@@ -78,6 +78,9 @@ _extract_pyvals(PyObject *ref, const char *name, int *bufsize,
 static int
 assign_reduce_identity_zero(PyArrayObject *result, void *data);
 
+static int
+assign_reduce_identity_minusone(PyArrayObject *result, void *data);
+
 static int
 assign_reduce_identity_one(PyArrayObject *result, void *data);
 
@@ -2414,6 +2417,9 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc,
                     case PyUFunc_One:
                         assign_reduce_identity_one(op[i], NULL);
                         break;
+                    case PyUFunc_MinusOne:
+                        assign_reduce_identity_minusone(op[i], NULL);
+                        break;
                     case PyUFunc_None:
                     case PyUFunc_ReorderableNone:
                         PyErr_Format(PyExc_ValueError,
@@ -2830,6 +2836,19 @@ assign_reduce_identity_one(PyArrayObject *result, void *NPY_UNUSED(data))
     return PyArray_FillWithScalar(result, PyArrayScalar_True);
 }
 
+static int
+assign_reduce_identity_minusone(PyArrayObject *result, void *NPY_UNUSED(data))
+{
+    static PyObject *MinusOne = NULL;
+
+    if (MinusOne == NULL) {
+        if ((MinusOne = PyInt_FromLong(-1)) == NULL) {
+            return -1;
+        }
+    }
+    return PyArray_FillWithScalar(result, MinusOne);
+}
+
 static int
 reduce_loop(NpyIter *iter, char **dataptrs, npy_intp *strides,
             npy_intp *countptr, NpyIter_IterNextFunc *iternext,
@@ -2983,6 +3002,18 @@ PyUFunc_Reduce(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out,
                 assign_identity = NULL;
             }
             break;
+        case PyUFunc_MinusOne:
+            assign_identity = &assign_reduce_identity_minusone;
+            reorderable = 1;
+            /*
+             * The identity for a dynamic dtype like
+             * object arrays can't be used in general
+             */
+            if (PyArray_ISOBJECT(arr) && PyArray_SIZE(arr) != 0) {
+                assign_identity = NULL;
+            }
+            break;
+
         case PyUFunc_None:
             reorderable = 0;
             break;
@@ -5572,6 +5603,8 @@ ufunc_get_identity(PyUFuncObject *ufunc)
         return PyInt_FromLong(1);
     case PyUFunc_Zero:
         return PyInt_FromLong(0);
+    case PyUFunc_MinusOne:
+        return PyInt_FromLong(-1);
     }
     Py_RETURN_NONE;
 }

numpy/core/tests/test_umath.py

@@ -991,6 +991,91 @@ def test_truth_table_bitwise(self):
         assert_equal(np.bitwise_xor(arg1, arg2), out)
 
 
+class TestBitwiseUFuncs(TestCase):
+
+    bitwise_types = [np.dtype(c) for c in '?' + 'bBhHiIlLqQ' + 'O']
+
+    def test_values(self):
+        for dt in self.bitwise_types:
+            zeros = np.array([0], dtype=dt)
+            ones = np.array([-1], dtype=dt)
+            msg = "dt = '%s'" % dt.char
+
+            assert_equal(np.bitwise_not(zeros), ones, err_msg=msg)
+            assert_equal(np.bitwise_not(ones), zeros, err_msg=msg)
+
+            assert_equal(np.bitwise_or(zeros, zeros), zeros, err_msg=msg)
+            assert_equal(np.bitwise_or(zeros, ones), ones, err_msg=msg)
+            assert_equal(np.bitwise_or(ones, zeros), ones, err_msg=msg)
+            assert_equal(np.bitwise_or(ones, ones), ones, err_msg=msg)
+
+            assert_equal(np.bitwise_xor(zeros, zeros), zeros, err_msg=msg)
+            assert_equal(np.bitwise_xor(zeros, ones), ones, err_msg=msg)
+            assert_equal(np.bitwise_xor(ones, zeros), ones, err_msg=msg)
+            assert_equal(np.bitwise_xor(ones, ones), zeros, err_msg=msg)
+
+            assert_equal(np.bitwise_and(zeros, zeros), zeros, err_msg=msg)
+            assert_equal(np.bitwise_and(zeros, ones), zeros, err_msg=msg)
+            assert_equal(np.bitwise_and(ones, zeros), zeros, err_msg=msg)
+            assert_equal(np.bitwise_and(ones, ones), ones, err_msg=msg)
+
+    def test_types(self):
+        for dt in self.bitwise_types:
+            zeros = np.array([0], dtype=dt)
+            ones = np.array([-1], dtype=dt)
+            msg = "dt = '%s'" % dt.char
+
+            assert_(np.bitwise_not(zeros).dtype == dt, msg)
+            assert_(np.bitwise_or(zeros, zeros).dtype == dt, msg)
+            assert_(np.bitwise_xor(zeros, zeros).dtype == dt, msg)
+            assert_(np.bitwise_and(zeros, zeros).dtype == dt, msg)
+
+
+    def test_identity(self):
+        assert_(np.bitwise_or.identity == 0, 'bitwise_or')
+        assert_(np.bitwise_xor.identity == 0, 'bitwise_xor')
+        assert_(np.bitwise_and.identity == -1, 'bitwise_and')
+
+    def test_reduction(self):
+        binary_funcs = (np.bitwise_or, np.bitwise_xor, np.bitwise_and)
+
+        for dt in self.bitwise_types:
+            zeros = np.array([0], dtype=dt)
+            ones = np.array([-1], dtype=dt)
+            for f in binary_funcs:
+                msg = "dt: '%s', f: '%s'" % (dt, f)
+                assert_equal(f.reduce(zeros), zeros, err_msg=msg)
+                assert_equal(f.reduce(ones), ones, err_msg=msg)
+
+        # Test empty reduction, no object dtype
+        for dt in self.bitwise_types[:-1]:
+            # No object array types
+            empty = np.array([], dtype=dt)
+            for f in binary_funcs:
+                msg = "dt: '%s', f: '%s'" % (dt, f)
+                tgt = np.array(f.identity, dtype=dt)
+                res = f.reduce(empty)
+                assert_equal(res, tgt, err_msg=msg)
+                assert_(res.dtype == tgt.dtype, msg)
+
+        # Empty object arrays use the identity.  Note that the types may
+        # differ, the actual type used is determined by the assign_identity
+        # function and is not the same as the type returned by the identity
+        # method.
+        for f in binary_funcs:
+            msg = "dt: '%s'" % (f,)
+            empty = np.array([], dtype=object)
+            tgt = f.identity
+            res = f.reduce(empty)
+            assert_equal(res, tgt, err_msg=msg)
+
+        # Non-empty object arrays do not use the identity
+        for f in binary_funcs:
+            msg = "dt: '%s'" % (f,)
+            btype = np.array([True], dtype=object)
+            assert_(type(f.reduce(btype)) is bool, msg)
+
+
 class TestInt(TestCase):
     def test_logical_not(self):
         x = np.ones(10, dtype=np.int16)