bpo-27923: Minor API improvements for binary sequences (PEP 467)

Doc/library/stdtypes.rst

@@ -2303,8 +2303,11 @@ data and are closely related to string objects in a variety of other ways.
    other ways:
 
    * A zero-filled bytes object of a specified length: ``bytes(10)``
+     (deprecated in version 3.7)
    * From an iterable of integers: ``bytes(range(20))``
    * Copying existing binary data via the buffer protocol:  ``bytes(obj)``
+   * Using the :meth:`~bytes.zeros` constructor: ``bytes.zeros(10)``
+   * Using the :meth:`~bytes.byte` single-byte constructor: ``bytes.byte(65)``
 
    Also see the :ref:`bytes <func-bytes>` built-in.
 
@@ -2375,8 +2378,11 @@ objects.
 
    * Creating an empty instance: ``bytearray()``
    * Creating a zero-filled instance with a given length: ``bytearray(10)``
+     (deprecated in version 3.7)
    * From an iterable of integers: ``bytearray(range(20))``
    * Copying existing binary data via the buffer protocol:  ``bytearray(b'Hi!')``
+   * Using the :meth:`~bytearray.zeros` constructor: ``bytearray.zeros(10)``
+   * Using the :meth:`~bytearray.byte` single-byte constructor: ``bytearray.byte(65)``
 
    As bytearray objects are mutable, they support the
    :ref:`mutable <typesseq-mutable>` sequence operations in addition to the
@@ -2547,6 +2553,14 @@ arbitrary binary data.
       Also accept an integer in the range 0 to 255 as the subsequence.
 
 
+.. classmethod:: bytes.iterbytes()
+                 bytearray.iterbytes()
+
+   Return an iterator that produce :class:`bytes` objects of length 1.
+
+   .. versionadded:: 3.7
+
+
 .. method:: bytes.join(iterable)
             bytearray.join(iterable)
 
@@ -3681,6 +3695,12 @@ copying.
       .. versionchanged:: 3.5
          The source format is no longer restricted when casting to a byte view.
 
+   .. classmethod:: memoryview.iterbytes()
+
+      Return an iterator that produce :class:`bytes` objects of length 1.
+
+      .. versionadded:: 3.7
+
    There are also several readonly attributes available:
 
    .. attribute:: obj

Doc/whatsnew/3.7.rst

@@ -108,6 +108,23 @@ locale remains active when the core interpreter is initialized.
        PEP written and implemented by Nick Coghlan.
 
 
+.. _whatsnew37-pep467:
+
+PEP 467: Minor API improvements for binary sequences
+----------------------------------------------------
+
+:pep:`467` adds four small adjustments to the APIs of the :class:`bytes`, :class:`bytearray` and :class:`memoryview` types to make it easier to operate entirely in the binary domain:
+
+* Deprecate passing single integer values to :class:`bytes` and :class:`bytearray`
+* Add :meth:`bytes.zeros` and :meth:`bytearray.zeros` alternative constructors
+* Add :meth:`bytes.byte` and :meth:`bytearray.byte` alternative constructors
+* Add :meth:`bytes.iterbytes`, :meth:`bytearray.iterbytes` and :meth:`memoryview.iterbytes` alternative iterators
+
+.. seealso::
+
+    :pep:`467` -- Minor API improvements for binary sequences
+
+
 Other Language Changes
 ======================
 

Include/bytes_methods.h

@@ -41,6 +41,7 @@ extern const char _Py_isdigit__doc__[];
 extern const char _Py_islower__doc__[];
 extern const char _Py_isupper__doc__[];
 extern const char _Py_istitle__doc__[];
+extern const char _Py_iterbytes__doc__[];
 extern const char _Py_lower__doc__[];
 extern const char _Py_upper__doc__[];
 extern const char _Py_title__doc__[];

Lib/sre_compile.py

@@ -262,7 +262,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None):
     # internal: optimize character set
     out = []
     tail = []
-    charmap = bytearray(256)
+    charmap = bytearray.zeros(256)
     hascased = False
     for op, av in charset:
         while True:
@@ -373,7 +373,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None):
 
     charmap = bytes(charmap) # should be hashable
     comps = {}
-    mapping = bytearray(256)
+    mapping = bytearray.zeros(256)
     block = 0
     data = bytearray()
     for i in range(0, 65536, 256):

Lib/test/test_bytes.py

@@ -95,15 +95,38 @@ def __index__(self):
         self.assertEqual(self.type2test(B(b"foobar")), b"foobar")
 
     def test_from_ssize(self):
-        self.assertEqual(self.type2test(0), b'')
-        self.assertEqual(self.type2test(1), b'\x00')
-        self.assertEqual(self.type2test(5), b'\x00\x00\x00\x00\x00')
+        with self.assertWarns(DeprecationWarning):
+            self.assertEqual(self.type2test(0), b'')
+        with self.assertWarns(DeprecationWarning):
+            self.assertEqual(self.type2test(1), b'\x00')
+        with self.assertWarns(DeprecationWarning):
+            self.assertEqual(self.type2test(5), b'\x00\x00\x00\x00\x00')
+        with self.assertWarns(DeprecationWarning):
+            self.assertEqual(self.type2test(10), self.type2test([0]*10))
+        with self.assertWarns(DeprecationWarning):
+            self.assertEqual(self.type2test(10000), self.type2test([0]*10000))
+
         self.assertRaises(ValueError, self.type2test, -1)
 
         self.assertEqual(self.type2test('0', 'ascii'), b'0')
         self.assertEqual(self.type2test(b'0'), b'0')
         self.assertRaises(OverflowError, self.type2test, sys.maxsize + 1)
 
+    def test_zeros(self):
+        self.assertEqual(self.type2test.zeros(0), b'')
+        self.assertEqual(self.type2test.zeros(1), b'\x00')
+        self.assertEqual(self.type2test.zeros(5), b'\x00\x00\x00\x00\x00')
+        self.assertRaises(ValueError, self.type2test.zeros, -1)
+
+    def test_single_byte(self):
+        self.assertEqual(self.type2test.byte(0), b'\x00')
+        self.assertEqual(self.type2test.byte(65), b'A')
+        self.assertEqual(self.type2test.byte(255), b'\xff')
+        self.assertEqual(self.type2test.byte(False), b'\x00')
+        self.assertEqual(self.type2test.byte(True), b'\x01')
+        self.assertRaises(ValueError, self.type2test.byte, 256)
+        self.assertRaises(ValueError, self.type2test.byte, -1)
+
     def test_constructor_type_errors(self):
         self.assertRaises(TypeError, self.type2test, 0.0)
         class C:
@@ -242,14 +265,6 @@ def test_decode(self):
         # Default encoding is utf-8
         self.assertEqual(self.type2test(b'\xe2\x98\x83').decode(), '\u2603')
 
-    def test_from_int(self):
-        b = self.type2test(0)
-        self.assertEqual(b, self.type2test())
-        b = self.type2test(10)
-        self.assertEqual(b, self.type2test([0]*10))
-        b = self.type2test(10000)
-        self.assertEqual(b, self.type2test([0]*10000))
-
     def test_concat(self):
         b1 = self.type2test(b"abc")
         b2 = self.type2test(b"def")
@@ -504,6 +519,14 @@ def test_rindex(self):
         self.assertEqual(b.rindex(i, 3, 9), 7)
         self.assertRaises(ValueError, b.rindex, w, 1, 3)
 
+    def test_iterbytes(self):
+        b = self.type2test(b'foo')
+        it = b.iterbytes()
+        self.assertEqual(next(it), b'f')
+        self.assertEqual(next(it), b'o')
+        self.assertEqual(next(it), b'o')
+        self.assertRaises(StopIteration, next, it)
+
     def test_mod(self):
         b = self.type2test(b'hello, %b!')
         orig = b

Lib/test/test_doctest.py

@@ -659,7 +659,7 @@ def non_Python_modules(): r"""
 
     >>> import builtins
     >>> tests = doctest.DocTestFinder().find(builtins)
-    >>> 790 < len(tests) < 810 # approximate number of objects with docstrings
+    >>> 800 < len(tests) < 850 # approximate number of objects with docstrings
     True
     >>> real_tests = [t for t in tests if len(t.examples) > 0]
     >>> len(real_tests) # objects that actually have doctests

Lib/test/test_memoryview.py

@@ -383,6 +383,15 @@ def test_issue22668(self):
         self.assertEqual(c.format, "H")
         self.assertEqual(d.format, "H")
 
+    def test_iterbytes(self):
+        for tp in self._types:
+            b = tp(self._source)
+            m = self._view(b)
+            it = m.iterbytes()
+            for byte in m:
+                self.assertEqual(next(it), bytes([byte]))
+            self.assertRaises(StopIteration, next, it)
+
 
 # Variations on source objects for the buffer: bytes-like objects, then arrays
 # with itemsize > 1.

Misc/ACKS

@@ -1759,6 +1759,7 @@ Masazumi Yoshikawa
 Arnaud Ysmal
 Bernard Yue
 Moshe Zadka
+Elias Zamaria
 Milan Zamazal
 Artur Zaprzala
 Mike Zarnstorff

Misc/NEWS.d/next/Core and Builtins/2017-08-29-11-51-08.bpo-27923.y1IldO.rst

@@ -0,0 +1 @@
+Minor API improvements for binary sequences (PEP 467)

Objects/bytearrayobject.c

@@ -807,6 +807,13 @@ bytearray_init(PyByteArrayObject *self, PyObject *args, PyObject *kwds)
                 PyErr_SetString(PyExc_ValueError, "negative count");
                 return -1;
             }
+            if (count >= 0) {
+                if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
+                                     "Passing an integer to the bytearray "
+                                     "constructor is deprecated. Use "
+                                     "bytearray.zeros(%zd) instead.\n", count) < 0)
+                    return NULL;
+            }
             if (count > 0) {
                 if (PyByteArray_Resize((PyObject *)self, count))
                     return -1;
@@ -1995,6 +2002,67 @@ bytearray_fromhex_impl(PyTypeObject *type, PyObject *string)
     return result;
 }
 
+/*[clinic input]
+@classmethod
+bytearray.zeros
+
+    size: int
+    /
+
+Create a bytearray object of size given by the parameter initialized with null bytes.
+
+Parameter must be 0 or a positive integer.
+Example: bytearray.zeros(3) -> bytearray(b\'\\x00\\x00\\x00')
+[clinic start generated code]*/
+
+static PyObject *
+bytearray_zeros_impl(PyTypeObject *type, int size)
+/*[clinic end generated code: output=483e961ce69e50dc input=b6e4556bd3095de2]*/
+{
+    if (size == -1 && PyErr_Occurred()) {
+        return NULL;
+    }
+    if (size < 0) {
+        PyErr_SetString(PyExc_ValueError, "negative count");
+        return NULL;
+    }
+    if (size >= 0) {
+        PyObject *result = PyByteArray_FromStringAndSize(NULL, size);
+        if (PyByteArray_Resize((PyObject *)result, size))
+            return NULL;
+        memset(PyByteArray_AS_STRING(result), 0, size);
+        return result;
+    }
+}
+
+/*[clinic input]
+@classmethod
+bytearray.byte
+
+    x: int
+    /
+
+Create a bytearray object, consisting of a single byte.
+
+Parameter must be in range(0, 256)
+bytearray.byte(x) is equivalent to bytearray([x])
+Example: bytearray.byte(3) -> bytearray(b'\x03')
+[clinic start generated code]*/
+
+static PyObject *
+bytearray_byte_impl(PyTypeObject *type, int x)
+/*[clinic end generated code: output=756fefaafb00a523 input=d36148c99214b551]*/
+{
+    char byte;
+    if (x < 0 || x > 255) {
+        PyErr_Format(PyExc_ValueError, "bytes must be in range(0, 256)");
+        return NULL;
+    }
+    byte = (char)x;
+    return PyByteArray_FromStringAndSize(&byte, 1);
+}
+
+
 PyDoc_STRVAR(hex__doc__,
 "B.hex() -> string\n\
 \n\
@@ -2113,13 +2181,16 @@ static PyBufferProcs bytearray_as_buffer = {
     (releasebufferproc)bytearray_releasebuffer,
 };
 
+static PyObject *bytearray_iterbytes(PyObject *seq);
+
 static PyMethodDef
 bytearray_methods[] = {
     {"__alloc__", (PyCFunction)bytearray_alloc, METH_NOARGS, alloc_doc},
     BYTEARRAY_REDUCE_METHODDEF
     BYTEARRAY_REDUCE_EX_METHODDEF
     BYTEARRAY_SIZEOF_METHODDEF
     BYTEARRAY_APPEND_METHODDEF
+    BYTEARRAY_BYTE_METHODDEF
     {"capitalize", (PyCFunction)stringlib_capitalize, METH_NOARGS,
      _Py_capitalize__doc__},
     {"center", (PyCFunction)stringlib_center, METH_VARARGS, _Py_center__doc__},
@@ -2153,6 +2224,8 @@ bytearray_methods[] = {
      _Py_istitle__doc__},
     {"isupper", (PyCFunction)stringlib_isupper, METH_NOARGS,
      _Py_isupper__doc__},
+    {"iterbytes", (PyCFunction)bytearray_iterbytes, METH_NOARGS,
+     _Py_iterbytes__doc__},
     BYTEARRAY_JOIN_METHODDEF
     {"ljust", (PyCFunction)stringlib_ljust, METH_VARARGS, _Py_ljust__doc__},
     {"lower", (PyCFunction)stringlib_lower, METH_NOARGS, _Py_lower__doc__},
@@ -2179,6 +2252,7 @@ bytearray_methods[] = {
     {"title", (PyCFunction)stringlib_title, METH_NOARGS, _Py_title__doc__},
     BYTEARRAY_TRANSLATE_METHODDEF
     {"upper", (PyCFunction)stringlib_upper, METH_NOARGS, _Py_upper__doc__},
+    BYTEARRAY_ZEROS_METHODDEF
     {"zfill", (PyCFunction)stringlib_zfill, METH_VARARGS, _Py_zfill__doc__},
     {NULL}
 };
@@ -2411,3 +2485,67 @@ bytearray_iter(PyObject *seq)
     _PyObject_GC_TRACK(it);
     return (PyObject *)it;
 }
+
+/****************** bytearray_iterbytes object ***********************/
+
+static PyObject *
+bytearray_iterbytes_next(bytesiterobject *it)
+{
+    PyObject *item = bytearrayiter_next(it);
+    if (item != NULL) {
+        return Py_BuildValue("c", (unsigned char)PyLong_AsLong(item));
+    }
+    return item;
+}
+
+PyTypeObject PyByteArrayIterBytes_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    "bytearray_iterator",                        /* tp_name */
+    sizeof(bytesiterobject),          /* tp_basicsize */
+    0,                                           /* tp_itemsize */
+    /* methods */
+    (destructor)bytearrayiter_dealloc,           /* tp_dealloc */
+    0,                                           /* tp_print */
+    0,                                           /* tp_getattr */
+    0,                                           /* tp_setattr */
+    0,                                           /* tp_reserved */
+    0,                                           /* tp_repr */
+    0,                                           /* tp_as_number */
+    0,                                           /* tp_as_sequence */
+    0,                                           /* tp_as_mapping */
+    0,                                           /* tp_hash */
+    0,                                           /* tp_call */
+    0,                                           /* tp_str */
+    PyObject_GenericGetAttr,                     /* tp_getattro */
+    0,                                           /* tp_setattro */
+    0,                                           /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,     /* tp_flags */
+    0,                                           /* tp_doc */
+    (traverseproc)bytearrayiter_traverse,        /* tp_traverse */
+    0,                                           /* tp_clear */
+    0,                                           /* tp_richcompare */
+    0,                                           /* tp_weaklistoffset */
+    PyObject_SelfIter,                           /* tp_iter */
+    (iternextfunc)bytearray_iterbytes_next,      /* tp_iternext */
+    bytearrayiter_methods,                       /* tp_methods */
+    0,
+};
+
+static PyObject *
+bytearray_iterbytes(PyObject *seq)
+{
+    bytesiterobject *it;
+
+    if (!PyByteArray_Check(seq)) {
+        PyErr_BadInternalCall();
+        return NULL;
+    }
+    it = PyObject_GC_New(bytesiterobject, &PyByteArrayIterBytes_Type);
+    if (it == NULL)
+        return NULL;
+    it->it_index = 0;
+    Py_INCREF(seq);
+    it->it_seq = (PyByteArrayObject *)seq;
+    _PyObject_GC_TRACK(it);
+    return (PyObject *)it;
+}