Skip to content

Commit

Permalink
gh-101819: Adapt _io types to heap types, batch 1 (GH-101949)
Browse files Browse the repository at this point in the history 
Adapt StringIO, TextIOWrapper, FileIO, Buffered*, and BytesIO types.

Automerge-Triggered-By: GH:erlend-aasland
  • Loading branch information
@erlend-aasland
erlend-aasland committed Feb 20, 2023
1 parent 2713631 commit c00faf7
Show file tree
Hide file tree
Showing 10 changed files with 463 additions and 478 deletions.
91 changes: 90 additions & 1 deletion Lib/test/test_io.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1042,6 +1042,95 @@ def close(self):
support.gc_collect()
self.assertIsNone(wr(), wr)

@support.cpython_only
class TestIOCTypes(unittest.TestCase):
def setUp(self):
_io = import_helper.import_module("_io")
self.types = [
_io.BufferedRWPair,
_io.BufferedRandom,
_io.BufferedReader,
_io.BufferedWriter,
_io.BytesIO,
_io.FileIO,
_io.IncrementalNewlineDecoder,
_io.StringIO,
_io.TextIOWrapper,
_io._BufferedIOBase,
_io._BytesIOBuffer,
_io._IOBase,
_io._RawIOBase,
_io._TextIOBase,
]
if sys.platform == "win32":
self.types.append(_io._WindowsConsoleIO)
self._io = _io

def test_immutable_types(self):
for tp in self.types:
with self.subTest(tp=tp):
with self.assertRaisesRegex(TypeError, "immutable"):
tp.foo = "bar"

def test_class_hierarchy(self):
def check_subs(types, base):
for tp in types:
with self.subTest(tp=tp, base=base):
self.assertTrue(issubclass(tp, base))

def recursive_check(d):
for k, v in d.items():
if isinstance(v, dict):
recursive_check(v)
elif isinstance(v, set):
check_subs(v, k)
else:
self.fail("corrupt test dataset")

_io = self._io
hierarchy = {
_io._IOBase: {
_io._BufferedIOBase: {
_io.BufferedRWPair,
_io.BufferedRandom,
_io.BufferedReader,
_io.BufferedWriter,
_io.BytesIO,
},
_io._RawIOBase: {
_io.FileIO,
},
_io._TextIOBase: {
_io.StringIO,
_io.TextIOWrapper,
},
},
}
if sys.platform == "win32":
hierarchy[_io._IOBase][_io._RawIOBase].add(_io._WindowsConsoleIO)

recursive_check(hierarchy)

def test_subclassing(self):
_io = self._io
dataset = {k: True for k in self.types}
dataset[_io._BytesIOBuffer] = False

for tp, is_basetype in dataset.items():
with self.subTest(tp=tp, is_basetype=is_basetype):
name = f"{tp.__name__}_subclass"
bases = (tp,)
if is_basetype:
_ = type(name, bases, {})
else:
msg = "not an acceptable base type"
with self.assertRaisesRegex(TypeError, msg):
_ = type(name, bases, {})

def test_disallow_instantiation(self):
_io = self._io
support.check_disallow_instantiation(self, _io._BytesIOBuffer)

class PyIOTest(IOTest):
pass

Expand Down Expand Up @@ -4671,7 +4760,7 @@ def load_tests(loader, tests, pattern):
CIncrementalNewlineDecoderTest, PyIncrementalNewlineDecoderTest,
CTextIOWrapperTest, PyTextIOWrapperTest,
CMiscIOTest, PyMiscIOTest,
CSignalsTest, PySignalsTest,
CSignalsTest, PySignalsTest, TestIOCTypes,
)

# Put the namespaces of the IO module we are testing and some useful mock
Expand Down
99 changes: 62 additions & 37 deletions Modules/_io/_iomodule.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include "_iomodule.h"
#include "pycore_moduleobject.h" // _PyModule_GetState()
#include "pycore_pystate.h" // _PyInterpreterState_GET()

#ifdef HAVE_SYS_TYPES_H
Expand Down Expand Up @@ -315,8 +314,9 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode,
}

/* Create the Raw file stream */
_PyIO_State *state = get_io_state(module);
{
PyObject *RawIO_class = (PyObject *)&PyFileIO_Type;
PyObject *RawIO_class = (PyObject *)state->PyFileIO_Type;
#ifdef MS_WINDOWS
const PyConfig *config = _Py_GetConfig();
if (!config->legacy_windows_stdio && _PyIO_get_console_type(path_or_fd) != '\0') {
Expand Down Expand Up @@ -390,12 +390,15 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode,
{
PyObject *Buffered_class;

if (updating)
Buffered_class = (PyObject *)&PyBufferedRandom_Type;
else if (creating || writing || appending)
Buffered_class = (PyObject *)&PyBufferedWriter_Type;
else if (reading)
Buffered_class = (PyObject *)&PyBufferedReader_Type;
if (updating) {
Buffered_class = (PyObject *)state->PyBufferedRandom_Type;
}
else if (creating || writing || appending) {
Buffered_class = (PyObject *)state->PyBufferedWriter_Type;
}
else if (reading) {
Buffered_class = (PyObject *)state->PyBufferedReader_Type;
}
else {
PyErr_Format(PyExc_ValueError,
"unknown mode: '%s'", mode);
Expand All @@ -417,7 +420,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode,
}

/* wraps into a TextIOWrapper */
wrapper = PyObject_CallFunction((PyObject *)&PyTextIOWrapper_Type,
wrapper = PyObject_CallFunction((PyObject *)state->PyTextIOWrapper_Type,
"OsssO",
buffer,
encoding, errors, newline,
Expand Down Expand Up @@ -558,14 +561,6 @@ PyNumber_AsOff_t(PyObject *item, PyObject *err)
return result;
}

static inline _PyIO_State*
get_io_state(PyObject *module)
{
void *state = _PyModule_GetState(module);
assert(state != NULL);
return (_PyIO_State *)state;
}

_PyIO_State *
_PyIO_get_module_state(void)
{
Expand All @@ -587,6 +582,15 @@ iomodule_traverse(PyObject *mod, visitproc visit, void *arg) {
return 0;
Py_VISIT(state->locale_module);
Py_VISIT(state->unsupported_operation);

Py_VISIT(state->PyBufferedRWPair_Type);
Py_VISIT(state->PyBufferedRandom_Type);
Py_VISIT(state->PyBufferedReader_Type);
Py_VISIT(state->PyBufferedWriter_Type);
Py_VISIT(state->PyBytesIO_Type);
Py_VISIT(state->PyFileIO_Type);
Py_VISIT(state->PyStringIO_Type);
Py_VISIT(state->PyTextIOWrapper_Type);
return 0;
}

Expand All @@ -599,6 +603,15 @@ iomodule_clear(PyObject *mod) {
if (state->locale_module != NULL)
Py_CLEAR(state->locale_module);
Py_CLEAR(state->unsupported_operation);

Py_CLEAR(state->PyBufferedRWPair_Type);
Py_CLEAR(state->PyBufferedRandom_Type);
Py_CLEAR(state->PyBufferedReader_Type);
Py_CLEAR(state->PyBufferedWriter_Type);
Py_CLEAR(state->PyBytesIO_Type);
Py_CLEAR(state->PyFileIO_Type);
Py_CLEAR(state->PyStringIO_Type);
Py_CLEAR(state->PyTextIOWrapper_Type);
return 0;
}

Expand All @@ -612,7 +625,9 @@ iomodule_free(PyObject *mod) {
* Module definition
*/

#define clinic_state() (get_io_state(module))
#include "clinic/_iomodule.c.h"
#undef clinic_state

static PyMethodDef module_methods[] = {
_IO_OPEN_METHODDEF
Expand Down Expand Up @@ -644,23 +659,11 @@ static PyTypeObject* static_types[] = {
&PyRawIOBase_Type,
&PyTextIOBase_Type,

// PyBufferedIOBase_Type(PyIOBase_Type) subclasses
&PyBytesIO_Type,
&PyBufferedReader_Type,
&PyBufferedWriter_Type,
&PyBufferedRWPair_Type,
&PyBufferedRandom_Type,

// PyRawIOBase_Type(PyIOBase_Type) subclasses
&PyFileIO_Type,
&_PyBytesIOBuffer_Type,
#ifdef MS_WINDOWS
&PyWindowsConsoleIO_Type,
#endif

// PyTextIOBase_Type(PyIOBase_Type) subclasses
&PyStringIO_Type,
&PyTextIOWrapper_Type,
};


Expand All @@ -673,6 +676,17 @@ _PyIO_Fini(void)
}
}

#define ADD_TYPE(module, type, spec, base) \
do { \
type = (PyTypeObject *)PyType_FromModuleAndSpec(module, spec, \
(PyObject *)base); \
if (type == NULL) { \
goto fail; \
} \
if (PyModule_AddType(module, type) < 0) { \
goto fail; \
} \
} while (0)

PyMODINIT_FUNC
PyInit__io(void)
Expand Down Expand Up @@ -705,17 +719,9 @@ PyInit__io(void)
}

// Set type base classes
PyFileIO_Type.tp_base = &PyRawIOBase_Type;
PyBytesIO_Type.tp_base = &PyBufferedIOBase_Type;
PyStringIO_Type.tp_base = &PyTextIOBase_Type;
#ifdef MS_WINDOWS
PyWindowsConsoleIO_Type.tp_base = &PyRawIOBase_Type;
#endif
PyBufferedReader_Type.tp_base = &PyBufferedIOBase_Type;
PyBufferedWriter_Type.tp_base = &PyBufferedIOBase_Type;
PyBufferedRWPair_Type.tp_base = &PyBufferedIOBase_Type;
PyBufferedRandom_Type.tp_base = &PyBufferedIOBase_Type;
PyTextIOWrapper_Type.tp_base = &PyTextIOBase_Type;

// Add types
for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) {
Expand All @@ -725,6 +731,25 @@ PyInit__io(void)
}
}

// PyBufferedIOBase_Type(PyIOBase_Type) subclasses
ADD_TYPE(m, state->PyBytesIO_Type, &bytesio_spec, &PyBufferedIOBase_Type);
ADD_TYPE(m, state->PyBufferedWriter_Type, &bufferedwriter_spec,
&PyBufferedIOBase_Type);
ADD_TYPE(m, state->PyBufferedReader_Type, &bufferedreader_spec,
&PyBufferedIOBase_Type);
ADD_TYPE(m, state->PyBufferedRWPair_Type, &bufferedrwpair_spec,
&PyBufferedIOBase_Type);
ADD_TYPE(m, state->PyBufferedRandom_Type, &bufferedrandom_spec,
&PyBufferedIOBase_Type);

// PyRawIOBase_Type(PyIOBase_Type) subclasses
ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, &PyRawIOBase_Type);

// PyTextIOBase_Type(PyIOBase_Type) subclasses
ADD_TYPE(m, state->PyStringIO_Type, &stringio_spec, &PyTextIOBase_Type);
ADD_TYPE(m, state->PyTextIOWrapper_Type, &textiowrapper_spec,
&PyTextIOBase_Type);

state->initialized = 1;

return m;
Expand Down
47 changes: 39 additions & 8 deletions Modules/_io/_iomodule.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,28 @@

#include "exports.h"

#include "pycore_moduleobject.h" // _PyModule_GetState()
#include "structmember.h"

/* ABCs */
extern PyTypeObject PyIOBase_Type;
extern PyTypeObject PyRawIOBase_Type;
extern PyTypeObject PyBufferedIOBase_Type;
extern PyTypeObject PyTextIOBase_Type;

/* Concrete classes */
extern PyTypeObject PyFileIO_Type;
extern PyTypeObject PyBytesIO_Type;
extern PyTypeObject PyStringIO_Type;
extern PyTypeObject PyBufferedReader_Type;
extern PyTypeObject PyBufferedWriter_Type;
extern PyTypeObject PyBufferedRWPair_Type;
extern PyTypeObject PyBufferedRandom_Type;
extern PyTypeObject PyTextIOWrapper_Type;
extern PyTypeObject PyIncrementalNewlineDecoder_Type;

/* Type specs */
extern PyType_Spec bufferedrandom_spec;
extern PyType_Spec bufferedreader_spec;
extern PyType_Spec bufferedrwpair_spec;
extern PyType_Spec bufferedwriter_spec;
extern PyType_Spec bytesio_spec;
extern PyType_Spec fileio_spec;
extern PyType_Spec stringio_spec;
extern PyType_Spec textiowrapper_spec;

#ifdef MS_WINDOWS
extern PyTypeObject PyWindowsConsoleIO_Type;
#endif /* MS_WINDOWS */
Expand Down Expand Up @@ -140,11 +145,37 @@ typedef struct {
PyObject *locale_module;

PyObject *unsupported_operation;

/* Types */
PyTypeObject *PyBufferedRWPair_Type;
PyTypeObject *PyBufferedRandom_Type;
PyTypeObject *PyBufferedReader_Type;
PyTypeObject *PyBufferedWriter_Type;
PyTypeObject *PyBytesIO_Type;
PyTypeObject *PyFileIO_Type;
PyTypeObject *PyStringIO_Type;
PyTypeObject *PyTextIOWrapper_Type;
} _PyIO_State;

#define IO_MOD_STATE(mod) ((_PyIO_State *)PyModule_GetState(mod))
#define IO_STATE() _PyIO_get_module_state()

static inline _PyIO_State *
get_io_state(PyObject *module)
{
void *state = _PyModule_GetState(module);
assert(state != NULL);
return (_PyIO_State *)state;
}

static inline _PyIO_State *
find_io_state_by_def(PyTypeObject *type)
{
PyObject *mod = PyType_GetModuleByDef(type, &_PyIO_Module);
assert(mod != NULL);
return get_io_state(mod);
}

extern _PyIO_State *_PyIO_get_module_state(void);

#ifdef MS_WINDOWS
Expand Down

0 comments on commit c00faf7

Please sign in to comment.