Merge pull request #1934 from wiredfool/incremental_removal

Rewrite of Jpeg2k data handling

PIL/ImageFile.py

@@ -185,53 +185,59 @@ def load(self):
             except AttributeError:
                 prefix = b""
 
-            for d, e, o, a in self.tile:
-                d = Image._getdecoder(self.mode, d, a, self.decoderconfig)
-                seek(o)
+            for decoder_name, extents, offset, args in self.tile:
+                decoder = Image._getdecoder(self.mode, decoder_name,
+                                      args, self.decoderconfig)
+                seek(offset)
                 try:
-                    d.setimage(self.im, e)
+                    decoder.setimage(self.im, extents)
                 except ValueError:
                     continue
-                b = prefix
-                while True:
-                    try:
-                        s = read(self.decodermaxblock)
-                    except (IndexError, struct.error):  # truncated png/gif
-                        if LOAD_TRUNCATED_IMAGES:
-                            break
-                        else:
-                            raise IOError("image file is truncated")
-
-                    if not s and not d.handles_eof:  # truncated jpeg
-                        self.tile = []
-
-                        # JpegDecode needs to clean things up here either way
-                        # If we don't destroy the decompressor,
-                        # we have a memory leak.
-                        d.cleanup()
-
-                        if LOAD_TRUNCATED_IMAGES:
+                if decoder.pulls_fd:
+                    decoder.setfd(self.fp)
+                    status, err_code = decoder.decode(b"")
+                else:
+                    b = prefix
+                    while True:
+                        try:
+                            s = read(self.decodermaxblock)
+                        except (IndexError, struct.error):  # truncated png/gif
+                            if LOAD_TRUNCATED_IMAGES:
+                                break
+                            else:
+                                raise IOError("image file is truncated")
+
+                        if not s and not decoder.handles_eof:  # truncated jpeg
+                            self.tile = []
+
+                            # JpegDecode needs to clean things up here either way
+                            # If we don't destroy the decompressor,
+                            # we have a memory leak.
+                            decoder.cleanup()
+
+                            if LOAD_TRUNCATED_IMAGES:
+                                break
+                            else:
+                                raise IOError("image file is truncated "
+                                              "(%d bytes not processed)" % len(b))
+
+                        b = b + s
+                        n, err_code = decoder.decode(b)
+                        if n < 0:
                             break
-                        else:
-                            raise IOError("image file is truncated "
-                                          "(%d bytes not processed)" % len(b))
-
-                    b = b + s
-                    n, e = d.decode(b)
-                    if n < 0:
-                        break
-                    b = b[n:]
+                        b = b[n:]
+
                 # Need to cleanup here to prevent leaks in PyPy
-                d.cleanup()
+                decoder.cleanup()
 
         self.tile = []
         self.readonly = readonly
 
         self.fp = None  # might be shared
 
-        if not self.map and not LOAD_TRUNCATED_IMAGES and e < 0:
+        if not self.map and not LOAD_TRUNCATED_IMAGES and err_code < 0:
             # still raised if decoder fails to return anything
-            raise_ioerror(e)
+            raise_ioerror(err_code)
 
         # post processing
         if hasattr(self, "tile_post_rotate"):
@@ -465,11 +471,15 @@ def _save(im, fp, tile, bufsize=0):
             if o > 0:
                 fp.seek(o, 0)
             e.setimage(im.im, b)
-            while True:
-                l, s, d = e.encode(bufsize)
-                fp.write(d)
-                if s:
-                    break
+            if e.pushes_fd:
+                e.setfd(fp)
+                l,s = e.encode_to_pyfd()
+            else:
+                while True:
+                    l, s, d = e.encode(bufsize)
+                    fp.write(d)
+                    if s:
+                        break
             if s < 0:
                 raise IOError("encoder error %d when writing image file" % s)
             e.cleanup()
@@ -480,7 +490,11 @@ def _save(im, fp, tile, bufsize=0):
             if o > 0:
                 fp.seek(o, 0)
             e.setimage(im.im, b)
-            s = e.encode_to_file(fh, bufsize)
+            if e.pushes_fd:
+                e.setfd(fp)
+                l,s = e.encode_to_pyfd()
+            else:
+                s = e.encode_to_file(fh, bufsize)
             if s < 0:
                 raise IOError("encoder error %d when writing image file" % s)
             e.cleanup()

PIL/Jpeg2KImagePlugin.py

@@ -192,7 +192,7 @@ def _open(self):
                 length = -1
 
         self.tile = [('jpeg2k', (0, 0) + self.size, 0,
-                      (self.codec, self.reduce, self.layers, fd, length))]
+                      (self.codec, self.reduce, self.layers, fd, length, self.fp))]
 
     def load(self):
         if self.reduce:

decode.c

@@ -52,7 +52,8 @@ typedef struct {
     struct ImagingCodecStateInstance state;
     Imaging im;
     PyObject* lock;
-    int     handles_eof;
+    int handles_eof;
+    int pulls_fd;
 } ImagingDecoderObject;
 
 static PyTypeObject ImagingDecoderType;
@@ -97,6 +98,10 @@ PyImaging_DecoderNew(int contextsize)
     /* Most decoders don't want to handle EOF themselves */
     decoder->handles_eof = 0;
 
+    /* set if the decoder needs to pull data from the fd, instead of
+       having it pushed */
+    decoder->pulls_fd = 0;
+
     return decoder;
 }
 
@@ -108,6 +113,7 @@ _dealloc(ImagingDecoderObject* decoder)
     free(decoder->state.buffer);
     free(decoder->state.context);
     Py_XDECREF(decoder->lock);
+    Py_XDECREF(decoder->state.fd);
     PyObject_Del(decoder);
 }
 
@@ -121,11 +127,15 @@ _decode(ImagingDecoderObject* decoder, PyObject* args)
     if (!PyArg_ParseTuple(args, PY_ARG_BYTES_LENGTH, &buffer, &bufsize))
         return NULL;
 
-    ImagingSectionEnter(&cookie);
+    if (!decoder->pulls_fd) {
+        ImagingSectionEnter(&cookie);
+    }
 
     status = decoder->decode(decoder->im, &decoder->state, buffer, bufsize);
 
-    ImagingSectionLeave(&cookie);
+    if (!decoder->pulls_fd) {
+        ImagingSectionLeave(&cookie);
+    }
 
     return Py_BuildValue("ii", status, decoder->state.errcode);
 }
@@ -210,23 +220,52 @@ _setimage(ImagingDecoderObject* decoder, PyObject* args)
     return Py_None;
 }
 
+static PyObject*
+_setfd(ImagingDecoderObject* decoder, PyObject* args)
+{
+    PyObject* fd;
+    ImagingCodecState state;
+
+    if (!PyArg_ParseTuple(args, "O", &fd))
+        return NULL;
+
+    state = &decoder->state;
+
+    Py_XINCREF(fd);
+    state->fd = fd;
+
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+
 static PyObject *
 _get_handles_eof(ImagingDecoderObject *decoder)
 {
     return PyBool_FromLong(decoder->handles_eof);
 }
 
+static PyObject *
+_get_pulls_fd(ImagingDecoderObject *decoder)
+{
+    return PyBool_FromLong(decoder->pulls_fd);
+}
+
 static struct PyMethodDef methods[] = {
     {"decode", (PyCFunction)_decode, 1},
     {"cleanup", (PyCFunction)_decode_cleanup, 1},
     {"setimage", (PyCFunction)_setimage, 1},
+    {"setfd", (PyCFunction)_setfd, 1},
     {NULL, NULL} /* sentinel */
 };
 
 static struct PyGetSetDef getseters[] = {
     {"handles_eof", (getter)_get_handles_eof, NULL,
      "True if this decoder expects to handle EOF itself.",
      NULL},
+   {"pulls_fd", (getter)_get_pulls_fd, NULL,
+     "True if this decoder expects to pull from self.fd itself.",
+     NULL},
     {NULL, NULL, NULL, NULL, NULL} /* sentinel */
 };
 
@@ -811,6 +850,7 @@ PyImaging_Jpeg2KDecoderNew(PyObject* self, PyObject* args)
     int layers = 0;
     int fd = -1;
     PY_LONG_LONG length = -1;
+
     if (!PyArg_ParseTuple(args, "ss|iiiL", &mode, &format,
                           &reduce, &layers, &fd, &length))
         return NULL;
@@ -829,11 +869,12 @@ PyImaging_Jpeg2KDecoderNew(PyObject* self, PyObject* args)
         return NULL;
 
     decoder->handles_eof = 1;
+    decoder->pulls_fd = 1;
     decoder->decode = ImagingJpeg2KDecode;
     decoder->cleanup = ImagingJpeg2KDecodeCleanup;
 
     context = (JPEG2KDECODESTATE *)decoder->state.context;
-
+    
     context->fd = fd;
     context->length = (off_t)length;
     context->format = codec_format;
@@ -843,3 +884,4 @@ PyImaging_Jpeg2KDecoderNew(PyObject* self, PyObject* args)
     return (PyObject*) decoder;
 }
 #endif /* HAVE_OPENJPEG */
+