bpo-36694: Do not memoize temporary objects in the C implementation of pickle.

Lib/test/pickletester.py

@@ -572,6 +572,8 @@ def create_dynamic_class(name, bases):
 
 # xrange(5) pickled from 2.x with protocol 2
 DATA_XRANGE = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.'
+# optimized pickle produced by the C implementation in 3.8
+DATA_XRANGE_X = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87Rq\x01.'
 
 # a SimpleCookie() object pickled from 2.x with protocol 2
 DATA_COOKIE = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key'
@@ -583,6 +585,8 @@ def create_dynamic_class(name, bases):
 
 # set([3]) pickled from 2.x with protocol 2
 DATA_SET2 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.'
+# optimized pickle produced by the C implementation in 3.8
+DATA_SET2_X = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85Rq\x02.'
 
 python2_exceptions_without_args = (
     ArithmeticError,
@@ -2050,9 +2054,9 @@ def test_pickle_to_2x(self):
         # NOTE: this test is a bit too strong since we can produce different
         # bytecode that 2.x will still understand.
         dumped = self.dumps(range(5), 2)
-        self.assertEqual(dumped, DATA_XRANGE)
+        self.assertIn(dumped, (DATA_XRANGE, DATA_XRANGE_X))
         dumped = self.dumps(set([3]), 2)
-        self.assertEqual(dumped, DATA_SET2)
+        self.assertIn(dumped, (DATA_SET2, DATA_SET2_X))
 
     def test_large_pickles(self):
         # Test the correctness of internal buffering routines when handling

Misc/NEWS.d/next/Library/2019-05-01-15-15-51.bpo-36694.e4BH1A.rst

@@ -0,0 +1,3 @@
+The C implementation of :mod:`pickle` no longer memoizes temporary objects.
+This produces more optimal pickle data and reduces memory consumption on
+pickling and unpickling.

Modules/_pickle.c

@@ -1601,15 +1601,15 @@ memo_get(PicklerObject *self, PyObject *key)
 /* Store an object in the memo, assign it a new unique ID based on the number
    of objects currently stored in the memo and generate a PUT opcode. */
 static int
-memo_put(PicklerObject *self, PyObject *obj)
+memo_put(PicklerObject *self, PyObject *obj, int opt)
 {
     char pdata[30];
     Py_ssize_t len;
     Py_ssize_t idx;
 
     const char memoize_op = MEMOIZE;
 
-    if (self->fast)
+    if (self->fast || (opt && Py_REFCNT(obj) == 1))
         return 0;
 
     idx = PyMemoTable_Size(self->memo);
@@ -2289,7 +2289,7 @@ save_bytes(PicklerObject *self, PyObject *obj)
             return -1;
         }
 
-        if (memo_put(self, obj) < 0)
+        if (memo_put(self, obj, 1) < 0)
             return -1;
 
         return 0;
@@ -2457,7 +2457,7 @@ save_unicode(PicklerObject *self, PyObject *obj)
         if (_Pickler_Write(self, "\n", 1) < 0)
             return -1;
     }
-    if (memo_put(self, obj) < 0)
+    if (memo_put(self, obj, 1) < 0)
         return -1;
 
     return 0;
@@ -2583,7 +2583,7 @@ save_tuple(PicklerObject *self, PyObject *obj)
     }
 
   memoize:
-    if (memo_put(self, obj) < 0)
+    if (memo_put(self, obj, 1) < 0)
         return -1;
 
     return 0;
@@ -2784,7 +2784,7 @@ save_list(PicklerObject *self, PyObject *obj)
     if ((len = PyList_Size(obj)) < 0)
         goto error;
 
-    if (memo_put(self, obj) < 0)
+    if (memo_put(self, obj, len == 0) < 0)
         goto error;
 
     if (len != 0) {
@@ -3041,10 +3041,11 @@ save_dict(PicklerObject *self, PyObject *obj)
     if (_Pickler_Write(self, header, len) < 0)
         goto error;
 
-    if (memo_put(self, obj) < 0)
+    len = PyDict_GET_SIZE(obj);
+    if (memo_put(self, obj, len == 0) < 0)
         goto error;
 
-    if (PyDict_GET_SIZE(obj)) {
+    if (len != 0) {
         /* Save the dict items. */
         if (PyDict_CheckExact(obj) && self->proto > 0) {
             /* We can take certain shortcuts if we know this is a dict and
@@ -3119,10 +3120,10 @@ save_set(PicklerObject *self, PyObject *obj)
     if (_Pickler_Write(self, &empty_set_op, 1) < 0)
         return -1;
 
-    if (memo_put(self, obj) < 0)
+    set_size = PySet_GET_SIZE(obj);
+    if (memo_put(self, obj, set_size == 0) < 0)
         return -1;
 
-    set_size = PySet_GET_SIZE(obj);
     if (set_size == 0)
         return 0;  /* nothing to do */
 
@@ -3224,7 +3225,7 @@ save_frozenset(PicklerObject *self, PyObject *obj)
 
     if (_Pickler_Write(self, &frozenset_op, 1) < 0)
         return -1;
-    if (memo_put(self, obj) < 0)
+    if (memo_put(self, obj, 1) < 0)
         return -1;
 
     return 0;
@@ -3533,7 +3534,7 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name)
                 goto error;
         }
         /* Memoize the object. */
-        if (memo_put(self, obj) < 0)
+        if (memo_put(self, obj, 1) < 0)
             goto error;
     }
 
@@ -3922,7 +3923,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj)
 
             return 0;
         }
-        else if (memo_put(self, obj) < 0)
+        else if (memo_put(self, obj, 1) < 0)
             return -1;
     }
 
@@ -4113,6 +4114,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save)
             }
         }
     }
+    Py_DECREF(reduce_func);
 
     if (reduce_value == NULL)
         goto error;
@@ -4138,7 +4140,6 @@ save(PicklerObject *self, PyObject *obj, int pers_save)
   done:
 
     Py_LeaveRecursiveCall();
-    Py_XDECREF(reduce_func);
     Py_XDECREF(reduce_value);
 
     return status;