Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Merge branch 'copy-refcount-bug' into devel

  • Loading branch information...
commit e9a485e30bf74c598536417819f196476e0dc406 2 parents dd7ee70 + b6e710b
Daniele Varrazzo authored June 07, 2011
2  NEWS
@@ -13,6 +13,8 @@ What's new in psycopg 2.4.2
13 13
     support was built (ticket #53).
14 14
   - Fixed escape for negative numbers prefixed by minus operator
15 15
     (ticket #57).
  16
+  - Fixed refcount issue during copy.  Reported and fixed by Dave
  17
+    Malcolm (ticket #58, Red Hat Bug 711095).
16 18
   - Trying to execute concurrent operations on the same connection
17 19
     through concurrent green thread results in an error instead of a
18 20
     deadlock.
41  psycopg/cursor_type.c
@@ -1220,8 +1220,12 @@ _psyco_curs_has_read_check(PyObject* o, void* var)
1220 1220
 {
1221 1221
     if (PyObject_HasAttrString(o, "readline")
1222 1222
         && PyObject_HasAttrString(o, "read")) {
1223  
-        /* It's OK to store a borrowed reference, because it is only held for
1224  
-         * the duration of psyco_curs_copy_from. */
  1223
+        /* This routine stores a borrowed reference.  Although it is only held
  1224
+         * for the duration of psyco_curs_copy_from, nested invocations of
  1225
+         * Py_BEGIN_ALLOW_THREADS could surrender control to another thread,
  1226
+         * which could invoke the garbage collector.  We thus need an
  1227
+         * INCREF/DECREF pair if we store this pointer in a GC object, such as
  1228
+         * a cursorObject */
1225 1229
         *((PyObject**)var) = o;
1226 1230
         return 1;
1227 1231
     }
@@ -1311,6 +1315,7 @@ psyco_curs_copy_from(cursorObject *self, PyObject *args, PyObject *kwargs)
1311 1315
     Dprintf("psyco_curs_copy_from: query = %s", query);
1312 1316
 
1313 1317
     self->copysize = bufsize;
  1318
+    Py_INCREF(file);
1314 1319
     self->copyfile = file;
1315 1320
 
1316 1321
     if (pq_execute(self, query, 0) == 1) {
@@ -1319,6 +1324,7 @@ psyco_curs_copy_from(cursorObject *self, PyObject *args, PyObject *kwargs)
1319 1324
     }
1320 1325
 
1321 1326
     self->copyfile = NULL;
  1327
+    Py_DECREF(file);
1322 1328
 
1323 1329
 exit:
1324 1330
     PyMem_Free(quoted_delimiter);
@@ -1337,8 +1343,6 @@ static int
1337 1343
 _psyco_curs_has_write_check(PyObject* o, void* var)
1338 1344
 {
1339 1345
     if (PyObject_HasAttrString(o, "write")) {
1340  
-        /* It's OK to store a borrowed reference, because it is only held for
1341  
-         * the duration of psyco_curs_copy_to. */
1342 1346
         *((PyObject**)var) = o;
1343 1347
         return 1;
1344 1348
     }
@@ -1424,12 +1428,15 @@ psyco_curs_copy_to(cursorObject *self, PyObject *args, PyObject *kwargs)
1424 1428
     Dprintf("psyco_curs_copy_to: query = %s", query);
1425 1429
 
1426 1430
     self->copysize = 0;
  1431
+    Py_INCREF(file);
1427 1432
     self->copyfile = file;
1428 1433
 
1429 1434
     if (pq_execute(self, query, 0) == 1) {
1430 1435
         res = Py_None;
1431 1436
         Py_INCREF(Py_None);
1432 1437
     }
  1438
+
  1439
+    Py_DECREF(file);
1433 1440
     self->copyfile = NULL;
1434 1441
 
1435 1442
 exit:
@@ -1471,18 +1478,18 @@ psyco_curs_copy_expert(cursorObject *self, PyObject *args, PyObject *kwargs)
1471 1478
     EXC_IF_TPC_PREPARED(self->conn, copy_expert);
1472 1479
 
1473 1480
     sql = _psyco_curs_validate_sql_basic(self, sql);
1474  
-    
1475  
-    /* Any failure from here forward should 'goto fail' rather than
  1481
+
  1482
+    /* Any failure from here forward should 'goto exit' rather than
1476 1483
        'return NULL' directly. */
1477  
-    
1478  
-    if (sql == NULL) { goto fail; }
  1484
+
  1485
+    if (sql == NULL) { goto exit; }
1479 1486
 
1480 1487
     /* This validation of file is rather weak, in that it doesn't enforce the
1481 1488
        assocation between "COPY FROM" -> "read" and "COPY TO" -> "write".
1482 1489
        However, the error handling in _pq_copy_[in|out] must be able to handle
1483 1490
        the case where the attempt to call file.read|write fails, so no harm
1484 1491
        done. */
1485  
-    
  1492
+
1486 1493
     if (   !PyObject_HasAttrString(file, "read")
1487 1494
         && !PyObject_HasAttrString(file, "write")
1488 1495
       )
@@ -1490,26 +1497,22 @@ psyco_curs_copy_expert(cursorObject *self, PyObject *args, PyObject *kwargs)
1490 1497
         PyErr_SetString(PyExc_TypeError, "file must be a readable file-like"
1491 1498
             " object for COPY FROM; a writeable file-like object for COPY TO."
1492 1499
           );
1493  
-        goto fail;
  1500
+        goto exit;
1494 1501
     }
1495 1502
 
1496 1503
     self->copysize = bufsize;
  1504
+    Py_INCREF(file);
1497 1505
     self->copyfile = file;
1498 1506
 
1499 1507
     /* At this point, the SQL statement must be str, not unicode */
1500  
-    if (pq_execute(self, Bytes_AS_STRING(sql), 0) != 1) { goto fail; }
  1508
+    if (pq_execute(self, Bytes_AS_STRING(sql), 0) != 1) { goto exit; }
1501 1509
 
1502 1510
     res = Py_None;
1503 1511
     Py_INCREF(res);
1504  
-    goto cleanup;
1505  
- fail:
1506  
-    if (res != NULL) {
1507  
-        Py_DECREF(res);
1508  
-        res = NULL;
1509  
-    }
1510  
-    /* Fall through to cleanup */
1511  
- cleanup:
  1512
+
  1513
+exit:
1512 1514
     self->copyfile = NULL;
  1515
+    Py_XDECREF(file);
1513 1516
     Py_XDECREF(sql);
1514 1517
 
1515 1518
     return res;
75  scripts/ticket58.py
... ...
@@ -0,0 +1,75 @@
  1
+"""
  2
+A script to reproduce the race condition described in ticket #58
  3
+
  4
+from https://bugzilla.redhat.com/show_bug.cgi?id=711095
  5
+
  6
+Results in the error:
  7
+
  8
+  python: Modules/gcmodule.c:277: visit_decref: Assertion `gc->gc.gc_refs != 0'
  9
+  failed.
  10
+
  11
+on unpatched library.
  12
+"""
  13
+
  14
+import threading
  15
+import gc
  16
+import time
  17
+
  18
+import psycopg2
  19
+from StringIO import StringIO
  20
+
  21
+done = 0
  22
+
  23
+class GCThread(threading.Thread):
  24
+    # A thread that sits in an infinite loop, forcing the garbage collector
  25
+    # to run
  26
+    def run(self):
  27
+        global done
  28
+        while not done:
  29
+            gc.collect()
  30
+            time.sleep(0.1) # give the other thread a chance to run
  31
+
  32
+gc_thread = GCThread()
  33
+
  34
+
  35
+# This assumes a pre-existing db named "test", with:
  36
+#   "CREATE TABLE test (id serial PRIMARY KEY, num integer, data varchar);"
  37
+
  38
+conn = psycopg2.connect("dbname=test user=postgres")
  39
+cur = conn.cursor()
  40
+
  41
+# Start the other thread, running the GC regularly
  42
+gc_thread.start()
  43
+
  44
+# Now do lots of "cursor.copy_from" calls:
  45
+print "copy_from"
  46
+for i in range(1000):
  47
+    f = StringIO("42\tfoo\n74\tbar\n")
  48
+    cur.copy_from(f, 'test', columns=('num', 'data'))
  49
+    # Assuming the other thread gets a chance to run during this call, expect a
  50
+    # build of python (with assertions enabled) to bail out here with:
  51
+    #    python: Modules/gcmodule.c:277: visit_decref: Assertion `gc->gc.gc_refs != 0' failed.
  52
+
  53
+# Also exercise the copy_to code path
  54
+print "copy_to"
  55
+cur.execute("truncate test")
  56
+f = StringIO("42\tfoo\n74\tbar\n")
  57
+cur.copy_from(f, 'test', columns=('num', 'data'))
  58
+for i in range(1000):
  59
+    f = StringIO()
  60
+    cur.copy_to(f, 'test', columns=('num', 'data'))
  61
+
  62
+# And copy_expert too
  63
+print "copy_expert"
  64
+cur.execute("truncate test")
  65
+for i in range(1000):
  66
+    f = StringIO("42\tfoo\n74\tbar\n")
  67
+    cur.copy_expert("copy test to stdout", f)
  68
+
  69
+# Terminate the GC thread's loop:
  70
+done = 1
  71
+
  72
+cur.close()
  73
+conn.close()
  74
+
  75
+

0 notes on commit e9a485e

Please sign in to comment.
Something went wrong with that request. Please try again.