Skip to content

Commit 3859219

Browse files
author
Jakub Łopuszański
committed
Bug #29882690 UNDETECTED DEADLOCK WITH GAP AND INSERT INTENTION LOCKS IS POSSIBLE
This patch moves the task of detecting a dealock cycle among data locks from the transaction thread, to a dedicated background thread. This patch also fixes Bug #21278148 DEADLOCKS MISSED BY INFORMATION_SCHEMA.INNODB_METRICS LOCK_DEADLOCKS COUNTER The idea is that for each waiting transaction, we store a pointer to a transaction which is blocking it. (Note, that in general there might be multiple such transactions, but we pick any of them). We let the dedicated deadlock detector thread to peek the contents of these pointers for transansactions which are sleeping in slots. This creates a small snapshot of a subset of the big conceptual wait-for-graph, restricted only to blocked transactions, and only to one edge per transaction. This does not even have to be a consistent snapshot, as the only thing we want to find in it are deadlock cycles, which are stable by definition. As the graph has out-degree=1, it has a shape of forests possibly leading to cycles, and thus finding those cycles is trivial. All this can be done almost lock-free, as long as care is taken not to derefernce pointers without making sure that trx is still in the slot. For this purpose we store reservation_no on each slot, and compare current slot->reservation_no to the one in the snapshot. Reviewed-by: Debarun Banerjee <debarun.banerjee@oracle.com> RB:22374
1 parent 7afb630 commit 3859219

23 files changed

+1534
-731
lines changed

‎mysql-test/suite/innodb/r/deadlock_stats.result‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#
2+
# Bug #21278148 DEADLOCKS MISSED BY INFORMATION_SCHEMA.INNODB_METRICS
3+
# LOCK_DEADLOCKS COUNTER
4+
#
5+
CREATE TABLE t(a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
6+
INSERT INTO t VALUES (2,1), (1,2);
7+
# Scenario 1: the con1 will be chosen as victim
8+
SELECT `count` INTO @deadlocks_before_test
9+
FROM INFORMATION_SCHEMA.INNODB_METRICS
10+
WHERE NAME="lock_deadlocks";
11+
BEGIN;
12+
SELECT b FROM t WHERE a=1 FOR UPDATE;
13+
b
14+
2
15+
BEGIN;
16+
INSERT INTO t VALUES (3,3), (4,4), (5,5);
17+
SELECT b FROM t WHERE a=2 FOR UPDATE;
18+
b
19+
1
20+
SET DEBUG_SYNC='lock_wait_will_wait SIGNAL con1_will_wait';
21+
SELECT b FROM t WHERE a=2 FOR UPDATE;;
22+
SET DEBUG_SYNC='now WAIT_FOR con1_will_wait';
23+
SELECT b FROM t WHERE a=1 FOR UPDATE;
24+
b
25+
2
26+
ROLLBACK;
27+
ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
28+
SELECT `count` - @deadlocks_before_test AS deadlocks_during_test
29+
FROM INFORMATION_SCHEMA.INNODB_METRICS
30+
WHERE NAME="lock_deadlocks";
31+
deadlocks_during_test
32+
1
33+
# Scenario 2: the con2 will be chosen as victim
34+
SELECT `count` INTO @deadlocks_before_test
35+
FROM INFORMATION_SCHEMA.INNODB_METRICS
36+
WHERE NAME="lock_deadlocks";
37+
BEGIN;
38+
SELECT b FROM t WHERE a=1 FOR UPDATE;
39+
b
40+
2
41+
BEGIN;
42+
SELECT b FROM t WHERE a=2 FOR UPDATE;
43+
b
44+
1
45+
SET DEBUG_SYNC='lock_wait_will_wait SIGNAL con1_will_wait';
46+
SELECT b FROM t WHERE a=2 FOR UPDATE;;
47+
SET DEBUG_SYNC='now WAIT_FOR con1_will_wait';
48+
SELECT b FROM t WHERE a=1 FOR UPDATE;
49+
ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
50+
b
51+
1
52+
ROLLBACK;
53+
SELECT `count` - @deadlocks_before_test AS deadlocks_during_test
54+
FROM INFORMATION_SCHEMA.INNODB_METRICS
55+
WHERE NAME="lock_deadlocks";
56+
deadlocks_during_test
57+
1
58+
DROP TABLE t;

‎mysql-test/suite/innodb/r/long_deadlock_cycle.result‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ Record lock, heap no 2 PHYSICAL RECORD: n_fields 3; compact format; info bits 0
6464
*** (2) TRANSACTION:
6565
TRANSACTION %NUMBER%, ACTIVE %NUMBER% sec starting index read
6666
mysql tables in use 1, locked 1
67-
3 lock struct(s), heap size %NUMBER%, 2 row lock(s)
67+
LOCK WAIT 3 lock struct(s), heap size %NUMBER%, 2 row lock(s)
6868
MySQL thread id %NUMBER%, OS thread handle %NUMBER%, query id %NUMBER% %ADDRESS% %USER% statistics
6969
SELECT * FROM t1 WHERE id = 4 FOR UPDATE
7070

@@ -206,7 +206,7 @@ Record lock, heap no 2 PHYSICAL RECORD: n_fields 3; compact format; info bits 0
206206
*** (2) TRANSACTION:
207207
TRANSACTION %NUMBER%, ACTIVE %NUMBER% sec starting index read
208208
mysql tables in use 1, locked 1
209-
5 lock struct(s), heap size %NUMBER%, 8 row lock(s)
209+
LOCK WAIT 5 lock struct(s), heap size %NUMBER%, 8 row lock(s)
210210
MySQL thread id %NUMBER%, OS thread handle %NUMBER%, query id %NUMBER% %ADDRESS% %USER% statistics
211211
SELECT * FROM t1 WHERE id = 4 FOR UPDATE
212212

‎mysql-test/suite/innodb/r/monitor.result‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ metadata_table_handles_opened disabled
55
metadata_table_handles_closed disabled
66
metadata_table_reference_count disabled
77
lock_deadlocks disabled
8+
lock_deadlock_false_positives disabled
9+
lock_deadlock_rounds disabled
10+
lock_threads_waiting disabled
811
lock_timeouts disabled
912
lock_rec_lock_waits disabled
1013
lock_table_lock_waits disabled
@@ -320,6 +323,9 @@ select name, status from information_schema.innodb_metrics
320323
where name like "%lock%";
321324
name status
322325
lock_deadlocks disabled
326+
lock_deadlock_false_positives disabled
327+
lock_deadlock_rounds disabled
328+
lock_threads_waiting disabled
323329
lock_timeouts disabled
324330
lock_rec_lock_waits disabled
325331
lock_table_lock_waits disabled

‎mysql-test/suite/innodb/r/undetected_deadlock.result‎

Lines changed: 294 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,294 @@
1+
#
2+
# Bug #29882690 UNDETECTED DEADLOCK WITH GAP AND INSERT INTENTION LOCKS
3+
# IS POSSIBLE
4+
#
5+
SET @innodb_lock_wait_timeout_saved = @@global.innodb_lock_wait_timeout;
6+
SET @@global.innodb_lock_wait_timeout = 20;
7+
CREATE TABLE t0 (id INT NOT NULL PRIMARY KEY);
8+
INSERT INTO t0 VALUES (1), (2), (3);
9+
CREATE PROCEDURE show_locks ()
10+
BEGIN
11+
SELECT engine_transaction_id INTO @con1
12+
FROM performance_schema.data_locks
13+
WHERE object_name = 't0' AND lock_data = '1';
14+
SELECT engine_transaction_id INTO @con2
15+
FROM performance_schema.data_locks
16+
WHERE object_name = 't0' AND lock_data = '2';
17+
SELECT engine_transaction_id INTO @con3
18+
FROM performance_schema.data_locks
19+
WHERE object_name = 't0' AND lock_data = '3';
20+
SELECT
21+
CASE engine_transaction_id
22+
WHEN @con1 THEN "con1"
23+
WHEN @con2 THEN "con2"
24+
WHEN @con3 THEN "con3"
25+
ELSE "unknown"
26+
END connection,index_name,lock_type,lock_mode,lock_status,lock_data
27+
FROM performance_schema.data_locks
28+
WHERE object_name = 't1'
29+
ORDER BY 1,2,3,4,5,6;
30+
END //
31+
CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, val VARCHAR(200));
32+
INSERT INTO t1 (id,val) VALUES (1, "a"), (3, "c"), (4, "d");
33+
BEGIN;
34+
SELECT * FROM t0 WHERE id=1 FOR UPDATE;
35+
id
36+
1
37+
BEGIN;
38+
SELECT * FROM t0 WHERE id=2 FOR UPDATE;
39+
id
40+
2
41+
BEGIN;
42+
SELECT * FROM t0 WHERE id=3 FOR UPDATE;
43+
id
44+
3
45+
# 1. con1 obtains a GRANTED LOCK_GAP on gap before row id=3
46+
SELECT * FROM t1 WHERE id=2 FOR UPDATE;
47+
id val
48+
CALL show_locks();
49+
connection index_name lock_type lock_mode lock_status lock_data
50+
con1 NULL TABLE IX GRANTED NULL
51+
con1 PRIMARY RECORD X,GAP GRANTED 3
52+
# 2. con2 obtains a GRANTED LOCK_X on row id=4
53+
SELECT * FROM t1 WHERE id=4 FOR UPDATE;
54+
id val
55+
4 d
56+
CALL show_locks();
57+
connection index_name lock_type lock_mode lock_status lock_data
58+
con1 NULL TABLE IX GRANTED NULL
59+
con1 PRIMARY RECORD X,GAP GRANTED 3
60+
con2 NULL TABLE IX GRANTED NULL
61+
con2 PRIMARY RECORD X,REC_NOT_GAP GRANTED 4
62+
# 3. con2 enqueues a waiting LOCK_INSERT_INTENTION into the gap
63+
# before id=3
64+
SET DEBUG_SYNC = 'lock_wait_will_wait SIGNAL con2_will_wait';
65+
INSERT INTO t1 (id, val) VALUES (2, "b");
66+
SET DEBUG_SYNC = 'now WAIT_FOR con2_will_wait';
67+
CALL show_locks();
68+
connection index_name lock_type lock_mode lock_status lock_data
69+
con1 NULL TABLE IX GRANTED NULL
70+
con1 PRIMARY RECORD X,GAP GRANTED 3
71+
con2 NULL TABLE IX GRANTED NULL
72+
con2 PRIMARY RECORD X,GAP,INSERT_INTENTION WAITING 3
73+
con2 PRIMARY RECORD X,REC_NOT_GAP GRANTED 4
74+
# 4. con3 obtains a GRANTED LOCK_GAP on gap before row id=3
75+
SELECT * FROM t1 WHERE id=2 FOR UPDATE;
76+
id val
77+
CALL show_locks();
78+
connection index_name lock_type lock_mode lock_status lock_data
79+
con1 NULL TABLE IX GRANTED NULL
80+
con1 PRIMARY RECORD X,GAP GRANTED 3
81+
con2 NULL TABLE IX GRANTED NULL
82+
con2 PRIMARY RECORD X,GAP,INSERT_INTENTION WAITING 3
83+
con2 PRIMARY RECORD X,REC_NOT_GAP GRANTED 4
84+
con3 NULL TABLE IX GRANTED NULL
85+
con3 PRIMARY RECORD X,GAP GRANTED 3
86+
# 5. con3 enqueues a waiting lock on same row as in step 2.
87+
SET DEBUG_SYNC = 'lock_wait_will_wait SIGNAL con3_will_wait';
88+
SELECT * FROM t1 WHERE id=4 FOR UPDATE;
89+
SET DEBUG_SYNC = 'now WAIT_FOR con3_will_wait';
90+
CALL show_locks();
91+
connection index_name lock_type lock_mode lock_status lock_data
92+
con1 NULL TABLE IX GRANTED NULL
93+
con1 PRIMARY RECORD X,GAP GRANTED 3
94+
con2 NULL TABLE IX GRANTED NULL
95+
con2 PRIMARY RECORD X,GAP,INSERT_INTENTION WAITING 3
96+
con2 PRIMARY RECORD X,REC_NOT_GAP GRANTED 4
97+
con3 NULL TABLE IX GRANTED NULL
98+
con3 PRIMARY RECORD X,GAP GRANTED 3
99+
con3 PRIMARY RECORD X,REC_NOT_GAP WAITING 4
100+
# 6. con1 resizes the row with id=3 ("c"->"cccccccccccccccc") causing
101+
# locks to be moved
102+
UPDATE t1 SET val="cccccccccccccccc" WHERE id=3;
103+
# 7. it depends on implementation of con3's LOCK_GAP lands is in front
104+
# of con2's waiting II or not
105+
# 8. con1 commits
106+
COMMIT;
107+
ROLLBACK;
108+
ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
109+
ROLLBACK;
110+
DROP TABLE t1;
111+
CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, val VARCHAR(200));
112+
INSERT INTO t1 (id,val) VALUES (1, "a"), (3, "c"), (4, "d");
113+
BEGIN;
114+
SELECT * FROM t0 WHERE id=1 FOR UPDATE;
115+
id
116+
1
117+
BEGIN;
118+
SELECT * FROM t0 WHERE id=2 FOR UPDATE;
119+
id
120+
2
121+
BEGIN;
122+
SELECT * FROM t0 WHERE id=3 FOR UPDATE;
123+
id
124+
3
125+
# 1. con1 obtains a GRANTED LOCK_GAP on gap before row id=3
126+
SELECT * FROM t1 WHERE id=2 FOR UPDATE;
127+
id val
128+
CALL show_locks();
129+
connection index_name lock_type lock_mode lock_status lock_data
130+
con1 NULL TABLE IX GRANTED NULL
131+
con1 PRIMARY RECORD X,GAP GRANTED 3
132+
# 2. con2 obtains a GRANTED LOCK_X on row id=4
133+
SELECT * FROM t1 WHERE id=4 FOR UPDATE;
134+
id val
135+
4 d
136+
CALL show_locks();
137+
connection index_name lock_type lock_mode lock_status lock_data
138+
con1 NULL TABLE IX GRANTED NULL
139+
con1 PRIMARY RECORD X,GAP GRANTED 3
140+
con2 NULL TABLE IX GRANTED NULL
141+
con2 PRIMARY RECORD X,REC_NOT_GAP GRANTED 4
142+
# 3. con2 enqueues a waiting LOCK_INSERT_INTENTION into the gap
143+
# before id=3
144+
SET DEBUG_SYNC = 'lock_wait_will_wait SIGNAL con2_will_wait';
145+
INSERT INTO t1 (id, val) VALUES (2, "b");
146+
SET DEBUG_SYNC = 'now WAIT_FOR con2_will_wait';
147+
CALL show_locks();
148+
connection index_name lock_type lock_mode lock_status lock_data
149+
con1 NULL TABLE IX GRANTED NULL
150+
con1 PRIMARY RECORD X,GAP GRANTED 3
151+
con2 NULL TABLE IX GRANTED NULL
152+
con2 PRIMARY RECORD X,GAP,INSERT_INTENTION WAITING 3
153+
con2 PRIMARY RECORD X,REC_NOT_GAP GRANTED 4
154+
# 4. con3 obtains a GRANTED LOCK_GAP on gap before row id=3
155+
SELECT * FROM t1 WHERE id=2 FOR UPDATE;
156+
id val
157+
CALL show_locks();
158+
connection index_name lock_type lock_mode lock_status lock_data
159+
con1 NULL TABLE IX GRANTED NULL
160+
con1 PRIMARY RECORD X,GAP GRANTED 3
161+
con2 NULL TABLE IX GRANTED NULL
162+
con2 PRIMARY RECORD X,GAP,INSERT_INTENTION WAITING 3
163+
con2 PRIMARY RECORD X,REC_NOT_GAP GRANTED 4
164+
con3 NULL TABLE IX GRANTED NULL
165+
con3 PRIMARY RECORD X,GAP GRANTED 3
166+
# 5. con3 enqueues a waiting lock on same row as in step 2.
167+
SET DEBUG_SYNC = 'lock_wait_will_wait SIGNAL con3_will_wait';
168+
SELECT * FROM t1 WHERE id=4 FOR UPDATE;
169+
SET DEBUG_SYNC = 'now WAIT_FOR con3_will_wait';
170+
CALL show_locks();
171+
connection index_name lock_type lock_mode lock_status lock_data
172+
con1 NULL TABLE IX GRANTED NULL
173+
con1 PRIMARY RECORD X,GAP GRANTED 3
174+
con2 NULL TABLE IX GRANTED NULL
175+
con2 PRIMARY RECORD X,GAP,INSERT_INTENTION WAITING 3
176+
con2 PRIMARY RECORD X,REC_NOT_GAP GRANTED 4
177+
con3 NULL TABLE IX GRANTED NULL
178+
con3 PRIMARY RECORD X,GAP GRANTED 3
179+
con3 PRIMARY RECORD X,REC_NOT_GAP WAITING 4
180+
# We enable CATS algorithm, to force placing GRANTED locks at the
181+
# front of queue
182+
# establishing connection cats0
183+
# establishing connection cats1
184+
# establishing connection cats2
185+
# establishing connection cats3
186+
# establishing connection cats4
187+
# establishing connection cats5
188+
# establishing connection cats6
189+
# establishing connection cats7
190+
# establishing connection cats8
191+
# establishing connection cats9
192+
# establishing connection cats10
193+
# establishing connection cats11
194+
# establishing connection cats12
195+
# establishing connection cats13
196+
# establishing connection cats14
197+
# establishing connection cats15
198+
# establishing connection cats16
199+
# establishing connection cats17
200+
# establishing connection cats18
201+
# establishing connection cats19
202+
# establishing connection cats20
203+
# establishing connection cats21
204+
# establishing connection cats22
205+
# establishing connection cats23
206+
# establishing connection cats24
207+
# establishing connection cats25
208+
# establishing connection cats26
209+
# establishing connection cats27
210+
# establishing connection cats28
211+
# establishing connection cats29
212+
# establishing connection cats30
213+
# establishing connection cats31
214+
# establishing connection cats32
215+
# wating for cats1
216+
# wating for cats2
217+
# wating for cats3
218+
# wating for cats4
219+
# wating for cats5
220+
# wating for cats6
221+
# wating for cats7
222+
# wating for cats8
223+
# wating for cats9
224+
# wating for cats10
225+
# wating for cats11
226+
# wating for cats12
227+
# wating for cats13
228+
# wating for cats14
229+
# wating for cats15
230+
# wating for cats16
231+
# wating for cats17
232+
# wating for cats18
233+
# wating for cats19
234+
# wating for cats20
235+
# wating for cats21
236+
# wating for cats22
237+
# wating for cats23
238+
# wating for cats24
239+
# wating for cats25
240+
# wating for cats26
241+
# wating for cats27
242+
# wating for cats28
243+
# wating for cats29
244+
# wating for cats30
245+
# wating for cats31
246+
# wating for cats32
247+
# 6. con1 resizes the row with id=3 ("c"->"cccccccccccccccc") causing
248+
# locks to be moved
249+
UPDATE t1 SET val="cccccccccccccccc" WHERE id=3;
250+
# 7. it depends on implementation of con3's LOCK_GAP lands is in front
251+
# of con2's waiting II or not
252+
# 8. con1 commits
253+
COMMIT;
254+
ROLLBACK;
255+
ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
256+
ROLLBACK;
257+
# Turn of CATS
258+
# cleaning up connection cats0
259+
# cleaning up connection cats1
260+
# cleaning up connection cats2
261+
# cleaning up connection cats3
262+
# cleaning up connection cats4
263+
# cleaning up connection cats5
264+
# cleaning up connection cats6
265+
# cleaning up connection cats7
266+
# cleaning up connection cats8
267+
# cleaning up connection cats9
268+
# cleaning up connection cats10
269+
# cleaning up connection cats11
270+
# cleaning up connection cats12
271+
# cleaning up connection cats13
272+
# cleaning up connection cats14
273+
# cleaning up connection cats15
274+
# cleaning up connection cats16
275+
# cleaning up connection cats17
276+
# cleaning up connection cats18
277+
# cleaning up connection cats19
278+
# cleaning up connection cats20
279+
# cleaning up connection cats21
280+
# cleaning up connection cats22
281+
# cleaning up connection cats23
282+
# cleaning up connection cats24
283+
# cleaning up connection cats25
284+
# cleaning up connection cats26
285+
# cleaning up connection cats27
286+
# cleaning up connection cats28
287+
# cleaning up connection cats29
288+
# cleaning up connection cats30
289+
# cleaning up connection cats31
290+
# cleaning up connection cats32
291+
DROP TABLE t1;
292+
DROP PROCEDURE show_locks;
293+
DROP TABLE t0;
294+
SET @@global.innodb_lock_wait_timeout = @innodb_lock_wait_timeout_saved ;

0 commit comments

Comments
 (0)