Test for deadlock

src/backend/utils/gp/segadmin.c

@@ -34,6 +34,7 @@
 #include "postmaster/primary_mirror_mode.h"
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
+#include "utils/faultinjector.h"
 
 #define MASTER_ONLY 0x1
 #define UTILITY_MODE 0x2
@@ -1480,6 +1481,8 @@ gp_add_segment_persistent_entries(PG_FUNCTION_ARGS)
 	seg.db.dbid = mirdbid;
 	seg.db.role = SEGMENT_ROLE_MIRROR;
 
+	SIMPLE_FAULT_INJECTOR(AddSegmentPersistentEntries);
+
 	add_segment_persistent_entries(dbid, &seg, fsmap);
 
 	/* tell filerep to start a full resync */

src/backend/utils/misc/faultinjector.c

@@ -371,6 +371,8 @@ FaultInjectorIdentifierEnumToString[] = {
 		/* inject fault inside dynamic index scan after context reset */
 	_("dynamic_index_scan_context_reset"),
 		/* inject fault when creating new TOAST tables, to modify the chunk size */
+	_("add_segment_persistent_entries"),
+		/* inject fault when add segment persistent entries */
 	_("not recognized"),
 };
 

src/include/utils/faultinjector.h

@@ -252,6 +252,8 @@ typedef enum FaultInjectorIdentifier_e {
 
 	DynamicIndexScanContextReset,
 
+	AddSegmentPersistentEntries,
+
 	/* INSERT has to be done before that line */
 	FaultInjectorIdMax,
 

src/test/regress/expected/fts_deadlock.out

@@ -0,0 +1,72 @@
+--
+-- Test queries that can lead to deadlock on the gp_segment_configuration table
+-- between the QD backend and FTS. The queries could be gp_add_segment_mirror()
+-- and gp_remove_segment_mirror(), both are called during gprecoverseg.
+--
+-- start_ignore
+create language plpythonu;
+create language plpgsql;
+CREATE EXTENSION IF NOT EXISTS gp_inject_fault;
+-- end_ignore
+create or replace function stop_segment(datadir text)
+returns text as $$
+    import subprocess
+    cmd = 'pg_ctl -l postmaster.log -D %s -w -m immediate stop' % datadir
+    return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).replace('.', '')
+$$ language plpythonu;
+-- Wait for content 0 to assume specified mode
+create or replace function wait_for_content0(target_mode char) /*in func*/
+returns void as $$ /*in func*/
+declare /*in func*/
+    iterations int := 0; /*in func*/
+begin /*in func*/
+    while iterations < 120 loop /*in func*/
+        perform pg_sleep(1); /*in func*/
+        if exists (select * from gp_segment_configuration where content = 0 and mode = target_mode) then /*in func*/
+                return; /*in func*/
+        end if; /*in func*/
+        iterations := iterations + 1; /*in func*/
+    end loop; /*in func*/
+end $$ /*in func*/
+language plpgsql;
+-- Stop content 0 mirror
+select stop_segment(fselocation) from pg_filespace_entry fe, gp_segment_configuration c, pg_filespace f
+where fe.fsedbid = c.dbid and c.content=0 and c.role='m' and f.oid = fe.fsefsoid and f.fsname = 'pg_system';
+             stop_segment             
+--------------------------------------
+ waiting for server to shut down done 
+ server stopped                       
+
+(1 row)
+
+select wait_for_content0('c');
+ wait_for_content0 
+-------------------
+
+(1 row)
+
+SELECT gp_inject_fault('add_segment_persistent_entries', 'sleep', '', '', '', 1, 70,
+(select dbid from gp_segment_configuration where content = 0 and role = 'p'));
+NOTICE:  Success:
+ gp_inject_fault 
+-----------------
+ t
+(1 row)
+
+-- start_ignore
+\! gprecoverseg -a -F;
+-- end_ignore
+select wait_for_content0('s');
+ wait_for_content0 
+-------------------
+
+(1 row)
+
+select gp_inject_fault('add_segment_persistent_entries', 'reset',
+(select dbid from gp_segment_configuration where content = 0 and role = 'p'));
+NOTICE:  Success:
+ gp_inject_fault 
+-----------------
+ t
+(1 row)
+

src/test/regress/sql/fts_deadlock.sql

@@ -0,0 +1,52 @@
+--
+-- Test queries that can lead to deadlock on the gp_segment_configuration table
+-- between the QD backend and FTS. The queries could be gp_add_segment_mirror()
+-- and gp_remove_segment_mirror(), both are called during gprecoverseg.
+--
+
+-- start_ignore
+create language plpythonu;
+create language plpgsql;
+CREATE EXTENSION IF NOT EXISTS gp_inject_fault;
+-- end_ignore
+
+create or replace function stop_segment(datadir text)
+returns text as $$
+    import subprocess
+    cmd = 'pg_ctl -l postmaster.log -D %s -w -m immediate stop' % datadir
+    return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).replace('.', '')
+$$ language plpythonu;
+
+-- Wait for content 0 to assume specified mode
+create or replace function wait_for_content0(target_mode char) /*in func*/
+returns void as $$ /*in func*/
+declare /*in func*/
+    iterations int := 0; /*in func*/
+begin /*in func*/
+    while iterations < 120 loop /*in func*/
+        perform pg_sleep(1); /*in func*/
+        if exists (select * from gp_segment_configuration where content = 0 and mode = target_mode) then /*in func*/
+                return; /*in func*/
+        end if; /*in func*/
+        iterations := iterations + 1; /*in func*/
+    end loop; /*in func*/
+end $$ /*in func*/
+language plpgsql;
+
+-- Stop content 0 mirror
+select stop_segment(fselocation) from pg_filespace_entry fe, gp_segment_configuration c, pg_filespace f
+where fe.fsedbid = c.dbid and c.content=0 and c.role='m' and f.oid = fe.fsefsoid and f.fsname = 'pg_system';
+
+select wait_for_content0('c');
+
+SELECT gp_inject_fault('add_segment_persistent_entries', 'sleep', '', '', '', 1, 70,
+(select dbid from gp_segment_configuration where content = 0 and role = 'p'));
+
+-- start_ignore
+\! gprecoverseg -a -F;
+-- end_ignore
+
+select wait_for_content0('s');
+
+select gp_inject_fault('add_segment_persistent_entries', 'reset',
+(select dbid from gp_segment_configuration where content = 0 and role = 'p'));