Take stronger lock before creating a series partition

Stupidly create table ... partition of X first takes an access share lock
and then an access exclusive lock. Such an upgrade can, and does, cause
deadlocks. Taking the stronger lock beforehand prevents such deadlocks.

pkg/pgmodel/end_to_end_tests/concurrent_sql_test.go

@@ -167,11 +167,11 @@ func TestConcurrentSQL(t *testing.T) {
 	})
 }
 
-func testConcurrentInsertSimple(t testing.TB, db *pgxpool.Pool) {
+func testConcurrentInsertSimple(t testing.TB, db *pgxpool.Pool, metric string) {
 	metrics := []prompb.TimeSeries{
 		{
 			Labels: []prompb.Label{
-				{Name: MetricNameLabelName, Value: "cpu_usage"},
+				{Name: MetricNameLabelName, Value: metric},
 			},
 			Samples: []prompb.Sample{
 				{Timestamp: 10, Value: 0.5},
@@ -250,16 +250,29 @@ func TestConcurrentInsert(t *testing.T) {
 
 	withDB(t, *testDatabase, func(db *pgxpool.Pool, t testing.TB) {
 		var wg sync.WaitGroup
-		wg.Add(2)
-		go func() {
-			defer wg.Done()
-			testConcurrentInsertSimple(t, db)
-		}()
-		go func() {
-			defer wg.Done()
-			testConcurrentInsertSimple(t, db)
-		}()
-		wg.Wait()
+		for i := 0; i < 10; i++ {
+			wg.Add(2)
+			go func() {
+				defer wg.Done()
+				testConcurrentInsertSimple(t, db, fmt.Sprintf("metric_%d", i))
+			}()
+			go func() {
+				defer wg.Done()
+				testConcurrentInsertSimple(t, db, fmt.Sprintf("metric_%d", i))
+			}()
+			wg.Wait()
+
+			wg.Add(2)
+			go func() {
+				defer wg.Done()
+				testConcurrentInsertSimple(t, db, fmt.Sprintf("metric_1_%d", i))
+			}()
+			go func() {
+				defer wg.Done()
+				testConcurrentInsertSimple(t, db, fmt.Sprintf("metric_2_%d", i))
+			}()
+			wg.Wait()
+		}
 
 		wg.Add(2)
 		go func() {

pkg/pgmodel/migrations/sql/1_base_schema.up.sql

@@ -193,6 +193,12 @@ BEGIN
     SELECT SCHEMA_CATALOG.get_or_create_label_id('__name__', NEW.metric_name)
     INTO STRICT label_id;
 
+
+    -- stupidly create table ... partition of X first takes an access share lock
+    -- and then an access exclusive lock. Such an upgrade can, and does, cause
+    -- deadlocks. Taking the stronger lock beforehand prevents such deadlocks.
+    LOCK TABLE _prom_catalog.series in ACCESS EXCLUSIVE mode;
+
     --note that because labels[1] is unique across partitions and UNIQUE(labels) inside partition, labels are guaranteed globally unique
     EXECUTE format($$
         CREATE TABLE SCHEMA_DATA_SERIES.%1$I PARTITION OF SCHEMA_CATALOG.series (