Add some more randomness to chunk assignment

Previously the assignment of data nodes to chunks had a bit
of a thundering-herd problem for multiple hypertables
without space partions: the data node assigned for the
first chunk was always the same across hypertables.
We fix this by adding the hypertable_id to the
index into the datanode array. This de-synchronizes
across hypertables but maintains consistency for any
given hypertable.

We could make this consistent for space partitioned tables
as well but avoid doing so now to prevent partitions
jumping nodes due to this change.

This also effects tablespace selection in the same way.

src/hypertable.c

@@ -1193,26 +1193,37 @@ ts_hypertable_has_tablespace(Hypertable *ht, Oid tspc_oid)
 }
 
 static int
-hypertable_get_chunk_slice_ordinal(const Hypertable *ht, const Hypercube *hc)
+hypertable_get_chunk_round_robin_index(const Hypertable *ht, const Hypercube *hc)
 {
 	Dimension *dim;
 	DimensionSlice *slice;
+	int offset = 0;
 
 	Assert(NULL != ht);
 	Assert(NULL != hc);
 
 	dim = hyperspace_get_closed_dimension(ht->space, 0);
 
 	if (NULL == dim)
+	{
 		dim = hyperspace_get_open_dimension(ht->space, 0);
+		/* Add some randomness between hypertables so that
+		 * if there is no space partitions, but multiple hypertables
+		 * the initial index is different for different hypertables.
+		 * This protects against creating a lot of chunks on the same
+		 * data node when many hypertables are created at roughly
+		 * the same time, e.g., from a bootstrap script.
+		 */
+		offset = (int) ht->fd.id;
+	}
 
 	Assert(NULL != dim);
 
 	slice = ts_hypercube_get_slice_by_dimension_id(hc, dim->fd.id);
 
 	Assert(NULL != slice);
 
-	return ts_dimension_get_slice_ordinal(dim, slice);
+	return ts_dimension_get_slice_ordinal(dim, slice) + offset;
 }
 
 /*
@@ -1234,7 +1245,7 @@ ts_hypertable_select_tablespace(Hypertable *ht, Chunk *chunk)
 	if (NULL == tspcs || tspcs->num_tablespaces == 0)
 		return NULL;
 
-	i = hypertable_get_chunk_slice_ordinal(ht, chunk->cube);
+	i = hypertable_get_chunk_round_robin_index(ht, chunk->cube);
 
 	/* Use the index of the slice to find the tablespace */
 	return &tspcs->tablespaces[i % tspcs->num_tablespaces];
@@ -2574,7 +2585,7 @@ ts_hypertable_assign_chunk_data_nodes(const Hypertable *ht, const Hypercube *cub
 	int num_assigned = MIN(ht->fd.replication_factor, list_length(available_nodes));
 	int n, i;
 
-	n = hypertable_get_chunk_slice_ordinal(ht, cube);
+	n = hypertable_get_chunk_round_robin_index(ht, cube);
 
 	for (i = 0; i < num_assigned; i++)
 	{

tsl/test/expected/dist_compression.out

@@ -677,7 +677,7 @@ from chunk_compression_stats('conditions') where compression_status like 'Compre
       chunk_name       |       node_name       | before_total | after_total 
 -----------------------+-----------------------+--------------+-------------
  _dist_hyper_2_6_chunk | db_dist_compression_1 | 32 kB        | 32 kB
- _dist_hyper_2_6_chunk | db_dist_compression_2 | 32 kB        | 32 kB
+ _dist_hyper_2_6_chunk | db_dist_compression_3 | 32 kB        | 32 kB
 (2 rows)
 
 SELECT * FROM _timescaledb_catalog.chunk ORDER BY id;

tsl/test/expected/dist_triggers.out

@@ -152,6 +152,17 @@ FROM trigger_events
 GROUP BY 1,2,3,4
 ORDER BY 1,2,3,4
 NOTICE:  [db_dist_triggers_1]:
+tg_when|tg_level|tg_op|tg_name|count
+-------+--------+-----+-------+-----
+(0 rows)
+
+
+NOTICE:  [db_dist_triggers_2]: 
+SELECT tg_when, tg_level, tg_op, tg_name, count(*)
+FROM trigger_events
+GROUP BY 1,2,3,4
+ORDER BY 1,2,3,4
+NOTICE:  [db_dist_triggers_2]:
 tg_when|tg_level |tg_op |tg_name                               |count
 -------+---------+------+--------------------------------------+-----
 AFTER  |ROW      |INSERT|_0_test_trigger_insert_after          |    1
@@ -164,17 +175,6 @@ BEFORE |STATEMENT|INSERT|_0_test_trigger_insert_s_before       |    1
 (7 rows)
 
 
-NOTICE:  [db_dist_triggers_2]: 
-SELECT tg_when, tg_level, tg_op, tg_name, count(*)
-FROM trigger_events
-GROUP BY 1,2,3,4
-ORDER BY 1,2,3,4
-NOTICE:  [db_dist_triggers_2]:
-tg_when|tg_level|tg_op|tg_name|count
--------+--------+-----+-------+-----
-(0 rows)
-
-
 NOTICE:  [db_dist_triggers_3]: 
 SELECT tg_when, tg_level, tg_op, tg_name, count(*)
 FROM trigger_events
@@ -225,22 +225,6 @@ FROM trigger_events
 GROUP BY 1,2,3,4
 ORDER BY 1,2,3,4
 NOTICE:  [db_dist_triggers_1]:
-tg_when|tg_level|tg_op |tg_name                     |count
--------+--------+------+----------------------------+-----
-AFTER  |ROW     |UPDATE|_0_test_trigger_update_after|    1
-AFTER  |ROW     |UPDATE|z_test_trigger_all_after    |    1
-BEFORE |ROW     |DELETE|_0_test_trigger_delete      |    1
-BEFORE |ROW     |UPDATE|_0_test_trigger_update      |    1
-BEFORE |ROW     |UPDATE|z_test_trigger_all          |    1
-(5 rows)
-
-
-NOTICE:  [db_dist_triggers_2]: 
-SELECT tg_when, tg_level, tg_op, tg_name, count(*)
-FROM trigger_events
-GROUP BY 1,2,3,4
-ORDER BY 1,2,3,4
-NOTICE:  [db_dist_triggers_2]:
 tg_when|tg_level |tg_op |tg_name                        |count
 -------+---------+------+-------------------------------+-----
 AFTER  |ROW      |INSERT|_0_test_trigger_insert_after   |    1
@@ -257,6 +241,22 @@ BEFORE |STATEMENT|INSERT|_0_test_trigger_insert_s_before|    1
 (11 rows)
 
 
+NOTICE:  [db_dist_triggers_2]: 
+SELECT tg_when, tg_level, tg_op, tg_name, count(*)
+FROM trigger_events
+GROUP BY 1,2,3,4
+ORDER BY 1,2,3,4
+NOTICE:  [db_dist_triggers_2]:
+tg_when|tg_level|tg_op |tg_name                     |count
+-------+--------+------+----------------------------+-----
+AFTER  |ROW     |UPDATE|_0_test_trigger_update_after|    1
+AFTER  |ROW     |UPDATE|z_test_trigger_all_after    |    1
+BEFORE |ROW     |DELETE|_0_test_trigger_delete      |    1
+BEFORE |ROW     |UPDATE|_0_test_trigger_update      |    1
+BEFORE |ROW     |UPDATE|z_test_trigger_all          |    1
+(5 rows)
+
+
 NOTICE:  [db_dist_triggers_3]: 
 SELECT tg_when, tg_level, tg_op, tg_name, count(*)
 FROM trigger_events