congestion: Smooth traffic light variations (near#11074)

These variants are to explore different design points, optimizing for
delay or utilization.

I tried many different angles and distances to find a good set of
parameter combinations. I can't say for sure if these are optimal, but I
think they are reasonable representations of how we can change
parameters to get better delays or utilization.

tools/congestion-model/src/main.rs

@@ -206,6 +206,33 @@ fn strategy(strategy_name: &str, num_shards: usize) -> Vec<Box<dyn CongestionStr
             "New TX last" => Box::<NewTxLast>::default(),
             "Traffic Light" => Box::<TrafficLight>::default(),
             "Smooth Traffic Light" => Box::<SmoothTrafficLight>::default(),
+            // Trade essentially unbounded outgoing delays for higher
+            // utilization. If run for long enough, it will still fill the
+            // buffer and hit a memory limit, but full throughput can be
+            // sustained for a long time this way.
+            "STL_MAX_UTIL" => Box::new(
+                SmoothTrafficLight::default()
+                    .with_smooth_slow_down(false)
+                    .with_gas_limits(50 * PGAS, u64::MAX)
+                    .with_tx_reject_threshold(0.125),
+            ),
+            "STL_HIGH_UTIL" => Box::new(
+                SmoothTrafficLight::default()
+                    .with_smooth_slow_down(false)
+                    .with_gas_limits(50 * PGAS, 50 * PGAS)
+                    .with_tx_reject_threshold(0.125),
+            ),
+            // Keep queues short enough that the can be emptied in one round.
+            "STL_MIN_DELAY" => Box::new(
+                SmoothTrafficLight::default()
+                    .with_gas_limits(1300 * TGAS, 1 * PGAS)
+                    .with_tx_reject_threshold(0.95),
+            ),
+            "STL_LOW_DELAY" => Box::new(
+                SmoothTrafficLight::default()
+                    .with_gas_limits(5 * PGAS, 1 * PGAS)
+                    .with_tx_reject_threshold(0.5),
+            ),
             "NEP" => Box::<NepStrategy>::default(),
             "NEP 200MB" => Box::new(
                 NepStrategy::default().with_memory_limits(ByteSize::mb(100), ByteSize::mb(100)),
@@ -331,6 +358,10 @@ fn parse_strategy_names(strategy_name: &str) -> Vec<String> {
         "New TX last".to_string(),
         "Traffic Light".to_string(),
         "Smooth Traffic Light".to_string(),
+        "STL_MAX_UTIL".to_string(),
+        "STL_HIGH_UTIL".to_string(),
+        "STL_MIN_DELAY".to_string(),
+        "STL_LOW_DELAY".to_string(),
         "NEP".to_string(),
         "NEP 200MB".to_string(),
         "NEP 450/50MB".to_string(),

tools/congestion-model/src/strategy/smooth_traffic_light.rs

@@ -24,6 +24,7 @@ pub struct SmoothTrafficLight {
     pub min_send_limit_amber: GGas,
     pub max_send_limit: GGas,
     pub red_send_limit: GGas,
+    pub smooth_slow_down: bool,
 
     // queue limits to calculate congestion level
     pub red_incoming_gas: GGas,
@@ -43,6 +44,7 @@ impl Default for SmoothTrafficLight {
             min_send_limit_amber: 1 * PGAS,
             max_send_limit: 300 * PGAS,
             red_send_limit: 1 * PGAS,
+            smooth_slow_down: true,
 
             // queue limits to calculate congestion level
             red_incoming_gas: 20 * PGAS,
@@ -209,33 +211,25 @@ impl SmoothTrafficLight {
         let outgoing_gas_congestion = self.outgoing_gas_congestion(ctx);
         let memory_congestion = self.memory_congestion(ctx);
 
-        // Initial traffic light:
-        // Usually, signal other shards to slow down based on our incoming gas congestion only.
-        // All other limits (outgoing gas, memory) are ignored until they hit a red light.
-        // The goal is to never hit red, unless we have to prevent the unbounded queues vs deadlocks tradeoff.
-        // If we hit red, it slows down incoming traffic massively, which should bring us out of red soon.
-        // let red =
-        //     incoming_gas_congestion.max(outgoing_gas_congestion).max(memory_congestion) >= 1.0;
-        // let info = if red {
-        //     CongestedShardsInfo {
-        //         congestion_level: 1.0,
-        //         allowed_shard: Some(self.round_robin_shard(ctx.block_height() as usize)),
-        //     }
-        // } else {
-        //     CongestedShardsInfo { congestion_level: incoming_gas_congestion, allowed_shard: None }
-        // };
-
-        // Simplified and smoothed. Requires a larger max memory limit but will
-        // smoothly reduce based on size, which can even lead to smaller peak
-        // buffer size.
         let max_congestion =
             incoming_gas_congestion.max(outgoing_gas_congestion).max(memory_congestion);
-        let info = if max_congestion >= 1.0 {
+        let red = max_congestion >= 1.0;
+        let info = if red {
             CongestedShardsInfo {
                 congestion_level: 1.0,
                 allowed_shard: Some(self.round_robin_shard(ctx.block_height() as usize)),
             }
+        } else if !self.smooth_slow_down {
+            // Initial traffic light:
+            // Usually, signal other shards to slow down based on our incoming gas congestion only.
+            // All other limits (outgoing gas, memory) are ignored until they hit a red light.
+            // The goal is to never hit red, unless we have to prevent the unbounded queues vs deadlocks tradeoff.
+            // If we hit red, it slows down incoming traffic massively, which should bring us out of red soon.
+            CongestedShardsInfo { congestion_level: incoming_gas_congestion, allowed_shard: None }
         } else {
+            // Simplified and smoothed.
+            // Requires a larger max memory limit but will smoothly reduce based
+            // on size, which can even lead to smaller peak buffer size.
             CongestedShardsInfo { congestion_level: max_congestion, allowed_shard: None }
         };
 
@@ -325,6 +319,20 @@ impl SmoothTrafficLight {
         self
     }
 
+    /// Define at what congestion level new transactions to a shard must be rejected.
+    pub fn with_tx_reject_threshold(mut self, threshold: f64) -> Self {
+        self.reject_tx_congestion_limit = threshold;
+        self
+    }
+
+    /// Set to false to use a less smooth strategy for slowing down, only
+    /// looking at memory and outgoing congestion once it is at the threshold.
+    /// This can give higher utilization but will lead to larger buffers.
+    pub fn with_smooth_slow_down(mut self, yes: bool) -> Self {
+        self.smooth_slow_down = yes;
+        self
+    }
+
     /// Gas spent on new transactions.
     pub fn with_tx_gas_limit_range(mut self, min: GGas, max: GGas) -> Self {
         self.min_tx_gas = min;