Add Fallback Response Route Config for Standard Ensemblers

docs/how-to/create-a-router/configure-ensembler.md

@@ -13,6 +13,14 @@ The router will return a response from the route configured to act as the final
 
 ![](../../.gitbook/assets/nop_ensembler_config.png)
 
+## Standard Ensembler
+
+For experiment engines configured to work with standard ensemblers (i.e., Standard Experiment Engines with experiment selection enabled), the router will return a response from one of the routes based on the configured mapping between routes and experiment treatments. At run time, the treatment returned by the engine will be used to select the corresponding route’s response.
+
+In addition, a fallback route may be configured whose results will be used at runtime when the call to the experiment engine fails or if a route mapping for the treatment generated by the experiment engine does not exist.
+
+![](../../.gitbook/assets/standard_ensembler_config.png)
+
 ## Docker
 Turing will deploy specified image as a post-processor and will send the original request, responses from all routes, and the treatment configuration (if a Experiment Engine is selected, in Configure Experiment Engine), for ensembling. To configure a Docker ensembler, there are 3 sections to be filled.
 

engines/router/missionctl/fiberapi/routing_strategy.go

@@ -64,23 +64,20 @@ func (r *DefaultTuringRoutingStrategy) SelectRoute(
 		close(expResultCh)
 	}
 
-	// If error, return it
+	// If error, log it and return the fallback(s)
 	if expErr != nil {
 		log.WithContext(ctx).Errorf(expErr.Error())
-		return nil, fallbacks, createFiberError(expErr)
+		return nil, fallbacks, nil
 	}
 
-	// selectedRoute is the route that should be visited first based on the the experiment treatment and mappings (if any)
-	// selectedRoute will be nil if the experiment treatment has no corresponding mappings
-	var selectedRoute fiber.Component
-
 	for _, m := range r.experimentMappings {
 		if m.Experiment == expPlan.ExperimentName && m.Treatment == expPlan.Name {
-			selectedRoute = routes[m.Route]
-			// stop matching on first match because only 1 selected route is required
-			break
+			// Stop matching on first match because only 1 route is required. Don't send in fallbacks,
+			// because we do not want to suppress the error from the preferred route.
+			return routes[m.Route], []fiber.Component{}, nil
 		}
 	}
 
-	return selectedRoute, fallbacks, nil
+	// primary route will be nil if there are no matching treatments in the mapping
+	return nil, fallbacks, nil
 }

engines/router/missionctl/fiberapi/routing_strategy_test.go

@@ -161,11 +161,17 @@ func TestDefaultRoutingStrategy(t *testing.T) {
 			expectedRoute:     nil,
 			expectedFallbacks: []fiber.Component{},
 		},
-		"simulate experiment engine returns error when calling GetTreatmentForRequest()": {
-			endpoints:               []string{"treatment-B", "treatment-C"},
+		"error when calling GetTreatmentForRequest() should fallback to default route": {
+			endpoints:               []string{"treatment-B", "control"},
 			experimentRunnerWantErr: true,
 			expectedRoute:           nil,
-			expectedFallbacks:       []fiber.Component{},
+			defaultRoute:            "control",
+			expectedFallbacks: []fiber.Component{
+				fiber.NewProxy(
+					fiber.NewBackend("control", ""),
+					tfu.NewFiberCallerWithHTTPDispatcher(t, "control"),
+				),
+			},
 		},
 	}
 
@@ -194,9 +200,7 @@ func TestDefaultRoutingStrategy(t *testing.T) {
 			}
 			route, fallbacks, err := strategy.SelectRoute(context.Background(), fiberReq, routes)
 
-			if (err != nil) != data.experimentRunnerWantErr {
-				t.Errorf("SelectRoute() error = %v, wantErr %v", err, data.experimentRunnerWantErr)
-			}
+			assert.NoError(t, err)
 			assert.Equal(t, data.expectedRoute, route)
 			assert.Equal(t, data.expectedFallbacks, fallbacks)
 		})

sdk/tests/router/config/router_ensembler_config_test.py

@@ -6,6 +6,7 @@
 from turing.router.config.route import InvalidRouteException
 from turing.router.config.router_ensembler_config import (RouterEnsemblerConfig,
                                                           EnsemblerNopConfig,
+                                                          EnsemblerStandardConfig,
                                                           NopRouterEnsemblerConfig,
                                                           PyfuncRouterEnsemblerConfig,
                                                           DockerRouterEnsemblerConfig,
@@ -17,7 +18,7 @@
         pytest.param(
             1,
             "standard",
-            turing.generated.models.EnsemblerStandardConfig(
+            EnsemblerStandardConfig(
                 experiment_mappings=[
                     turing.generated.models.EnsemblerStandardConfigExperimentMappings(
                         experiment="experiment-1",
@@ -29,7 +30,8 @@
                         treatment="treatment-2",
                         route="route-2"
                     )
-                ]
+                ],
+                fallback_response_route_id="route-1"
             ),
             None,
             "generic_standard_router_ensembler_config"
@@ -324,7 +326,7 @@ def test_create_docker_router_ensembler_config_with_invalid_env(
 
 
 @pytest.mark.parametrize(
-    "experiment_mappings,expected", [
+    "experiment_mappings,fallback_response_route_id,expected", [
         pytest.param(
             [
                 {
@@ -338,18 +340,21 @@ def test_create_docker_router_ensembler_config_with_invalid_env(
                     "route": "route-2"
                 },
             ],
+            "route-1",
             "generic_standard_router_ensembler_config"
         )
     ])
-def test_create_standard_router_ensembler_config(experiment_mappings, expected, request):
+def test_create_standard_router_ensembler_config(experiment_mappings, fallback_response_route_id, expected, request):
     actual = StandardRouterEnsemblerConfig(
-        experiment_mappings=experiment_mappings
-    ).to_open_api()
-    assert actual == request.getfixturevalue(expected)
+        experiment_mappings=experiment_mappings,
+        fallback_response_route_id=fallback_response_route_id,
+    )
+    assert actual.to_open_api() == request.getfixturevalue(expected)
+    assert actual.standard_config.fallback_response_route_id == fallback_response_route_id
 
 
 @pytest.mark.parametrize(
-    "new_experiment_mappings,experiment_mappings,expected", [
+    "new_experiment_mappings,experiment_mappings,fallback_response_route_id,expected", [
         pytest.param(
             [
                 {
@@ -368,6 +373,7 @@ def test_create_standard_router_ensembler_config(experiment_mappings, expected,
                     "route": "route-2"
                 },
             ],
+            "route-1",
             InvalidExperimentMappingException
         ),
         pytest.param(
@@ -390,22 +396,25 @@ def test_create_standard_router_ensembler_config(experiment_mappings, expected,
                     "route": "route-2"
                 },
             ],
+            "route-1",
             InvalidExperimentMappingException
         )
     ])
 def test_set_standard_router_ensembler_config_with_invalid_experiment_mappings(
         new_experiment_mappings,
         experiment_mappings,
+        fallback_response_route_id,
         expected):
     actual = StandardRouterEnsemblerConfig(
-        experiment_mappings=experiment_mappings
+        experiment_mappings=experiment_mappings,
+        fallback_response_route_id=fallback_response_route_id
     )
     with pytest.raises(expected):
         actual.experiment_mappings = new_experiment_mappings
 
 
 @pytest.mark.parametrize(
-    "new_experiment_mappings,experiment_mappings,expected", [
+    "new_experiment_mappings,experiment_mappings,fallback_response_route_id,expected", [
         pytest.param(
             [
                 {
@@ -426,19 +435,23 @@ def test_set_standard_router_ensembler_config_with_invalid_experiment_mappings(
                     "route": "wrong-route"
                 }
             ],
+            "route-1",
             "generic_standard_router_ensembler_config"
         )
     ])
 def test_set_standard_router_ensembler_config_with_valid_experiment_mappings(
         new_experiment_mappings,
         experiment_mappings,
+        fallback_response_route_id,
         expected,
         request):
     actual = StandardRouterEnsemblerConfig(
-        experiment_mappings=experiment_mappings
+        experiment_mappings=experiment_mappings,
+        fallback_response_route_id=fallback_response_route_id
     )
     actual.experiment_mappings = new_experiment_mappings
     assert actual.to_open_api() == request.getfixturevalue(expected)
+    assert actual.standard_config.fallback_response_route_id == fallback_response_route_id
 
 
 @pytest.mark.parametrize(
@@ -454,8 +467,8 @@ def test_create_nop_router_ensembler_config(
     nop_config,
     expected):
     ensembler = NopRouterEnsemblerConfig(final_response_route_id=final_response_route_id)
-    assert ensembler.nop_config == nop_config
     assert ensembler.to_open_api() == expected
+    assert ensembler.nop_config == nop_config
 
 @pytest.mark.parametrize(
     "router_config,ensembler_config", [
@@ -478,6 +491,30 @@ def test_copy_nop_ensembler_default_route(
     expected = router.to_open_api()
     assert actual == expected
 
+@pytest.mark.parametrize(
+    "router_config,ensembler_config", [
+        pytest.param(
+            "generic_router_config",
+           StandardRouterEnsemblerConfig(
+               experiment_mappings=[],
+               fallback_response_route_id="model-b",
+           )
+        )
+    ])
+def test_copy_standard_ensembler_default_route(
+    router_config,
+    ensembler_config,
+    request):
+    router = request.getfixturevalue(router_config)
+    # Check precondition
+    assert router.default_route_id != ensembler_config.fallback_response_route_id
+
+    router.ensembler = ensembler_config
+    actual = router.to_open_api()
+    router.default_route_id = ensembler_config.fallback_response_route_id
+    expected = router.to_open_api()
+    assert actual == expected
+
 @pytest.mark.parametrize(
     "router_config,ensembler_config,expected", [
         pytest.param(

sdk/turing/router/config/router_config.py

@@ -10,7 +10,7 @@
 from turing.router.config.resource_request import ResourceRequest
 from turing.router.config.log_config import LogConfig, ResultLoggerType
 from turing.router.config.enricher import Enricher
-from turing.router.config.router_ensembler_config import RouterEnsemblerConfig, NopRouterEnsemblerConfig
+from turing.router.config.router_ensembler_config import RouterEnsemblerConfig, NopRouterEnsemblerConfig, StandardRouterEnsemblerConfig
 from turing.router.config.experiment_config import ExperimentConfig
 
 
@@ -73,8 +73,8 @@ def __init__(self,
         self.timeout = timeout
         self.log_config = log_config
         self.enricher = enricher
-        # Init nop ensembler config if ensembler is not set
-        self.ensembler = ensembler or NopRouterEnsemblerConfig(final_response_route_id=default_route_id)
+        # Init ensembler after the default route has been initialized
+        self.ensembler = ensembler
 
     @property
     def environment_name(self) -> str:
@@ -198,9 +198,15 @@ def ensembler(self) -> RouterEnsemblerConfig:
 
     @ensembler.setter
     def ensembler(self, ensembler: Union[RouterEnsemblerConfig, Dict]):
-        if isinstance(ensembler, RouterEnsemblerConfig):
+        if ensembler is None:
+            # Init nop ensembler config if ensembler is not set
+            self._ensembler = NopRouterEnsemblerConfig(final_response_route_id=self.default_route_id)
+        elif isinstance(ensembler, RouterEnsemblerConfig):
             self._ensembler = ensembler
         elif isinstance(ensembler, dict):
+            # Set fallback_response_route_id into standard ensembler config
+            if ensembler["type"] == "standard" and "fallback_response_route_id" not in ensembler["standard_config"]:
+                ensembler["standard_config"]["fallback_response_route_id"] = self.default_route_id
             self._ensembler = RouterEnsemblerConfig(**ensembler)
         else:
             self._ensembler = ensembler
@@ -239,10 +245,11 @@ def to_open_api(self) -> OpenApiModel:
     def _get_default_route_id(self):
         default_route_id = self.default_route_id
         # If nop config is set, use the final_response_route_id as the default
-        if (self.ensembler.type == "nop" and
-            self.ensembler.nop_config is not None and
-            self.ensembler.nop_config.final_response_route_id is not None):
-            default_route_id = self.ensembler.nop_config.final_response_route_id
+        if isinstance(self.ensembler, NopRouterEnsemblerConfig):
+            default_route_id = self.ensembler.final_response_route_id
+        # Or, if standard config is set, use the fallback_response_route_id as the default
+        elif isinstance(self.ensembler, StandardRouterEnsemblerConfig):
+            default_route_id = self.ensembler.fallback_response_route_id
         self._verify_default_route_exists(default_route_id)
         return default_route_id