Implements the search cutoff

assets/grafana-dashboard.json

@@ -238,6 +238,70 @@
       "title": "Total Searches (1h)",
       "type": "gauge"
     },
+    {
+      "datasource": {
+        "type": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 8,
+        "y": 1
+      },
+      "id": 26,
+      "options": {
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "text": {}
+      },
+      "pluginVersion": "9.5.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus"
+          },
+          "editorMode": "builder",
+          "exemplar": true,
+          "expr": "round(increase(meilisearch_degraded_search_requests{job=\"$job\"}[1h]))",
+          "interval": "",
+          "legendFormat": "",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Total Degraded Searches (1h)",
+      "type": "gauge"
+    },
     {
       "datasource": {
         "type": "prometheus"

dump/src/lib.rs

@@ -277,6 +277,7 @@ pub(crate) mod test {
             }),
             pagination: Setting::NotSet,
             embedders: Setting::NotSet,
+            search_cutoff_ms: Setting::NotSet,
             _kind: std::marker::PhantomData,
         };
         settings.check()

dump/src/reader/compat/v5_to_v6.rs

@@ -379,6 +379,7 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
                 v5::Setting::NotSet => v6::Setting::NotSet,
             },
             embedders: v6::Setting::NotSet,
+            search_cutoff_ms: v6::Setting::NotSet,
             _kind: std::marker::PhantomData,
         }
     }

meilisearch-types/src/error.rs

@@ -259,6 +259,7 @@ InvalidSettingsProximityPrecision     , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsFaceting               , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsFilterableAttributes   , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsPagination             , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsSearchCutoffMs           , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsEmbedders              , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsRankingRules           , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSearchableAttributes   , InvalidRequest       , BAD_REQUEST ;

meilisearch-types/src/settings.rs

@@ -202,6 +202,9 @@ pub struct Settings<T> {
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsEmbedders>)]
     pub embedders: Setting<BTreeMap<String, Setting<milli::vector::settings::EmbeddingSettings>>>,
+    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
+    #[deserr(default, error = DeserrJsonError<InvalidSettingsSearchCutoffMs>)]
+    pub search_cutoff_ms: Setting<u64>,
 
     #[serde(skip)]
     #[deserr(skip)]
@@ -227,6 +230,7 @@ impl Settings<Checked> {
             faceting: Setting::Reset,
             pagination: Setting::Reset,
             embedders: Setting::Reset,
+            search_cutoff_ms: Setting::Reset,
             _kind: PhantomData,
         }
     }
@@ -249,6 +253,7 @@ impl Settings<Checked> {
             faceting,
             pagination,
             embedders,
+            search_cutoff_ms,
             ..
         } = self;
 
@@ -269,6 +274,7 @@ impl Settings<Checked> {
             faceting,
             pagination,
             embedders,
+            search_cutoff_ms,
             _kind: PhantomData,
         }
     }
@@ -315,6 +321,7 @@ impl Settings<Unchecked> {
             faceting: self.faceting,
             pagination: self.pagination,
             embedders: self.embedders,
+            search_cutoff_ms: self.search_cutoff_ms,
             _kind: PhantomData,
         }
     }
@@ -347,87 +354,108 @@ pub fn apply_settings_to_builder(
     settings: &Settings<Checked>,
     builder: &mut milli::update::Settings,
 ) {
-    match settings.searchable_attributes {
+    let Settings {
+        displayed_attributes,
+        searchable_attributes,
+        filterable_attributes,
+        sortable_attributes,
+        ranking_rules,
+        stop_words,
+        non_separator_tokens,
+        separator_tokens,
+        dictionary,
+        synonyms,
+        distinct_attribute,
+        proximity_precision,
+        typo_tolerance,
+        faceting,
+        pagination,
+        embedders,
+        search_cutoff_ms,
+        _kind,
+    } = settings;
+
+    match searchable_attributes {
         Setting::Set(ref names) => builder.set_searchable_fields(names.clone()),
         Setting::Reset => builder.reset_searchable_fields(),
         Setting::NotSet => (),
     }
 
-    match settings.displayed_attributes {
+    match displayed_attributes {
         Setting::Set(ref names) => builder.set_displayed_fields(names.clone()),
         Setting::Reset => builder.reset_displayed_fields(),
         Setting::NotSet => (),
     }
 
-    match settings.filterable_attributes {
+    match filterable_attributes {
         Setting::Set(ref facets) => {
             builder.set_filterable_fields(facets.clone().into_iter().collect())
         }
         Setting::Reset => builder.reset_filterable_fields(),
         Setting::NotSet => (),
     }
 
-    match settings.sortable_attributes {
+    match sortable_attributes {
         Setting::Set(ref fields) => builder.set_sortable_fields(fields.iter().cloned().collect()),
         Setting::Reset => builder.reset_sortable_fields(),
         Setting::NotSet => (),
     }
 
-    match settings.ranking_rules {
+    match ranking_rules {
         Setting::Set(ref criteria) => {
             builder.set_criteria(criteria.iter().map(|c| c.clone().into()).collect())
         }
         Setting::Reset => builder.reset_criteria(),
         Setting::NotSet => (),
     }
 
-    match settings.stop_words {
+    match stop_words {
         Setting::Set(ref stop_words) => builder.set_stop_words(stop_words.clone()),
         Setting::Reset => builder.reset_stop_words(),
         Setting::NotSet => (),
     }
 
-    match settings.non_separator_tokens {
+    match non_separator_tokens {
         Setting::Set(ref non_separator_tokens) => {
             builder.set_non_separator_tokens(non_separator_tokens.clone())
         }
         Setting::Reset => builder.reset_non_separator_tokens(),
         Setting::NotSet => (),
     }
 
-    match settings.separator_tokens {
+    match separator_tokens {
         Setting::Set(ref separator_tokens) => {
             builder.set_separator_tokens(separator_tokens.clone())
         }
         Setting::Reset => builder.reset_separator_tokens(),
         Setting::NotSet => (),
     }
 
-    match settings.dictionary {
+    match dictionary {
         Setting::Set(ref dictionary) => builder.set_dictionary(dictionary.clone()),
         Setting::Reset => builder.reset_dictionary(),
         Setting::NotSet => (),
     }
 
-    match settings.synonyms {
+    match synonyms {
         Setting::Set(ref synonyms) => builder.set_synonyms(synonyms.clone().into_iter().collect()),
         Setting::Reset => builder.reset_synonyms(),
         Setting::NotSet => (),
     }
 
-    match settings.distinct_attribute {
+    match distinct_attribute {
         Setting::Set(ref attr) => builder.set_distinct_field(attr.clone()),
         Setting::Reset => builder.reset_distinct_field(),
         Setting::NotSet => (),
     }
 
-    match settings.proximity_precision {
+    match proximity_precision {
         Setting::Set(ref precision) => builder.set_proximity_precision((*precision).into()),
         Setting::Reset => builder.reset_proximity_precision(),
         Setting::NotSet => (),
     }
 
-    match settings.typo_tolerance {
+    match typo_tolerance {
         Setting::Set(ref value) => {
             match value.enabled {
                 Setting::Set(val) => builder.set_autorize_typos(val),
@@ -482,7 +510,7 @@ pub fn apply_settings_to_builder(
         Setting::NotSet => (),
     }
 
-    match &settings.faceting {
+    match faceting {
         Setting::Set(FacetingSettings { max_values_per_facet, sort_facet_values_by }) => {
             match max_values_per_facet {
                 Setting::Set(val) => builder.set_max_values_per_facet(*val),
@@ -504,7 +532,7 @@ pub fn apply_settings_to_builder(
         Setting::NotSet => (),
     }
 
-    match settings.pagination {
+    match pagination {
         Setting::Set(ref value) => match value.max_total_hits {
             Setting::Set(val) => builder.set_pagination_max_total_hits(val),
             Setting::Reset => builder.reset_pagination_max_total_hits(),
@@ -514,11 +542,17 @@ pub fn apply_settings_to_builder(
         Setting::NotSet => (),
     }
 
-    match settings.embedders.clone() {
-        Setting::Set(value) => builder.set_embedder_settings(value),
+    match embedders {
+        Setting::Set(value) => builder.set_embedder_settings(value.clone()),
         Setting::Reset => builder.reset_embedder_settings(),
         Setting::NotSet => (),
     }
+
+    match search_cutoff_ms {
+        Setting::Set(cutoff) => builder.set_search_cutoff(*cutoff),
+        Setting::Reset => builder.reset_search_cutoff(),
+        Setting::NotSet => (),
+    }
 }
 
 pub fn settings(
@@ -607,6 +641,8 @@ pub fn settings(
         .collect();
     let embedders = if embedders.is_empty() { Setting::NotSet } else { Setting::Set(embedders) };
 
+    let search_cutoff_ms = index.search_cutoff(rtxn)?;
+
     Ok(Settings {
         displayed_attributes: match displayed_attributes {
             Some(attrs) => Setting::Set(attrs),
@@ -633,6 +669,10 @@ pub fn settings(
         faceting: Setting::Set(faceting),
         pagination: Setting::Set(pagination),
         embedders,
+        search_cutoff_ms: match search_cutoff_ms {
+            Some(cutoff) => Setting::Set(cutoff),
+            None => Setting::Reset,
+        },
         _kind: PhantomData,
     })
 }
@@ -783,6 +823,7 @@ pub(crate) mod test {
             faceting: Setting::NotSet,
             pagination: Setting::NotSet,
             embedders: Setting::NotSet,
+            search_cutoff_ms: Setting::NotSet,
             _kind: PhantomData::<Unchecked>,
         };
 
@@ -809,6 +850,7 @@ pub(crate) mod test {
             faceting: Setting::NotSet,
             pagination: Setting::NotSet,
             embedders: Setting::NotSet,
+            search_cutoff_ms: Setting::NotSet,
             _kind: PhantomData::<Unchecked>,
         };
 

meilisearch/src/analytics/segment_analytics.rs

@@ -579,6 +579,7 @@ pub struct SearchAggregator {
     // requests
     total_received: usize,
     total_succeeded: usize,
+    total_degraded: usize,
     time_spent: BinaryHeap<usize>,
 
     // sort
@@ -758,9 +759,13 @@ impl SearchAggregator {
             hits_info: _,
             facet_distribution: _,
             facet_stats: _,
+            degraded,
         } = result;
 
         self.total_succeeded = self.total_succeeded.saturating_add(1);
+        if *degraded {
+            self.total_degraded = self.total_degraded.saturating_add(1);
+        }
         self.time_spent.push(*processing_time_ms as usize);
     }
 
@@ -802,6 +807,7 @@ impl SearchAggregator {
             semantic_ratio,
             embedder,
             hybrid,
+            total_degraded,
         } = other;
 
         if self.timestamp.is_none() {
@@ -816,6 +822,7 @@ impl SearchAggregator {
         // request
         self.total_received = self.total_received.saturating_add(total_received);
         self.total_succeeded = self.total_succeeded.saturating_add(total_succeeded);
+        self.total_degraded = self.total_degraded.saturating_add(total_degraded);
         self.time_spent.append(time_spent);
 
         // sort
@@ -921,6 +928,7 @@ impl SearchAggregator {
             semantic_ratio,
             embedder,
             hybrid,
+            total_degraded,
         } = self;
 
         if total_received == 0 {
@@ -940,6 +948,7 @@ impl SearchAggregator {
                     "total_succeeded": total_succeeded,
                     "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics
                     "total_received": total_received,
+                    "total_degraded": total_degraded,
                 },
                 "sort": {
                     "with_geoPoint": sort_with_geo_point,

meilisearch/src/metrics.rs

@@ -22,6 +22,11 @@ lazy_static! {
         &["method", "path"]
     )
     .expect("Can't create a metric");
+    pub static ref MEILISEARCH_DEGRADED_SEARCH_REQUESTS: IntGauge = register_int_gauge!(opts!(
+        "meilisearch_degraded_search_requests",
+        "Meilisearch number of degraded search requests"
+    ))
+    .expect("Can't create a metric");
     pub static ref MEILISEARCH_DB_SIZE_BYTES: IntGauge =
         register_int_gauge!(opts!("meilisearch_db_size_bytes", "Meilisearch DB Size In Bytes"))
             .expect("Can't create a metric");