nexus: revamp live migration saga

common/src/api/external/error.rs

@@ -61,6 +61,9 @@ pub enum Error {
 
     #[error("Type version mismatch! {internal_message}")]
     TypeVersionMismatch { internal_message: String },
+
+    #[error("Conflict: {internal_message}")]
+    Conflict { internal_message: String },
 }
 
 /// Indicates how an object was looked up (for an `ObjectNotFound` error)
@@ -118,7 +121,8 @@ impl Error {
             | Error::Forbidden
             | Error::MethodNotAllowed { .. }
             | Error::InternalError { .. }
-            | Error::TypeVersionMismatch { .. } => false,
+            | Error::TypeVersionMismatch { .. }
+            | Error::Conflict { .. } => false,
         }
     }
 
@@ -174,6 +178,18 @@ impl Error {
         Error::TypeVersionMismatch { internal_message: message.to_owned() }
     }
 
+    /// Generates an [`Error::Conflict`] with a specific message.
+    ///
+    /// This is used in cases where a request cannot proceed because the target
+    /// resource is currently in a state that's incompatible with that request,
+    /// but where the request might succeed if it is retried or modified and
+    /// retried. The internal message should provide more information about the
+    /// source of the conflict and possible actions the caller can take to
+    /// resolve it (if any).
+    pub fn conflict(message: &str) -> Error {
+        Error::Conflict { internal_message: message.to_owned() }
+    }
+
     /// Given an [`Error`] with an internal message, return the same error with
     /// `context` prepended to it to provide more context
     ///
@@ -223,6 +239,9 @@ impl Error {
                     ),
                 }
             }
+            Error::Conflict { internal_message } => Error::Conflict {
+                internal_message: format!("{}: {}", context, internal_message),
+            },
         }
     }
 }
@@ -317,6 +336,14 @@ impl From<Error> for HttpError {
             Error::TypeVersionMismatch { internal_message } => {
                 HttpError::for_internal_error(internal_message)
             }
+
+            Error::Conflict { internal_message } => {
+                HttpError::for_client_error(
+                    Some(String::from("Conflict")),
+                    http::StatusCode::CONFLICT,
+                    internal_message,
+                )
+            }
         }
     }
 }

nexus/db-model/src/instance.rs

@@ -98,7 +98,10 @@ impl DatastoreAttachTargetConfig<Disk> for Instance {
     Serialize,
     Deserialize,
 )]
-#[diesel(table_name = instance)]
+// N.B. Setting `treat_none_as_null` is required for these fields to be cleared
+//      properly during live migrations. See the documentation for
+//      `diesel::prelude::AsChangeset`.
+#[diesel(table_name = instance, treat_none_as_null = true)]
 pub struct InstanceRuntimeState {
     /// The instance's current user-visible instance state.
     ///

nexus/db-model/src/sled.rs

@@ -143,3 +143,54 @@ impl DatastoreCollectionConfig<super::Service> for Sled {
     type CollectionTimeDeletedColumn = sled::dsl::time_deleted;
     type CollectionIdColumn = service::dsl::sled_id;
 }
+
+/// A set of constraints that can be placed on operations that select a sled.
+#[derive(Debug)]
+pub struct SledReservationConstraints {
+    must_select_from: Vec<Uuid>,
+}
+
+impl SledReservationConstraints {
+    /// Creates a constraint set with no constraints in it.
+    pub fn none() -> Self {
+        Self { must_select_from: Vec::new() }
+    }
+
+    /// If the constraints include a set of sleds that the caller must select
+    /// from, returns `Some` and a slice containing the members of that set.
+    ///
+    /// If no "must select from these" constraint exists, returns None.
+    pub fn must_select_from(&self) -> Option<&[Uuid]> {
+        if self.must_select_from.is_empty() {
+            None
+        } else {
+            Some(&self.must_select_from)
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct SledReservationConstraintBuilder {
+    constraints: SledReservationConstraints,
+}
+
+impl SledReservationConstraintBuilder {
+    pub fn new() -> Self {
+        SledReservationConstraintBuilder {
+            constraints: SledReservationConstraints::none(),
+        }
+    }
+
+    /// Adds a "must select from the following sled IDs" constraint. If such a
+    /// constraint already exists, appends the supplied sled IDs to the "must
+    /// select from" list.
+    pub fn must_select_from(mut self, sled_ids: &[Uuid]) -> Self {
+        self.constraints.must_select_from.extend(sled_ids);
+        self
+    }
+
+    /// Builds a set of constraints from this builder's current state.
+    pub fn build(self) -> SledReservationConstraints {
+        self.constraints
+    }
+}

nexus/db-queries/src/db/datastore/sled.rs

@@ -76,6 +76,7 @@ impl DataStore {
         resource_id: Uuid,
         resource_kind: db::model::SledResourceKind,
         resources: db::model::Resources,
+        constraints: db::model::SledReservationConstraints,
     ) -> CreateResult<db::model::SledResource> {
         #[derive(Debug)]
         enum SledReservationError {
@@ -120,10 +121,10 @@ impl DataStore {
                         resource_dsl::rss_ram::NAME
                     )) + resources.rss_ram)
                         .le(sled_dsl::usable_physical_ram);
-                sql_function!(fn random() -> diesel::sql_types::Float);
-                let sled_targets = sled_dsl::sled
-                    // LEFT JOIN so we can observe sleds with no
-                    // currently-allocated resources as potential targets
+
+                // Generate a query describing all of the sleds that have space
+                // for this reservation.
+                let mut sled_targets = sled_dsl::sled
                     .left_join(
                         resource_dsl::sled_resource
                             .on(resource_dsl::sled_id.eq(sled_dsl::id)),
@@ -135,6 +136,17 @@ impl DataStore {
                     )
                     .filter(sled_dsl::time_deleted.is_null())
                     .select(sled_dsl::id)
+                    .into_boxed();
+
+                // Further constrain the sled IDs according to any caller-
+                // supplied constraints.
+                if let Some(must_select_from) = constraints.must_select_from() {
+                    sled_targets = sled_targets
+                        .filter(sled_dsl::id.eq_any(must_select_from.to_vec()));
+                }
+
+                sql_function!(fn random() -> diesel::sql_types::Float);
+                let sled_targets = sled_targets
                     .order(random())
                     .limit(1)
                     .get_results_async::<Uuid>(&conn)