Acknowledge network teardowns

members/nullnet-client/src/control_channel.rs

@@ -34,6 +34,33 @@ fn fire_event(grpc: &NullnetGrpcInterface, kind: AgentEventKind) {
     });
 }
 
+/// Confirm a completed teardown so the server can return the net id to its
+/// pool. Must be called only once the teardown has actually run — the whole
+/// point of the ack is that the id stays out of circulation until this edge's
+/// kernel state is gone.
+///
+/// `msg_id` is absent when the server predates the ack field; there is then
+/// nothing to confirm and the server falls back to its grace timer.
+async fn ack_teardown(
+    outbound: &Sender<MsgId>,
+    msg_id: Option<MsgId>,
+    grpc: &NullnetGrpcInterface,
+    message_type: &str,
+) {
+    let Some(msg_id) = msg_id else {
+        return;
+    };
+    if outbound.send(msg_id.clone()).await.is_err() {
+        fire_event(
+            grpc,
+            AgentEventKind::ControlChannelAckFailed(AgentControlChannelAckFailed {
+                msg_id: msg_id.id,
+                message_type: message_type.to_string(),
+            }),
+        );
+    }
+}
+
 #[allow(clippy::too_many_arguments)]
 pub(crate) async fn control_channel(
     server: NullnetGrpcInterface,
@@ -87,6 +114,7 @@ pub(crate) async fn control_channel(
                         vlan_teardown,
                         rtnetlink_handle,
                         peers,
+                        outbound,
                         host_mappings_state,
                         server,
                         firewall_peers,
@@ -118,12 +146,14 @@ pub(crate) async fn control_channel(
                     handle_vxlan_teardown(
                         vxlan_teardown,
                         triggers_state,
+                        outbound,
                         host_mappings_state,
                         server,
                         firewall_peers,
                         firewall_vxlan_ports,
                         egress_state,
-                    );
+                    )
+                    .await;
                 });
             }
             Some(net_message::Message::ContainerSuspend(container_suspend)) => {
@@ -290,10 +320,12 @@ async fn handle_vlan_teardown(
     message: VlanTeardown,
     rtnetlink_handle: RtNetLinkHandle,
     peers: Arc<RwLock<Peers>>,
+    outbound: Sender<MsgId>,
     host_mappings_state: Arc<HostMappingsState>,
     grpc: NullnetGrpcInterface,
     firewall_peers: Arc<FirewallPeers>,
 ) -> Result<(), Error> {
+    let ack_id = message.msg_id.clone();
     let vlan_id = u16::try_from(message.vlan_id)
         .handle_err(location!())
         .inspect_err(|e| {
@@ -327,6 +359,10 @@ async fn handle_vlan_teardown(
         let _ = remove_host_mapping(&host_mapping, None);
     }
 
+    // Acked last: the server frees the net id on this, so everything above must
+    // already be undone.
+    ack_teardown(&outbound, ack_id, &grpc, "vlan_teardown").await;
+
     Ok(())
 }
 
@@ -624,15 +660,18 @@ async fn handle_vxlan_setup(
     Ok(())
 }
 
-fn handle_vxlan_teardown(
+#[allow(clippy::too_many_arguments)]
+async fn handle_vxlan_teardown(
     message: VxlanTeardown,
     triggers_state: Arc<TriggersState>,
+    outbound: Sender<MsgId>,
     host_mappings_state: Arc<HostMappingsState>,
     grpc: NullnetGrpcInterface,
     firewall_peers: Arc<FirewallPeers>,
     firewall_vxlan_ports: Arc<FirewallVxlanPorts>,
     egress_state: Arc<EgressState>,
 ) {
+    let ack_id = message.msg_id.clone();
     // reverse egress steering/interception if this was an egress edge
     if let Some(rec) = egress_state.take(message.vxlan_id) {
         match rec {
@@ -719,6 +758,11 @@ fn handle_vxlan_teardown(
         "VXLAN teardown completed in {} ms",
         init_t.elapsed().as_millis()
     );
+
+    // Acked last: the server frees the net id on this, so every kernel object
+    // named after it — bridge, veth/macsec pair, XFRM SA, DNAT — must already
+    // be gone.
+    ack_teardown(&outbound, ack_id, &grpc, "vxlan_teardown").await;
 }
 
 /// Pause an idle container. Fire-and-forget: the server marks the replica

members/nullnet-grpc-lib/proto/nullnet_grpc.proto

@@ -146,6 +146,11 @@ message VlanSetup {
 
 message VlanTeardown {
   uint32 vlan_id = 1;
+  // Acked once the teardown has actually run, so the server can hold the net id
+  // out of the pool until this edge is really gone. Optional: a client that
+  // predates this field simply never acks, and the server frees on its grace
+  // timer instead.
+  optional MsgId msg_id = 2;
 }
 
 message VxlanSetup {
@@ -199,6 +204,11 @@ message VxlanTeardown {
   string local_ip = 5;
   string remote_ip = 6;
   uint32 dstport = 7;
+  // Acked once the teardown has actually run, so the server can hold the net id
+  // out of the pool until this edge is really gone. Optional: a client that
+  // predates this field simply never acks, and the server frees on its grace
+  // timer instead.
+  optional MsgId msg_id = 8;
 }
 
 message MsgId {

members/nullnet-grpc-lib/src/proto/nullnet_grpc.rs

@@ -99,10 +99,16 @@ pub struct VlanSetup {
     #[prost(bool, tag = "9")]
     pub encrypted: bool,
 }
-#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct VlanTeardown {
     #[prost(uint32, tag = "1")]
     pub vlan_id: u32,
+    /// Acked once the teardown has actually run, so the server can hold the net id
+    /// out of the pool until this edge is really gone. Optional: a client that
+    /// predates this field simply never acks, and the server frees on its grace
+    /// timer instead.
+    #[prost(message, optional, tag = "2")]
+    pub msg_id: ::core::option::Option<MsgId>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct VxlanSetup {
@@ -179,6 +185,12 @@ pub struct VxlanTeardown {
     pub remote_ip: ::prost::alloc::string::String,
     #[prost(uint32, tag = "7")]
     pub dstport: u32,
+    /// Acked once the teardown has actually run, so the server can hold the net id
+    /// out of the pool until this edge is really gone. Optional: a client that
+    /// predates this field simply never acks, and the server frees on its grace
+    /// timer instead.
+    #[prost(message, optional, tag = "8")]
+    pub msg_id: ::core::option::Option<MsgId>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct MsgId {

members/nullnet-server/src/events.rs

@@ -85,6 +85,14 @@ pub(crate) enum Event {
         client_ip: String,
         timestamp: u64,
     },
+    /// An endpoint never confirmed a teardown, so the net id went back to the
+    /// pool unverified. Its kernel state may still exist on that node, and a
+    /// later edge reusing the id would collide with it.
+    NetTeardownUnconfirmed {
+        net_id: u32,
+        node_ip: String,
+        timestamp: u64,
+    },
     ConfigReloaded {
         stack: String,
         timestamp: u64,
@@ -350,6 +358,7 @@ impl Event {
             Self::SetupTimeout { .. } => "setup_timeout",
             Self::SessionCreated { .. } => "session_created",
             Self::SessionTornDown { .. } => "session_torn_down",
+            Self::NetTeardownUnconfirmed { .. } => "net_teardown_unconfirmed",
             Self::ConfigReloaded { .. } => "config_reloaded",
             Self::ConfigStackRemoved { .. } => "config_stack_removed",
             Self::PortMappingConflict { .. } => "port_mapping_conflict",
@@ -428,6 +437,7 @@ impl Event {
             | Self::MaxNetworksLimitEnforced { .. }
             | Self::BackendTriggerSetupBailed { .. }
             | Self::ControlChannelClosed { .. }
+            | Self::NetTeardownUnconfirmed { .. }
             | Self::CertificateRemoved { .. } => Severity::Warning,
 
             Self::SetupTimeout { .. }
@@ -549,6 +559,14 @@ impl Event {
         }
     }
 
+    pub(crate) fn net_teardown_unconfirmed(net_id: u32, node_ip: String) -> Self {
+        Self::NetTeardownUnconfirmed {
+            net_id,
+            node_ip,
+            timestamp: now_secs(),
+        }
+    }
+
     pub(crate) fn config_reloaded(stack: String) -> Self {
         Self::ConfigReloaded {
             stack,

members/nullnet-server/src/net.rs

@@ -48,6 +48,7 @@ pub(crate) trait NetExt {
         local_ip: IpAddr,
         remote_ip: IpAddr,
         dstport: Option<u16>,
+        msg_id: String,
     ) -> NetMessage;
 }
 
@@ -100,11 +101,14 @@ impl NetExt for Net {
         local_ip: IpAddr,
         remote_ip: IpAddr,
         dstport: Option<u16>,
+        msg_id: String,
     ) -> NetMessage {
+        let msg_id = Some(MsgId { id: msg_id });
         match self {
             Net::Vlan => NetMessage {
                 message: Some(net_message::Message::VlanTeardown(VlanTeardown {
                     vlan_id: net_id,
+                    msg_id,
                 })),
             },
             Net::Vxlan => NetMessage {
@@ -116,6 +120,7 @@ impl NetExt for Net {
                     local_ip: local_ip.to_string(),
                     remote_ip: remote_ip.to_string(),
                     dstport: u32::from(dstport.unwrap_or(DEFAULT_VXLAN_DSTPORT)),
+                    msg_id,
                 })),
             },
         }