OS-8000 viona ring reset can race with Rx packets

Reviewed by: Mike Zeller <mike.zeller@joyent.com>
Reviewed by: Patrick Mooney <pmooney@pfmooney.com>
Reviewed by: Dan McDonald <danmcd@joyent.com>
Approved by: Mike Zeller <mike.zeller@joyent.com>

usr/src/uts/i86pc/io/viona/viona_impl.h

@@ -70,6 +70,7 @@ enum viona_ring_state {
 	VRS_SETUP	= 0x1,	/* addrs setup and starting worker thread */
 	VRS_INIT	= 0x2,	/* worker thread started & waiting to run */
 	VRS_RUN		= 0x3,	/* running work routine */
+	VRS_STOP	= 0x4,	/* worker is exiting */
 };
 enum viona_ring_state_flags {
 	VRSF_REQ_START	= 0x1,	/* start running from INIT state */

usr/src/uts/i86pc/io/viona/viona_main.c

@@ -133,8 +133,14 @@
  *  +-----------+
  *        |						^
  *        |---* ioctl(VNA_IOC_RING_RESET) issued	|
- *        |	(or bhyve process begins exit)		|
- *        V						|
+ *        |	(or bhyve process begins exit)		^
+ *        |
+ *  +-----------+	The worker thread associated with the ring is in the
+ *  | VRS_STOP  |	process of exiting. All outstanding TX and RX
+ *  +-----------+	requests are allowed to complete, but new requests
+ *        |		must be ignored.
+ *        |						^
+ *        |						|
  *        +-------------------------------------------->+
  *
  *

usr/src/uts/i86pc/io/viona/viona_ring.c

@@ -389,6 +389,8 @@ viona_worker(void *arg)
 		panic("unexpected ring: %p", (void *)ring);
 	}
 
+	VERIFY3U(ring->vr_state, ==, VRS_STOP);
+
 cleanup:
 	if (ring->vr_txdesb != NULL) {
 		/*

usr/src/uts/i86pc/io/viona/viona_rx.c

@@ -123,6 +123,17 @@ viona_worker_rx(viona_vring_t *ring, viona_link_t *link)
 		(void) cv_wait_sig(&ring->vr_cv, &ring->vr_lock);
 	} while (!VRING_NEED_BAIL(ring, p));
 
+	ring->vr_state = VRS_STOP;
+
+	/*
+	 * The RX ring is stopping, before we start tearing it down it
+	 * is imperative that we perform an RX barrier so that
+	 * incoming packets are dropped at viona_rx_classified().
+	 */
+	mutex_exit(&ring->vr_lock);
+	mac_rx_barrier(link->l_mch);
+	mutex_enter(&ring->vr_lock);
+
 	*ring->vr_used_flags &= ~VRING_USED_F_NO_NOTIFY;
 }
 

usr/src/uts/i86pc/io/viona/viona_tx.c

@@ -283,6 +283,7 @@ viona_worker_tx(viona_vring_t *ring, viona_link_t *link)
 
 	ASSERT(MUTEX_HELD(&ring->vr_lock));
 
+	ring->vr_state = VRS_STOP;
 	viona_tx_wait_outstanding(ring);
 }