DAOS-11092 bio: restrict inflight ios

src/bio/bio_buffer.c

@@ -1062,9 +1062,7 @@ nvme_rw(struct bio_desc *biod, struct bio_rsrvd_region *rg)
 	pg_cnt -= pg_idx;
 
 	while (pg_cnt > 0) {
-		/* NVMe poll needs be scheduled */
-		if (bio_need_nvme_poll(xs_ctxt))
-			bio_yield();
+		drain_inflight_ios(xs_ctxt);
 
 		biod->bd_inflights++;
 		xs_ctxt->bxc_blob_rw++;

src/bio/bio_context.c

@@ -725,8 +725,9 @@ bio_blob_unmap(struct bio_io_context *ioctxt, uint64_t off, uint64_t len)
 	return rc;
 }
 
-int
-bio_blob_unmap_sgl(struct bio_io_context *ioctxt, d_sg_list_t *unmap_sgl, uint32_t blk_sz)
+static int
+blob_unmap_sgl(struct bio_io_context *ioctxt, d_sg_list_t *unmap_sgl, uint32_t blk_sz,
+	       unsigned int start_idx, unsigned int unmap_cnt)
 {
 	struct bio_xs_context	*xs_ctxt;
 	struct blob_msg_arg	 bma = { 0 };
@@ -736,7 +737,6 @@ bio_blob_unmap_sgl(struct bio_io_context *ioctxt, d_sg_list_t *unmap_sgl, uint32
 	uint64_t		 pg_off, pg_cnt;
 	int			 i, rc;
 
-	D_ASSERT(blk_sz >= ioctxt->bic_io_unit && (blk_sz & (ioctxt->bic_io_unit - 1)) == 0);
 	xs_ctxt = ioctxt->bic_xs_ctxt;
 	D_ASSERT(xs_ctxt != NULL);
 	channel = xs_ctxt->bxc_io_channel;
@@ -754,12 +754,14 @@ bio_blob_unmap_sgl(struct bio_io_context *ioctxt, d_sg_list_t *unmap_sgl, uint32
 	bma.bma_ioc = ioctxt;
 	ioctxt->bic_inflight_dmas++;
 	ba->bca_inflights = 1;
-	for (i = 0; i < unmap_sgl->sg_nr_out; i++) {
+
+	i = start_idx;
+	while (unmap_cnt > 0) {
 		unmap_iov = &unmap_sgl->sg_iovs[i];
+		i++;
+		unmap_cnt--;
 
-		/* NVMe poll needs be scheduled */
-		if (bio_need_nvme_poll(xs_ctxt))
-			bio_yield();
+		drain_inflight_ios(xs_ctxt);
 
 		ba->bca_inflights++;
 		xs_ctxt->bxc_blob_rw++;
@@ -777,7 +779,8 @@ bio_blob_unmap_sgl(struct bio_io_context *ioctxt, d_sg_list_t *unmap_sgl, uint32
 	}
 	ba->bca_inflights--;
 
-	blob_wait_completion(xs_ctxt, ba);
+	if (ba->bca_inflights > 0)
+		blob_wait_completion(xs_ctxt, ba);
 	rc = ba->bca_rc;
 	ioctxt->bic_inflight_dmas--;
 
@@ -801,6 +804,30 @@ bio_blob_unmap_sgl(struct bio_io_context *ioctxt, d_sg_list_t *unmap_sgl, uint32
 	return rc;
 }
 
+int
+bio_blob_unmap_sgl(struct bio_io_context *ioctxt, d_sg_list_t *unmap_sgl, uint32_t blk_sz)
+{
+	unsigned int	start_idx, tot_unmap_cnt, unmap_cnt;
+	int		rc = 0;
+
+	D_ASSERT(blk_sz >= ioctxt->bic_io_unit && (blk_sz & (ioctxt->bic_io_unit - 1)) == 0);
+
+	tot_unmap_cnt = unmap_sgl->sg_nr_out;
+	start_idx = 0;
+	while (tot_unmap_cnt > 0) {
+		unmap_cnt = min(tot_unmap_cnt, bio_spdk_max_unmap_cnt);
+
+		rc = blob_unmap_sgl(ioctxt, unmap_sgl, blk_sz, start_idx, unmap_cnt);
+		if (rc)
+			break;
+
+		tot_unmap_cnt -= unmap_cnt;
+		start_idx += unmap_cnt;
+	}
+
+	return rc;
+}
+
 int
 bio_write_blob_hdr(struct bio_io_context *ioctxt, struct bio_blob_hdr *bio_bh)
 {

src/bio/bio_internal.h

@@ -508,6 +508,7 @@ extern bool		bio_spdk_inited;
 extern unsigned int	bio_chk_sz;
 extern unsigned int	bio_chk_cnt_max;
 extern unsigned int	bio_numa_node;
+extern unsigned int	bio_spdk_max_unmap_cnt;
 int xs_poll_completion(struct bio_xs_context *ctxt, unsigned int *inflights,
 		       uint64_t timeout);
 void bio_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev,
@@ -529,6 +530,7 @@ void setup_bio_bdev(void *arg);
 void destroy_bio_bdev(struct bio_bdev *d_bdev);
 void replace_bio_bdev(struct bio_bdev *old_dev, struct bio_bdev *new_dev);
 bool bypass_health_collect(void);
+void drain_inflight_ios(struct bio_xs_context *ctxt);
 
 /* bio_buffer.c */
 void dma_buffer_destroy(struct bio_dma_buffer *buf);

src/bio/bio_xstream.c

@@ -37,6 +37,8 @@
 #define BIO_BS_MAX_CHANNEL_OPS	(4096)
 /* Schedule a NVMe poll when so many blob IOs queued for an io channel */
 #define BIO_BS_POLL_WATERMARK	(2048)
+/* Stop issuing new IO when queued blob IOs reach a threshold */
+#define BIO_BS_STOP_WATERMARK	(4000)
 
 /* Chunk size of DMA buffer in pages */
 unsigned int bio_chk_sz;
@@ -52,6 +54,8 @@ bool bio_scm_rdma;
 bool bio_spdk_inited;
 /* SPDK subsystem fini timeout */
 unsigned int bio_spdk_subsys_timeout = 9000;	/* ms */
+/* How many blob unmap calls can be called in a row */
+unsigned int bio_spdk_max_unmap_cnt = 32;
 
 struct bio_nvme_data {
 	ABT_mutex		 bd_mutex;
@@ -202,6 +206,11 @@ bio_nvme_init(const char *nvme_conf, int numa_node, unsigned int mem_size,
 	d_getenv_int("DAOS_SPDK_SUBSYS_TIMEOUT", &bio_spdk_subsys_timeout);
 	D_INFO("SPDK subsystem fini timeout is %u ms\n", bio_spdk_subsys_timeout);
 
+	d_getenv_int("DAOS_SPDK_MAX_UNMAP_CNT", &bio_spdk_max_unmap_cnt);
+	if (bio_spdk_max_unmap_cnt == 0)
+		bio_spdk_max_unmap_cnt = UINT32_MAX;
+	D_INFO("SPDK batch blob unmap call count is %u\n", bio_spdk_max_unmap_cnt);
+
 	/* Hugepages disabled */
 	if (mem_size == 0) {
 		D_INFO("Set per-xstream DMA buffer upper bound to %u %uMB chunks\n",
@@ -343,6 +352,20 @@ bio_need_nvme_poll(struct bio_xs_context *ctxt)
 	return ctxt->bxc_blob_rw > BIO_BS_POLL_WATERMARK;
 }
 
+void
+drain_inflight_ios(struct bio_xs_context *ctxt)
+{
+	if (ctxt == NULL || ctxt->bxc_blob_rw <= BIO_BS_POLL_WATERMARK)
+		return;
+
+	do {
+		if (ctxt->bxc_self_polling)
+			spdk_thread_poll(ctxt->bxc_thread, 0, 0);
+		else
+			bio_yield();
+	} while (ctxt->bxc_blob_rw >= BIO_BS_STOP_WATERMARK);
+}
+
 struct common_cp_arg {
 	unsigned int		 cca_inflights;
 	int			 cca_rc;