[host] switch to dynamically allocated defrag buffers

Signed-off-by: Fabio M. Di Nitto <fdinitto@redhat.com>

libknet/handle.c

@@ -628,6 +628,13 @@ knet_handle_t knet_handle_new(knet_node_id_t host_id,
 
 	log_info(knet_h, KNET_SUB_HANDLE, "Default onwire version: %u", knet_h->onwire_ver);
 
+	/*
+	 * set default buffers
+	 */
+
+	knet_h->min_defrag_bufs = KNET_DEFAULT_MIN_DEFRAG_BUFFERS;
+	knet_h->max_defrag_bufs = KNET_DEFAULT_MAX_DEFRAG_BUFFERS;
+
 	/*
 	 * init global shared bits
 	 */

libknet/host.c

@@ -68,6 +68,22 @@ int knet_host_add(knet_handle_t knet_h, knet_node_id_t host_id)
 
 	memset(host, 0, sizeof(struct knet_host));
 
+	host->defrag_bufs = malloc(knet_h->min_defrag_bufs * sizeof(struct knet_host_defrag_buf));
+	if (!host->defrag_bufs) {
+		err = -1;
+		savederrno = errno;
+		log_err(knet_h, KNET_SUB_HOST, "Unable to allocate memory for host %u defrag buffers: %s",
+			host_id, strerror(savederrno));
+		goto exit_unlock;
+	}
+
+	host->allocated_defrag_bufs = knet_h->min_defrag_bufs;
+
+	memset(host->defrag_bufs, 0, host->allocated_defrag_bufs * sizeof(struct knet_host_defrag_buf));
+
+	log_debug(knet_h, KNET_SUB_HOST, "Allocated %u defrag buffers for host %u",
+		  host->allocated_defrag_bufs, host_id);
+
 	/*
 	 * set host_id
 	 */
@@ -113,6 +129,9 @@ int knet_host_add(knet_handle_t knet_h, knet_node_id_t host_id)
 exit_unlock:
 	pthread_rwlock_unlock(&knet_h->global_rwlock);
 	if (err < 0) {
+		if (host) {
+			free(host->defrag_bufs);
+		}
 		free(host);
 	}
 	errno = err ? savederrno : 0;
@@ -180,6 +199,10 @@ int knet_host_remove(knet_handle_t knet_h, knet_node_id_t host_id)
 	}
 
 	knet_h->host_index[host_id] = NULL;
+
+	if (removed) {
+		free(removed->defrag_bufs);
+	}
 	free(removed);
 
 	_host_list_update(knet_h);
@@ -535,8 +558,8 @@ static void _clear_cbuffers(struct knet_host *host, seq_num_t rx_seq_num)
 
 	memset(host->circular_buffer_defrag, 0, KNET_CBUFFER_SIZE);
 
-	for (i = 0; i < KNET_DEFRAG_BUFFERS; i++) {
-		memset(&host->defrag_buf[i], 0, sizeof(struct knet_host_defrag_buf));
+	for (i = 0; i < host->allocated_defrag_bufs; i++) {
+		memset(&host->defrag_bufs[i], 0, sizeof(struct knet_host_defrag_buf));
 	}
 }
 
@@ -546,23 +569,23 @@ static void _reclaim_old_defrag_bufs(knet_handle_t knet_h, struct knet_host *hos
 	int i;
 
 	head = seq_num + 1;
-	tail = seq_num - (KNET_DEFRAG_BUFFERS + 1);
+	tail = seq_num - (host->allocated_defrag_bufs + 1);
 
 	/*
 	 * expire old defrag buffers
 	 */
-	for (i = 0; i < KNET_DEFRAG_BUFFERS; i++) {
-		if (host->defrag_buf[i].in_use) {
+	for (i = 0; i < host->allocated_defrag_bufs; i++) {
+		if (host->defrag_bufs[i].in_use) {
 			/*
 			 * head has done a rollover to 0+
 			 */
 			if (tail > head) {
-				if ((host->defrag_buf[i].pckt_seq >= head) && (host->defrag_buf[i].pckt_seq <= tail)) {
-					host->defrag_buf[i].in_use = 0;
+				if ((host->defrag_bufs[i].pckt_seq >= head) && (host->defrag_bufs[i].pckt_seq <= tail)) {
+					host->defrag_bufs[i].in_use = 0;
 				}
 			} else {
-				if ((host->defrag_buf[i].pckt_seq >= head) || (host->defrag_buf[i].pckt_seq <= tail)){
-					host->defrag_buf[i].in_use = 0;
+				if ((host->defrag_bufs[i].pckt_seq >= head) || (host->defrag_bufs[i].pckt_seq <= tail)){
+					host->defrag_bufs[i].in_use = 0;
 				}
 			}
 		}

libknet/internals.h

@@ -96,7 +96,9 @@ struct knet_link {
 	uint8_t has_valid_mtu;
 };
 
-#define KNET_DEFRAG_BUFFERS 32
+#define KNET_DEFAULT_MIN_DEFRAG_BUFFERS 32
+#define KNET_DEFAULT_MAX_DEFRAG_BUFFERS 1024
+
 #define KNET_CBUFFER_SIZE 4096
 
 struct knet_host_defrag_buf {
@@ -131,7 +133,8 @@ struct knet_host {
 	seq_num_t timed_rx_seq_num;
 	uint8_t got_data;
 	/* defrag/reassembly buffers */
-	struct knet_host_defrag_buf defrag_buf[KNET_DEFRAG_BUFFERS];
+	struct knet_host_defrag_buf *defrag_bufs;
+	uint8_t allocated_defrag_bufs;
 	char circular_buffer_defrag[KNET_CBUFFER_SIZE];
 	/* link stuff */
 	struct knet_link link[KNET_MAX_LINK];
@@ -247,6 +250,8 @@ struct knet_handle {
 	unsigned char *send_to_links_buf_compress;
 	seq_num_t tx_seq_num;
 	pthread_mutex_t tx_seq_num_mutex;
+	uint16_t min_defrag_bufs;
+	uint16_t max_defrag_bufs;
 	uint8_t has_loop_link;
 	uint8_t loop_link;
 	void *dst_host_filter_fn_private_data;

libknet/threads_rx.c

@@ -71,9 +71,9 @@ static int _find_pckt_defrag_buf(knet_handle_t knet_h, struct knet_host *src_hos
 	/*
 	 * check if there is a buffer already in use handling the same seq_num
 	 */
-	for (i = 0; i < KNET_DEFRAG_BUFFERS; i++) {
-		if (src_host->defrag_buf[i].in_use) {
-			if (src_host->defrag_buf[i].pckt_seq == seq_num) {
+	for (i = 0; i < src_host->allocated_defrag_bufs; i++) {
+		if (src_host->defrag_bufs[i].in_use) {
+			if (src_host->defrag_bufs[i].pckt_seq == seq_num) {
 				return i;
 			}
 		}
@@ -99,8 +99,8 @@ static int _find_pckt_defrag_buf(knet_handle_t knet_h, struct knet_host *src_hos
 	/*
 	 * see if there is a free buffer
 	 */
-	for (i = 0; i < KNET_DEFRAG_BUFFERS; i++) {
-		if (!src_host->defrag_buf[i].in_use) {
+	for (i = 0; i < src_host->allocated_defrag_bufs; i++) {
+		if (!src_host->defrag_bufs[i].in_use) {
 			return i;
 		}
 	}
@@ -113,12 +113,12 @@ static int _find_pckt_defrag_buf(knet_handle_t knet_h, struct knet_host *src_hos
 
 	oldest = 0;
 
-	for (i = 0; i < KNET_DEFRAG_BUFFERS; i++) {
-		if (_timecmp(src_host->defrag_buf[i].last_update, src_host->defrag_buf[oldest].last_update) < 0) {
+	for (i = 0; i < src_host->allocated_defrag_bufs; i++) {
+		if (_timecmp(src_host->defrag_bufs[i].last_update, src_host->defrag_bufs[oldest].last_update) < 0) {
 			oldest = i;
 		}
 	}
-	src_host->defrag_buf[oldest].in_use = 0;
+	src_host->defrag_bufs[oldest].in_use = 0;
 	return oldest;
 }
 
@@ -132,7 +132,7 @@ static int _pckt_defrag(knet_handle_t knet_h, struct knet_host *src_host, seq_nu
 		return 1;
 	}
 
-	defrag_buf = &src_host->defrag_buf[defrag_buf_idx];
+	defrag_buf = &src_host->defrag_bufs[defrag_buf_idx];
 
 	/*
 	 * if the buf is not is use, then make sure it's clean