MRP-2853 lnet: Bulk IO write error

different aligment on both sides issue has been fixed by
large memory cache alloc on the router side

Signed-off-by: Alyona Romanenko <alyona.romanenko@seagate.com>
Change-Id: Idfb5046bd61bfcc360b7d2146cd0a9a6bdabd85b
Reviewed-on: http://es-gerrit.xyus.xyratex.com:8080/8247
Tested-by: Jenkins
Reviewed-by: Alexey Leonidovich Lyashkov <alexey.lyashkov@seagate.com>
Reviewed-by: Sergey Cheremencev <sergey.cheremencev@seagate.com>
Tested-by: Elena V. Gryaznova <elena.gryaznova@seagate.com>

libcfs/include/libcfs/linux/linux-mem.h

@@ -99,8 +99,11 @@ extern void *cfs_cpt_malloc(struct cfs_cpt_table *cptab, int cpt,
 			    size_t nr_bytes, gfp_t flags);
 extern void *cfs_cpt_vzalloc(struct cfs_cpt_table *cptab, int cpt,
 			     size_t nr_bytes);
-extern struct page *cfs_page_cpt_alloc(struct cfs_cpt_table *cptab,
-				      int cpt, gfp_t flags);
+extern struct page *cfs_pages_cpt_alloc(struct cfs_cpt_table *cptab,
+				      int cpt, gfp_t flags, unsigned order);
+#define cfs_page_cpt_alloc(cptab, cpt, flags) \
+		cfs_pages_cpt_alloc(cptab, cpt, flags, 0)
+
 extern void *cfs_mem_cache_cpt_alloc(struct kmem_cache *cachep,
 				     struct cfs_cpt_table *cptab,
 				     int cpt, gfp_t flags);

libcfs/libcfs/linux/linux-mem.c

@@ -66,11 +66,12 @@ cfs_cpt_vzalloc(struct cfs_cpt_table *cptab, int cpt, size_t nr_bytes)
 EXPORT_SYMBOL(cfs_cpt_vzalloc);
 
 struct page *
-cfs_page_cpt_alloc(struct cfs_cpt_table *cptab, int cpt, gfp_t flags)
+cfs_pages_cpt_alloc(struct cfs_cpt_table *cptab, int cpt, gfp_t flags,
+			unsigned order)
 {
-	return alloc_pages_node(cfs_cpt_spread_node(cptab, cpt), flags, 0);
+	return alloc_pages_node(cfs_cpt_spread_node(cptab, cpt), flags, order);
 }
-EXPORT_SYMBOL(cfs_page_cpt_alloc);
+EXPORT_SYMBOL(cfs_pages_cpt_alloc);
 
 void *
 cfs_mem_cache_cpt_alloc(struct kmem_cache *cachep, struct cfs_cpt_table *cptab,

lnet/lnet/lib-move.c

@@ -1003,8 +1003,8 @@ lnet_post_routed_recv_locked (lnet_msg_t *msg, int do_recv)
 	rb = list_entry(rbp->rbp_bufs.next, lnet_rtrbuf_t, rb_list);
 	list_del(&rb->rb_list);
 
-        msg->msg_niov = rbp->rbp_npages;
-        msg->msg_kiov = &rb->rb_kiov[0];
+	msg->msg_niov = 1;
+	msg->msg_kiov = &rb->rb_kiov[0];
 
         if (do_recv) {
 		int cpt = msg->msg_rx_cpt;

lnet/lnet/router.c

@@ -1342,46 +1342,36 @@ lnet_router_checker(void *arg)
 void
 lnet_destroy_rtrbuf(lnet_rtrbuf_t *rb, int npages)
 {
-        int sz = offsetof(lnet_rtrbuf_t, rb_kiov[npages]);
+	int sz = offsetof(lnet_rtrbuf_t, rb_kiov[1]);
+	int order = fls(npages);
 
-        while (--npages >= 0)
-		__free_page(rb->rb_kiov[npages].kiov_page);
+	__free_pages(rb->rb_kiov[0].kiov_page, order);
 
-        LIBCFS_FREE(rb, sz);
+	LIBCFS_FREE(rb, sz);
 }
 
 lnet_rtrbuf_t *
 lnet_new_rtrbuf(lnet_rtrbufpool_t *rbp, int cpt)
 {
-        int            npages = rbp->rbp_npages;
-        int            sz = offsetof(lnet_rtrbuf_t, rb_kiov[npages]);
-        struct page   *page;
-        lnet_rtrbuf_t *rb;
-        int            i;
+	int            order = fls(rbp->rbp_npages);
+	int            sz = offsetof(lnet_rtrbuf_t, rb_kiov[1]);
+	lnet_rtrbuf_t *rb;
 
 	LIBCFS_CPT_ALLOC(rb, lnet_cpt_table(), cpt, sz);
 	if (rb == NULL)
 		return NULL;
 
-	rb->rb_pool = rbp;
-
-	for (i = 0; i < npages; i++) {
-		page = cfs_page_cpt_alloc(lnet_cpt_table(), cpt,
-					  __GFP_ZERO | GFP_IOFS);
-                if (page == NULL) {
-                        while (--i >= 0)
-				__free_page(rb->rb_kiov[i].kiov_page);
-
-                        LIBCFS_FREE(rb, sz);
-                        return NULL;
-                }
-
-		rb->rb_kiov[i].kiov_len = PAGE_CACHE_SIZE;
-                rb->rb_kiov[i].kiov_offset = 0;
-                rb->rb_kiov[i].kiov_page = page;
-        }
+	rb->rb_kiov[0].kiov_page = alloc_pages(GFP_KERNEL, order);
+	if (rb->rb_kiov[0].kiov_page != NULL) {
+		rb->rb_pool = rbp;
+		rb->rb_kiov[0].kiov_len = rbp->rbp_npages * PAGE_SIZE;
+		rb->rb_kiov[0].kiov_offset = 0;
+	} else {
+		LIBCFS_FREE(rb, sz);
+		rb = NULL;
+	}
 
-        return rb;
+	return rb;
 }
 
 void