From e2eecff0455a4fc719f8646b2f601ca4cc4ee74a Mon Sep 17 00:00:00 2001 From: Alexander Gall Date: Mon, 23 Jul 2018 12:44:49 +0200 Subject: [PATCH] connectx4.lua: fix page sizes in CREATE_{C,R}Q Set the page size parameters for CQs and RQs so that the entire queue fits in a single page. The alternative would be to select a fixed page size and add as many PAS entries as necessary. It is unclear whether there is a difference between the methods (e.g. for perfomance). --- src/apps/mellanox/connectx4.lua | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/apps/mellanox/connectx4.lua b/src/apps/mellanox/connectx4.lua index 2cd2345798..4c9cf10728 100644 --- a/src/apps/mellanox/connectx4.lua +++ b/src/apps/mellanox/connectx4.lua @@ -916,7 +916,9 @@ end function HCA:create_cq (entries, uar_page, eqn, collapsed) local doorbell, doorbell_phy = memory.dma_alloc(16) -- Memory for completion queue entries - local cqe, cqe_phy = memory.dma_alloc(entries * 64, 4096) + local size = entries * 64 + local cqe, cqe_phy = memory.dma_alloc(size, 4096) + local log_page_size = log2size(math.ceil(size/4096)) ffi.fill(cqe, entries * 64, 0xFF) self:command("CREATE_CQ", 0x114, 0x0C) :input("opcode", 0x00, 31, 16, 0x400) @@ -925,7 +927,7 @@ function HCA:create_cq (entries, uar_page, eqn, collapsed) :input("log_cq_size", 0x10 + 0x0C, 28, 24, log2size(entries)) :input("uar_page", 0x10 + 0x0C, 23, 0, uar_page) :input("c_eqn", 0x10 + 0x14, 7, 0, eqn) - :input("log_page_size", 0x10 + 0x18, 28, 24, 4) + :input("log_page_size", 0x10 + 0x18, 28, 24, log_page_size) :input("db_addr high", 0x10 + 0x38, 31, 0, ptrbits(doorbell_phy, 63, 32)) :input("db_addr_low", 0x10 + 0x3C, 31, 0, ptrbits(doorbell_phy, 31, 0)) :input("pas[0] high", 0x110, 31, 0, ptrbits(cqe_phy, 63, 32)) @@ -941,6 +943,7 @@ function HCA:create_rq (cqn, pd, size, doorbell, rwq) local log_wq_size = log2size(size) local db_phy = memory.virtual_to_physical(doorbell) local rwq_phy = memory.virtual_to_physical(rwq) + local log_page_size = log2size(math.ceil(size * 64/4096)) self:command("CREATE_RQ", 0x20 + 0x30 + 0xC4, 0x0C) :input("opcode", 0x00, 31, 16, 0x908) :input("rlkey", 0x20 + 0x00, 31, 31, 1) @@ -951,7 +954,7 @@ function HCA:create_rq (cqn, pd, size, doorbell, rwq) :input("dbr_addr high", 0x20 + 0x30 + 0x10, 31, 0, ptrbits(db_phy, 63, 32)) :input("dbr_addr low", 0x20 + 0x30 + 0x14, 31, 0, ptrbits(db_phy, 31, 0)) :input("log_wq_stride", 0x20 + 0x30 + 0x20, 19, 16, 4) - :input("page_size", 0x20 + 0x30 + 0x20, 12, 8, 4) -- XXX one big page? + :input("log_page_size", 0x20 + 0x30 + 0x20, 12, 8, log_page_size) :input("log_wq_size", 0x20 + 0x30 + 0x20, 4 , 0, log_wq_size) :input("pas[0] high", 0x20 + 0x30 + 0xC0, 63, 32, ptrbits(rwq_phy, 63, 32)) :input("pas[0] low", 0x20 + 0x30 + 0xC4, 31, 0, ptrbits(rwq_phy, 31, 0))