root@ser141:/freeflow/ffrouter# ./router router1
[INFO] FreeFlowRouter Init
[ERROR] Missing HOST_IP or HOST_IP_PREFIX. Socket may not work.
[DEBUG] Pathname for Unix domain socket: /freeflow/router1
open device cmd_fd is 3 
[DEBUG] RDMA Dev: dev.name=mlx4_0, dev.dev_name=uverbs0
[INFO] FreeFlowRouter Starting... 
[DEBUG] result of start_udp_server --> 0
[INFO] Start the control channel loop.
[DEBUG] Accepting new clients... 
[TRACE] New client with sock 7.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 7.
[TRACE] Start to read from sock 7
[TRACE] Get request cmd 1
[DEBUG] GET_CONTEXT
[TRACE] write rsp 8 bytes to sock 7
[TRACE] Start to read from sock 7
[TRACE] New client with sock 8.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 8.
[TRACE] Start to read from sock 8
[TRACE] Get request cmd 4
[DEBUG] QUERY_PORT client_id=8
[TRACE] write rsp 48 bytes to sock 8
[TRACE] Start to read from sock 8
[TRACE] New client with sock 7.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 7.
[TRACE] Start to read from sock 7
[TRACE] Get request cmd 2
[DEBUG] QUERY_DEV client_id=7
[DEBUG] Finished QUERY_DEV client_id=7
[TRACE] write rsp 232 bytes to sock 7
[TRACE] Start to read from sock 7
[TRACE] New client with sock 8.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 8.
[TRACE] Start to read from sock 8
[TRACE] Get request cmd 4
[DEBUG] QUERY_PORT client_id=8
[TRACE] write rsp 48 bytes to sock 8
[TRACE] Start to read from sock 8
[TRACE] New client with sock 7.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 7.
[TRACE] Start to read from sock 7
[TRACE] Get request cmd 2
[DEBUG] QUERY_DEV client_id=7
[DEBUG] Finished QUERY_DEV client_id=7
[TRACE] write rsp 232 bytes to sock 7
[TRACE] Start to read from sock 7
[TRACE] New client with sock 8.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 8.
[TRACE] Start to read from sock 8
[TRACE] Get request cmd 3
[DEBUG] EXP_QUERY_DEV client_id=8 cmd_fd=3
[TRACE] write rsp 424 bytes to sock 8
[TRACE] Start to read from sock 8
[TRACE] New client with sock 7.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 7.
[TRACE] Start to read from sock 7
[TRACE] Get request cmd 5
[DEBUG] ALLOC_PD
[DEBUG] Return pd_handle 0for client_id 10
[TRACE] write rsp 4 bytes to sock 7
[TRACE] Start to read from sock 7
[TRACE] New client with sock 8.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 8.
[TRACE] Start to read from sock 8
[TRACE] Get request cmd 21
[DEBUG] REG_MR
[TRACE] Create a shared memory piece for client 10 with size 131072
[TRACE] create shm from client id and count.
[TRACE] Looking for PD with pd_handle 0
[DEBUG] Registering a MR ptr=0x7fa70ad96000, size=131072
[TRACE] mr->handle=0
[TRACE] mr->lkey=939589888
[TRACE] mr->rkey=939589888
[TRACE] shm_name=client-10-memsize-131072-index-0
[TRACE] write rsp 112 bytes to sock 8
[TRACE] Start to read from sock 8
[TRACE] New client with sock 7.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 7.
[TRACE] Start to read from sock 7
[TRACE] Get request cmd 22
[DEBUG] REG_MR_MAPPING
[TRACE] Sent MR mapping to 192.168.10.140
[INFO] ## ACK from 192.168.10.140/192.168.10.140:11232ack-rkey=ack-939589888 rkey= 939589888
[TRACE] Sent MR mapping to 192.168.10.141
[DEBUG] Receive MR Mapping: rkey=805372160 mr=140367240683520 shm=140600085131264 from 192.168.10.140:44614
[DEBUG] Receive MR Mapping: rkey=939589888 mr=139958609211392 shm=140355418284032 from 192.168.10.141:51066
[INFO] ## ACK from 192.168.10.141/192.168.10.141:11232ack-rkey=ack-939589888 rkey= 939589888
[TRACE] write rsp 4 bytes to sock 7
[TRACE] Start to read from sock 7
[TRACE] New client with sock 8.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 8.
[TRACE] Start to read from sock 8
[TRACE] Get request cmd 15
[INFO] CREATE_CQ, body_size=12
[DEBUG] Create CQ: cqe=255 handle=0
[TRACE] write rsp 108 bytes to sock 8
[TRACE] Start to read from sock 8
[TRACE] New client with sock 7.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 7.
[TRACE] Start to read from sock 7
[TRACE] Get request cmd 15
[INFO] CREATE_CQ, body_size=12
[DEBUG] Create CQ: cqe=1023 handle=1
[TRACE] write rsp 108 bytes to sock 7
[TRACE] Start to read from sock 7
[TRACE] New client with sock 8.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 8.
[TRACE] Start to read from sock 8
[TRACE] Get request cmd 7
[DEBUG] CREATE_QP
[TRACE] init_attr.qp_type=2
[TRACE] init_attr.sq_sig_all=0
[TRACE] init_attr.srq=0
[TRACE] init_attr.send_cq=0
[TRACE] init_attr.recv_cq=1
[TRACE] init_attr.cap.max_send_wr=128
[TRACE] init_attr.cap.max_recv_wr=512
[TRACE] init_attr.cap.max_send_sge=1
[TRACE] init_attr.cap.max_recv_sge=1
[TRACE] init_attr.cap.max_inline_data=0
[TRACE] Get pd 0x7fa7002008e0from pd_handle 0
[INFO] Create a qp for client=10 with rate limit=40000Mbps
[TRACE] qp->qp_num=539
[TRACE] qp->handle=0
[TRACE] write rsp 128 bytes to sock 8
[TRACE] Start to read from sock 8
[TRACE] New client with sock 7.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 7.
[TRACE] Start to read from sock 7
[TRACE] Get request cmd 4
[DEBUG] QUERY_PORT client_id=7
[TRACE] write rsp 48 bytes to sock 7
[TRACE] Start to read from sock 7
[TRACE] New client with sock 8.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 8.
[TRACE] Start to read from sock 8
[TRACE] Get request cmd 8
[TRACE] MODIFY_QP
[TRACE] QP handle to modify: 0
[DEBUG] ---------- QP=0 -----------
[DEBUG] attr.qp_state=1
[DEBUG] attr.cur_qp_state=0
[DEBUG] attr.path_mtu=0
[DEBUG] attr.path_mig_state=0
[DEBUG] attr.qkey=0
[DEBUG] attr.rq_psn=0
[DEBUG] attr.sq_psn=0
[DEBUG] attr.dest_qp_num=0
[DEBUG] attr.qp_access_flags=3
[DEBUG] attr.cap.max_send_wr=0
[DEBUG] attr.cap.max_recv_wr=0
[DEBUG] attr.cap.max_send_sge=0
[DEBUG] attr.cap.max_recv_sge=0
[DEBUG] attr.cap.max_inline_data=0
[DEBUG] attr.ah_attr.global.subnet_prefix=0
[DEBUG] attr.ah_attr.global.interface_id=0
[DEBUG] attr.ah_attr.flow_label=0
[DEBUG] attr.ah_attr.sgid_index=0
[DEBUG] attr.ah_attr.hop_limit=0
[DEBUG] attr.ah_attr.traffic_class=0
[TRACE] write rsp 8 bytes to sock 8
[TRACE] Start to read from sock 8
[TRACE] New client with sock 7.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 7.
[TRACE] Start to read from sock 7
[TRACE] Get request cmd 4
[DEBUG] QUERY_PORT client_id=7
[TRACE] write rsp 48 bytes to sock 7
[TRACE] Start to read from sock 7
[TRACE] New client with sock 8.
[TRACE] result of pthread_create --> 0
[TRACE] Start to handle the request from client sock 8.
[TRACE] Start to read from sock 8
[TRACE] Get request cmd 8
[TRACE] MODIFY_QP
[TRACE] QP handle to modify: 0
[DEBUG] ---------- QP=0 -----------
[DEBUG] attr.qp_state=2
[DEBUG] attr.cur_qp_state=0
[DEBUG] attr.path_mtu=4
[DEBUG] attr.path_mig_state=0
[DEBUG] attr.qkey=0
[DEBUG] attr.rq_psn=4386641
[DEBUG] attr.sq_psn=0
[DEBUG] attr.dest_qp_num=595
[DEBUG] attr.qp_access_flags=0
[DEBUG] attr.cap.max_send_wr=0
[DEBUG] attr.cap.max_recv_wr=0
[DEBUG] attr.cap.max_send_sge=0
[DEBUG] attr.cap.max_recv_sge=0
[DEBUG] attr.cap.max_inline_data=0
[DEBUG] attr.ah_attr.global.subnet_prefix=0
[DEBUG] attr.ah_attr.global.interface_id=0
[DEBUG] attr.ah_attr.flow_label=0
[DEBUG] attr.ah_attr.sgid_index=0
[DEBUG] attr.ah_attr.hop_limit=0
[DEBUG] attr.ah_attr.traffic_class=0
[TRACE] write rsp 8 bytes to sock 8
[TRACE] Start to read from sock 8