From 5f7df9650f164d2e46462611b7b5d11e7a5265aa Mon Sep 17 00:00:00 2001 From: Howard Pritchard Date: Sat, 14 Feb 2015 15:30:10 -0700 Subject: [PATCH 1/2] prov/gni: initial domain_open implementation add method to get ptag/cookie from apshepherd, daemon, add support in configury for finding alps header files, etc. Add new nic type to be used on for connection management. There will need to be a lot more refinement of the way that fi fabric/domain map to GNI CDM and NIC constructs. Signed-off-by: Howard Pritchard --- Makefile.am | 3 + configure.ac | 3 + prov/gni/configure.m4 | 15 ++- prov/gni/src/gnix.h | 86 ++++++++++++++--- prov/gni/src/gnix_av.c | 50 ++++++++++ prov/gni/src/gnix_cq.c | 50 ++++++++++ prov/gni/src/gnix_dom.c | 192 ++++++++++++++++++++++++++++++++++++- prov/gni/src/gnix_ep.c | 6 ++ prov/gni/src/gnix_fabric.c | 20 ++-- prov/gni/src/gnix_mr.c | 50 ++++++++++ prov/gni/src/gnix_util.c | 100 +++++++++++++++++++ prov/gni/src/gnix_util.h | 6 ++ 12 files changed, 554 insertions(+), 27 deletions(-) create mode 100644 prov/gni/src/gnix_av.c create mode 100644 prov/gni/src/gnix_cq.c create mode 100644 prov/gni/src/gnix_mr.c diff --git a/Makefile.am b/Makefile.am index 4d0723f5b17..5bee470f035 100644 --- a/Makefile.am +++ b/Makefile.am @@ -284,6 +284,9 @@ _gni_files = \ prov/gni/src/gnix_fabric.c \ prov/gni/src/gnix_dom.c \ prov/gni/src/gnix_ep.c \ + prov/gni/src/gnix_cq.c \ + prov/gni/src/gnix_av.c \ + prov/gni/src/gnix_mr.c \ prov/gni/src/gnix_ep_rdm.c \ prov/gni/src/gnix_nameserver.c \ prov/gni/src/gnix_util.c diff --git a/configure.ac b/configure.ac index 77ed9303e92..ac060602ecb 100644 --- a/configure.ac +++ b/configure.ac @@ -74,6 +74,9 @@ AC_PROG_CC_C99 dnl Checks for header files. AC_HEADER_STDC +dnl Check for compiler features +AC_C_TYPEOF + LT_INIT dnl dlopen support is optional diff --git a/prov/gni/configure.m4 b/prov/gni/configure.m4 index 1c48a9ba69f..b109215cf64 100644 --- a/prov/gni/configure.m4 +++ b/prov/gni/configure.m4 @@ -26,7 +26,20 @@ m4_include([config/pkg.m4]) LDFLAGS="$CRAY_GNI_HEADER_LIBS $LDFLAGS" ], [gni_header_happy=0]) + PKG_CHECK_MODULES([CRAY_ALPS_LLI], [cray-alpslli], + [alps_lli_happy=1 + CPPFLAGS="$CRAY_ALPS_LLI_CFLAGS $CPPFLAGS" + LDFLAGS="$CRAY_ALPS_LLI_LIBS $LDFLAGS" + ], + [alps_lli_happy=0]) + PKG_CHECK_MODULES([CRAY_ALPS_UTIL], [cray-alpsutil], + [alps_util_happy=1 + CPPFLAGS="$CRAY_ALPS_UTIL_CFLAGS $CPPFLAGS" + LDFLAGS="$CRAY_ALPS_UTIL_LIBS $LDFLAGS" + ], + [alps_util_happy=0]) ]) - AS_IF([test $gni_header_happy -eq 1 -a $ugni_lib_happy -eq 1], [$1], [$2]) + AS_IF([test $gni_header_happy -eq 1 -a $ugni_lib_happy -eq 1 \ + -a $alps_lli_happy -eq 1 -a $alps_util_happy -eq 1], [$1], [$2]) ]) diff --git a/prov/gni/src/gnix.h b/prov/gni/src/gnix.h index 930cb5d7621..adc3176066f 100644 --- a/prov/gni/src/gnix.h +++ b/prov/gni/src/gnix.h @@ -165,6 +165,7 @@ struct gnix_ep_name { struct { uint32_t name_type : 8; uint32_t unused : 24; + uint32_t cookie; }; uint64_t reserved[4]; }; @@ -180,9 +181,10 @@ enum gnix_progress_type { /* * simple struct for gnix fabric, may add more stuff here later */ + struct gnix_fabric { struct fid_fabric fab_fid; - atomic_t ref; + struct list_head cdm_list; /* llist of cdm's opened from fabric */ }; /* @@ -190,13 +192,20 @@ struct gnix_fabric { * since a single cdm with a given cookie/cdm_id can only * be bound once to a given physical aries nic */ + struct gnix_domain { struct fid_domain domain_fid; - struct gnix_cdm *cdm; - struct gnix_nic *nic; - struct list_head domain_wq; - uint32_t device_id; - uint32_t device_addr; + /* used for fabric object llist of domains*/ + struct list_node list; + /* list nics this domain is attached to, TODO: thread safety */ + struct list_head nic_list; + /* cm nic bound to this domain */ + struct gnix_cm_nic *cm_nic; + uint8_t ptag; + uint32_t cookie; + /* work queue for domain */ + struct list_head domain_wq; + int ref_cnt; }; struct gnix_cdm { @@ -212,8 +221,33 @@ struct gnix_cdm { int ref_cnt; }; +/* + * gnix cm nic struct - to be used only for GNI_EpPostData, etc. + */ + +struct gnix_cm_nic { + struct list_node list; + gni_cdm_handle_t gni_cdm_hndl; + gni_nic_handle_t gni_nic_hndl; + /* free list of datagrams */ + struct list_head datagram_free_list; + /* list of active wc datagrams */ + struct list_head wc_datagram_active_list; + /* free list of wc datagrams */ + struct list_head wc_datagram_free_list; + /* pointer to domain this nic is attached to */ + struct gnix_domain *domain; + struct gnix_datagram *datagram_base; + uint32_t inst_id; + uint32_t device_id; + uint32_t device_addr; + int ref_cnt; +}; + + struct gnix_nic { struct list_node list; + gni_cdm_handle_t gni_cdm_hndl; gni_nic_handle_t gni_nic_hndl; /* receive completion queue for hndl */ gni_cq_handle_t rx_cq; @@ -227,7 +261,7 @@ struct gnix_nic { struct list_head wqe_active_list; /* list for managing wqe's */ struct gnix_wqe_list *wqe_list; - /* list of active smsg req's */ + struct gnix_domain *domain; /* pointer to domain this nic is attached to */ struct list_head smsg_active_req_list; /* list for managing smsg req's */ struct gnix_smsg_req_list *smsg_req_list; @@ -302,17 +336,41 @@ struct gnix_rdm_ep { * globals */ extern const char const gnix_fab_name[]; +extern const char const gnix_dom_name[]; +extern uint32_t gnix_cdm_modes; + +/* + * linked list helpers + */ + +static inline void gnix_list_node_init(struct list_node *node) +{ + node->prev = node->next = NULL; +} + +static inline void gnix_list_del_init(struct list_node *node) +{ + list_del(node); + node->prev = node->next = node; +} /* - * prototypes + * prototypes */ int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info, - struct fid_domain **domain, void *context); -int gnix_rdm_getinfo(uint32_t version, const char *node, const char *service, - uint64_t flags, struct fi_info *hints, - struct fi_info **info); -struct fi_info *gnix_fi_info(enum fi_ep_type ep_type, struct fi_info *hints); -int gnix_verify_domain_attr(struct fi_domain_attr *attr); + struct fid_domain **domain, void *context); +int gnix_av_open(struct fid_domain *domain, struct fi_av_attr *attr, + struct fid_av **av, void *context); +int gnix_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, + struct fid_cq **cq, void *context); +int gnix_ep_open(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, void *context); + +int gnix_mr_reg(struct fid_domain *domain, const void *buf, size_t len, + uint64_t access, uint64_t offset, uint64_t requested_key, + uint64_t flags, struct fid_mr **mr, void *context); + + #ifdef __cplusplus } /* extern "C" */ diff --git a/prov/gni/src/gnix_av.c b/prov/gni/src/gnix_av.c new file mode 100644 index 00000000000..2995cadf176 --- /dev/null +++ b/prov/gni/src/gnix_av.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2015 Cray Inc. All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. Allrights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// +// Address vector common code +// + +#include +#include + +#include "gnix.h" +#include "gnix_util.h" + +int gnix_av_open(struct fid_domain *domain, struct fi_av_attr *attr, + struct fid_av **av, void *context) +{ + return -FI_ENOSYS; /* TODO: implement this puppy */ +} + + diff --git a/prov/gni/src/gnix_cq.c b/prov/gni/src/gnix_cq.c new file mode 100644 index 00000000000..674f2f59876 --- /dev/null +++ b/prov/gni/src/gnix_cq.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2015 Cray Inc. All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. Allrights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// +// CQ common code +// + +#include +#include + +#include "gnix.h" +#include "gnix_util.h" + +int gnix_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, + struct fid_cq **cq, void *context) +{ + return -FI_ENOSYS; /* TODO: implement this puppy */ +} + + diff --git a/prov/gni/src/gnix_dom.c b/prov/gni/src/gnix_dom.c index b652180eb3b..79495b7fa5c 100644 --- a/prov/gni/src/gnix_dom.c +++ b/prov/gni/src/gnix_dom.c @@ -1,5 +1,4 @@ /* - * Copyright (c) 2014 Intel Corporation, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. Allrights reserved. * Copyright (c) 2015 Cray Inc. All rights reserved. * @@ -37,14 +36,199 @@ #endif /* HAVE_CONFIG_H */ #include -#include #include "gnix.h" #include "gnix_util.h" +static int gnix_domain_close(fid_t fid) +{ + int ret=FI_SUCCESS; + struct gnix_domain *domain; + struct gnix_cm_nic *cm_nic; + struct gnix_nic *p,*next; + gni_return_t status; + + domain = container_of(fid, struct gnix_domain, domain_fid.fid); + if (domain->ref_cnt) { + --domain->ref_cnt; + } + +#if 0 + fprintf(stderr,"gnix_domain_close invoked\n"); +#endif + + if (domain->ref_cnt == 0) { + + if (domain->cm_nic) { + cm_nic = domain->cm_nic; + if (cm_nic->gni_cdm_hndl != NULL) { + status = GNI_CdmDestroy(cm_nic->gni_cdm_hndl); + if (status != GNI_RC_SUCCESS) { + fprintf(stderr,"oops, cdm destroy" + " failed\n"); + } + } + free(domain->cm_nic); + } + + list_for_each_safe(&domain->nic_list, p, next, list) { + list_del(&p->list); + gnix_list_node_init(&p->list); + /* TODO: free nic here */ + } + + /* + * remove from the list of cdms attached to fabric + */ + + gnix_list_del_init(&domain->list); + + memset(domain,0,sizeof *domain); + free(domain); + } + +#if 0 + fprintf(stderr,"gnix_domain_close invoked returning %d\n",ret); +#endif + return ret; +} + +static struct fi_ops gnix_fi_ops = { + .size = sizeof(struct fi_ops), + .close = gnix_domain_close, + .bind = fi_no_bind, + .control = fi_no_control, +}; + +static struct fi_ops_domain gnix_domain_ops = { + .size = sizeof(struct fi_ops_domain), + .av_open = gnix_av_open, + .cq_open = gnix_cq_open, + .endpoint = gnix_ep_open, + .cntr_open = fi_no_cntr_open, /* TODO: no cntrs for now in gnix */ + .poll_open = fi_no_poll_open, + .stx_ctx = fi_no_stx_context, + .srx_ctx = fi_no_srx_context, +}; + +static struct fi_ops_mr gnix_domain_mr_ops = { + .size = sizeof(struct fi_ops_mr), + .reg = gnix_mr_reg, +}; + + int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info, struct fid_domain **dom, void *context) { - /* TODO: need to implement */ - return -FI_ENOSYS; + struct gnix_domain *domain = NULL; + int ret = FI_SUCCESS; + uint8_t ptag; + uint32_t cookie, device_addr; + struct gnix_cm_nic *cm_nic = NULL; + struct gnix_fabric *fabric_priv; + gni_return_t status; + +#if 0 + GNIX_DEBUG("%s\n", __func__); +#endif + + fabric_priv = container_of(fabric, struct gnix_fabric, fab_fid); + if (!info->domain_attr->name || strncmp(info->domain_attr->name, + gnix_dom_name, + strlen(gnix_dom_name))) { + return -FI_EINVAL; + } + + /* + * check cookie/ptag credentials - for FI_EP_MSG we may be creating a domain + * using a cookie supplied being used by the server. Otherwise, we use + * use the cookie/ptag supplied by the job launch system. + */ + + if (info->dest_addr) { + ret = gnixu_get_rdma_credentials(info->dest_addr,&ptag,&cookie); + if (ret) { + goto fn_err; + } + } else { + ret = gnixu_get_rdma_credentials(NULL,&ptag,&cookie); + } + +#if 0 + fprintf(stderr,"gnix rdma credentials returned ptag %d cookie " + "0x%x\n",ptag,cookie); +#endif + domain = calloc(1, sizeof *domain); + if (domain == NULL) { + ret = -FI_ENOMEM; + goto fn_err; + } + + list_head_init(&domain->nic_list); + gnix_list_node_init(&domain->list); + + list_add_tail(&fabric_priv->cdm_list,&domain->list); + + list_head_init(&domain->domain_wq); + + /* + * Set up the connection management nic for this domain + */ + + cm_nic = (struct gnix_cm_nic *)calloc(1, sizeof *cm_nic); + if( cm_nic == NULL) { + ret = -FI_ENOMEM; + goto fn_err; + } + + gnix_list_node_init(&cm_nic->list); + list_head_init(&cm_nic->datagram_free_list); + list_head_init(&cm_nic->wc_datagram_active_list); + list_head_init(&cm_nic->wc_datagram_free_list); + + status = GNI_CdmCreate(getpid(), ptag, cookie, gnix_cdm_modes, + &cm_nic->gni_cdm_hndl); + if (status != GNI_RC_SUCCESS) { + fprintf(stderr,"GNI_CdmCreate returned %s\n", + gni_err_str[status]); + ret = -FI_EACCES; /* TODO: need a translater from gni to fi + errors */ + goto fn_err; + } + + /* + * Okay, now go for the attach + */ + + status = GNI_CdmAttach(cm_nic->gni_cdm_hndl, 0, &device_addr, + &cm_nic->gni_nic_hndl); + if (status != GNI_RC_SUCCESS) { + fprintf(stderr,"GNI_CdmAttach returned %s\n", + gni_err_str[status]); + ret = -FI_EACCES; + goto fn_err; + } + + domain->cm_nic = cm_nic; + domain->ptag = ptag; + domain->cookie = cookie; + domain->ref_cnt = 1; + cm_nic->domain = domain; + + domain->domain_fid.fid.fclass = FI_CLASS_DOMAIN; + domain->domain_fid.fid.context = context; + domain->domain_fid.fid.ops = &gnix_fi_ops; + domain->domain_fid.ops = &gnix_domain_ops; + domain->domain_fid.mr = &gnix_domain_mr_ops; + + *dom = &domain->domain_fid; + return FI_SUCCESS; + +fn_err: + if (cm_nic && cm_nic->gni_cdm_hndl) { + GNI_CdmDestroy(cm_nic->gni_cdm_hndl); + } + if (domain != NULL) free(domain); + if (cm_nic != NULL) free(cm_nic); + return ret; } diff --git a/prov/gni/src/gnix_ep.c b/prov/gni/src/gnix_ep.c index 3d23392e2e3..7c656f1d756 100644 --- a/prov/gni/src/gnix_ep.c +++ b/prov/gni/src/gnix_ep.c @@ -38,3 +38,9 @@ #include "gnix.h" #include "gnix_util.h" + +int gnix_ep_open(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, void *context) +{ + return -FI_ENOSYS; /* TODO: implement this puppy */ +} diff --git a/prov/gni/src/gnix_fabric.c b/prov/gni/src/gnix_fabric.c index ab122d17562..87c94876fe2 100644 --- a/prov/gni/src/gnix_fabric.c +++ b/prov/gni/src/gnix_fabric.c @@ -54,8 +54,13 @@ #include "gnix_util.h" #include "gnix_nameserver.h" -const const char gnix_fab_name[] = "gni"; -const const char gnix_dom_name[] = "/sys/class/gni/kgni0"; +const char gnix_fab_name[] = "gni"; +const char gnix_dom_name[] = "/sys/class/gni/kgni0"; +uint32_t gnix_cdm_modes = (GNI_CDM_MODE_FAST_DATAGRAM_POLL | \ + GNI_CDM_MODE_FMA_SHARED | \ + GNI_CDM_MODE_FMA_SMALL_WINDOW | \ + GNI_CDM_MODE_FORK_PARTCOPY | \ + GNI_CDM_MODE_ERR_NO_KILL); const struct fi_fabric_attr gnix_fabric_attr = { .fabric = NULL, @@ -80,12 +85,12 @@ static int gnix_fabric_close(fid_t fid) struct gnix_fabric *fab; fab = container_of(fid, struct gnix_fabric, fab_fid); - if (atomic_get(&fab->ref)) { + if(!list_empty(&fab->cdm_list)) { return -FI_EBUSY; } free(fab); - return 0; + return FI_SUCCESS; } static struct fi_ops gnix_fab_fi_ops = { @@ -117,11 +122,10 @@ static int gnix_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, fab->fab_fid.fid.context = context; fab->fab_fid.fid.ops = &gnix_fab_fi_ops; fab->fab_fid.ops = &gnix_fab_ops; + list_head_init(&fab->cdm_list); *fabric = &fab->fab_fid; -#if 0 - atomic_init(&fab->ref, 0); -#endif - return 0; + + return FI_SUCCESS; } static int gnix_getinfo(uint32_t version, const char *node, const char *service, diff --git a/prov/gni/src/gnix_mr.c b/prov/gni/src/gnix_mr.c new file mode 100644 index 00000000000..a1f1d0f1f56 --- /dev/null +++ b/prov/gni/src/gnix_mr.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2015 Cray Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// +// memory registration common code +// + +#include +#include + +#include "gnix.h" +#include "gnix_util.h" + +int gnix_mr_reg(struct fid_domain *domain, const void *buf, size_t len, + uint64_t access, uint64_t offset, uint64_t requested_key, + uint64_t flags, struct fid_mr **mr, void *context) +{ + return -FI_ENOSYS; /* TODO: implement this puppy */ +} + + diff --git a/prov/gni/src/gnix_util.c b/prov/gni/src/gnix_util.c index 915937822e4..5f36894f928 100644 --- a/prov/gni/src/gnix_util.c +++ b/prov/gni/src/gnix_util.c @@ -53,5 +53,105 @@ #include #include +#include "alps/alps.h" +#include "alps/alps_toolAssist.h" +#include "alps/libalpsutil.h" +#include "alps/libalpslli.h" + #include "gnix.h" #include "gnix_util.h" + +int gnixu_get_rdma_credentials(void *addr, uint8_t *ptag, uint32_t *cookie) +{ + int ret = FI_SUCCESS; + int alps_status=0; + uint64_t apid; + size_t alps_count; + alpsAppLLIGni_t *rdmacred_rsp=NULL; + alpsAppGni_t *rdmacred_buf; + + if((ptag == NULL) || (cookie == NULL)) { + ret = -FI_EINVAL; + goto err; + } + + /* + * TODO: need to handle non null addr differently at some point, + * a non-NULL addr can be used to acquire RDMA credentials other than + * those assigned by ALPS/nativized slurm. + */ + + ret = alps_app_lli_lock(); /* lli_lock doesn't return anything useful */ + + /* + * First get our apid + */ + + ret = alps_app_lli_put_request(ALPS_APP_LLI_ALPS_REQ_APID, NULL, 0); + if (ret != ALPS_APP_LLI_ALPS_STAT_OK) { + fprintf(stderr,"lli put failed\n"); + ret = -FI_EIO; + goto err; + } + + ret = alps_app_lli_get_response (&alps_status, &alps_count); + if (alps_status != ALPS_APP_LLI_ALPS_STAT_OK) { + fprintf(stderr,"lli get response failed failed ret =" + "%s\n",strerror(ret)); + ret = -FI_EIO; + goto err; + } + + ret = alps_app_lli_get_response_bytes (&apid, sizeof(apid)); + if (ret != ALPS_APP_LLI_ALPS_STAT_OK) { + fprintf(stderr,"lli get response bytes failed\n"); + ret = -FI_EIO; + goto err; + } + + /* + * now get the GNI rdma credentials info + */ + + ret = alps_app_lli_put_request(ALPS_APP_LLI_ALPS_REQ_GNI, NULL, 0); + if (ret != ALPS_APP_LLI_ALPS_STAT_OK) { + ret = -FI_EIO; + goto err; + } + + ret = alps_app_lli_get_response(&alps_status, &alps_count); + if (alps_status != ALPS_APP_LLI_ALPS_STAT_OK) { + ret = -FI_EIO; + goto err; + } + + rdmacred_rsp = (alpsAppLLIGni_t *)malloc(alps_count); + if (rdmacred_rsp == NULL) { + ret = -FI_ENOMEM; + goto err; + } + + memset(rdmacred_rsp,0,alps_count); + + ret = alps_app_lli_get_response_bytes(rdmacred_rsp, alps_count); + if (ret != ALPS_APP_LLI_ALPS_STAT_OK) { + ret = -FI_EIO; + goto err; + } + + rdmacred_buf = (alpsAppGni_t *)(rdmacred_rsp->u.buf); + + /* + * just use the first ptag/cookie for now + */ + + *ptag = rdmacred_buf[0].ptag; + *cookie = rdmacred_buf[0].cookie; +err: + alps_app_lli_unlock(); + if (rdmacred_rsp != NULL) + free(rdmacred_rsp); + return ret; + + +} diff --git a/prov/gni/src/gnix_util.h b/prov/gni/src/gnix_util.h index e0ddae0e7ee..db7988882c4 100644 --- a/prov/gni/src/gnix_util.h +++ b/prov/gni/src/gnix_util.h @@ -49,4 +49,10 @@ extern const char gnix_dom_name[]; #define GNIX_LOG_WARN(...) FI_WARN(gnix_fab_name, __VA_ARGS__) #define GNIX_LOG_ERROR(...) FI_WARN(gnix_fab_name, __VA_ARGS__) +/* + * prototypes + */ + +int gnixu_get_rdma_credentials(void *addr, uint8_t *ptag, uint32_t *cookie); + #endif From 19d30781233c70787ebd8f93841bc26e9751a135 Mon Sep 17 00:00:00 2001 From: Ben Turrubiates Date: Thu, 19 Feb 2015 14:24:50 -0700 Subject: [PATCH 2/2] Fix minor formatting issues and update to use logging macros. - Conform to Linux Kernel style guide - Remove extra const declarations - Replace all #if 0 statements with GNIX_LOG_INFO - Replace all fprintf with GNIX_LOG_ERROR - Remove cast of pointer returned by malloc Signed-off-by: Ben Turrubiates --- prov/gni/src/gnix.h | 49 ++++++------ prov/gni/src/gnix_av.c | 8 +- prov/gni/src/gnix_cq.c | 8 +- prov/gni/src/gnix_dom.c | 151 ++++++++++++++++++------------------- prov/gni/src/gnix_ep.c | 5 +- prov/gni/src/gnix_fabric.c | 22 +++--- prov/gni/src/gnix_mr.c | 6 +- prov/gni/src/gnix_util.c | 117 ++++++++++++++-------------- prov/gni/src/gnix_util.h | 3 +- 9 files changed, 181 insertions(+), 188 deletions(-) diff --git a/prov/gni/src/gnix.h b/prov/gni/src/gnix.h index adc3176066f..b773a4aa344 100644 --- a/prov/gni/src/gnix.h +++ b/prov/gni/src/gnix.h @@ -181,10 +181,10 @@ enum gnix_progress_type { /* * simple struct for gnix fabric, may add more stuff here later */ - struct gnix_fabric { struct fid_fabric fab_fid; - struct list_head cdm_list; /* llist of cdm's opened from fabric */ + /* llist of cdm's opened from fabric */ + struct list_head cdm_list; }; /* @@ -192,7 +192,6 @@ struct gnix_fabric { * since a single cdm with a given cookie/cdm_id can only * be bound once to a given physical aries nic */ - struct gnix_domain { struct fid_domain domain_fid; /* used for fabric object llist of domains*/ @@ -204,8 +203,8 @@ struct gnix_domain { uint8_t ptag; uint32_t cookie; /* work queue for domain */ - struct list_head domain_wq; - int ref_cnt; + struct list_head domain_wq; + int ref_cnt; }; struct gnix_cdm { @@ -226,28 +225,27 @@ struct gnix_cdm { */ struct gnix_cm_nic { - struct list_node list; - gni_cdm_handle_t gni_cdm_hndl; - gni_nic_handle_t gni_nic_hndl; + struct list_node list; + gni_cdm_handle_t gni_cdm_hndl; + gni_nic_handle_t gni_nic_hndl; /* free list of datagrams */ - struct list_head datagram_free_list; + struct list_head datagram_free_list; /* list of active wc datagrams */ - struct list_head wc_datagram_active_list; + struct list_head wc_datagram_active_list; /* free list of wc datagrams */ - struct list_head wc_datagram_free_list; + struct list_head wc_datagram_free_list; /* pointer to domain this nic is attached to */ - struct gnix_domain *domain; - struct gnix_datagram *datagram_base; - uint32_t inst_id; - uint32_t device_id; - uint32_t device_addr; - int ref_cnt; + struct gnix_domain *domain; + struct gnix_datagram *datagram_base; + uint32_t inst_id; + uint32_t device_id; + uint32_t device_addr; + int ref_cnt; }; - struct gnix_nic { struct list_node list; - gni_cdm_handle_t gni_cdm_hndl; + gni_cdm_handle_t gni_cdm_hndl; gni_nic_handle_t gni_nic_hndl; /* receive completion queue for hndl */ gni_cq_handle_t rx_cq; @@ -261,7 +259,8 @@ struct gnix_nic { struct list_head wqe_active_list; /* list for managing wqe's */ struct gnix_wqe_list *wqe_list; - struct gnix_domain *domain; /* pointer to domain this nic is attached to */ + /* pointer to domain this nic is attached to */ + struct gnix_domain *domain; struct list_head smsg_active_req_list; /* list for managing smsg req's */ struct gnix_smsg_req_list *smsg_req_list; @@ -335,8 +334,8 @@ struct gnix_rdm_ep { /* * globals */ -extern const char const gnix_fab_name[]; -extern const char const gnix_dom_name[]; +extern const char gnix_fab_name[]; +extern const char gnix_dom_name[]; extern uint32_t gnix_cdm_modes; /* @@ -355,10 +354,10 @@ static inline void gnix_list_del_init(struct list_node *node) } /* - * prototypes + * prototypes */ int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info, - struct fid_domain **domain, void *context); + struct fid_domain **domain, void *context); int gnix_av_open(struct fid_domain *domain, struct fi_av_attr *attr, struct fid_av **av, void *context); int gnix_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, @@ -370,8 +369,6 @@ int gnix_mr_reg(struct fid_domain *domain, const void *buf, size_t len, uint64_t access, uint64_t offset, uint64_t requested_key, uint64_t flags, struct fid_mr **mr, void *context); - - #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/prov/gni/src/gnix_av.c b/prov/gni/src/gnix_av.c index 2995cadf176..d61fc0000e1 100644 --- a/prov/gni/src/gnix_av.c +++ b/prov/gni/src/gnix_av.c @@ -34,7 +34,6 @@ // // Address vector common code // - #include #include @@ -42,9 +41,8 @@ #include "gnix_util.h" int gnix_av_open(struct fid_domain *domain, struct fi_av_attr *attr, - struct fid_av **av, void *context) + struct fid_av **av, void *context) { - return -FI_ENOSYS; /* TODO: implement this puppy */ + /* TODO: implement this puppy */ + return -FI_ENOSYS; } - - diff --git a/prov/gni/src/gnix_cq.c b/prov/gni/src/gnix_cq.c index 674f2f59876..6ecabe36d8e 100644 --- a/prov/gni/src/gnix_cq.c +++ b/prov/gni/src/gnix_cq.c @@ -34,7 +34,6 @@ // // CQ common code // - #include #include @@ -42,9 +41,8 @@ #include "gnix_util.h" int gnix_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, - struct fid_cq **cq, void *context) + struct fid_cq **cq, void *context) { - return -FI_ENOSYS; /* TODO: implement this puppy */ + /* TODO: implement this puppy */ + return -FI_ENOSYS; } - - diff --git a/prov/gni/src/gnix_dom.c b/prov/gni/src/gnix_dom.c index 79495b7fa5c..05a14e49845 100644 --- a/prov/gni/src/gnix_dom.c +++ b/prov/gni/src/gnix_dom.c @@ -42,10 +42,10 @@ static int gnix_domain_close(fid_t fid) { - int ret=FI_SUCCESS; + int ret = FI_SUCCESS; struct gnix_domain *domain; struct gnix_cm_nic *cm_nic; - struct gnix_nic *p,*next; + struct gnix_nic *p, *next; gni_return_t status; domain = container_of(fid, struct gnix_domain, domain_fid.fid); @@ -53,25 +53,23 @@ static int gnix_domain_close(fid_t fid) --domain->ref_cnt; } -#if 0 - fprintf(stderr,"gnix_domain_close invoked\n"); -#endif + GNIX_LOG_INFO("gnix_domain_close invoked.\n"); if (domain->ref_cnt == 0) { - if (domain->cm_nic) { cm_nic = domain->cm_nic; - if (cm_nic->gni_cdm_hndl != NULL) { + if (cm_nic->gni_cdm_hndl) { status = GNI_CdmDestroy(cm_nic->gni_cdm_hndl); if (status != GNI_RC_SUCCESS) { - fprintf(stderr,"oops, cdm destroy" - " failed\n"); + GNIX_LOG_ERROR("oops, cdm destroy" + "failed\n"); } } free(domain->cm_nic); } - list_for_each_safe(&domain->nic_list, p, next, list) { + list_for_each_safe(&domain->nic_list, p, next, list) + { list_del(&p->list); gnix_list_node_init(&p->list); /* TODO: free nic here */ @@ -80,16 +78,13 @@ static int gnix_domain_close(fid_t fid) /* * remove from the list of cdms attached to fabric */ - gnix_list_del_init(&domain->list); - memset(domain,0,sizeof *domain); + memset(domain, 0, sizeof *domain); free(domain); } -#if 0 - fprintf(stderr,"gnix_domain_close invoked returning %d\n",ret); -#endif + GNIX_LOG_INFO("gnix_domain_close invoked returning %d\n", ret); return ret; } @@ -97,26 +92,26 @@ static struct fi_ops gnix_fi_ops = { .size = sizeof(struct fi_ops), .close = gnix_domain_close, .bind = fi_no_bind, - .control = fi_no_control, + .control = fi_no_control }; static struct fi_ops_domain gnix_domain_ops = { - .size = sizeof(struct fi_ops_domain), - .av_open = gnix_av_open, - .cq_open = gnix_cq_open, - .endpoint = gnix_ep_open, - .cntr_open = fi_no_cntr_open, /* TODO: no cntrs for now in gnix */ - .poll_open = fi_no_poll_open, - .stx_ctx = fi_no_stx_context, - .srx_ctx = fi_no_srx_context, + .size = sizeof(struct fi_ops_domain), + .av_open = gnix_av_open, + .cq_open = gnix_cq_open, + .endpoint = gnix_ep_open, + /* TODO: no cntrs for now in gnix */ + .cntr_open = fi_no_cntr_open, + .poll_open = fi_no_poll_open, + .stx_ctx = fi_no_stx_context, + .srx_ctx = fi_no_srx_context }; static struct fi_ops_mr gnix_domain_mr_ops = { - .size = sizeof(struct fi_ops_mr), - .reg = gnix_mr_reg, + .size = sizeof(struct fi_ops_mr), + .reg = gnix_mr_reg }; - int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info, struct fid_domain **dom, void *context) { @@ -128,107 +123,109 @@ int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info, struct gnix_fabric *fabric_priv; gni_return_t status; -#if 0 - GNIX_DEBUG("%s\n", __func__); -#endif + GNIX_LOG_INFO("%s\n", __func__); fabric_priv = container_of(fabric, struct gnix_fabric, fab_fid); - if (!info->domain_attr->name || strncmp(info->domain_attr->name, - gnix_dom_name, - strlen(gnix_dom_name))) { - return -FI_EINVAL; - } - - /* - * check cookie/ptag credentials - for FI_EP_MSG we may be creating a domain + if (!info->domain_attr->name || + strncmp(info->domain_attr->name, gnix_dom_name, + strlen(gnix_dom_name))) { + ret = -FI_EINVAL; + goto fn_err; + } + + /* + * check cookie/ptag credentials - for FI_EP_MSG we may be creating a + * domain * using a cookie supplied being used by the server. Otherwise, we use * use the cookie/ptag supplied by the job launch system. - */ - + */ if (info->dest_addr) { - ret = gnixu_get_rdma_credentials(info->dest_addr,&ptag,&cookie); + ret = + gnixu_get_rdma_credentials(info->dest_addr, &ptag, &cookie); if (ret) { + GNIX_LOG_ERROR("gnixu_get_rdma_credentials returned" + "ptag %d cookie 0x%x\n", ptag, cookie); goto fn_err; } } else { - ret = gnixu_get_rdma_credentials(NULL,&ptag,&cookie); - } + ret = gnixu_get_rdma_credentials(NULL, &ptag, &cookie); + } -#if 0 - fprintf(stderr,"gnix rdma credentials returned ptag %d cookie " - "0x%x\n",ptag,cookie); -#endif + GNIX_LOG_INFO("gnix rdma credentials returned ptag %d cookie 0x%x\n", + ptag, cookie); domain = calloc(1, sizeof *domain); if (domain == NULL) { ret = -FI_ENOMEM; goto fn_err; } - list_head_init(&domain->nic_list); - gnix_list_node_init(&domain->list); + list_head_init(&domain->nic_list); + gnix_list_node_init(&domain->list); - list_add_tail(&fabric_priv->cdm_list,&domain->list); + list_add_tail(&fabric_priv->cdm_list, &domain->list); - list_head_init(&domain->domain_wq); + list_head_init(&domain->domain_wq); /* * Set up the connection management nic for this domain */ - cm_nic = (struct gnix_cm_nic *)calloc(1, sizeof *cm_nic); - if( cm_nic == NULL) { + if (cm_nic == NULL) { ret = -FI_ENOMEM; goto fn_err; } - gnix_list_node_init(&cm_nic->list); - list_head_init(&cm_nic->datagram_free_list); - list_head_init(&cm_nic->wc_datagram_active_list); - list_head_init(&cm_nic->wc_datagram_free_list); + gnix_list_node_init(&cm_nic->list); + list_head_init(&cm_nic->datagram_free_list); + list_head_init(&cm_nic->wc_datagram_active_list); + list_head_init(&cm_nic->wc_datagram_free_list); status = GNI_CdmCreate(getpid(), ptag, cookie, gnix_cdm_modes, &cm_nic->gni_cdm_hndl); if (status != GNI_RC_SUCCESS) { - fprintf(stderr,"GNI_CdmCreate returned %s\n", - gni_err_str[status]); - ret = -FI_EACCES; /* TODO: need a translater from gni to fi - errors */ + GNIX_LOG_ERROR("GNI_CdmCreate returned %s\n", + gni_err_str[status]); + /* TODO: need a translater from gni to fi errors */ + ret = -FI_EACCES; goto fn_err; } - /* - * Okay, now go for the attach + /* + * Okay, now go for the attach */ - status = GNI_CdmAttach(cm_nic->gni_cdm_hndl, 0, &device_addr, &cm_nic->gni_nic_hndl); if (status != GNI_RC_SUCCESS) { - fprintf(stderr,"GNI_CdmAttach returned %s\n", - gni_err_str[status]); + GNIX_LOG_ERROR("GNI_CdmAttach returned %s\n", + gni_err_str[status]); ret = -FI_EACCES; goto fn_err; } domain->cm_nic = cm_nic; - domain->ptag = ptag; - domain->cookie = cookie; + domain->ptag = ptag; + domain->cookie = cookie; domain->ref_cnt = 1; cm_nic->domain = domain; - domain->domain_fid.fid.fclass = FI_CLASS_DOMAIN; - domain->domain_fid.fid.context = context; - domain->domain_fid.fid.ops = &gnix_fi_ops; - domain->domain_fid.ops = &gnix_domain_ops; - domain->domain_fid.mr = &gnix_domain_mr_ops; + domain->domain_fid.fid.fclass = FI_CLASS_DOMAIN; + domain->domain_fid.fid.context = context; + domain->domain_fid.fid.ops = &gnix_fi_ops; + domain->domain_fid.ops = &gnix_domain_ops; + domain->domain_fid.mr = &gnix_domain_mr_ops; - *dom = &domain->domain_fid; + *dom = &domain->domain_fid; return FI_SUCCESS; fn_err: if (cm_nic && cm_nic->gni_cdm_hndl) { GNI_CdmDestroy(cm_nic->gni_cdm_hndl); } - if (domain != NULL) free(domain); - if (cm_nic != NULL) free(cm_nic); - return ret; + if (domain != NULL) { + free(domain); + } + if (cm_nic != NULL) { + free(cm_nic); + } + return ret; } diff --git a/prov/gni/src/gnix_ep.c b/prov/gni/src/gnix_ep.c index 7c656f1d756..f687a35771f 100644 --- a/prov/gni/src/gnix_ep.c +++ b/prov/gni/src/gnix_ep.c @@ -40,7 +40,8 @@ #include "gnix_util.h" int gnix_ep_open(struct fid_domain *domain, struct fi_info *info, - struct fid_ep **ep, void *context) + struct fid_ep **ep, void *context) { - return -FI_ENOSYS; /* TODO: implement this puppy */ + /* TODO: implement this puppy */ + return -FI_ENOSYS; } diff --git a/prov/gni/src/gnix_fabric.c b/prov/gni/src/gnix_fabric.c index 87c94876fe2..b868bf4e9cc 100644 --- a/prov/gni/src/gnix_fabric.c +++ b/prov/gni/src/gnix_fabric.c @@ -56,11 +56,11 @@ const char gnix_fab_name[] = "gni"; const char gnix_dom_name[] = "/sys/class/gni/kgni0"; -uint32_t gnix_cdm_modes = (GNI_CDM_MODE_FAST_DATAGRAM_POLL | \ - GNI_CDM_MODE_FMA_SHARED | \ - GNI_CDM_MODE_FMA_SMALL_WINDOW | \ - GNI_CDM_MODE_FORK_PARTCOPY | \ - GNI_CDM_MODE_ERR_NO_KILL); + +uint32_t gnix_cdm_modes = + (GNI_CDM_MODE_FAST_DATAGRAM_POLL | GNI_CDM_MODE_FMA_SHARED | + GNI_CDM_MODE_FMA_SMALL_WINDOW | GNI_CDM_MODE_FORK_PARTCOPY | + GNI_CDM_MODE_ERR_NO_KILL); const struct fi_fabric_attr gnix_fabric_attr = { .fabric = NULL, @@ -85,7 +85,7 @@ static int gnix_fabric_close(fid_t fid) struct gnix_fabric *fab; fab = container_of(fid, struct gnix_fabric, fab_fid); - if(!list_empty(&fab->cdm_list)) { + if (!list_empty(&fab->cdm_list)) { return -FI_EBUSY; } @@ -122,7 +122,7 @@ static int gnix_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, fab->fab_fid.fid.context = context; fab->fab_fid.fid.ops = &gnix_fab_fi_ops; fab->fab_fid.ops = &gnix_fab_ops; - list_head_init(&fab->cdm_list); + list_head_init(&fab->cdm_list); *fabric = &fab->fab_fid; return FI_SUCCESS; @@ -135,7 +135,9 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service, int ret = 0; int mode = GNIX_FAB_MODES; struct fi_info *gnix_info; - struct gnix_ep_name *dest_addr = NULL, *src_addr = NULL, *addr = NULL; + struct gnix_ep_name *dest_addr = NULL; + struct gnix_ep_name *src_addr = NULL; + struct gnix_ep_name *addr = NULL; /* * the code below for resolving a node/service to what @@ -300,7 +302,7 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service, gnix_info->tx_attr->caps = gnix_info->caps; gnix_info->tx_attr->mode = gnix_info->mode; - if(hints && hints->tx_attr && hints->tx_attr->op_flags) { + if (hints && hints->tx_attr && hints->tx_attr->op_flags) { gnix_info->tx_attr->op_flags = hints->tx_attr->op_flags; } else { gnix_info->tx_attr->op_flags = GNIX_EP_OP_FLAGS; @@ -316,7 +318,7 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service, gnix_info->rx_attr->caps = gnix_info->caps; gnix_info->rx_attr->mode = gnix_info->mode; - if(hints && hints->rx_attr && hints->rx_attr->op_flags) { + if (hints && hints->rx_attr && hints->rx_attr->op_flags) { gnix_info->rx_attr->op_flags = hints->rx_attr->op_flags; } else { gnix_info->rx_attr->op_flags = GNIX_EP_OP_FLAGS; diff --git a/prov/gni/src/gnix_mr.c b/prov/gni/src/gnix_mr.c index a1f1d0f1f56..bf86bd33ca2 100644 --- a/prov/gni/src/gnix_mr.c +++ b/prov/gni/src/gnix_mr.c @@ -33,7 +33,6 @@ // // memory registration common code // - #include #include @@ -44,7 +43,6 @@ int gnix_mr_reg(struct fid_domain *domain, const void *buf, size_t len, uint64_t access, uint64_t offset, uint64_t requested_key, uint64_t flags, struct fid_mr **mr, void *context) { - return -FI_ENOSYS; /* TODO: implement this puppy */ + /* TODO: implement this puppy */ + return -FI_ENOSYS; } - - diff --git a/prov/gni/src/gnix_util.c b/prov/gni/src/gnix_util.c index 5f36894f928..7478a855461 100644 --- a/prov/gni/src/gnix_util.c +++ b/prov/gni/src/gnix_util.c @@ -64,94 +64,97 @@ int gnixu_get_rdma_credentials(void *addr, uint8_t *ptag, uint32_t *cookie) { int ret = FI_SUCCESS; - int alps_status=0; + int alps_status = 0; uint64_t apid; size_t alps_count; - alpsAppLLIGni_t *rdmacred_rsp=NULL; - alpsAppGni_t *rdmacred_buf; + alpsAppLLIGni_t *rdmacred_rsp = NULL; + alpsAppGni_t *rdmacred_buf = NULL; - if((ptag == NULL) || (cookie == NULL)) { + if ((ptag == NULL) || (cookie == NULL)) { ret = -FI_EINVAL; goto err; } - /* - * TODO: need to handle non null addr differently at some point, + /* + * TODO: need to handle non null addr differently at some point, * a non-NULL addr can be used to acquire RDMA credentials other than * those assigned by ALPS/nativized slurm. - */ - - ret = alps_app_lli_lock(); /* lli_lock doesn't return anything useful */ - - /* - * First get our apid - */ + */ + /* lli_lock doesn't return anything useful */ + ret = alps_app_lli_lock(); - ret = alps_app_lli_put_request(ALPS_APP_LLI_ALPS_REQ_APID, NULL, 0); + /* + * First get our apid + */ + ret = alps_app_lli_put_request(ALPS_APP_LLI_ALPS_REQ_APID, NULL, 0); if (ret != ALPS_APP_LLI_ALPS_STAT_OK) { - fprintf(stderr,"lli put failed\n"); - ret = -FI_EIO; - goto err; - } + GNIX_LOG_ERROR("lli put failed, ret=%s\n", strerror(ret)); + ret = -FI_EIO; + goto err; + } - ret = alps_app_lli_get_response (&alps_status, &alps_count); + ret = alps_app_lli_get_response(&alps_status, &alps_count); if (alps_status != ALPS_APP_LLI_ALPS_STAT_OK) { - fprintf(stderr,"lli get response failed failed ret =" - "%s\n",strerror(ret)); - ret = -FI_EIO; - goto err; - } + GNIX_LOG_ERROR("lli get response failed, ret=%s\n", + strerror(ret)); + ret = -FI_EIO; + goto err; + } - ret = alps_app_lli_get_response_bytes (&apid, sizeof(apid)); + ret = alps_app_lli_get_response_bytes(&apid, sizeof(apid)); if (ret != ALPS_APP_LLI_ALPS_STAT_OK) { - fprintf(stderr,"lli get response bytes failed\n"); - ret = -FI_EIO; - goto err; - } - - /* - * now get the GNI rdma credentials info - */ + GNIX_LOG_ERROR("lli get response failed, ret=%s\n", + strerror(ret)); + ret = -FI_EIO; + goto err; + } - ret = alps_app_lli_put_request(ALPS_APP_LLI_ALPS_REQ_GNI, NULL, 0); + /* + * now get the GNI rdma credentials info + */ + ret = alps_app_lli_put_request(ALPS_APP_LLI_ALPS_REQ_GNI, NULL, 0); if (ret != ALPS_APP_LLI_ALPS_STAT_OK) { - ret = -FI_EIO; - goto err; - } + GNIX_LOG_ERROR("lli put failed, ret=%s\n", strerror(ret)); + ret = -FI_EIO; + goto err; + } - ret = alps_app_lli_get_response(&alps_status, &alps_count); + ret = alps_app_lli_get_response(&alps_status, &alps_count); if (alps_status != ALPS_APP_LLI_ALPS_STAT_OK) { - ret = -FI_EIO; - goto err; - } + GNIX_LOG_ERROR("lli get response failed, ret=%s\n", + strerror(ret)); + ret = -FI_EIO; + goto err; + } - rdmacred_rsp = (alpsAppLLIGni_t *)malloc(alps_count); - if (rdmacred_rsp == NULL) { - ret = -FI_ENOMEM; - goto err; - } + rdmacred_rsp = malloc(alps_count); + if (rdmacred_rsp == NULL) { + ret = -FI_ENOMEM; + goto err; + } - memset(rdmacred_rsp,0,alps_count); + memset(rdmacred_rsp, 0, alps_count); - ret = alps_app_lli_get_response_bytes(rdmacred_rsp, alps_count); + ret = alps_app_lli_get_response_bytes(rdmacred_rsp, alps_count); if (ret != ALPS_APP_LLI_ALPS_STAT_OK) { - ret = -FI_EIO; - goto err; - } + GNIX_LOG_ERROR("lli get response failed, ret=%s\n", + strerror(ret)); + ret = -FI_EIO; + goto err; + } - rdmacred_buf = (alpsAppGni_t *)(rdmacred_rsp->u.buf); + rdmacred_buf = (alpsAppGni_t *) rdmacred_rsp->u.buf; - /* + /* * just use the first ptag/cookie for now */ *ptag = rdmacred_buf[0].ptag; *cookie = rdmacred_buf[0].cookie; err: - alps_app_lli_unlock(); - if (rdmacred_rsp != NULL) + alps_app_lli_unlock(); + if (rdmacred_rsp != NULL) { free(rdmacred_rsp); + } return ret; - - } diff --git a/prov/gni/src/gnix_util.h b/prov/gni/src/gnix_util.h index db7988882c4..c115f03e075 100644 --- a/prov/gni/src/gnix_util.h +++ b/prov/gni/src/gnix_util.h @@ -52,7 +52,6 @@ extern const char gnix_dom_name[]; /* * prototypes */ - int gnixu_get_rdma_credentials(void *addr, uint8_t *ptag, uint32_t *cookie); - + #endif