/lustre/scratch5/treddy/march_april_2024_testing/libfabric_install_custom/bin/fi_info fi_info -l: libfabric:21222:1714406479::core:core:fi_param_get_():372 variable perf_cntr= libfabric:21222:1714406479::core:core:fi_param_get_():372 variable hook= libfabric:21222:1714406479::core:core:fi_param_get_():381 read string var hmem=cuda libfabric:21222:1714406479::core:core:cuda_hmem_verify_devices():563 Skipping check for NVIDIA devices with NVML routines libfabric:21222:1714406480::core:core:cuda_hmem_verify_devices():588 Number of CUDA devices detected: 4 libfabric:21222:1714406480::core:core:cuda_hmem_detect_p2p_access_support():638 Peer access from CUDA device 0 -> CUDA device 1 : Yes libfabric:21222:1714406480::core:core:cuda_hmem_detect_p2p_access_support():638 Peer access from CUDA device 1 -> CUDA device 2 : Yes libfabric:21222:1714406480::core:core:cuda_hmem_detect_p2p_access_support():638 Peer access from CUDA device 2 -> CUDA device 3 : Yes libfabric:21222:1714406480::core:core:fi_param_get_():372 variable hmem_cuda_use_gdrcopy= libfabric:21222:1714406480::core:core:fi_param_get_():372 variable hmem_disable_p2p= libfabric:21222:1714406480::core:mr:ofi_monitors_init():221 Initializing memory monitor uffd libfabric:21222:1714406480::core:mr:ofi_monitors_init():221 Initializing memory monitor memhooks libfabric:21222:1714406480::core:mr:ofi_monitors_init():221 Initializing memory monitor cuda libfabric:21222:1714406480::core:mr:ofi_monitors_init():221 Initializing memory monitor cuda_ipc libfabric:21222:1714406480::core:mr:ofi_monitors_init():221 Initializing memory monitor rocr libfabric:21222:1714406480::core:mr:ofi_monitors_init():221 Initializing memory monitor rocr_ipc libfabric:21222:1714406480::core:mr:ofi_monitors_init():221 Initializing memory monitor xpmem libfabric:21222:1714406480::core:mr:ofi_monitors_init():221 Initializing memory monitor ze libfabric:21222:1714406480::core:mr:ofi_monitors_init():221 Initializing memory monitor import libfabric:21222:1714406480::core:core:fi_param_get_():372 variable mr_cache_max_size= libfabric:21222:1714406480::core:core:fi_param_get_():372 variable mr_cache_max_count= libfabric:21222:1714406480::core:core:fi_param_get_():372 variable mr_cache_monitor= libfabric:21222:1714406480::core:core:fi_param_get_():372 variable mr_cuda_cache_monitor_enabled= libfabric:21222:1714406480::core:core:fi_param_get_():372 variable mr_rocr_cache_monitor_enabled= libfabric:21222:1714406480::core:core:fi_param_get_():372 variable mr_ze_cache_monitor_enabled= libfabric:21222:1714406480::core:mr:ofi_default_cache_size():82 default cache size=1053667248 libfabric:21222:1714406480::core:mr:ofi_monitors_init():305 Default memory monitor is: memhooks libfabric:21222:1714406480::core:core:fi_param_get_():372 variable provider= libfabric:21222:1714406480::core:core:fi_param_get_():372 variable universe_size= libfabric:21222:1714406480::core:core:fi_param_get_():372 variable av_remove_cleanup= libfabric:21222:1714406480::core:core:fi_param_get_():372 variable offload_coll_provider= libfabric:21222:1714406480::core:core:fi_param_get_():372 variable provider_path= libfabric:21222:1714406480::psm3:core:fi_psm3_ini():928 nid001212:rank0: build options: VERSION=305.1010=3.5.1.1, HAVE_PSM3_SRC=1, PSM3_CUDA=0 libfabric:21222:1714406480::psm3:core:psmx3_param_get_bool():94 nid001212:rank0: variable FI_PSM3_NAME_SERVER= libfabric:21222:1714406480::psm3:core:psmx3_param_get_bool():94 nid001212:rank0: variable FI_PSM3_TAGGED_RMA= libfabric:21222:1714406480::psm3:core:psmx3_param_get_str():124 nid001212:rank0: variable FI_PSM3_UUID= libfabric:21222:1714406480::psm3:core:psmx3_param_get_int():109 nid001212:rank0: variable FI_PSM3_DELAY= libfabric:21222:1714406480::psm3:core:psmx3_param_get_int():109 nid001212:rank0: variable FI_PSM3_TIMEOUT= libfabric:21222:1714406480::psm3:core:psmx3_param_get_int():109 nid001212:rank0: variable FI_PSM3_PROG_INTERVAL= libfabric:21222:1714406480::psm3:core:psmx3_param_get_str():124 nid001212:rank0: variable FI_PSM3_PROG_AFFINITY= libfabric:21222:1714406480::psm3:core:psmx3_param_get_int():109 nid001212:rank0: variable FI_PSM3_INJECT_SIZE= libfabric:21222:1714406480::psm3:core:psmx3_param_get_int():109 nid001212:rank0: variable FI_PSM3_LOCK_LEVEL= libfabric:21222:1714406480::psm3:core:psmx3_param_get_bool():94 nid001212:rank0: variable FI_PSM3_LAZY_CONN= libfabric:21222:1714406480::psm3:core:psmx3_param_get_int():109 nid001212:rank0: variable FI_PSM3_CONN_TIMEOUT= libfabric:21222:1714406480::psm3:core:psmx3_param_get_bool():94 nid001212:rank0: variable FI_PSM3_DISCONNECT= libfabric:21222:1714406480::psm3:core:psmx3_param_get_str():124 nid001212:rank0: variable FI_PSM3_TAG_LAYOUT= libfabric:21222:1714406480::psm3:core:psmx3_param_get_bool():94 nid001212:rank0: variable FI_PSM3_YIELD_MODE= libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: psm3 (305.1010) libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable rget_tc= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable rdzv_aligned_sw_rget= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable enable_trig_op_limit= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable disable_non_inject_msg_idc= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable disable_host_register= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable enable_unrestricted_end_ro= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable odp= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable force_odp= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable ats= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable iotlb= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable ats_mlock_mode= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable device_name= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable telemetry= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable telemetry_rgid= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable rx_match_mode= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable rdzv_threshold= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable rdzv_get_min= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable rdzv_eager_size= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable oflow_buf_size= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable oflow_buf_count= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable oflow_buf_min_posted= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable oflow_buf_max_cached= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable safe_devmem_copy_threshold= libfabric:21222:1714406480::cxi:core:fi_param_get_():399 read bool var optimized_mrs=0 libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable mr_match_events= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable prov_key_cache= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable llring_mode= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable zbcoll_radix= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable cq_policy= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable default_vni= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable eq_ack_batch_size= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable msg_lossless= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable req_buf_size= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable req_buf_min_posted= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable req_buf_max_count= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable req_buf_max_cached= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable hybrid_preemptive= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable hybrid_recv_preemptive= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable hybrid_posted_recv_preemptive= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable hybrid_unexpected_msg_preemptive= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable fc_retry_usec_delay= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable sw_rx_tx_init_max= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable ctrl_rx_eq_max_size= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable default_cq_size= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable disable_cq_hugetlb= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable disable_eq_hugetlb= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable cq_fill_percent= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable coll_job_id= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable coll_job_step_id= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable coll_fabric_mgr_url= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable coll_mcast_token= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable coll_use_dma_put= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable hwcoll_addrs_per_job= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable hwcoll_min_nodes= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable coll_retry_usec= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable coll_timeout_usec= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable default_tx_size= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable default_rx_size= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable disable_hmem_dev_register= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable force_ze_hmem_support= libfabric:21222:1714406480::cxi:core:fi_param_get_():372 variable rdzv_proto= libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi0 libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi0 libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi0 libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi0 libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi1 libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi1 libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi1 libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21222:1714406480::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi1 libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: cxi (0.1) libfabric:21222:1714406480::shm:core:fi_param_get_():372 variable sar_threshold= libfabric:21222:1714406480::shm:core:fi_param_get_():372 variable tx_size= libfabric:21222:1714406480::shm:core:fi_param_get_():372 variable rx_size= libfabric:21222:1714406480::shm:core:fi_param_get_():372 variable disable_cma= libfabric:21222:1714406480::shm:core:fi_param_get_():372 variable use_dsa_sar= libfabric:21222:1714406480::shm:core:fi_param_get_():372 variable use_xpmem= libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: shm (120.0) libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: sm2 (120.0) libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: verbs (120.0) libfabric:21222:1714406480::ofi_mrail:core:fi_param_get_():372 variable config= libfabric:21222:1714406480::ofi_mrail:core:fi_param_get_():372 variable addr= libfabric:21222:1714406480::ofi_mrail:core:fi_param_get_():372 variable addr_strc= libfabric:21222:1714406480::ofi_mrail:core:mrail_parse_env_vars():115 unable to read FI_OFI_MRAIL_ADDR env variable libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: ofi_mrail (120.0) libfabric:21222:1714406480::ofi_rxd:core:fi_param_get_():372 variable spin_count= libfabric:21222:1714406480::ofi_rxd:core:fi_param_get_():372 variable retry= libfabric:21222:1714406480::ofi_rxd:core:fi_param_get_():372 variable max_peers= libfabric:21222:1714406480::ofi_rxd:core:fi_param_get_():372 variable max_unacked= libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: ofi_rxd (120.0) libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: udp (120.0) libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable prov_name= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable port_high_range= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable port_low_range= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable tx_size= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable rx_size= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable max_inject= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable max_saved= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable max_saved_size= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable max_rx_size= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable nodelay= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable staging_sbuf_size= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable prefetch_rbuf_size= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable zerocopy_size= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable trace_msg= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable disable_auto_progress= libfabric:21222:1714406480::tcp:core:fi_param_get_():372 variable io_uring= libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: tcp (120.0) libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: ofi_hook_perf (120.0) libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: ofi_hook_trace (120.0) libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: ofi_hook_debug (120.0) libfabric:21222:1714406480::core:core:fi_param_get_():381 read string var hmem=cuda libfabric:21222:1714406480::core:core:cuda_hmem_verify_devices():563 Skipping check for NVIDIA devices with NVML routines libfabric:21222:1714406480::core:core:cuda_hmem_verify_devices():588 Number of CUDA devices detected: 4 libfabric:21222:1714406480::core:core:cuda_hmem_detect_p2p_access_support():638 Peer access from CUDA device 0 -> CUDA device 1 : Yes libfabric:21222:1714406480::core:core:cuda_hmem_detect_p2p_access_support():638 Peer access from CUDA device 1 -> CUDA device 2 : Yes libfabric:21222:1714406480::core:core:cuda_hmem_detect_p2p_access_support():638 Peer access from CUDA device 2 -> CUDA device 3 : Yes libfabric:21222:1714406480::core:core:fi_param_get_():372 variable hmem_cuda_use_gdrcopy= libfabric:21222:1714406480::core:core:fi_param_get_():372 variable hmem_disable_p2p= libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: ofi_hook_hmem (120.0) libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: ofi_hook_dmabuf_peer_mem (120.0) libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: ofi_hook_noop (120.0) libfabric:21222:1714406480::core:core:ofi_register_provider():504 registering provider: off_coll (120.0) libfabric:21222:1714406480::psm3:core:psmx3_fini():887 nid001212:rank0: libfabric:21222:1714406480::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor uffd libfabric:21222:1714406480::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor memhooks libfabric:21222:1714406480::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor cuda libfabric:21222:1714406480::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor cuda_ipc libfabric:21222:1714406480::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor rocr libfabric:21222:1714406480::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor rocr_ipc libfabric:21222:1714406480::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor xpmem libfabric:21222:1714406480::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor ze libfabric:21222:1714406480::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor import verbs: version: 120.0 cxi: version: 0.1 psm3: version: 305.1010 ofi_rxd: version: 120.0 shm: version: 120.0 udp: version: 120.0 tcp: version: 120.0 ofi_hook_perf: version: 120.0 ofi_hook_trace: version: 120.0 ofi_hook_debug: version: 120.0 ofi_hook_noop: version: 120.0 ofi_hook_hmem: version: 120.0 ofi_hook_dmabuf_peer_mem: version: 120.0 off_coll: version: 120.0 sm2: version: 120.0 ofi_mrail: version: 120.0 fi_info -p cxi: libfabric:21233:1714406481::core:core:fi_param_get_():372 variable perf_cntr= libfabric:21233:1714406481::core:core:fi_param_get_():372 variable hook= libfabric:21233:1714406481::core:core:fi_param_get_():381 read string var hmem=cuda libfabric:21233:1714406481::core:core:cuda_hmem_verify_devices():563 Skipping check for NVIDIA devices with NVML routines libfabric:21233:1714406482::core:core:cuda_hmem_verify_devices():588 Number of CUDA devices detected: 4 libfabric:21233:1714406482::core:core:cuda_hmem_detect_p2p_access_support():638 Peer access from CUDA device 0 -> CUDA device 1 : Yes libfabric:21233:1714406482::core:core:cuda_hmem_detect_p2p_access_support():638 Peer access from CUDA device 1 -> CUDA device 2 : Yes libfabric:21233:1714406482::core:core:cuda_hmem_detect_p2p_access_support():638 Peer access from CUDA device 2 -> CUDA device 3 : Yes libfabric:21233:1714406482::core:core:fi_param_get_():372 variable hmem_cuda_use_gdrcopy= libfabric:21233:1714406482::core:core:fi_param_get_():372 variable hmem_disable_p2p= libfabric:21233:1714406482::core:mr:ofi_monitors_init():221 Initializing memory monitor uffd libfabric:21233:1714406482::core:mr:ofi_monitors_init():221 Initializing memory monitor memhooks libfabric:21233:1714406482::core:mr:ofi_monitors_init():221 Initializing memory monitor cuda libfabric:21233:1714406482::core:mr:ofi_monitors_init():221 Initializing memory monitor cuda_ipc libfabric:21233:1714406482::core:mr:ofi_monitors_init():221 Initializing memory monitor rocr libfabric:21233:1714406482::core:mr:ofi_monitors_init():221 Initializing memory monitor rocr_ipc libfabric:21233:1714406482::core:mr:ofi_monitors_init():221 Initializing memory monitor xpmem libfabric:21233:1714406482::core:mr:ofi_monitors_init():221 Initializing memory monitor ze libfabric:21233:1714406482::core:mr:ofi_monitors_init():221 Initializing memory monitor import libfabric:21233:1714406482::core:core:fi_param_get_():372 variable mr_cache_max_size= libfabric:21233:1714406482::core:core:fi_param_get_():372 variable mr_cache_max_count= libfabric:21233:1714406482::core:core:fi_param_get_():372 variable mr_cache_monitor= libfabric:21233:1714406482::core:core:fi_param_get_():372 variable mr_cuda_cache_monitor_enabled= libfabric:21233:1714406482::core:core:fi_param_get_():372 variable mr_rocr_cache_monitor_enabled= libfabric:21233:1714406482::core:core:fi_param_get_():372 variable mr_ze_cache_monitor_enabled= libfabric:21233:1714406482::core:mr:ofi_default_cache_size():82 default cache size=1053667248 libfabric:21233:1714406482::core:mr:ofi_monitors_init():305 Default memory monitor is: memhooks libfabric:21233:1714406482::core:core:fi_param_get_():372 variable provider= libfabric:21233:1714406482::core:core:fi_param_get_():372 variable universe_size= libfabric:21233:1714406482::core:core:fi_param_get_():372 variable av_remove_cleanup= libfabric:21233:1714406482::core:core:fi_param_get_():372 variable offload_coll_provider= libfabric:21233:1714406482::core:core:fi_param_get_():372 variable provider_path= libfabric:21233:1714406482::psm3:core:fi_psm3_ini():928 nid001212:rank0: build options: VERSION=305.1010=3.5.1.1, HAVE_PSM3_SRC=1, PSM3_CUDA=0 libfabric:21233:1714406482::psm3:core:psmx3_param_get_bool():94 nid001212:rank0: variable FI_PSM3_NAME_SERVER= libfabric:21233:1714406482::psm3:core:psmx3_param_get_bool():94 nid001212:rank0: variable FI_PSM3_TAGGED_RMA= libfabric:21233:1714406482::psm3:core:psmx3_param_get_str():124 nid001212:rank0: variable FI_PSM3_UUID= libfabric:21233:1714406482::psm3:core:psmx3_param_get_int():109 nid001212:rank0: variable FI_PSM3_DELAY= libfabric:21233:1714406482::psm3:core:psmx3_param_get_int():109 nid001212:rank0: variable FI_PSM3_TIMEOUT= libfabric:21233:1714406482::psm3:core:psmx3_param_get_int():109 nid001212:rank0: variable FI_PSM3_PROG_INTERVAL= libfabric:21233:1714406482::psm3:core:psmx3_param_get_str():124 nid001212:rank0: variable FI_PSM3_PROG_AFFINITY= libfabric:21233:1714406482::psm3:core:psmx3_param_get_int():109 nid001212:rank0: variable FI_PSM3_INJECT_SIZE= libfabric:21233:1714406482::psm3:core:psmx3_param_get_int():109 nid001212:rank0: variable FI_PSM3_LOCK_LEVEL= libfabric:21233:1714406482::psm3:core:psmx3_param_get_bool():94 nid001212:rank0: variable FI_PSM3_LAZY_CONN= libfabric:21233:1714406482::psm3:core:psmx3_param_get_int():109 nid001212:rank0: variable FI_PSM3_CONN_TIMEOUT= libfabric:21233:1714406482::psm3:core:psmx3_param_get_bool():94 nid001212:rank0: variable FI_PSM3_DISCONNECT= libfabric:21233:1714406482::psm3:core:psmx3_param_get_str():124 nid001212:rank0: variable FI_PSM3_TAG_LAYOUT= libfabric:21233:1714406482::psm3:core:psmx3_param_get_bool():94 nid001212:rank0: variable FI_PSM3_YIELD_MODE= libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: psm3 (305.1010) libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable rget_tc= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable rdzv_aligned_sw_rget= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable enable_trig_op_limit= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable disable_non_inject_msg_idc= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable disable_host_register= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable enable_unrestricted_end_ro= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable odp= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable force_odp= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable ats= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable iotlb= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable ats_mlock_mode= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable device_name= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable telemetry= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable telemetry_rgid= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable rx_match_mode= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable rdzv_threshold= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable rdzv_get_min= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable rdzv_eager_size= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable oflow_buf_size= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable oflow_buf_count= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable oflow_buf_min_posted= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable oflow_buf_max_cached= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable safe_devmem_copy_threshold= libfabric:21233:1714406482::cxi:core:fi_param_get_():399 read bool var optimized_mrs=0 libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable mr_match_events= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable prov_key_cache= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable llring_mode= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable zbcoll_radix= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable cq_policy= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable default_vni= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable eq_ack_batch_size= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable msg_lossless= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable req_buf_size= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable req_buf_min_posted= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable req_buf_max_count= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable req_buf_max_cached= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable hybrid_preemptive= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable hybrid_recv_preemptive= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable hybrid_posted_recv_preemptive= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable hybrid_unexpected_msg_preemptive= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable fc_retry_usec_delay= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable sw_rx_tx_init_max= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable ctrl_rx_eq_max_size= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable default_cq_size= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable disable_cq_hugetlb= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable disable_eq_hugetlb= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable cq_fill_percent= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable coll_job_id= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable coll_job_step_id= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable coll_fabric_mgr_url= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable coll_mcast_token= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable coll_use_dma_put= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable hwcoll_addrs_per_job= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable hwcoll_min_nodes= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable coll_retry_usec= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable coll_timeout_usec= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable default_tx_size= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable default_rx_size= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable disable_hmem_dev_register= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable force_ze_hmem_support= libfabric:21233:1714406482::cxi:core:fi_param_get_():372 variable rdzv_proto= libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi0 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi0 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi0 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi0 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi1 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi1 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi1 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi1 libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: cxi (0.1) libfabric:21233:1714406482::shm:core:fi_param_get_():372 variable sar_threshold= libfabric:21233:1714406482::shm:core:fi_param_get_():372 variable tx_size= libfabric:21233:1714406482::shm:core:fi_param_get_():372 variable rx_size= libfabric:21233:1714406482::shm:core:fi_param_get_():372 variable disable_cma= libfabric:21233:1714406482::shm:core:fi_param_get_():372 variable use_dsa_sar= libfabric:21233:1714406482::shm:core:fi_param_get_():372 variable use_xpmem= libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: shm (120.0) libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: sm2 (120.0) libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: verbs (120.0) libfabric:21233:1714406482::ofi_mrail:core:fi_param_get_():372 variable config= libfabric:21233:1714406482::ofi_mrail:core:fi_param_get_():372 variable addr= libfabric:21233:1714406482::ofi_mrail:core:fi_param_get_():372 variable addr_strc= libfabric:21233:1714406482::ofi_mrail:core:mrail_parse_env_vars():115 unable to read FI_OFI_MRAIL_ADDR env variable libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: ofi_mrail (120.0) libfabric:21233:1714406482::ofi_rxd:core:fi_param_get_():372 variable spin_count= libfabric:21233:1714406482::ofi_rxd:core:fi_param_get_():372 variable retry= libfabric:21233:1714406482::ofi_rxd:core:fi_param_get_():372 variable max_peers= libfabric:21233:1714406482::ofi_rxd:core:fi_param_get_():372 variable max_unacked= libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: ofi_rxd (120.0) libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: udp (120.0) libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable prov_name= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable port_high_range= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable port_low_range= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable tx_size= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable rx_size= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable max_inject= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable max_saved= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable max_saved_size= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable max_rx_size= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable nodelay= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable staging_sbuf_size= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable prefetch_rbuf_size= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable zerocopy_size= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable trace_msg= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable disable_auto_progress= libfabric:21233:1714406482::tcp:core:fi_param_get_():372 variable io_uring= libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: tcp (120.0) libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: ofi_hook_perf (120.0) libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: ofi_hook_trace (120.0) libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: ofi_hook_debug (120.0) libfabric:21233:1714406482::core:core:fi_param_get_():381 read string var hmem=cuda libfabric:21233:1714406482::core:core:cuda_hmem_verify_devices():563 Skipping check for NVIDIA devices with NVML routines libfabric:21233:1714406482::core:core:cuda_hmem_verify_devices():588 Number of CUDA devices detected: 4 libfabric:21233:1714406482::core:core:cuda_hmem_detect_p2p_access_support():638 Peer access from CUDA device 0 -> CUDA device 1 : Yes libfabric:21233:1714406482::core:core:cuda_hmem_detect_p2p_access_support():638 Peer access from CUDA device 1 -> CUDA device 2 : Yes libfabric:21233:1714406482::core:core:cuda_hmem_detect_p2p_access_support():638 Peer access from CUDA device 2 -> CUDA device 3 : Yes libfabric:21233:1714406482::core:core:fi_param_get_():372 variable hmem_cuda_use_gdrcopy= libfabric:21233:1714406482::core:core:fi_param_get_():372 variable hmem_disable_p2p= libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: ofi_hook_hmem (120.0) libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: ofi_hook_dmabuf_peer_mem (120.0) libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: ofi_hook_noop (120.0) libfabric:21233:1714406482::core:core:ofi_register_provider():504 registering provider: off_coll (120.0) libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi0 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi0 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi0 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi0 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi1 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi1 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi1 libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_ss_env_get_vni():23 nid001212: SLINGSHOT_VNIS not found libfabric:21233:1714406482::cxi:fabric:cxip_nic_get_best_rgroup_vni():235 nid001212: Found (1:1) for cxi1 libfabric:21233:1714406482::core:core:ofi_layering_ok():1220 Skipping cxi;ofi_rxd layering libfabric:21233:1714406482::core:core:ofi_layering_ok():1220 Skipping cxi;ofi_mrail layering libfabric:21233:1714406482::psm3:core:psmx3_fini():887 nid001212:rank0: libfabric:21233:1714406482::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor uffd libfabric:21233:1714406482::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor memhooks libfabric:21233:1714406482::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor cuda libfabric:21233:1714406482::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor cuda_ipc libfabric:21233:1714406482::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor rocr libfabric:21233:1714406482::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor rocr_ipc libfabric:21233:1714406482::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor xpmem libfabric:21233:1714406482::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor ze libfabric:21233:1714406482::core:mr:ofi_monitors_cleanup():328 Cleaning up memory monitor import provider: cxi fabric: cxi domain: cxi0 version: 0.1 type: FI_EP_RDM protocol: FI_PROTO_CXI provider: cxi fabric: cxi domain: cxi1 version: 0.1 type: FI_EP_RDM protocol: FI_PROTO_CXI rm -rf cufftmp_r2c_c2r_slabs_GROMACS /lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/nvhpc-23.11-tkgy4rjxilbay253jj65msbj2vrbq673/Linux_x86_64/23.11/cuda/12.3/bin/../bin/nvcc cufftmp_r2c_c2r_slabs_GROMACS.cu -o cufftmp_r2c_c2r_slabs_GROMACS -std=c++17 --generate-code arch=compute_70,code=sm_70 --generate-code arch=compute_80,code=sm_80 --generate-code arch=compute_90,code=sm_90 -I/lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/nvhpc-23.11-tkgy4rjxilbay253jj65msbj2vrbq673/Linux_x86_64/23.11/math_libs/12.3/targets/x86_64-linux/include/cufftmp -I/lustre/scratch5/treddy/march_april_2024_testing/custom_nvshmem_install/include -I/lustre/scratch5/treddy/march_april_2024_testing/ompi5_install/include -lcuda -L/lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/nvhpc-23.11-tkgy4rjxilbay253jj65msbj2vrbq673/Linux_x86_64/23.11/math_libs/12.3/targets/x86_64-linux/lib -L/lustre/scratch5/treddy/march_april_2024_testing/custom_nvshmem_install/lib -lcufftMp -lnvshmem_device -lnvshmem_host -L/lustre/scratch5/treddy/march_april_2024_testing/ompi5_install/lib -lmpi LD_LIBRARY_PATH="/lustre/scratch5/treddy/march_april_2024_testing/custom_nvshmem_install/lib:/lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/nvhpc-23.11-tkgy4rjxilbay253jj65msbj2vrbq673/Linux_x86_64/23.11/math_libs/12.3/targets/x86_64-linux/lib:/usr/projects/hpcsoft/cos2/chicoma/cuda-compat/12.0/:lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/nvhpc-23.11-tkgy4rjxilbay253jj65msbj2vrbq673/Linux_x86_64/23.11/cuda/12.3/targets/x86_64-linux/lib:/lustre/scratch5/treddy/march_april_2024_testing/custom_nvshmem_install/lib:lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/nvhpc-23.11-tkgy4rjxilbay253jj65msbj2vrbq673/Linux_x86_64/23.11/cuda/12.3/targets/x86_64-linux/lib:/lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/ucx-1.15.0-hc255f5j4fcqhtufeisjj3pytrkv4dqt/lib/ucx:/lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/ucx-1.15.0-hc255f5j4fcqhtufeisjj3pytrkv4dqt/lib:/lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/nvhpc-23.11-tkgy4rjxilbay253jj65msbj2vrbq673/Linux_x86_64/23.11/comm_libs/12.3/nccl/lib:/lustre/scratch5/treddy/march_april_2024_testing/ompi5_install/lib:/lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/nvhpc-23.11-tkgy4rjxilbay253jj65msbj2vrbq673/Linux_x86_64/23.11/cuda/12.3/lib64:/lustre/scratch5/treddy/march_april_2024_testing/libfabric_install_custom/lib:/opt/cray/pe/papi/7.0.0.2/lib64:/opt/cray/libfabric/1.15.2.0/lib64:/lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/pmix-4.2.9-7kfa4s6dwyd5wlayw24vx7jai7d4oi4x/lib" /lustre/scratch5/treddy/march_april_2024_testing/ompi5_install/bin/mpirun -oversubscribe -n 8 -N 4 cufftmp_r2c_c2r_slabs_GROMACS Hello from rank 6/8 using GPU 2 Hello from rank 4/8 using GPU 0 Hello from rank 5/8 using GPU 1 Hello from rank 7/8 using GPU 3 Hello from rank 3/8 using GPU 3 Hello from rank 2/8 using GPU 2 Hello from rank 1/8 using GPU 1 Hello from rank 0/8 using GPU 0 NVSHMEM configuration: CUDA API 12030 CUDA Runtime 12030 CUDA Driver 12000 Build Timestamp Apr 25 2024 16:28:15 Build Variables NVSHMEM_DEBUG=OFF NVSHMEM_DEVEL=OFF NVSHMEM_DEFAULT_PMI2=OFF NVSHMEM_DEFAULT_PMIX=OFF NVSHMEM_DEFAULT_UCX=OFF NVSHMEM_DISABLE_COLL_POLL=ON NVSHMEM_ENABLE_ALL_DEVICE_INLINING=OFF NVSHMEM_ENV_ALL=OFF NVSHMEM_GPU_COLL_USE_LDST=OFF NVSHMEM_IBGDA_SUPPORT=OFF NVSHMEM_IBGDA_SUPPORT_GPUMEM_ONLY=OFF NVSHMEM_IBDEVX_SUPPORT=OFF NVSHMEM_IBRC_SUPPORT=ON NVSHMEM_LIBFABRIC_SUPPORT=ON NVSHMEM_MPI_SUPPORT=ON NVSHMEM_NVTX=ON NVSHMEM_PMIX_SUPPORT=OFF NVSHMEM_SHMEM_SUPPORT=OFF NVSHMEM_TEST_STATIC_LIB=OFF NVSHMEM_TIMEOUT_DEVICE_POLLING=OFF NVSHMEM_TRACE=OFF NVSHMEM_UCX_SUPPORT=OFF NVSHMEM_USE_DLMALLOC=OFF NVSHMEM_USE_NCCL=OFF NVSHMEM_USE_GDRCOPY=ON NVSHMEM_VERBOSE=OFF CUDA_HOME=/lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/nvhpc-23.11-tkgy4rjxilbay253jj65msbj2vrbq673/Linux_x86_64/23.11/cuda/12.3 GDRCOPY_HOME=/lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/gdrcopy-2.3-ftyzikjaithdoznahhzpuecguynyqqyv LIBFABRIC_HOME=/lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/libfabric-1.18.1-wjlmzgm3owaum6fxdhb45tpbfunbs33w MPI_HOME=/lustre/scratch5/treddy/march_april_2024_testing/ompi5_install NCCL_HOME=/usr/local/nccl NVSHMEM_PREFIX=/lustre/scratch5/treddy/march_april_2024_testing/custom_nvshmem_install PMIX_HOME=/usr SHMEM_HOME=/lustre/scratch5/treddy/march_april_2024_testing/ompi5_install UCX_HOME=/lustre/scratch5/.mdt1/treddy/march_april_2024_testing/github_projects/spack/opt/spack/linux-sles15-zen2/gcc-12.2.0/ucx-1.15.0-hc255f5j4fcqhtufeisjj3pytrkv4dqt nid001212:21306:21306 [1] NVSHMEM INFO PE 1 (process) affinity to 128 CPUs: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 nid001212:21307:21307 [2] NVSHMEM INFO PE 2 (process) affinity to 128 CPUs: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 nid001212:21308:21308 [3] NVSHMEM INFO PE 3 (process) affinity to 128 CPUs: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 nid001212:21305:21305 [0] NVSHMEM INFO PE 0 (process) affinity to 128 CPUs: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 nid001213:9631:9631 [2] NVSHMEM INFO PE 6 (process) affinity to 128 CPUs: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 nid001213:9630:9630 [1] NVSHMEM INFO PE 5 (process) affinity to 128 CPUs: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 nid001213:9629:9629 [0] NVSHMEM INFO PE 4 (process) affinity to 128 CPUs: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 nid001213:9632:9632 [3] NVSHMEM INFO PE 7 (process) affinity to 128 CPUs: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 nid001212:21305:21305 [0] NVSHMEM INFO cudaDriverVersion 12000 nid001212:21305:21305 [0] NVSHMEM INFO NVSHMEM symmetric heap kind = DEVICE selected nid001212:21305:21305 [0] NVSHMEM INFO [0] nvshmemi_get_cucontext->cuCtxSynchronize->CUDA_SUCCESS) my_stream (nil) nid001212:21305:21305 [0] NVSHMEM INFO in get_cucontext, queried and saved context for device: 0 context: 0x244c880 nid001212:21306:21306 [1] NVSHMEM INFO cudaDriverVersion 12000 nid001212:21306:21306 [1] NVSHMEM INFO NVSHMEM symmetric heap kind = DEVICE selected nid001212:21306:21306 [1] NVSHMEM INFO [1] nvshmemi_get_cucontext->cuCtxSynchronize->CUDA_SUCCESS) my_stream (nil) nid001212:21306:21306 [1] NVSHMEM INFO in get_cucontext, queried and saved context for device: 1 context: 0x24444c0 nid001212:21308:21308 [3] NVSHMEM INFO cudaDriverVersion 12000 nid001212:21308:21308 [3] NVSHMEM INFO NVSHMEM symmetric heap kind = DEVICE selected nid001212:21308:21308 [3] NVSHMEM INFO [3] nvshmemi_get_cucontext->cuCtxSynchronize->CUDA_SUCCESS) my_stream (nil) nid001212:21308:21308 [3] NVSHMEM INFO in get_cucontext, queried and saved context for device: 3 context: 0x2433d40 nid001212:21307:21307 [2] NVSHMEM INFO cudaDriverVersion 12000 nid001212:21307:21307 [2] NVSHMEM INFO NVSHMEM symmetric heap kind = DEVICE selected nid001212:21307:21307 [2] NVSHMEM INFO [2] nvshmemi_get_cucontext->cuCtxSynchronize->CUDA_SUCCESS) my_stream (nil) nid001212:21307:21307 [2] NVSHMEM INFO in get_cucontext, queried and saved context for device: 2 context: 0x243c100 nid001213:9631:9631 [2] NVSHMEM INFO cudaDriverVersion 12000 nid001213:9631:9631 [2] NVSHMEM INFO NVSHMEM symmetric heap kind = DEVICE selected nid001213:9631:9631 [2] NVSHMEM INFO [6] nvshmemi_get_cucontext->cuCtxSynchronize->CUDA_SUCCESS) my_stream (nil) nid001213:9631:9631 [2] NVSHMEM INFO in get_cucontext, queried and saved context for device: 2 context: 0x243bff0 nid001212:21305:21305 [0] NVSHMEM INFO [0] nvshmemi_get_cucontext->cuCtxGetDevice->0(CUDA_ERROR_INVALID_CONTEXT 201) cuStreamCreateWithPriority my_stream 0x40c3560 nid001212:21306:21306 [1] NVSHMEM INFO [1] nvshmemi_get_cucontext->cuCtxGetDevice->0(CUDA_ERROR_INVALID_CONTEXT 201) cuStreamCreateWithPriority my_stream 0x40c27a0 nid001212:21308:21308 [3] NVSHMEM INFO [3] nvshmemi_get_cucontext->cuCtxGetDevice->0(CUDA_ERROR_INVALID_CONTEXT 201) cuStreamCreateWithPriority my_stream 0x40c2860 nid001212:21307:21307 [2] NVSHMEM INFO [2] nvshmemi_get_cucontext->cuCtxGetDevice->0(CUDA_ERROR_INVALID_CONTEXT 201) cuStreamCreateWithPriority my_stream 0x40c2d60 nid001213:9632:9632 [3] NVSHMEM INFO cudaDriverVersion 12000 nid001213:9632:9632 [3] NVSHMEM INFO NVSHMEM symmetric heap kind = DEVICE selected nid001213:9632:9632 [3] NVSHMEM INFO [7] nvshmemi_get_cucontext->cuCtxSynchronize->CUDA_SUCCESS) my_stream (nil) nid001213:9632:9632 [3] NVSHMEM INFO in get_cucontext, queried and saved context for device: 3 context: 0x2433c30 nid001213:9629:9629 [0] NVSHMEM INFO cudaDriverVersion 12000 nid001213:9629:9629 [0] NVSHMEM INFO NVSHMEM symmetric heap kind = DEVICE selected nid001213:9629:9629 [0] NVSHMEM INFO [4] nvshmemi_get_cucontext->cuCtxSynchronize->CUDA_SUCCESS) my_stream (nil) nid001213:9629:9629 [0] NVSHMEM INFO in get_cucontext, queried and saved context for device: 0 context: 0x244c770 nid001213:9630:9630 [1] NVSHMEM INFO cudaDriverVersion 12000 nid001213:9630:9630 [1] NVSHMEM INFO NVSHMEM symmetric heap kind = DEVICE selected nid001213:9630:9630 [1] NVSHMEM INFO [5] nvshmemi_get_cucontext->cuCtxSynchronize->CUDA_SUCCESS) my_stream (nil) nid001213:9630:9630 [1] NVSHMEM INFO in get_cucontext, queried and saved context for device: 1 context: 0x24443b0 nid001213:9632:9632 [3] NVSHMEM INFO [7] nvshmemi_get_cucontext->cuCtxGetDevice->0(CUDA_ERROR_INVALID_CONTEXT 201) cuStreamCreateWithPriority my_stream 0x40c23e0 nid001213:9630:9630 [1] NVSHMEM INFO [5] nvshmemi_get_cucontext->cuCtxGetDevice->0(CUDA_ERROR_INVALID_CONTEXT 201) cuStreamCreateWithPriority my_stream 0x40c2620 nid001213:9631:9631 [2] NVSHMEM INFO [6] nvshmemi_get_cucontext->cuCtxGetDevice->0(CUDA_ERROR_INVALID_CONTEXT 201) cuStreamCreateWithPriority my_stream 0x40c26a0 nid001213:9629:9629 [0] NVSHMEM INFO [4] nvshmemi_get_cucontext->cuCtxGetDevice->0(CUDA_ERROR_INVALID_CONTEXT 201) cuStreamCreateWithPriority my_stream 0x40c2d70 nid001212:21305:21305 [0] NVSHMEM INFO nvshmemi_setup_local_heap, heapextra = 285225000 nid001212:21307:21307 [2] NVSHMEM INFO nvshmemi_setup_local_heap, heapextra = 285225000 nid001212:21308:21308 [3] NVSHMEM INFO nvshmemi_setup_local_heap, heapextra = 285225000 nid001212:21306:21306 [1] NVSHMEM INFO nvshmemi_setup_local_heap, heapextra = 285225000 nid001213:9631:9631 [2] NVSHMEM INFO nvshmemi_setup_local_heap, heapextra = 285225000 nid001213:9629:9629 [0] NVSHMEM INFO nvshmemi_setup_local_heap, heapextra = 285225000 nid001213:9632:9632 [3] NVSHMEM INFO nvshmemi_setup_local_heap, heapextra = 285225000 nid001213:9630:9630 [1] NVSHMEM INFO nvshmemi_setup_local_heap, heapextra = 285225000 nid001213:9631:9631 [2] NVSHMEM INFO [6] heap base: 0x14c720000000 NVSHMEM_SYMMETRIC_SIZE 1073741824 total 2147483648 heapextra 285225000 nid001213:9631:9631 [2] NVSHMEM INFO [6] mspace ptr: 0x14c80f4f5340 nid001213:9629:9629 [0] NVSHMEM INFO [4] heap base: 0x14c9e0000000 NVSHMEM_SYMMETRIC_SIZE 1073741824 total 2147483648 heapextra 285225000 nid001213:9629:9629 [0] NVSHMEM INFO [4] mspace ptr: 0x14cacb31f340 nid001213:9632:9632 [3] NVSHMEM INFO [7] heap base: 0x14e500000000 NVSHMEM_SYMMETRIC_SIZE 1073741824 total 2147483648 heapextra 285225000 nid001213:9632:9632 [3] NVSHMEM INFO [7] mspace ptr: 0x14e5f5237340 nid001213:9630:9630 [1] NVSHMEM INFO [5] heap base: 0x1466a0000000 NVSHMEM_SYMMETRIC_SIZE 1073741824 total 2147483648 heapextra 285225000 nid001213:9630:9630 [1] NVSHMEM INFO [5] mspace ptr: 0x146793e93340 nid001212:21305:21305 [0] NVSHMEM INFO [0] heap base: 0x150ae0000000 NVSHMEM_SYMMETRIC_SIZE 1073741824 total 2147483648 heapextra 285225000 nid001212:21305:21305 [0] NVSHMEM INFO [0] mspace ptr: 0x150bc9299340 nid001212:21307:21307 [2] NVSHMEM INFO [2] heap base: 0x146ae0000000 NVSHMEM_SYMMETRIC_SIZE 1073741824 total 2147483648 heapextra 285225000 nid001212:21307:21307 [2] NVSHMEM INFO [2] mspace ptr: 0x146bddf2f340 nid001212:21308:21308 [3] NVSHMEM INFO [3] heap base: 0x14eb40000000 NVSHMEM_SYMMETRIC_SIZE 1073741824 total 2147483648 heapextra 285225000 nid001212:21308:21308 [3] NVSHMEM INFO [3] mspace ptr: 0x14ec3edbf340 nid001212:21306:21306 [1] NVSHMEM INFO [1] heap base: 0x14ff20000000 NVSHMEM_SYMMETRIC_SIZE 1073741824 total 2147483648 heapextra 285225000 nid001212:21306:21306 [1] NVSHMEM INFO [1] mspace ptr: 0x1500113ab340 nid001213:9630:9630 [1] NVSHMEM INFO IBRC transport skipped in favor of: libfabric nid001213:9629:9629 [0] NVSHMEM INFO IBRC transport skipped in favor of: libfabric nid001213:9631:9631 [2] NVSHMEM INFO IBRC transport skipped in favor of: libfabric nid001213:9632:9632 [3] NVSHMEM INFO IBRC transport skipped in favor of: libfabric nid001212:21308:21308 [3] NVSHMEM INFO IBRC transport skipped in favor of: libfabric nid001212:21305:21305 [0] NVSHMEM INFO IBRC transport skipped in favor of: libfabric nid001212:21306:21306 [1] NVSHMEM INFO IBRC transport skipped in favor of: libfabric nid001212:21307:21307 [2] NVSHMEM INFO IBRC transport skipped in favor of: libfabric /lustre/scratch5/.mdt1/treddy/march_april_2024_testing/nvshmem_custom_dl/nvshmem_src_2.10.1-3/src/modules/transport/libfabric/libfabric.cpp 1706 GDRCopy requested, but unused by transport. Disabling. [nid001213:9629 :0:9629] Caught signal 11 (Segmentation fault: address not mapped to object at address 0x18) /lustre/scratch5/.mdt1/treddy/march_april_2024_testing/nvshmem_custom_dl/nvshmem_src_2.10.1-3/src/modules/transport/libfabric/libfabric.cpp 1706 GDRCopy requested, but unused by transport. Disabling. [nid001213:9630 :0:9630] Caught signal 11 (Segmentation fault: address not mapped to object at address 0x18) /lustre/scratch5/.mdt1/treddy/march_april_2024_testing/nvshmem_custom_dl/nvshmem_src_2.10.1-3/src/modules/transport/libfabric/libfabric.cpp 1706 GDRCopy requested, but unused by transport. Disabling. [nid001213:9631 :0:9631] Caught signal 11 (Segmentation fault: address not mapped to object at address 0x18) /lustre/scratch5/.mdt1/treddy/march_april_2024_testing/nvshmem_custom_dl/nvshmem_src_2.10.1-3/src/modules/transport/libfabric/libfabric.cpp 1706 GDRCopy requested, but unused by transport. Disabling. [nid001213:9632 :0:9632] Caught signal 11 (Segmentation fault: address not mapped to object at address 0x18) /lustre/scratch5/.mdt1/treddy/march_april_2024_testing/nvshmem_custom_dl/nvshmem_src_2.10.1-3/src/modules/transport/libfabric/libfabric.cpp 1706 GDRCopy requested, but unused by transport. Disabling. [nid001212:21308:0:21308] Caught signal 11 (Segmentation fault: address not mapped to object at address 0x18) /lustre/scratch5/.mdt1/treddy/march_april_2024_testing/nvshmem_custom_dl/nvshmem_src_2.10.1-3/src/modules/transport/libfabric/libfabric.cpp 1706 GDRCopy requested, but unused by transport. Disabling. [nid001212:21305:0:21305] Caught signal 11 (Segmentation fault: address not mapped to object at address 0x18) /lustre/scratch5/.mdt1/treddy/march_april_2024_testing/nvshmem_custom_dl/nvshmem_src_2.10.1-3/src/modules/transport/libfabric/libfabric.cpp 1706 GDRCopy requested, but unused by transport. Disabling. [nid001212:21306:0:21306] Caught signal 11 (Segmentation fault: address not mapped to object at address 0x18) /lustre/scratch5/.mdt1/treddy/march_april_2024_testing/nvshmem_custom_dl/nvshmem_src_2.10.1-3/src/modules/transport/libfabric/libfabric.cpp 1706 GDRCopy requested, but unused by transport. Disabling. [nid001212:21307:0:21307] Caught signal 11 (Segmentation fault: address not mapped to object at address 0x18) ==== backtrace (tid: 21305) ==== 0 0x00000000000168c0 __funlockfile() ???:0 1 0x0000000000028acc fi_dupinfo_() :0 2 0x000000000002ea5c fi_dupinfo_1_3() :0 3 0x000000000002eb3c fi_getinfo_1_3() :0 4 0x000000000000d0db nvshmemt_init() ???:0 5 0x00000000000e6c89 nvshmemi_transport_init() :0 6 0x00000000000ec01c nvshmemi_common_init() :0 7 0x00000000000ede99 nvshmemi_try_common_init() :0 8 0x00000000000ee30e nvshmemx_host_init() ???:0 9 0x000000000052e635 cufftMpDestroyReshape() ???:0 10 0x000000000011b4c3 cufftXtSetCallbackSharedSize() ???:0 11 0x0000000000147450 cufftXtMakePlanGuru64() ???:0 12 0x0000000000148105 cufftXtMakePlanMany() ???:0 13 0x0000000000145d7d cufftMakePlanMany64() ???:0 14 0x00000000001461bf cufftMakePlanMany() ???:0 15 0x0000000000146386 cufftMakePlan3d() ???:0 16 0x00000000004067f9 run_r2c_c2r_slabs() ???:0 17 0x0000000000407ba7 main() ???:0 18 0x000000000003529d __libc_start_main() ???:0 19 0x000000000040591a _start() /home/abuild/rpmbuild/BUILD/glibc-2.31/csu/../sysdeps/x86_64/start.S:120 ================================= ==== backtrace (tid: 21307) ==== 0 0x00000000000168c0 __funlockfile() ???:0 1 0x0000000000028acc fi_dupinfo_() :0 2 0x000000000002ea5c fi_dupinfo_1_3() :0 3 0x000000000002eb3c fi_getinfo_1_3() :0 4 0x000000000000d0db nvshmemt_init() ???:0 5 0x00000000000e6c89 nvshmemi_transport_init() :0 6 0x00000000000ec01c nvshmemi_common_init() :0 7 0x00000000000ede99 nvshmemi_try_common_init() :0 8 0x00000000000ee30e nvshmemx_host_init() ???:0 9 0x000000000052e635 cufftMpDestroyReshape() ???:0 10 0x000000000011b4c3 cufftXtSetCallbackSharedSize() ???:0 11 0x0000000000147450 cufftXtMakePlanGuru64() ???:0 12 0x0000000000148105 cufftXtMakePlanMany() ???:0 13 0x0000000000145d7d cufftMakePlanMany64() ???:0 14 0x00000000001461bf cufftMakePlanMany() ???:0 15 0x0000000000146386 cufftMakePlan3d() ???:0 16 0x00000000004067f9 run_r2c_c2r_slabs() ???:0 17 0x0000000000407ba7 main() ???:0 18 0x000000000003529d __libc_start_main() ???:0 19 0x000000000040591a _start() /home/abuild/rpmbuild/BUILD/glibc-2.31/csu/../sysdeps/x86_64/start.S:120 ================================= ==== backtrace (tid: 21306) ==== 0 0x00000000000168c0 __funlockfile() ???:0 1 0x0000000000028acc fi_dupinfo_() :0 2 0x000000000002ea5c fi_dupinfo_1_3() :0 3 0x000000000002eb3c fi_getinfo_1_3() :0 4 0x000000000000d0db nvshmemt_init() ???:0 5 0x00000000000e6c89 nvshmemi_transport_init() :0 6 0x00000000000ec01c nvshmemi_common_init() :0 7 0x00000000000ede99 nvshmemi_try_common_init() :0 8 0x00000000000ee30e nvshmemx_host_init() ???:0 9 0x000000000052e635 cufftMpDestroyReshape() ???:0 10 0x000000000011b4c3 cufftXtSetCallbackSharedSize() ???:0 11 0x0000000000147450 cufftXtMakePlanGuru64() ???:0 12 0x0000000000148105 cufftXtMakePlanMany() ???:0 13 0x0000000000145d7d cufftMakePlanMany64() ???:0 14 0x00000000001461bf cufftMakePlanMany() ???:0 15 0x0000000000146386 cufftMakePlan3d() ???:0 16 0x00000000004067f9 run_r2c_c2r_slabs() ???:0 17 0x0000000000407ba7 main() ???:0 18 0x000000000003529d __libc_start_main() ???:0 19 0x000000000040591a _start() /home/abuild/rpmbuild/BUILD/glibc-2.31/csu/../sysdeps/x86_64/start.S:120 ================================= ==== backtrace (tid: 21308) ==== 0 0x00000000000168c0 __funlockfile() ???:0 1 0x0000000000028acc fi_dupinfo_() :0 2 0x000000000002ea5c fi_dupinfo_1_3() :0 3 0x000000000002eb3c fi_getinfo_1_3() :0 4 0x000000000000d0db nvshmemt_init() ???:0 5 0x00000000000e6c89 nvshmemi_transport_init() :0 6 0x00000000000ec01c nvshmemi_common_init() :0 7 0x00000000000ede99 nvshmemi_try_common_init() :0 8 0x00000000000ee30e nvshmemx_host_init() ???:0 9 0x000000000052e635 cufftMpDestroyReshape() ???:0 10 0x000000000011b4c3 cufftXtSetCallbackSharedSize() ???:0 11 0x0000000000147450 cufftXtMakePlanGuru64() ???:0 12 0x0000000000148105 cufftXtMakePlanMany() ???:0 13 0x0000000000145d7d cufftMakePlanMany64() ???:0 14 0x00000000001461bf cufftMakePlanMany() ???:0 15 0x0000000000146386 cufftMakePlan3d() ???:0 16 0x00000000004067f9 run_r2c_c2r_slabs() ???:0 17 0x0000000000407ba7 main() ???:0 18 0x000000000003529d __libc_start_main() ???:0 19 0x000000000040591a _start() /home/abuild/rpmbuild/BUILD/glibc-2.31/csu/../sysdeps/x86_64/start.S:120 ================================= ==== backtrace (tid: 9629) ==== 0 0x00000000000168c0 __funlockfile() ???:0 1 0x0000000000028acc fi_dupinfo_() :0 2 0x000000000002ea5c fi_dupinfo_1_3() :0 3 0x000000000002eb3c fi_getinfo_1_3() :0 4 0x000000000000d0db nvshmemt_init() ???:0 5 0x00000000000e6c89 nvshmemi_transport_init() :0 6 0x00000000000ec01c nvshmemi_common_init() :0 7 0x00000000000ede99 nvshmemi_try_common_init() :0 8 0x00000000000ee30e nvshmemx_host_init() ???:0 9 0x000000000052e635 cufftMpDestroyReshape() ???:0 10 0x000000000011b4c3 cufftXtSetCallbackSharedSize() ???:0 11 0x0000000000147450 cufftXtMakePlanGuru64() ???:0 12 0x0000000000148105 cufftXtMakePlanMany() ???:0 13 0x0000000000145d7d cufftMakePlanMany64() ???:0 14 0x00000000001461bf cufftMakePlanMany() ???:0 15 0x0000000000146386 cufftMakePlan3d() ???:0 16 0x00000000004067f9 run_r2c_c2r_slabs() ???:0 17 0x0000000000407ba7 main() ???:0 18 0x000000000003529d __libc_start_main() ???:0 19 0x000000000040591a _start() /home/abuild/rpmbuild/BUILD/glibc-2.31/csu/../sysdeps/x86_64/start.S:120 ================================= ==== backtrace (tid: 9632) ==== 0 0x00000000000168c0 __funlockfile() ???:0 1 0x0000000000028acc fi_dupinfo_() :0 2 0x000000000002ea5c fi_dupinfo_1_3() :0 3 0x000000000002eb3c fi_getinfo_1_3() :0 4 0x000000000000d0db nvshmemt_init() ???:0 5 0x00000000000e6c89 nvshmemi_transport_init() :0 6 0x00000000000ec01c nvshmemi_common_init() :0 7 0x00000000000ede99 nvshmemi_try_common_init() :0 8 0x00000000000ee30e nvshmemx_host_init() ???:0 9 0x000000000052e635 cufftMpDestroyReshape() ???:0 10 0x000000000011b4c3 cufftXtSetCallbackSharedSize() ???:0 11 0x0000000000147450 cufftXtMakePlanGuru64() ???:0 12 0x0000000000148105 cufftXtMakePlanMany() ???:0 13 0x0000000000145d7d cufftMakePlanMany64() ???:0 14 0x00000000001461bf cufftMakePlanMany() ???:0 15 0x0000000000146386 cufftMakePlan3d() ???:0 16 0x00000000004067f9 run_r2c_c2r_slabs() ???:0 17 0x0000000000407ba7 main() ???:0 18 0x000000000003529d __libc_start_main() ???:0 19 0x000000000040591a _start() /home/abuild/rpmbuild/BUILD/glibc-2.31/csu/../sysdeps/x86_64/start.S:120 ================================= ==== backtrace (tid: 9631) ==== 0 0x00000000000168c0 __funlockfile() ???:0 1 0x0000000000028acc fi_dupinfo_() :0 2 0x000000000002ea5c fi_dupinfo_1_3() :0 3 0x000000000002eb3c fi_getinfo_1_3() :0 4 0x000000000000d0db nvshmemt_init() ???:0 5 0x00000000000e6c89 nvshmemi_transport_init() :0 6 0x00000000000ec01c nvshmemi_common_init() :0 7 0x00000000000ede99 nvshmemi_try_common_init() :0 8 0x00000000000ee30e nvshmemx_host_init() ???:0 9 0x000000000052e635 cufftMpDestroyReshape() ???:0 10 0x000000000011b4c3 cufftXtSetCallbackSharedSize() ???:0 11 0x0000000000147450 cufftXtMakePlanGuru64() ???:0 12 0x0000000000148105 cufftXtMakePlanMany() ???:0 13 0x0000000000145d7d cufftMakePlanMany64() ???:0 14 0x00000000001461bf cufftMakePlanMany() ???:0 15 0x0000000000146386 cufftMakePlan3d() ???:0 16 0x00000000004067f9 run_r2c_c2r_slabs() ???:0 17 0x0000000000407ba7 main() ???:0 18 0x000000000003529d __libc_start_main() ???:0 19 0x000000000040591a _start() /home/abuild/rpmbuild/BUILD/glibc-2.31/csu/../sysdeps/x86_64/start.S:120 ================================= ==== backtrace (tid: 9630) ==== 0 0x00000000000168c0 __funlockfile() ???:0 1 0x0000000000028acc fi_dupinfo_() :0 2 0x000000000002ea5c fi_dupinfo_1_3() :0 3 0x000000000002eb3c fi_getinfo_1_3() :0 4 0x000000000000d0db nvshmemt_init() ???:0 5 0x00000000000e6c89 nvshmemi_transport_init() :0 6 0x00000000000ec01c nvshmemi_common_init() :0 7 0x00000000000ede99 nvshmemi_try_common_init() :0 8 0x00000000000ee30e nvshmemx_host_init() ???:0 9 0x000000000052e635 cufftMpDestroyReshape() ???:0 10 0x000000000011b4c3 cufftXtSetCallbackSharedSize() ???:0 11 0x0000000000147450 cufftXtMakePlanGuru64() ???:0 12 0x0000000000148105 cufftXtMakePlanMany() ???:0 13 0x0000000000145d7d cufftMakePlanMany64() ???:0 14 0x00000000001461bf cufftMakePlanMany() ???:0 15 0x0000000000146386 cufftMakePlan3d() ???:0 16 0x00000000004067f9 run_r2c_c2r_slabs() ???:0 17 0x0000000000407ba7 main() ???:0 18 0x000000000003529d __libc_start_main() ???:0 19 0x000000000040591a _start() /home/abuild/rpmbuild/BUILD/glibc-2.31/csu/../sysdeps/x86_64/start.S:120 ================================= -------------------------------------------------------------------------- This help section is empty because PRRTE was built without Sphinx. -------------------------------------------------------------------------- make: *** [Makefile:18: run] Error 139