nvmf_tgt seg fault

[jrgruher]

I'm running a dual socket Skylake server with P4510 NVMe and 100Gb Mellanox CX4 NIC. OS is Ubuntu 18.04 with kernel 4.18.16. SPDK version is 18.10, FIO version is 3.12. I'm running the SPDK NVMeoF target and exercising it from an initiator system (similar config to the target but with 50Gb NIC) using FIO with the bdev plugin. I find 128K sequential workloads reliably and immediately seg fault nvmf_tgt. I can run 4KB random workloads without experiencing the seg fault, so the problem seems tied to the block size and/or IO pattern. I can run the same IO pattern against a local PCIe device using SPDK without a problem, I only see the failure when running the NVMeoF target with FIO running the IO patter from an SPDK initiator system.

Steps to reproduce and seg fault output follow below.

Start the target:
sudo ~/install/spdk/app/nvmf_tgt/nvmf_tgt -m 0x0000F0 -r /var/tmp/spdk1.sock

Configure the target:
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d1 -t pcie -a 0000:1a:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d2 -t pcie -a 0000:1b:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d3 -t pcie -a 0000:1c:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d4 -t pcie -a 0000:1d:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d5 -t pcie -a 0000:3d:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d6 -t pcie -a 0000:3e:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d7 -t pcie -a 0000:3f:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d8 -t pcie -a 0000:40:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_raid_bdev -n raid1 -s 4 -r 0 -b "d1n1 d2n1 d3n1 d4n1 d5n1 d6n1 d7n1 d8n1"
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_store raid1 store1
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l1 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l2 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l3 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l4 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l5 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l6 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l7 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l8 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l9 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l10 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l11 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l12 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn1 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn2 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn3 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn4 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn5 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn6 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn7 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn8 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn9 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn10 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn11 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn12 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn1 store1/l1
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn2 store1/l2
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn3 store1/l3
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn4 store1/l4
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn5 store1/l5
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn6 store1/l6
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn7 store1/l7
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn8 store1/l8
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn9 store1/l9
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn10 store1/l10
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn11 store1/l11
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn12 store1/l12
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn1 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn2 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn3 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn4 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn5 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn6 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn7 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn8 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn9 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn10 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn11 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn12 -t rdma -a 10.5.0.202 -s 4420

FIO file on initiator:
[global]
rw=rw
rwmixread=100
numjobs=1
iodepth=32
bs=128k
direct=1
thread=1
time_based=1
ramp_time=10
runtime=10
ioengine=spdk_bdev
spdk_conf=/home/don/fio/nvmeof.conf
group_reporting=1
unified_rw_reporting=1
exitall=1
randrepeat=0
norandommap=1
cpus_allowed_policy=split
cpus_allowed=1-2
[job1]
filename=b0n1

Config file on initiator:
[Nvme]
TransportID "trtype:RDMA traddr:10.5.0.202 trsvcid:4420 subnqn:nqn.2018-11.io.spdk:nqn1 adrfam:IPv4" b0

Run FIO on initiator and nvmf_tgt seg faults immediate:
sudo LD_PRELOAD=/home/don/install/spdk/examples/bdev/fio_plugin/fio_plugin fio sr.ini

Seg fault looks like this:
mlx5: donsl202: got completion with error:
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000
00000001 00000000 00000000 00000000
00000000 9d005304 0800011b 0008d0d2
rdma.c:2698:spdk_nvmf_rdma_poller_poll: *WARNING*: CQ error on CQ 0x7f079c01d170, Request 0x139670660105216 (4): local protection error
rdma.c: 501:spdk_nvmf_rdma_set_ibv_state: *NOTICE*: IBV QP#1 changed to: IBV_QPS_ERR
rdma.c:2698:spdk_nvmf_rdma_poller_poll: *WARNING*: CQ error on CQ 0x7f079c01d170, Request 0x139670660105216 (5): Work Request Flushed Error
rdma.c: 501:spdk_nvmf_rdma_set_ibv_state: *NOTICE*: IBV QP#1 changed to: IBV_QPS_ERR
rdma.c:2698:spdk_nvmf_rdma_poller_poll: *WARNING*: CQ error on CQ 0x7f079c01d170, Request 0x139670660106280 (5): Work Request Flushed Error
rdma.c: 501:spdk_nvmf_rdma_set_ibv_state: *NOTICE*: IBV QP#1 changed to: IBV_QPS_ERR
rdma.c:2698:spdk_nvmf_rdma_poller_poll: *WARNING*: CQ error on CQ 0x7f079c01d170, Request 0x139670660106280 (5): Work Request Flushed Error
Segmentation fault

Adds this to dmesg:
[71561.859644] nvme nvme1: Connect rejected: status 8 (invalid service ID).
[71561.866466] nvme nvme1: rdma connection establishment failed (-104)
[71567.805288] reactor_7[9166]: segfault at 88 ip 00005630621e6580 sp 00007f07af5fc400 error 4 in nvmf_tgt[563062194000+df000]
[71567.805293] Code: 48 8b 30 e8 82 f7 ff ff e9 7d fe ff ff 0f 1f 44 00 00 41 81 f9 80 00 00 00 75 37 49 8b 07 4c 8b 70 40 48 c7 40 50 00 00 00 00 <49> 8b 96 88 00 00 00 48 89 50 58 49 8b 96 88 00 00 00 48 89 02 48

[peluse]

Didn't get the segfault yet as I was starting with a much more basic config to baseline things. Sinlge NVMe device on the target and on the init side tried your .fio file and for the error below on the target.

mlx5: pels-64: got completion with error:
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000
00000005 00000000 00000000 00000000
00000000 9d005304 08000188 0026c3d2

A few more notes:

• no observed errors with Q depth 32 and 4K IO
• no observed errors with Q depth 1 and 128K IO
• Same error as above with Q depth 32 or 16 and 128K IO
• the behavior above is the same wether using fio on the init side or the nvme perf tool
• tests conducted on master

Will look into size as a clue, Seth noted 128K is the max IO size for our implementation. Once the error above is understood we can determine next steps for the seg fault (could be the same root cause but manifesting itself differently here vs there due to the fact that I only have one NVMe target set up)

[jrgruher]

Good update Paul. By the way, I can provide access to my test setup for folks inside Intel, if that would help.

[peluse]

Yeah at some point soon I think that would be a huge help. I'll dig into this thing I can repro here a little bit today/tomorrow then Ben or someone will have to jump in to move on it a little more quickly, plus I'm off all next week for Turkey & family time....

[jrgruher]

For my testing, I do not install Mellanox OFED, just use the Ubuntu packages.

My original report used SPDK 18.10. I tried to reproduce on master today. When I start my FIO job I get a bunch of prints like this on the target, but it does not seg fault:

rdma.c:2487:spdk_nvmf_rdma_poller_poll: WARNING: CQ error on CQ 0x7f430c009250, Request 0x139925942219968 (4): local protection error
rdma.c:2487:spdk_nvmf_rdma_poller_poll: WARNING: CQ error on CQ 0x7f430c009250, Request 0x139925942219816 (5): Work Request Flushed Error
rdma.c:2487:spdk_nvmf_rdma_poller_poll: WARNING: CQ error on CQ 0x7f430c009250, Request 0x139925942210248 (5): Work Request Flushed Error
rdma.c:2487:spdk_nvmf_rdma_poller_poll: WARNING: CQ error on CQ 0x7f430c009250, Request 0x139925942210096 (5): Work Request Flushed Error
rdma.c:2487:spdk_nvmf_rdma_poller_poll: WARNING: CQ error on CQ 0x7f430c009250, Request 0x139925942211328 (5): Work Request Flushed Error

My FIO job also hangs up and doesn't exit. I think it is probably not getting any IO from the target.

[jrgruher]

Joe, Sasha - can you both try to reproduce the issue after having the NVMe-oF
target pre-allocate its memory? This is the '-s' option to the target. Set it to at
least 4GB to be safe. I'm concerned that this is a problem introduced by the
patches that enable dynamic memory allocation (which creates multiple ibv_mrs
and requires smarter splitting code that doesn't exist yet).

I can't seem to start the target with -s 4096, the most it will allow is 2048:

don@donsl202:~/install/spdk/scripts$ sudo ~/install/spdk/app/nvmf_tgt/nvmf_tgt -s 4096 -m 0x0F0000 -r /var/tmp/spdk2.sock
Starting SPDK v19.01-pre / DPDK 18.08.0 initialization...
[ DPDK EAL parameters: nvmf --no-shconf -c 0x0F0000 -m 4096 --base-virtaddr=0x200000000000 --file-prefix=spdk_pid24899 ]
EAL: Detected 24 lcore(s)
EAL: Detected 2 NUMA nodes
EAL: No free hugepages reported in hugepages-1048576kB
EAL: Probing VFIO support...
EAL: Not enough memory available! Requested: 4096MB, available: 2048MB
EAL: FATAL: Cannot init memory
EAL: Cannot init memory
Failed to initialize DPDK
Unable to initialize SPDK env

It if I use a value of 2048, with master I still get something like this when I launch FIO, and FIO does not report any IO:

rdma.c:2487:spdk_nvmf_rdma_poller_poll: WARNING: CQ error on CQ 0x7f68d0009250, Request 0x140088144468024 (5): Work Request Flushed Error

If I try it on 18.10 I still get my seg fault:

mlx5: donsl202: got completion with error:
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000
00000001 00000000 00000000 00000000
00000000 00008813 080000d2 00362ad2
rdma.c:2698:spdk_nvmf_rdma_poller_poll: WARNING: CQ error on CQ 0x7fcde409d050, Request 0x140522271589792 (10): remote access error
Segmentation fault

[jimharris]

I cloned the rdma-core git repo and started decoding these "got completion with error" dumps.

The dump is of the CQE (completion queue entry). The upper nibble (d) of the last byte (d2) is the opcode. In the mlx5 provider, this is MLX5_CQE_REQ_ERR. This is consistent in all of the logs I've seen so far (both of @jrgruher's, one from @peluse, three i've looked at from the test pool).

The syndrome is the last byte in 2nd dword of the last line. In @jgruher's last post, this would be (13) from dword (00008813). But all of the other cases it is (04).

04 = MLX5_CQE_SYNDROME_LOCAL_PROT_ERR
13 = MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR

Still a lot of work to do, but this narrows down where to look. @benlwalker just now suggested a couple of places to add instrumentation to get more details when these occur. I'm going to try those and get them merged so we can start collecting more data on this.

[peluse]

@jimharris I'm back from vacation, feel free to hand anything back off to me....

[Seth5141]

@jimharris @jrgruher
Just wanted to update this issue with some of the communication from the mailing list as well as my own findings this morning.

As far as the initial segfault/target crash goes, it sounds like that portion of the problem may already be solved. From a comment @sashakot made on the mailing list "Crash, after hit 'local protection error', is solved in 'master'. In 18.10, the crash is a result of wrong processing of op code in RDMA completion".

For the underlying issue, I believe that the patch Jim submitted on the mailing list is the desired solution to this problem.

On my system, I was able to apply three patches to fix a simplified instance of this error (one where I run nvmf_lvol locally.)

1. https://review.gerrithub.io/#/c/spdk/spdk/+/434895/ @jimharris - changes the way our memory hotplug callback handles memory allocation
2. https://review.gerrithub.io/#/c/spdk/spdk/+/435179/ @Seth5141 - adds callback functions to the NVMe-oF memory maps to check that two MR mappings are contiguous.
3. https://review.gerrithub.io/#/c/spdk/spdk/+/433076/ @darsto - fixes the calculation for contiguous memory in spdk_mem_map_translate

I believe that these three changes are sufficient to fix the overarching bug of buffers spanning memory regions. However, we should verify that this patch series fixes the initial testing scenario.

@jrgruher, please try it locally by checking out Darek's patch based on top of Jim's and my patches:
git fetch https://review.gerrithub.io/spdk/spdk refs/changes/76/433076/4 && git checkout FETCH_HEAD

[jrgruher]

The short version is that I cannot recreate the original seg fault issue, however, I see some new problems. Most concerning is a failure to create all my target namespaces when it that part of the process worked fine on 18.10 and master. Not sure if these issues are related to the new patches and we want to debug them here, or if we think the patches are OK as a fix for the original issue and we want to take the new problems to a separate issue.

Here's the long version. I did the following steps to reinstall SPDK on the initiator and target, hopefully that is correct to install with the requested patches:

sudo rm -rf spdk
git clone https://github.com/spdk/spdk.git
cd spdk
git fetch https://review.gerrithub.io/spdk/spdk refs/changes/76/433076/4 && git checkout FETCH_HEAD
git submodule update --init
sudo ./scripts/pkgdep.sh
sudo ./configure --with-rdma --with-fio=/home/don/install/fio/
sudo make -j24

Unit test passes on both systems. Scripts/setup.sh seems to run OK on the initiator system, but I get some output in dmesg on the target system, this is only a segment but it is all of this general form:

[  153.456521] pcieport 0000:b0:00.0: bridge window [io  0x1000-0x0fff] to [bus b1] add_size 1000
[  153.456793] pcieport 0000:b0:01.0: bridge window [io  0x1000-0x0fff] to [bus b2] add_size 1000
[  153.457066] pcieport 0000:b0:02.0: bridge window [io  0x1000-0x0fff] to [bus b3] add_size 1000
[  153.457341] pcieport 0000:b0:03.0: bridge window [io  0x1000-0x0fff] to [bus b4] add_size 1000
[  153.457616] pcieport 0000:b0:04.0: bridge window [io  0x1000-0x0fff] to [bus b5] add_size 1000
[  153.457892] pcieport 0000:b0:05.0: bridge window [io  0x1000-0x0fff] to [bus b6] add_size 1000
[  153.458170] pcieport 0000:b0:06.0: bridge window [io  0x1000-0x0fff] to [bus b7] add_size 1000
[  153.458462] pcieport 0000:b0:07.0: bridge window [io  0x1000-0x0fff] to [bus b8] add_size 1000
[  153.458711] pcieport 0000:af:00.0: bridge window [io  0x1000-0x0fff] to [bus b0-b8] add_size 8000
[  153.458717] pcieport 0000:ae:00.0: bridge window [io  0x1000-0x0fff] to [bus af-b8] add_size 8000
[  153.458721] pcieport 0000:ae:00.0: BAR 13: no space for [io  size 0x8000]
[  153.458722] pcieport 0000:ae:00.0: BAR 13: failed to assign [io  size 0x8000]
[  153.458724] pcieport 0000:ae:00.0: BAR 13: no space for [io  size 0x8000]
[  153.458725] pcieport 0000:ae:00.0: BAR 13: failed to assign [io  size 0x8000]
[  153.458726] pcieport 0000:af:00.0: BAR 13: no space for [io  size 0x8000]
[  153.458727] pcieport 0000:af:00.0: BAR 13: failed to assign [io  size 0x8000]
[  153.458729] pcieport 0000:af:00.0: BAR 13: no space for [io  size 0x8000]
[  153.458730] pcieport 0000:af:00.0: BAR 13: failed to assign [io  size 0x8000]
[  153.458732] pcieport 0000:b0:00.0: BAR 13: no space for [io  size 0x1000]

Possibly these are normal, but I don't recall seeing them in dmesg when testing on 18.10.

I'm still able to start nvmf_tgt on the target system without any obvious problems in the output or in dmesg.

When creating my target namespaces using rpc.py on the target, some seem to get created, but some are not formed correctly. Here's the discovery information, it has the correct numbers of namespaces, but you can see they don't all look happy:

Discovery Log Number of Records 12, Generation counter 25
=====Discovery Log Entry 0======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  0
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn1
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 1======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  1
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn2
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 2======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  2
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn3
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 3======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  3
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn4
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 4======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  4
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn5
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 5======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  5
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn6
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 6======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  6
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn7
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 7======
trtype:  unrecognized
adrfam:  pci
subtype: unrecognized
treq:    not specified
portid:  0
trsvcid:
subnqn:
traddr:
=====Discovery Log Entry 8======
trtype:  unrecognized
adrfam:  pci
subtype: unrecognized
treq:    not specified
portid:  0
trsvcid:
subnqn:
traddr:
=====Discovery Log Entry 9======
trtype:  unrecognized
adrfam:  pci
subtype: unrecognized
treq:    not specified
portid:  0
trsvcid:
subnqn:
traddr:
=====Discovery Log Entry 10======
trtype:  unrecognized
adrfam:  pci
subtype: unrecognized
treq:    not specified
portid:  0
trsvcid:
subnqn:
traddr:
=====Discovery Log Entry 11======
trtype:  unrecognized
adrfam:  pci
subtype: unrecognized
treq:    not specified
portid:  0
trsvcid:
subnqn:
traddr:

I also note this print from nvmf_tgt when I run the discovery operation:
rdma.c: 530:spdk_nvmf_rdma_set_ibv_state: *NOTICE*: IBV QP#0 changed to: IBV_QPS_ERR

I went ahead and tested two target namespaces that appear properly in the discover information from the initiatorm using the bdev plugin in FIO as before, to see if I could recreate the original seg fault issue here. This seems to work, no seg fault, and performance seems good, in a couple of very quick tests. It does seem to prompt more of these messages from nvmf_tgt:

rdma.c: 530:spdk_nvmf_rdma_set_ibv_state: *NOTICE*: IBV QP#1 changed to: IBV_QPS_ERR
rdma.c: 530:spdk_nvmf_rdma_set_ibv_state: *NOTICE*: IBV QP#1 changed to: IBV_QPS_ERR
rdma.c: 530:spdk_nvmf_rdma_set_ibv_state: *NOTICE*: IBV QP#0 changed to: IBV_QPS_ERR
rdma.c: 530:spdk_nvmf_rdma_set_ibv_state: *NOTICE*: IBV QP#1 changed to: IBV_QPS_ERR
rdma.c: 530:spdk_nvmf_rdma_set_ibv_state: *NOTICE*: IBV QP#1 changed to: IBV_QPS_ERR
rdma.c: 530:spdk_nvmf_rdma_set_ibv_state: *NOTICE*: IBV QP#0 changed to: IBV_QPS_ERR

So, in summary, I cannot recreate the original seg fault issue, however, I see some new problems, mainly the failure to create all my target namespaces. Not sure if those issues are related to the proposed patches and we want to debug them here, or if we want to take them to a separate issue.

[Seth5141]

@jrgruher
The following error:
rdma.c: 530:spdk_nvmf_rdma_set_ibv_state: NOTICE: IBV QP#0 changed to: IBV_QPS_ERR
Occurs every time a qpair is disconnected. It's not really an error, but looks confusing. I have actually been meaning to push a patch to remove that from the output.
From a high level, it looks like the namespaces not showing up is related to the dmesg output during setup.sh which would mean that it is unrelated to the new patches.
However, I think the only way to be 100% sure is if we figure out what is causing the setup.sh errors and fix that and then confirm that all of your drives are discoverable.
Is there any other information from dmesg, or output from the setup.sh script that may give us some more insight?
What debugging steps have you tried for the setup.sh error so far?

[jrgruher]

@Seth5141
I haven't attempted to debug the issue - that's a job for the SPDK developers, IMHO. I'm happy to gather data and test fixes, to the degree I have cycles available, and for the developers at Intel I can provide direct access to my systems if that would help. I can also report this didn't seem to happen on master (as of last week) or 18.10.

Regarding the output from setup.sh and dmesg, here's the full output, starting with a clean reboot of the system.

don@donsl202:~$ sudo ~/install/spdk/scripts/setup.sh
Active mountpoints on /dev/nvme0n1, so not binding PCI dev 0000:04:00.0
0000:1a:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:1b:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:1c:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:1d:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:3d:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:3e:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:3f:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:40:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:8c:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:8d:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:8e:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:8f:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:b5:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:b6:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:b7:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:b8:00.0 (8086 0a54): nvme -> uio_pci_generic
0000:00:04.0 (8086 2021): ioatdma -> uio_pci_generic
0000:00:04.1 (8086 2021): ioatdma -> uio_pci_generic
0000:00:04.2 (8086 2021): ioatdma -> uio_pci_generic
0000:00:04.3 (8086 2021): ioatdma -> uio_pci_generic
0000:00:04.4 (8086 2021): ioatdma -> uio_pci_generic
0000:00:04.5 (8086 2021): ioatdma -> uio_pci_generic
0000:00:04.6 (8086 2021): ioatdma -> uio_pci_generic
0000:00:04.7 (8086 2021): ioatdma -> uio_pci_generic
0000:80:04.0 (8086 2021): ioatdma -> uio_pci_generic
0000:80:04.1 (8086 2021): ioatdma -> uio_pci_generic
0000:80:04.2 (8086 2021): ioatdma -> uio_pci_generic
0000:80:04.3 (8086 2021): ioatdma -> uio_pci_generic
0000:80:04.4 (8086 2021): ioatdma -> uio_pci_generic
0000:80:04.5 (8086 2021): ioatdma -> uio_pci_generic
0000:80:04.6 (8086 2021): ioatdma -> uio_pci_generic
0000:80:04.7 (8086 2021): ioatdma -> uio_pci_generic

I'll attach the full dmesg output since it is quite long. Events caused by setup.sh start at 249.

Then I start the target:

don@donsl202:~$ sudo ~/install/spdk/app/nvmf_tgt/nvmf_tgt -m 0x0000F0 -r /var/tmp/spdk1.sock
Starting SPDK v19.01-pre / DPDK 18.08.0 initialization...
[ DPDK EAL parameters: nvmf --no-shconf -c 0x0000F0 --base-virtaddr=0x200000000000 --file-prefix=spdk_pid3461 ]
EAL: Detected 24 lcore(s)
EAL: Detected 2 NUMA nodes
EAL: No free hugepages reported in hugepages-1048576kB
EAL: Probing VFIO support...
app.c: 609:spdk_app_start: *NOTICE*: Total cores available: 4
reactor.c: 293:_spdk_reactor_run: *NOTICE*: Reactor started on core 5
reactor.c: 293:_spdk_reactor_run: *NOTICE*: Reactor started on core 6
reactor.c: 293:_spdk_reactor_run: *NOTICE*: Reactor started on core 4
reactor.c: 293:_spdk_reactor_run: *NOTICE*: Reactor started on core 7

I run this script to configure the target:

sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d1 -t pcie -a 0000:1a:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d2 -t pcie -a 0000:1b:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d3 -t pcie -a 0000:1c:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d4 -t pcie -a 0000:1d:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d5 -t pcie -a 0000:3d:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d6 -t pcie -a 0000:3e:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d7 -t pcie -a 0000:3f:00.0
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_nvme_bdev -b d8 -t pcie -a 0000:40:00.0

sudo ./rpc.py -s /var/tmp/spdk1.sock construct_raid_bdev -n raid1 -s 4 -r 0 -b "d1n1 d2n1 d3n1 d4n1 d5n1 d6n1 d7n1 d8n1"

sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_store raid1 store1

sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l1 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l2 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l3 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l4 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l5 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l6 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l7 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l8 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l9 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l10 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l11 1200000
sudo ./rpc.py -s /var/tmp/spdk1.sock construct_lvol_bdev -l store1 l12 1200000

sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_create_transport -t rdma

sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn1 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn2 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn3 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn4 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn5 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn6 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn7 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn8 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn9 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn10 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn11 -a
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_create nqn.2018-11.io.spdk:nqn12 -a

sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn1 store1/l1
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn2 store1/l2
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn3 store1/l3
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn4 store1/l4
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn5 store1/l5
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn6 store1/l6
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn7 store1/l7
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn8 store1/l8
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn9 store1/l9
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn10 store1/l10
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn11 store1/l11
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_ns nqn.2018-11.io.spdk:nqn12 store1/l12

sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn1 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn2 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn3 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn4 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn5 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn6 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn7 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn8 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn9 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn10 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn11 -t rdma -a 10.5.0.202 -s 4420
sudo ./rpc.py -s /var/tmp/spdk1.sock nvmf_subsystem_add_listener nqn.2018-11.io.spdk:nqn12 -t rdma -a 10.5.0.202 -s 4420

That gives me these new prints from nvmf_tgt, nothing new in dmesg:

EAL: PCI device 0000:1a:00.0 on NUMA socket 0
EAL:   probe driver: 8086:a54 spdk_nvme
blobstore.c:3093:_spdk_bs_load_super_cpl: *NOTICE*: Size mismatch, dev size: 2000398934016, blobstore size: 16003191472128
EAL: PCI device 0000:1b:00.0 on NUMA socket 0
EAL:   probe driver: 8086:a54 spdk_nvme
EAL: PCI device 0000:1c:00.0 on NUMA socket 0
EAL:   probe driver: 8086:a54 spdk_nvme
EAL: PCI device 0000:1d:00.0 on NUMA socket 0
EAL:   probe driver: 8086:a54 spdk_nvme
EAL: PCI device 0000:3d:00.0 on NUMA socket 0
EAL:   probe driver: 8086:a54 spdk_nvme
EAL: PCI device 0000:3e:00.0 on NUMA socket 0
EAL:   probe driver: 8086:a54 spdk_nvme
EAL: PCI device 0000:3f:00.0 on NUMA socket 0
EAL:   probe driver: 8086:a54 spdk_nvme
EAL: PCI device 0000:40:00.0 on NUMA socket 0
EAL:   probe driver: 8086:a54 spdk_nvme

Then I do a discovery and see some of my namespaces do not seem correct:

don@donsl202:~/install/spdk/scripts$ sudo nvme discover -t rdma -a 10.5.0.202

Discovery Log Number of Records 12, Generation counter 25
=====Discovery Log Entry 0======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  0
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn1
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 1======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  1
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn2
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 2======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  2
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn3
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 3======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  3
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn4
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 4======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  4
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn5
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 5======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  5
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn6
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 6======
trtype:  rdma
adrfam:  ipv4
subtype: nvme subsystem
treq:    not specified
portid:  6
trsvcid: 4420
subnqn:  nqn.2018-11.io.spdk:nqn7
traddr:  10.5.0.202
rdma_prtype: not specified
rdma_qptype: connected
rdma_cms:    rdma-cm
rdma_pkey: 0x0000
=====Discovery Log Entry 7======
trtype:  unrecognized
adrfam:  pci
subtype: unrecognized
treq:    not specified
portid:  0
trsvcid:
subnqn:
traddr:
=====Discovery Log Entry 8======
trtype:  unrecognized
adrfam:  pci
subtype: unrecognized
treq:    not specified
portid:  0
trsvcid:
subnqn:
traddr:
=====Discovery Log Entry 9======
trtype:  unrecognized
adrfam:  pci
subtype: unrecognized
treq:    not specified
portid:  0
trsvcid:
subnqn:
traddr:
=====Discovery Log Entry 10======
trtype:  unrecognized
adrfam:  pci
subtype: unrecognized
treq:    not specified
portid:  0
trsvcid:
subnqn:
traddr:
=====Discovery Log Entry 11======
trtype:  unrecognized
adrfam:  pci
subtype: unrecognized
treq:    not specified
portid:  0
trsvcid:
subnqn:
traddr:

The output of get_nvmf_subsystems seems OK (different from the discovery results)?

don@donsl202:~/install/spdk/scripts$ sudo ./rpc.py -s /var/tmp/spdk1.sock get_nvmf_subsystems
[
  {
    "nqn": "nqn.2014-08.org.nvmexpress.discovery",
    "subtype": "Discovery",
    "listen_addresses": [],
    "allow_any_host": true,
    "hosts": []
  },
  {
    "nqn": "nqn.2018-11.io.spdk:nqn1",
    "subtype": "NVMe",
    "listen_addresses": [
      {
        "transport": "RDMA",
        "trtype": "RDMA",
        "adrfam": "IPv4",
        "traddr": "10.5.0.202",
        "trsvcid": "4420"
      }
    ],
    "allow_any_host": true,
    "hosts": [],
    "serial_number": "00000000000000000000",
    "namespaces": [
      {
        "nsid": 1,
        "bdev_name": "79c95fc4-a953-4cd4-b0a1-e3fa2515adec",
        "name": "79c95fc4-a953-4cd4-b0a1-e3fa2515adec",
        "uuid": "79c95fc4-a953-4cd4-b0a1-e3fa2515adec"
      }
    ]
  },
  {
    "nqn": "nqn.2018-11.io.spdk:nqn2",
    "subtype": "NVMe",
    "listen_addresses": [
      {
        "transport": "RDMA",
        "trtype": "RDMA",
        "adrfam": "IPv4",
        "traddr": "10.5.0.202",
        "trsvcid": "4420"
      }
    ],
    "allow_any_host": true,
    "hosts": [],
    "serial_number": "00000000000000000000",
    "namespaces": [
      {
        "nsid": 1,
        "bdev_name": "15663c09-91ae-4f99-b6b5-20c1f8b9a53a",
        "name": "15663c09-91ae-4f99-b6b5-20c1f8b9a53a",
        "uuid": "15663c09-91ae-4f99-b6b5-20c1f8b9a53a"
      }
    ]
  },
  {
    "nqn": "nqn.2018-11.io.spdk:nqn3",
    "subtype": "NVMe",
    "listen_addresses": [
      {
        "transport": "RDMA",
        "trtype": "RDMA",
        "adrfam": "IPv4",
        "traddr": "10.5.0.202",
        "trsvcid": "4420"
      }
    ],
    "allow_any_host": true,
    "hosts": [],
    "serial_number": "00000000000000000000",
    "namespaces": [
      {
        "nsid": 1,
        "bdev_name": "83d5a5b0-bf0e-41a5-9c51-f6958edd299d",
        "name": "83d5a5b0-bf0e-41a5-9c51-f6958edd299d",
        "uuid": "83d5a5b0-bf0e-41a5-9c51-f6958edd299d"
      }
    ]
  },
  {
    "nqn": "nqn.2018-11.io.spdk:nqn4",
    "subtype": "NVMe",
    "listen_addresses": [
      {
        "transport": "RDMA",
        "trtype": "RDMA",
        "adrfam": "IPv4",
        "traddr": "10.5.0.202",
        "trsvcid": "4420"
      }
    ],
    "allow_any_host": true,
    "hosts": [],
    "serial_number": "00000000000000000000",
    "namespaces": [
      {
        "nsid": 1,
        "bdev_name": "a3478e3f-4217-45d4-9a85-c07c0267e3f7",
        "name": "a3478e3f-4217-45d4-9a85-c07c0267e3f7",
        "uuid": "a3478e3f-4217-45d4-9a85-c07c0267e3f7"
      }
    ]
  },
  {
    "nqn": "nqn.2018-11.io.spdk:nqn5",
    "subtype": "NVMe",
    "listen_addresses": [
      {
        "transport": "RDMA",
        "trtype": "RDMA",
        "adrfam": "IPv4",
        "traddr": "10.5.0.202",
        "trsvcid": "4420"
      }
    ],
    "allow_any_host": true,
    "hosts": [],
    "serial_number": "00000000000000000000",
    "namespaces": [
      {
        "nsid": 1,
        "bdev_name": "9bd72d7c-579b-4ffb-9ae0-2f962b95f3d2",
        "name": "9bd72d7c-579b-4ffb-9ae0-2f962b95f3d2",
        "uuid": "9bd72d7c-579b-4ffb-9ae0-2f962b95f3d2"
      }
    ]
  },
  {
    "nqn": "nqn.2018-11.io.spdk:nqn6",
    "subtype": "NVMe",
    "listen_addresses": [
      {
        "transport": "RDMA",
        "trtype": "RDMA",
        "adrfam": "IPv4",
        "traddr": "10.5.0.202",
        "trsvcid": "4420"
      }
    ],
    "allow_any_host": true,
    "hosts": [],
    "serial_number": "00000000000000000000",
    "namespaces": [
      {
        "nsid": 1,
        "bdev_name": "eee68937-4f74-489e-901a-f53e123652db",
        "name": "eee68937-4f74-489e-901a-f53e123652db",
        "uuid": "eee68937-4f74-489e-901a-f53e123652db"
      }
    ]
  },
  {
    "nqn": "nqn.2018-11.io.spdk:nqn7",
    "subtype": "NVMe",
    "listen_addresses": [
      {
        "transport": "RDMA",
        "trtype": "RDMA",
        "adrfam": "IPv4",
        "traddr": "10.5.0.202",
        "trsvcid": "4420"
      }
    ],
    "allow_any_host": true,
    "hosts": [],
    "serial_number": "00000000000000000000",
    "namespaces": [
      {
        "nsid": 1,
        "bdev_name": "14619338-a66a-4694-b87a-1f91aa5cbad2",
        "name": "14619338-a66a-4694-b87a-1f91aa5cbad2",
        "uuid": "14619338-a66a-4694-b87a-1f91aa5cbad2"
      }
    ]
  },
  {
    "nqn": "nqn.2018-11.io.spdk:nqn8",
    "subtype": "NVMe",
    "listen_addresses": [
      {
        "transport": "RDMA",
        "trtype": "RDMA",
        "adrfam": "IPv4",
        "traddr": "10.5.0.202",
        "trsvcid": "4420"
      }
    ],
    "allow_any_host": true,
    "hosts": [],
    "serial_number": "00000000000000000000",
    "namespaces": [
      {
        "nsid": 1,
        "bdev_name": "26a5a157-e92c-4dbd-bda9-05cf566686b5",
        "name": "26a5a157-e92c-4dbd-bda9-05cf566686b5",
        "uuid": "26a5a157-e92c-4dbd-bda9-05cf566686b5"
      }
    ]
  },
  {
    "nqn": "nqn.2018-11.io.spdk:nqn9",
    "subtype": "NVMe",
    "listen_addresses": [
      {
        "transport": "RDMA",
        "trtype": "RDMA",
        "adrfam": "IPv4",
        "traddr": "10.5.0.202",
        "trsvcid": "4420"
      }
    ],
    "allow_any_host": true,
    "hosts": [],
    "serial_number": "00000000000000000000",
    "namespaces": [
      {
        "nsid": 1,
        "bdev_name": "cefc8cbc-55fc-4ba4-8199-23bcb547b4d3",
        "name": "cefc8cbc-55fc-4ba4-8199-23bcb547b4d3",
        "uuid": "cefc8cbc-55fc-4ba4-8199-23bcb547b4d3"
      }
    ]
  },
  {
    "nqn": "nqn.2018-11.io.spdk:nqn10",
    "subtype": "NVMe",
    "listen_addresses": [
      {
        "transport": "RDMA",
        "trtype": "RDMA",
        "adrfam": "IPv4",
        "traddr": "10.5.0.202",
        "trsvcid": "4420"
      }
    ],
    "allow_any_host": true,
    "hosts": [],
    "serial_number": "00000000000000000000",
    "namespaces": [
      {
        "nsid": 1,
        "bdev_name": "d8e7c81f-be91-4ced-8a67-76b9b678a207",
        "name": "d8e7c81f-be91-4ced-8a67-76b9b678a207",
        "uuid": "d8e7c81f-be91-4ced-8a67-76b9b678a207"
      }
    ]
  },
  {
    "nqn": "nqn.2018-11.io.spdk:nqn11",
    "subtype": "NVMe",
    "listen_addresses": [
      {
        "transport": "RDMA",
        "trtype": "RDMA",
        "adrfam": "IPv4",
        "traddr": "10.5.0.202",
        "trsvcid": "4420"
      }
    ],
    "allow_any_host": true,
    "hosts": [],
    "serial_number": "00000000000000000000",
    "namespaces": [
      {
        "nsid": 1,
        "bdev_name": "1cc85562-bc10-484d-8062-16ad5b3f94d3",
        "name": "1cc85562-bc10-484d-8062-16ad5b3f94d3",
        "uuid": "1cc85562-bc10-484d-8062-16ad5b3f94d3"
      }
    ]
  },
  {
    "nqn": "nqn.2018-11.io.spdk:nqn12",
    "subtype": "NVMe",
    "listen_addresses": [
      {
        "transport": "RDMA",
        "trtype": "RDMA",
        "adrfam": "IPv4",
        "traddr": "10.5.0.202",
        "trsvcid": "4420"
      }
    ],
    "allow_any_host": true,
    "hosts": [],
    "serial_number": "00000000000000000000",
    "namespaces": [
      {
        "nsid": 1,
        "bdev_name": "0d55632c-1dcd-4624-a8cb-b3ded993402f",
        "name": "0d55632c-1dcd-4624-a8cb-b3ded993402f",
        "uuid": "0d55632c-1dcd-4624-a8cb-b3ded993402f"
      }
    ]
  }
]

Tried to do a get_subsystem_config but I can't figure out what it wants for the "name" parameter:

don@donsl202:~/install/spdk/scripts$ sudo ./rpc.py -s /var/tmp/spdk1.sock get_subsystem_config 79c95fc4-a953-4cd4-b0a1-e3fa2515adec
Got JSON-RPC error response
request:
{
  "jsonrpc": "2.0",
  "method": "get_subsystem_config",
  "id": 1,
  "params": {
    "name": "79c95fc4-a953-4cd4-b0a1-e3fa2515adec"
  }
}
response:
{
  "code": -32602,
  "message": "Subsystem '79c95fc4-a953-4cd4-b0a1-e3fa2515adec' not found"
}
don@donsl202:~/install/spdk/scripts$ sudo ./rpc.py -s /var/tmp/spdk1.sock get_subsystem_config nqn.2018-11.io.spdk:nqn1
Got JSON-RPC error response
request:
{
  "jsonrpc": "2.0",
  "method": "get_subsystem_config",
  "id": 1,
  "params": {
    "name": "nqn.2018-11.io.spdk:nqn1"
  }
}
response:
{
  "code": -32602,
  "message": "Subsystem 'nqn.2018-11.io.spdk:nqn1' not found"
}