Use PCIe full address to identify CUDA device

* Use PCIe full address (with domain) in order to locate CUDA device Signed-off-by: Moosa Baransi <moosab@nvidia.com>
linux-rdma · Dec 5, 2020 · 8e02a17 · 8e02a17
1 parent 1f01b41
commit 8e02a17
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 0 deletions.
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
@@ -475,6 +475,9 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
 		#ifdef HAVE_CUDA
 		printf("      --use_cuda=<cuda device id>");
 		printf(" Use CUDA specific device for GPUDirect RDMA testing\n");
+
+		printf("      --use_cuda_bus_id=<cuda full BUS id>");
+		printf(" Use CUDA specific device, based on its full PCIe address, for GPUDirect RDMA testing\n");
 		#endif
 
 		printf("      --use_hugepages ");
@@ -1873,6 +1876,7 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 	static int dont_xchg_versions_flag = 0;
 #ifdef HAVE_CUDA
 	static int use_cuda_flag = 0;
+	static int use_cuda_bus_id_flag = 0;
 #endif
 	static int disable_pcir_flag = 0;
 	static int mmap_file_flag = 0;
@@ -1999,6 +2003,7 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 			{ .name = "dont_xchg_versions",	.has_arg = 0, .flag = &dont_xchg_versions_flag, .val = 1},
 			#ifdef HAVE_CUDA
 			{ .name = "use_cuda",		.has_arg = 1, .flag = &use_cuda_flag, .val = 1},
+			{ .name = "use_cuda_bus_id",		.has_arg = 1, .flag = &use_cuda_bus_id_flag, .val = 1},
 			#endif
 			{ .name = "mmap",		.has_arg = 1, .flag = &mmap_file_flag, .val = 1},
 			{ .name = "mmap-offset",	.has_arg = 1, .flag = &mmap_offset_flag, .val = 1},
@@ -2375,6 +2380,12 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 					}
 					use_cuda_flag = 0;
 				}
+				if (use_cuda_bus_id_flag) {
+					user_param->use_cuda = 1;
+					user_param->cuda_device_bus_id = strdup(optarg);
+					printf("Got PCIe address of: %s\n", user_param->cuda_device_bus_id);
+					use_cuda_bus_id_flag = 0;
+				}
 #endif
 				if (flow_label_flag) {
 					user_param->flow_label = strtol(optarg,NULL,0);

diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h
@@ -456,6 +456,7 @@ struct perftest_parameters {
 #ifdef HAVE_CUDA
 	int				use_cuda;
 	int				cuda_device_id;
+	char				*cuda_device_bus_id;
 #endif
 	char				*mmap_file;
 	unsigned long			mmap_offset;

diff --git a/src/perftest_resources.c b/src/perftest_resources.c
@@ -1525,6 +1525,23 @@ int ctx_init(struct pingpong_context *ctx, struct perftest_parameters *user_para
 
 	#ifdef HAVE_CUDA
 	if (user_param->use_cuda) {
+		if (user_param->cuda_device_bus_id) {
+			int err;
+
+			printf("initializing CUDA\n");
+			CUresult error = cuInit(0);
+			if (error != CUDA_SUCCESS) {
+				printf("cuInit(0) returned %d\n", error);
+				exit(1);
+			}
+
+			printf("Finding PCIe BUS %s\n", user_param->cuda_device_bus_id);
+			err = cuDeviceGetByPCIBusId (&user_param->cuda_device_id, user_param->cuda_device_bus_id);
+			if (err != 0) {
+				fprintf(stderr, "We have an error from cuDeviceGetByPCIBusId: %d\n", err);
+			}
+			printf("Picking GPU number %d\n", user_param->cuda_device_id);
+		}
 		if (pp_init_gpu(ctx, user_param->cuda_device_id)) {
 			fprintf(stderr, "Couldn't init GPU context\n");
 			return FAILURE;