-
Notifications
You must be signed in to change notification settings - Fork 168
/
cuda.h
226 lines (198 loc) · 6.97 KB
/
cuda.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
/*
* Copyright © 2010-2023 Inria. All rights reserved.
* Copyright © 2010-2011 Université Bordeaux
* Copyright © 2011 Cisco Systems, Inc. All rights reserved.
* See COPYING in top-level directory.
*/
/** \file
* \brief Macros to help interaction between hwloc and the CUDA Driver API.
*
* Applications that use both hwloc and the CUDA Driver API may want to
* include this file so as to get topology information for CUDA devices.
*
*/
#ifndef HWLOC_CUDA_H
#define HWLOC_CUDA_H
#include "hwloc.h"
#include "hwloc/autogen/config.h"
#include "hwloc/helper.h"
#ifdef HWLOC_LINUX_SYS
#include "hwloc/linux.h"
#endif
#include <cuda.h>
#ifdef __cplusplus
extern "C" {
#endif
/** \defgroup hwlocality_cuda Interoperability with the CUDA Driver API
*
* This interface offers ways to retrieve topology information about
* CUDA devices when using the CUDA Driver API.
*
* @{
*/
/** \brief Return the domain, bus and device IDs of the CUDA device \p cudevice.
*
* Device \p cudevice must match the local machine.
*
* \return 0 on success.
* \return -1 on error, for instance if device information could not be found.
*/
static __hwloc_inline int
hwloc_cuda_get_device_pci_ids(hwloc_topology_t topology __hwloc_attribute_unused,
CUdevice cudevice, int *domain, int *bus, int *dev)
{
CUresult cres;
#if CUDA_VERSION >= 4000
cres = cuDeviceGetAttribute(domain, CU_DEVICE_ATTRIBUTE_PCI_DOMAIN_ID, cudevice);
if (cres != CUDA_SUCCESS) {
errno = ENOSYS;
return -1;
}
#else
*domain = 0;
#endif
cres = cuDeviceGetAttribute(bus, CU_DEVICE_ATTRIBUTE_PCI_BUS_ID, cudevice);
if (cres != CUDA_SUCCESS) {
errno = ENOSYS;
return -1;
}
cres = cuDeviceGetAttribute(dev, CU_DEVICE_ATTRIBUTE_PCI_DEVICE_ID, cudevice);
if (cres != CUDA_SUCCESS) {
errno = ENOSYS;
return -1;
}
return 0;
}
/** \brief Get the CPU set of processors that are physically
* close to device \p cudevice.
*
* Store in \p set the CPU-set describing the locality of the CUDA device \p cudevice.
*
* Topology \p topology and device \p cudevice must match the local machine.
* I/O devices detection and the CUDA component are not needed in the topology.
*
* The function only returns the locality of the device.
* If more information about the device is needed, OS objects should
* be used instead, see hwloc_cuda_get_device_osdev()
* and hwloc_cuda_get_device_osdev_by_index().
*
* This function is currently only implemented in a meaningful way for
* Linux; other systems will simply get a full cpuset.
*
* \return 0 on success.
* \return -1 on error, for instance if device information could not be found.
*/
static __hwloc_inline int
hwloc_cuda_get_device_cpuset(hwloc_topology_t topology __hwloc_attribute_unused,
CUdevice cudevice, hwloc_cpuset_t set)
{
#ifdef HWLOC_LINUX_SYS
/* If we're on Linux, use the sysfs mechanism to get the local cpus */
#define HWLOC_CUDA_DEVICE_SYSFS_PATH_MAX 128
char path[HWLOC_CUDA_DEVICE_SYSFS_PATH_MAX];
int domainid, busid, deviceid;
if (hwloc_cuda_get_device_pci_ids(topology, cudevice, &domainid, &busid, &deviceid))
return -1;
if (!hwloc_topology_is_thissystem(topology)) {
errno = EINVAL;
return -1;
}
sprintf(path, "/sys/bus/pci/devices/%04x:%02x:%02x.0/local_cpus", domainid, busid, deviceid);
if (hwloc_linux_read_path_as_cpumask(path, set) < 0
|| hwloc_bitmap_iszero(set))
hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology));
#else
/* Non-Linux systems simply get a full cpuset */
hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology));
#endif
return 0;
}
/** \brief Get the hwloc PCI device object corresponding to the
* CUDA device \p cudevice.
*
* \return The hwloc PCI device object describing the CUDA device \p cudevice.
* \return \c NULL if none could be found.
*
* Topology \p topology and device \p cudevice must match the local machine.
* I/O devices detection must be enabled in topology \p topology.
* The CUDA component is not needed in the topology.
*/
static __hwloc_inline hwloc_obj_t
hwloc_cuda_get_device_pcidev(hwloc_topology_t topology, CUdevice cudevice)
{
int domain, bus, dev;
if (hwloc_cuda_get_device_pci_ids(topology, cudevice, &domain, &bus, &dev))
return NULL;
return hwloc_get_pcidev_by_busid(topology, domain, bus, dev, 0);
}
/** \brief Get the hwloc OS device object corresponding to CUDA device \p cudevice.
*
* \return The hwloc OS device object that describes the given CUDA device \p cudevice.
* \return \c NULL if none could be found.
*
* Topology \p topology and device \p cudevice must match the local machine.
* I/O devices detection and the CUDA component must be enabled in the topology.
* If not, the locality of the object may still be found using
* hwloc_cuda_get_device_cpuset().
*
* \note This function cannot work if PCI devices are filtered out.
*
* \note The corresponding hwloc PCI device may be found by looking
* at the result parent pointer (unless PCI devices are filtered out).
*/
static __hwloc_inline hwloc_obj_t
hwloc_cuda_get_device_osdev(hwloc_topology_t topology, CUdevice cudevice)
{
hwloc_obj_t osdev = NULL;
int domain, bus, dev;
if (hwloc_cuda_get_device_pci_ids(topology, cudevice, &domain, &bus, &dev))
return NULL;
osdev = NULL;
while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) {
hwloc_obj_t pcidev = osdev->parent;
if (strncmp(osdev->name, "cuda", 4))
continue;
if (pcidev
&& pcidev->type == HWLOC_OBJ_PCI_DEVICE
&& (int) pcidev->attr->pcidev.domain == domain
&& (int) pcidev->attr->pcidev.bus == bus
&& (int) pcidev->attr->pcidev.dev == dev
&& pcidev->attr->pcidev.func == 0)
return osdev;
/* if PCI are filtered out, we need a info attr to match on */
}
return NULL;
}
/** \brief Get the hwloc OS device object corresponding to the
* CUDA device whose index is \p idx.
*
* \return The hwloc OS device object describing the CUDA device whose index is \p idx.
* \return \c NULL if none could be found.
*
* The topology \p topology does not necessarily have to match the current
* machine. For instance the topology may be an XML import of a remote host.
* I/O devices detection and the CUDA component must be enabled in the topology.
*
* \note The corresponding PCI device object can be obtained by looking
* at the OS device parent object (unless PCI devices are filtered out).
*
* \note This function is identical to hwloc_cudart_get_device_osdev_by_index().
*/
static __hwloc_inline hwloc_obj_t
hwloc_cuda_get_device_osdev_by_index(hwloc_topology_t topology, unsigned idx)
{
hwloc_obj_t osdev = NULL;
while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) {
if ((osdev->attr->osdev.type & (HWLOC_OBJ_OSDEV_GPU|HWLOC_OBJ_OSDEV_COPROC)) /* assume future CUDA devices will be at least GPU or COPROC */
&& osdev->name
&& !strncmp("cuda", osdev->name, 4)
&& atoi(osdev->name + 4) == (int) idx)
return osdev;
}
return NULL;
}
/** @} */
#ifdef __cplusplus
} /* extern "C" */
#endif
#endif /* HWLOC_CUDA_H */